diff --git a/src/lib.zig b/src/lib.zig index 6146ce1..56a869e 100644 --- a/src/lib.zig +++ b/src/lib.zig @@ -10,7 +10,7 @@ const Allocator = std.mem.Allocator; const Sha256 = std.crypto.hash.sha2.Sha256; /// Number of bytes per chunk. -const BYTES_PER_CHUNK = 32; +pub const BYTES_PER_CHUNK = 32; pub fn serializedFixedSize(T: type) !usize { const info = @typeInfo(T); @@ -417,17 +417,11 @@ pub fn deserialize(T: type, serialized: []const u8, out: *T, allocator: ?Allocat const enforce_min = !has_custom_decode or comptime std.meta.hasFn(T, "minInLength"); const enforce_max = !has_custom_decode or comptime std.meta.hasFn(T, "maxInLength"); - // Bounds check: ensure serialized length is within [minInLength, maxInLength] - const min_len: ?usize = if (enforce_min) blk: { - const m = minInLength(T) catch break :blk null; - break :blk m; - } else null; + // Bounds check: ensure serialized length is within [minInLength, maxInLength]. + const min_len: ?usize = comptime if (enforce_min) (minInLength(T) catch null) else null; if (min_len) |m| if (serialized.len < m) return error.PayloadTooSmall; - const max_len: ?usize = if (enforce_max) blk: { - const m = maxInLength(T) catch break :blk null; - break :blk m; - } else null; + const max_len: ?usize = comptime if (enforce_max) (maxInLength(T) catch null) else null; if (max_len) |m| if (serialized.len > m) return error.PayloadTooLarge; // shortcut if the type implements its own decode method @@ -734,8 +728,8 @@ pub fn chunkCount(T: type) usize { } } -const chunk = [BYTES_PER_CHUNK]u8; -const zero_chunk: chunk = [_]u8{0} ** BYTES_PER_CHUNK; +pub const chunk = [BYTES_PER_CHUNK]u8; +pub const zero_chunk: chunk = [_]u8{0} ** BYTES_PER_CHUNK; pub fn pack(T: type, values: T, l: *ArrayList(u8), allocator: Allocator) ![]chunk { try serialize(T, values, l, allocator); diff --git a/src/merkle_cache.zig b/src/merkle_cache.zig new file mode 100644 index 0000000..28bf28f --- /dev/null +++ b/src/merkle_cache.zig @@ -0,0 +1,358 @@ +const std = @import("std"); +const lib = @import("./lib.zig"); +const zeros = @import("./zeros.zig"); + +const BYTES_PER_CHUNK = lib.BYTES_PER_CHUNK; +const chunk = lib.chunk; +const zero_chunk = lib.zero_chunk; + +/// Depth at which a List/Bitlist of `chunk_limit` chunks must be merkleized, +pub fn targetDepth(comptime chunk_limit: usize) usize { + if (chunk_limit <= 1) return 0; + return @bitSizeOf(usize) - @clz(chunk_limit - 1); +} + +/// A grow-on-demand cached Merkle tree, generic over the hash function. +/// +/// Flat 0-indexed binary tree: root at nodes[0], children of i at 2i+1/2i+2, +/// leaves at nodes[capacity-1 .. 2*capacity-1]. +pub fn MerkleCache(comptime Hasher: type) type { + comptime std.debug.assert(Hasher.digest_length == BYTES_PER_CHUNK); + return struct { + const Self = @This(); + /// Zero-subtree roots for this hasher. Index i is the root of an + /// all-zero subtree with 2^i leaves. Depth 64 covers any usize limit. + const hashes_of_zero = zeros.buildHashesOfZero(Hasher, BYTES_PER_CHUNK, 64); + + nodes: []chunk, + capacity: usize, + dirty_low: ?usize, + dirty_high: ?usize, + cached_length: usize, + cached_root: chunk, + root_valid: bool, + + pub fn init() Self { + return .{ + .nodes = &.{}, + .capacity = 0, + .dirty_low = null, + .dirty_high = null, + .cached_length = 0, + .cached_root = zero_chunk, + .root_valid = false, + }; + } + + pub fn deinit(self: *Self, allocator: std.mem.Allocator) void { + if (self.capacity > 0) allocator.free(self.nodes); + self.* = init(); + } + + /// Ensure the tree can hold `required_chunks` leaves. Growing reallocates + /// the node array, but the existing tree is exactly the leftmost subtree + /// of the larger one, so we transplant its already-computed nodes into + /// their shifted positions instead of rehashing them. + /// + /// Old level `d` (indices [2^d-1, 2^d-1+2^d)) maps to new level `d+k` + /// where `k = new_depth - old_depth`. Only the newly exposed leaf slots + /// [old_cap, new_cap) are marked dirty; the transplanted subtree is + /// reused, and the content-addressed refresh dirties any old leaf that + /// actually changed. + pub fn ensureCapacity(self: *Self, allocator: std.mem.Allocator, required_chunks: usize) !void { + if (required_chunks <= self.capacity) return; + const new_cap = try std.math.ceilPowerOfTwo(usize, required_chunks); + if (new_cap == self.capacity) return; + const new_nodes = try allocator.alloc(chunk, 2 * new_cap - 1); + @memset(new_nodes, zero_chunk); + + const old_cap = self.capacity; + if (old_cap > 0) { + const old_depth = std.math.log2_int(usize, old_cap); + const new_depth = std.math.log2_int(usize, new_cap); + const k = new_depth - old_depth; + // Transplant the old tree level by level into the leftmost + // subtree of the new tree (no rehashing). + var d: usize = 0; + while (d <= old_depth) : (d += 1) { + const count = @as(usize, 1) << @intCast(d); + const old_start = count - 1; // 2^d - 1 + const new_start = (@as(usize, 1) << @intCast(d + k)) - 1; // 2^(d+k) - 1 + // The old tree is the leftmost subtree of the new one, so the + // old leaves land at new indices [new_cap-1, new_cap-1+old_cap) + // and every transplanted block stays within the new buffer. + std.debug.assert(new_start + count <= new_nodes.len); + @memcpy(new_nodes[new_start .. new_start + count], self.nodes[old_start .. old_start + count]); + } + allocator.free(self.nodes); + } + + self.nodes = new_nodes; + self.capacity = new_cap; + // Reuse the transplanted left subtree; only the newly exposed leaf + // slots need recomputing (plus any old leaf the refresh re-dirties). + self.dirty_low = if (old_cap > 0) old_cap else 0; + self.dirty_high = new_cap - 1; + self.root_valid = false; + } + + pub fn leafPtr(self: *Self, index: usize) *chunk { + std.debug.assert(index < self.capacity); + return &self.nodes[self.capacity - 1 + index]; + } + + pub fn markDirty(self: *Self, leaf_index: usize) void { + std.debug.assert(leaf_index < self.capacity); + if (self.dirty_low) |low| { + self.dirty_low = @min(low, leaf_index); + self.dirty_high = @max(self.dirty_high.?, leaf_index); + } else { + self.dirty_low = leaf_index; + self.dirty_high = leaf_index; + } + self.root_valid = false; + } + + fn hashPair(self: *Self, parent: usize, left: usize, right: usize) void { + var hasher = Hasher.init(Hasher.Options{}); + hasher.update(&self.nodes[left]); + hasher.update(&self.nodes[right]); + hasher.final(&self.nodes[parent]); + } + + /// Root of the materialized data subtree (over `capacity` leaves, with + /// padding leaves already zero). Returns zero_chunk when empty. Only + /// rehashes the paths from dirty leaves up to the root. + pub fn recompute(self: *Self) chunk { + if (self.capacity == 0) return zero_chunk; + const low_leaf = self.dirty_low orelse return self.nodes[0]; + var low = low_leaf + self.capacity - 1; + var high = self.dirty_high.? + self.capacity - 1; + + while (low > 0) { + // Expand the range to whole sibling pairs. The pulled-in sibling + // is a clean (possibly transplanted) node, read as a valid input + // without being recomputed. + const pair_low = if (low % 2 == 1) low else low - 1; + const pair_high = if (high % 2 == 0) high else high + 1; + + var i = pair_low; + while (i <= pair_high) : (i += 2) { + self.hashPair((i - 1) / 2, i, i + 1); + } + + low = (pair_low - 1) / 2; + high = (pair_high - 1) / 2; + } + + self.dirty_low = null; + self.dirty_high = null; + return self.nodes[0]; + } + + /// Full SSZ root: extend the materialized data subtree with zero padding + /// up to `target_depth`, then mix in the length. Caches the result keyed + /// by `length`. + pub fn recomputeWithLength(self: *Self, length: usize, comptime target_depth: usize) chunk { + comptime std.debug.assert(target_depth <= 64); + + // Hot path: nothing dirtied since the last identical-length root. + if (self.root_valid and self.cached_length == length) { + return self.cached_root; + } + + const data_root = self.recompute(); + + // Climb from the materialized depth to the spec depth, hashing the + // running root against the all-zero subtree at each level. + const current_depth: usize = if (self.capacity > 0) std.math.log2_int(usize, self.capacity) else 0; + var current = data_root; + var level = current_depth; + while (level < target_depth) : (level += 1) { + var hasher = Hasher.init(Hasher.Options{}); + hasher.update(¤t); + hasher.update(&hashes_of_zero[level]); + hasher.final(¤t); + } + + var length_buf: chunk = zero_chunk; + std.mem.writeInt(u64, length_buf[0..8], @intCast(length), .little); + + var hasher = Hasher.init(Hasher.Options{}); + hasher.update(¤t); + hasher.update(&length_buf); + hasher.final(&self.cached_root); + self.cached_length = length; + self.root_valid = true; + return self.cached_root; + } + }; +} + +// Tests +fn fillLeaves(comptime Hasher: type, cache: *MerkleCache(Hasher), chunks: []const chunk) void { + for (chunks, 0..) |c, i| { + cache.leafPtr(i).* = c; + cache.markDirty(i); + } +} + +test "MerkleCache.recompute matches merkleize over its capacity" { + const Sha256 = std.crypto.hash.sha2.Sha256; + const Cache = MerkleCache(Sha256); + + var cache = Cache.init(); + defer cache.deinit(std.testing.allocator); + + var chunks: [3]chunk = undefined; + for (0..3) |i| chunks[i] = [_]u8{@intCast(i + 1)} ** 32; + try cache.ensureCapacity(std.testing.allocator, 3); // capacity -> 4 + fillLeaves(Sha256, &cache, &chunks); + const root = cache.recompute(); + + var expected: chunk = undefined; + try lib.merkleize(Sha256, &chunks, cache.capacity, &expected); + try std.testing.expectEqualSlices(u8, &expected, &root); +} + +test "MerkleCache.recomputeWithLength matches merkleize + mixInLength (limit 8)" { + const Sha256 = std.crypto.hash.sha2.Sha256; + + var cache = MerkleCache(Sha256).init(); + defer cache.deinit(std.testing.allocator); + + var chunks: [3]chunk = undefined; + for (0..3) |i| chunks[i] = [_]u8{@intCast(i + 10)} ** 32; + try cache.ensureCapacity(std.testing.allocator, 3); + fillLeaves(Sha256, &cache, &chunks); + + const got = cache.recomputeWithLength(3, comptime targetDepth(8)); + + var data_root: chunk = undefined; + try lib.merkleize(Sha256, &chunks, 8, &data_root); + var expected: chunk = undefined; + lib.mixInLength2(Sha256, data_root, 3, &expected); + try std.testing.expectEqualSlices(u8, &expected, &got); +} + +test "MerkleCache empty list matches merkleize empty" { + const Sha256 = std.crypto.hash.sha2.Sha256; + + var cache = MerkleCache(Sha256).init(); + defer cache.deinit(std.testing.allocator); + + try std.testing.expect(cache.capacity == 0); + const got = cache.recomputeWithLength(0, comptime targetDepth(4)); + + var data_root: chunk = undefined; + const empty: []chunk = &.{}; + try lib.merkleize(Sha256, empty, 4, &data_root); + var expected: chunk = undefined; + lib.mixInLength2(Sha256, data_root, 0, &expected); + try std.testing.expectEqualSlices(u8, &expected, &got); +} + +test "MerkleCache incremental update matches full rebuild" { + const Sha256 = std.crypto.hash.sha2.Sha256; + const Cache = MerkleCache(Sha256); + + var cache = Cache.init(); + defer cache.deinit(std.testing.allocator); + + var chunks: [4]chunk = undefined; + for (0..4) |i| chunks[i] = [_]u8{@intCast(i + 1)} ** 32; + try cache.ensureCapacity(std.testing.allocator, 4); + fillLeaves(Sha256, &cache, &chunks); + _ = cache.recompute(); + + chunks[2] = [_]u8{0xFF} ** 32; + cache.leafPtr(2).* = chunks[2]; + cache.markDirty(2); + const incremental = cache.recompute(); + + var expected: chunk = undefined; + try lib.merkleize(Sha256, &chunks, 4, &expected); + try std.testing.expectEqualSlices(u8, &expected, &incremental); +} + +test "MerkleCache grow transplants and reuses the old subtree" { + const Sha256 = std.crypto.hash.sha2.Sha256; + const Cache = MerkleCache(Sha256); + var cache = Cache.init(); + defer cache.deinit(std.testing.allocator); + + const a: chunk = [_]u8{0xA1} ** 32; + const b: chunk = [_]u8{0xB2} ** 32; + try cache.ensureCapacity(std.testing.allocator, 2); + cache.leafPtr(0).* = a; + cache.markDirty(0); + cache.leafPtr(1).* = b; + cache.markDirty(1); + _ = cache.recompute(); + + // Grow to capacity 4. The old leaves must be transplanted, not zeroed, and + // we must NOT re-set them — only the new leaf gets written. + try cache.ensureCapacity(std.testing.allocator, 3); + try std.testing.expect(cache.capacity == 4); + try std.testing.expectEqualSlices(u8, &a, cache.leafPtr(0)); + try std.testing.expectEqualSlices(u8, &b, cache.leafPtr(1)); + + const c: chunk = [_]u8{0xC3} ** 32; + cache.leafPtr(2).* = c; + cache.markDirty(2); + const root = cache.recompute(); + + var chunks = [_]chunk{ a, b, c, zero_chunk }; + var expected: chunk = undefined; + try lib.merkleize(Sha256, &chunks, 4, &expected); + try std.testing.expectEqualSlices(u8, &expected, &root); +} + +test "MerkleCache multi-level grow reuses interior nodes (k>=2)" { + const Sha256 = std.crypto.hash.sha2.Sha256; + const Cache = MerkleCache(Sha256); + var cache = Cache.init(); + defer cache.deinit(std.testing.allocator); + + const a: chunk = [_]u8{0xA1} ** 32; + const b: chunk = [_]u8{0xB2} ** 32; + try cache.ensureCapacity(std.testing.allocator, 2); + cache.leafPtr(0).* = a; + cache.markDirty(0); + cache.leafPtr(1).* = b; + cache.markDirty(1); + _ = cache.recompute(); + + // Grow 2 -> 8 (k = 2). The old root H(a,b) becomes an *interior* node of the + // new tree (index 3) and must be reused without rehashing. + try cache.ensureCapacity(std.testing.allocator, 5); + try std.testing.expect(cache.capacity == 8); + + // The transplanted interior node holds the old root H(a,b). + var old_root: chunk = undefined; + var ab = [_]chunk{ a, b }; + try lib.merkleize(Sha256, &ab, 2, &old_root); + try std.testing.expectEqualSlices(u8, &old_root, &cache.nodes[3]); + + // Add new data c,d,e without touching the reused leaves 0,1. + const c: chunk = [_]u8{0xC3} ** 32; + const d: chunk = [_]u8{0xD4} ** 32; + const e: chunk = [_]u8{0xE5} ** 32; + cache.leafPtr(2).* = c; + cache.markDirty(2); + cache.leafPtr(3).* = d; + cache.markDirty(3); + cache.leafPtr(4).* = e; + cache.markDirty(4); + const root = cache.recompute(); + + // Interior node still the reused old root; leaves 0,1 still a,b. + try std.testing.expectEqualSlices(u8, &old_root, &cache.nodes[3]); + try std.testing.expectEqualSlices(u8, &a, cache.leafPtr(0)); + try std.testing.expectEqualSlices(u8, &b, cache.leafPtr(1)); + + var chunks = [_]chunk{ a, b, c, d, e, zero_chunk, zero_chunk, zero_chunk }; + var expected: chunk = undefined; + try lib.merkleize(Sha256, &chunks, 8, &expected); + try std.testing.expectEqualSlices(u8, &expected, &root); +} diff --git a/src/tests.zig b/src/tests.zig index f0abcaa..5379013 100644 --- a/src/tests.zig +++ b/src/tests.zig @@ -1192,6 +1192,31 @@ test "minInLength/maxInLength for struct with List field" { try expect(try libssz.maxInLength(S) == 4 + 4 + 8 * 1); } +test "minInLength/maxInLength fold at compile time" { + // The bounds depend only on the type, so they fold to compile-time + // constants usable directly in comptime contexts. + comptime { + std.debug.assert((libssz.maxInLength(u64) catch unreachable) == 8); + std.debug.assert((libssz.minInLength(u64) catch unreachable) == 8); + std.debug.assert((libssz.maxInLength([4]u8) catch unreachable) == 4); + std.debug.assert((libssz.minInLength(bool) catch unreachable) == 1); + } + + const S = struct { a: u32, b: u16 }; + // Used as an array length, proving the bound is a genuine compile-time constant. + const Buf = [libssz.maxInLength(S) catch unreachable]u8; + try expect(@typeInfo(Buf).array.len == 4 + 2); + + const ListU64 = utils.List(u64, 16); + try expect(try libssz.maxInLength(ListU64) == 16 * 8); + try expect(try libssz.minInLength(ListU64) == 0); + + // Dynamic/unbounded types still return a runtime error, not a compile error. + const VarS = struct { a: u32, b: []const u8 }; + try expectError(error.NoMaxInLengthAvailable, libssz.maxInLength(VarS)); + try expectError(error.NoMinInLengthAvailable, libssz.minInLength(VarS)); +} + test "deserialize rejects payload longer than maxInLength" { var out_u32: u32 = undefined; try expectError(error.PayloadTooLarge, deserialize(u32, &[_]u8{ 0x01, 0x02, 0x03, 0x04, 0x05 }, &out_u32, null)); @@ -2300,6 +2325,75 @@ test "roundtrip: [2][4]u32 (nested fixed array) preserves inner elements" { try expect(std.mem.eql(u32, &out[1], &.{ 5, 6, 7, 8 })); } +test "TreeHasher(List(u64)) matches uncached and updates on set" { + const CachedList = utils.TreeHasher(utils.List(u64, 1024), Sha256); + var h = try CachedList.init(std.testing.allocator); + defer h.deinit(); + + try h.append(1); + try h.append(2); + try h.append(3); + + var cached: [32]u8 = undefined; + try h.hashTreeRoot(Sha256, &cached, std.testing.allocator); + + // Truth: the uncached regular object. + var uncached: [32]u8 = undefined; + try h.inner.hashTreeRoot(Sha256, &uncached, std.testing.allocator); + try expect(std.mem.eql(u8, &cached, &uncached)); + + // Modify one element; cached recompute must match a fresh uncached list. + try h.set(1, 42); + var cached2: [32]u8 = undefined; + try h.hashTreeRoot(Sha256, &cached2, std.testing.allocator); + + var fresh = try utils.List(u64, 1024).init(std.testing.allocator); + defer fresh.deinit(); + try fresh.append(1); + try fresh.append(42); + try fresh.append(3); + var expected: [32]u8 = undefined; + try fresh.hashTreeRoot(Sha256, &expected, std.testing.allocator); + + try expect(std.mem.eql(u8, &cached2, &expected)); + try expect(!std.mem.eql(u8, &cached2, &uncached)); +} + +test "TreeHasher reflects direct backing-store mutation" { + // Content-addressed leaf refresh: mutating inner.items directly, bypassing + // set()/append(), must still produce a correct (non-stale) cached root. + const CachedList = utils.TreeHasher(utils.List(u64, 1024), Sha256); + var h = try CachedList.init(std.testing.allocator); + defer h.deinit(); + + try h.append(1); + try h.append(2); + try h.append(3); + + var hash_before: [32]u8 = undefined; + try h.hashTreeRoot(Sha256, &hash_before, std.testing.allocator); + + h.inner.inner.items[1] = 99; // direct mutation, no set() + var hash_after_first: [32]u8 = undefined; + try h.hashTreeRoot(Sha256, &hash_after_first, std.testing.allocator); + + h.inner.inner.items[1] = 42; + var hash_after_second: [32]u8 = undefined; + try h.hashTreeRoot(Sha256, &hash_after_second, std.testing.allocator); + + var fresh = try utils.List(u64, 1024).init(std.testing.allocator); + defer fresh.deinit(); + try fresh.append(1); + try fresh.append(42); + try fresh.append(3); + var expected: [32]u8 = undefined; + try fresh.hashTreeRoot(Sha256, &expected, std.testing.allocator); + + try expect(std.mem.eql(u8, &hash_after_second, &expected)); + try expect(!std.mem.eql(u8, &hash_before, &hash_after_first)); + try expect(!std.mem.eql(u8, &hash_after_first, &hash_after_second)); +} + test "deserialize struct: invalid first var offset returns error (no slice panic)" { const S = struct { a: u32, @@ -2574,6 +2668,270 @@ test "utils.List dynamic-item: offsets decrease" { try expectError(error.OffsetOrdering, L.sszDecode(&buf, &out, std.testing.allocator)); } +test "TreeHasher(List(*Point)) reflects pointee mutation" { + const Point = struct { x: u32, y: u32 }; + const CachedList = utils.TreeHasher(utils.List(*Point, 8), Sha256); + + var h = try CachedList.init(std.testing.allocator); + defer h.deinit(); + + var p1: Point = .{ .x = 1, .y = 2 }; + var p2: Point = .{ .x = 3, .y = 4 }; + try h.append(&p1); + try h.append(&p2); + + var hash_before: [32]u8 = undefined; + try h.hashTreeRoot(Sha256, &hash_before, std.testing.allocator); + + p2.x = 999; // mutate pointee, bypassing set() + var hash_after: [32]u8 = undefined; + try h.hashTreeRoot(Sha256, &hash_after, std.testing.allocator); + + var fresh_uncached: [32]u8 = undefined; + try h.inner.hashTreeRoot(Sha256, &fresh_uncached, std.testing.allocator); + + try expect(std.mem.eql(u8, &hash_after, &fresh_uncached)); + try expect(!std.mem.eql(u8, &hash_before, &hash_after)); +} + +test "TreeHasher zero-capacity List(u64,0) does not panic" { + const CachedList = utils.TreeHasher(utils.List(u64, 0), Sha256); + var h = try CachedList.init(std.testing.allocator); + defer h.deinit(); + + var cached: [32]u8 = undefined; + try h.hashTreeRoot(Sha256, &cached, std.testing.allocator); + var uncached: [32]u8 = undefined; + try h.inner.hashTreeRoot(Sha256, &uncached, std.testing.allocator); + try expect(std.mem.eql(u8, &cached, &uncached)); +} + +test "TreeHasher zero-capacity Bitlist(0) does not panic" { + const CachedBits = utils.TreeHasher(utils.Bitlist(0), Sha256); + var h = try CachedBits.init(std.testing.allocator); + defer h.deinit(); + + var cached: [32]u8 = undefined; + try h.hashTreeRoot(Sha256, &cached, std.testing.allocator); + var uncached: [32]u8 = undefined; + try h.inner.hashTreeRoot(Sha256, &uncached, std.testing.allocator); + try expect(std.mem.eql(u8, &cached, &uncached)); +} + +test "TreeHasher allocates proportional to data, not the SSZ limit" { + // A 2^40-element list must not try to allocate the full tree. Empty hashes + // instantly; adding a few items grows the cache by only a handful of nodes. + const Huge = utils.TreeHasher(utils.List(u32, 1 << 40), Sha256); + var h = try Huge.init(std.testing.allocator); + defer h.deinit(); + + var empty_cached: [32]u8 = undefined; + try h.hashTreeRoot(Sha256, &empty_cached, std.testing.allocator); + try expect(h.cache.capacity == 0); // zero allocation for empty + var empty_uncached: [32]u8 = undefined; + try h.inner.hashTreeRoot(Sha256, &empty_uncached, std.testing.allocator); + try expect(std.mem.eql(u8, &empty_cached, &empty_uncached)); + + // 10 u32 = 2 chunks -> capacity 2, i.e. 3 nodes, not 2^38 chunks. + for (0..10) |i| try h.append(@intCast(i)); + var cached: [32]u8 = undefined; + try h.hashTreeRoot(Sha256, &cached, std.testing.allocator); + try expect(h.cache.capacity == 2); + var uncached: [32]u8 = undefined; + try h.inner.hashTreeRoot(Sha256, &uncached, std.testing.allocator); + try expect(std.mem.eql(u8, &cached, &uncached)); +} + +test "TreeHasher persists its cache when nested in a Container" { + const Container = struct { + balances: utils.TreeHasher(utils.List(u64, 1024), Sha256), + }; + + var c: Container = .{ .balances = try utils.TreeHasher(utils.List(u64, 1024), Sha256).init(std.testing.allocator) }; + defer c.balances.deinit(); + + try c.balances.append(1); + try c.balances.append(2); + + var h1: [32]u8 = undefined; + try hashTreeRoot(Sha256, Container, c, &h1, std.testing.allocator); + // Cache persisted across the by-value container hash (not freed per call). + try expect(c.balances.cache.capacity > 0); + + var h2: [32]u8 = undefined; + try hashTreeRoot(Sha256, Container, c, &h2, std.testing.allocator); + try expect(std.mem.eql(u8, &h1, &h2)); + + try c.balances.set(0, 99); + var h3: [32]u8 = undefined; + try hashTreeRoot(Sha256, Container, c, &h3, std.testing.allocator); + try expect(!std.mem.eql(u8, &h2, &h3)); + + // Final root matches a fresh uncached container. + var fresh = try utils.List(u64, 1024).init(std.testing.allocator); + defer fresh.deinit(); + try fresh.append(99); + try fresh.append(2); + var expected_field: [32]u8 = undefined; + try fresh.hashTreeRoot(Sha256, &expected_field, std.testing.allocator); + var expected_container: [32]u8 = undefined; + // Container is a single-field struct: root = merkleize([field_root], null) + var chunks = [_][32]u8{expected_field}; + try libssz.merkleize(Sha256, &chunks, null, &expected_container); + try expect(std.mem.eql(u8, &h3, &expected_container)); +} + +const SweepPoint = struct { x: u32, y: u32 }; +fn mkU8(i: usize) u8 { + return @truncate(i *% 7 +% 1); +} +fn mkU64(i: usize) u64 { + return @intCast((i *% 1000) +% 1); +} +fn mkU256(i: usize) u256 { + return (@as(u256, @intCast(i)) << 200) | @as(u256, @intCast(i)); +} +fn mkPoint(i: usize) SweepPoint { + return .{ .x = @truncate(i), .y = @truncate(i *% 3) }; +} +fn mkBool(i: usize) bool { + return i % 3 == 0; +} + +fn sweepInner(comptime Inner: type, comptime makeItem: anytype) !void { + const alloc = std.testing.allocator; + const Cached = utils.TreeHasher(Inner, Sha256); + const counts = [_]usize{ 0, 1, 2, 3, 7, 8, 9, 16, 31, 32, 33 }; + for (counts) |cnt| { + var h = try Cached.init(alloc); + defer h.deinit(); + for (0..cnt) |i| try h.append(makeItem(i)); + + var c: [32]u8 = undefined; + try h.hashTreeRoot(Sha256, &c, alloc); + var u: [32]u8 = undefined; + try h.inner.hashTreeRoot(Sha256, &u, alloc); + try expect(std.mem.eql(u8, &c, &u)); + + if (cnt > 0) { + try h.set(0, makeItem(cnt + 3)); + try h.hashTreeRoot(Sha256, &c, alloc); + try h.inner.hashTreeRoot(Sha256, &u, alloc); + try expect(std.mem.eql(u8, &c, &u)); + } + } +} + +test "TreeHasher parity sweep: List(u8)" { + try sweepInner(utils.List(u8, 256), mkU8); +} +test "TreeHasher parity sweep: List(u64)" { + try sweepInner(utils.List(u64, 1024), mkU64); +} +test "TreeHasher parity sweep: List(u256)" { + try sweepInner(utils.List(u256, 256), mkU256); +} +test "TreeHasher parity sweep: List(composite)" { + try sweepInner(utils.List(SweepPoint, 256), mkPoint); +} +test "TreeHasher parity sweep: Bitlist" { + try sweepInner(utils.Bitlist(2048), mkBool); +} + +test "TreeHasher reflects backing-store shrink" { + const Cached = utils.TreeHasher(utils.List(u64, 64), Sha256); + var h = try Cached.init(std.testing.allocator); + defer h.deinit(); + for (0..20) |i| try h.append(@intCast(i)); // 20 u64 = 5 chunks -> capacity 8 + var c: [32]u8 = undefined; + try h.hashTreeRoot(Sha256, &c, std.testing.allocator); + + // Shrink the backing store directly to 4 items (1 chunk), bypassing any API. + h.inner.inner.items.len = 4; + try h.hashTreeRoot(Sha256, &c, std.testing.allocator); + var u: [32]u8 = undefined; + try h.inner.hashTreeRoot(Sha256, &u, std.testing.allocator); + try expect(std.mem.eql(u8, &c, &u)); +} + +test "TreeHasher parity across many growth doublings" { + const Cached = utils.TreeHasher(utils.List(u256, 1 << 12), Sha256); + var h = try Cached.init(std.testing.allocator); + defer h.deinit(); + + var i: usize = 0; + while (i < 600) : (i += 1) { + try h.append(@intCast(i)); + if (i % 7 == 0 or i + 1 == 600) { + var c: [32]u8 = undefined; + try h.hashTreeRoot(Sha256, &c, std.testing.allocator); + var u: [32]u8 = undefined; + try h.inner.hashTreeRoot(Sha256, &u, std.testing.allocator); + try expect(std.mem.eql(u8, &c, &u)); + + // Also mutate an earlier (reused-subtree) element and re-check. + if (i > 10) { + try h.set(i / 2, @intCast(i * 3)); + try h.hashTreeRoot(Sha256, &c, std.testing.allocator); + try h.inner.hashTreeRoot(Sha256, &u, std.testing.allocator); + try expect(std.mem.eql(u8, &c, &u)); + } + } + } +} + +// Sha256 that counts finalize calls, to prove the cache reuses nodes on growth. +var g_hash_calls: usize = 0; +const CountingSha256 = struct { + inner: Sha256, + pub const digest_length = Sha256.digest_length; + pub const Options = Sha256.Options; + pub fn init(o: Options) @This() { + return .{ .inner = Sha256.init(o) }; + } + pub fn update(self: *@This(), bytes: []const u8) void { + self.inner.update(bytes); + } + pub fn final(self: *@This(), out: *[digest_length]u8) void { + g_hash_calls += 1; + self.inner.final(out); + } + pub fn finalResult(self: *@This()) [digest_length]u8 { + // Used only at comptime (zero-hash table); must not touch runtime state. + return self.inner.finalResult(); + } +}; + +test "TreeHasher reuses unchanged nodes when growing" { + const Cached = utils.TreeHasher(utils.List(u256, 1 << 14), CountingSha256); + + // Build to capacity 512, then append one element to force a grow to 1024. + var grown = try Cached.init(std.testing.allocator); + defer grown.deinit(); + for (0..512) |i| try grown.append(@intCast(i)); + var out: [32]u8 = undefined; + try grown.hashTreeRoot(CountingSha256, &out, std.testing.allocator); + + try grown.append(512); + g_hash_calls = 0; + try grown.hashTreeRoot(CountingSha256, &out, std.testing.allocator); + const grow_count = g_hash_calls; + + // Build the same 513-chunk tree from scratch (full rebuild at capacity 1024). + var fresh = try Cached.init(std.testing.allocator); + defer fresh.deinit(); + for (0..513) |i| try fresh.append(@intCast(i)); + g_hash_calls = 0; + var out2: [32]u8 = undefined; + try fresh.hashTreeRoot(CountingSha256, &out2, std.testing.allocator); + const full_count = g_hash_calls; + + // Same root, but the grow path reused the transplanted left subtree and so + // performed strictly fewer hashes than the from-scratch rebuild. + try expect(std.mem.eql(u8, &out, &out2)); + try expect(grow_count < full_count); +} + test "utils.List.clone copies backing storage independently" { const L = utils.List([]const u8, 4); var data = try L.init(std.testing.allocator); @@ -2627,4 +2985,5 @@ test "utils.Bitlist.clone copies backing storage independently" { test { _ = @import("beacon_tests.zig"); + _ = @import("merkle_cache.zig"); } diff --git a/src/utils.zig b/src/utils.zig index d0eb5ef..126c441 100644 --- a/src/utils.zig +++ b/src/utils.zig @@ -7,12 +7,11 @@ const deserialize = lib.deserialize; const isFixedSizeObject = lib.isFixedSizeObject; const ArrayList = std.ArrayList; const Allocator = std.mem.Allocator; -const hashes_of_zero = @import("./zeros.zig").hashes_of_zero; +const merkle_cache = @import("./merkle_cache.zig"); -// SSZ specification constants -const BYTES_PER_CHUNK = 32; -const chunk = [BYTES_PER_CHUNK]u8; -const zero_chunk: chunk = [_]u8{0} ** BYTES_PER_CHUNK; +const BYTES_PER_CHUNK = lib.BYTES_PER_CHUNK; +const chunk = lib.chunk; +const zero_chunk = lib.zero_chunk; /// Implements the SSZ `List[N]` container. pub fn List(T: type, comptime N: usize) type { @@ -21,9 +20,17 @@ pub fn List(T: type, comptime N: usize) type { @compileError("List[bool, N] is not supported. Use Bitlist(" ++ std.fmt.comptimePrint("{}", .{N}) ++ ") instead for boolean lists."); } + // Compile-time check: integer items must be a supported SSZ width. + if (comptime @typeInfo(T) == .int) { + switch (@typeInfo(T).int.bits) { + 8, 16, 32, 64, 128, 256 => {}, + else => @compileError("List item type u" ++ std.fmt.comptimePrint("{d}", .{@typeInfo(T).int.bits}) ++ " is not a valid SSZ integer width; use u8, u16, u32, u64, u128, or u256"), + } + } + return struct { const Self = @This(); - const Item = T; + pub const Item = T; const Inner = std.ArrayList(T); const OFFSET_SIZE = 4; @@ -138,7 +145,7 @@ pub fn List(T: type, comptime N: usize) type { pub fn append(self: *Self, item: Self.Item) error{ Overflow, OutOfMemory }!void { if (self.inner.items.len >= N) return error.Overflow; - return self.inner.append(self.allocator, item); + try self.inner.append(self.allocator, item); } pub fn slice(self: *Self) []T { @@ -175,7 +182,7 @@ pub fn List(T: type, comptime N: usize) type { return lib.serializedSize(@TypeOf(inner_slice), inner_slice); } - pub fn hashTreeRoot(self: *const Self, Hasher: type, out: *[32]u8, allocator: Allocator) !void { + pub fn hashTreeRoot(self: *const Self, Hasher: type, out: *[Hasher.digest_length]u8, allocator: Allocator) !void { const items = self.constSlice(); switch (@typeInfo(Item)) { @@ -184,11 +191,8 @@ pub fn List(T: type, comptime N: usize) type { defer list.deinit(allocator); const chunks = try lib.pack([]const Item, items, &list, allocator); - const bytes_per_item = @sizeOf(Item); - const items_per_chunk = BYTES_PER_CHUNK / bytes_per_item; - const chunks_for_max_capacity = (N + items_per_chunk - 1) / items_per_chunk; var tmp: chunk = undefined; - try lib.merkleize(Hasher, chunks, chunks_for_max_capacity, &tmp); + try lib.merkleize(Hasher, chunks, chunkCountLimit(), &tmp); lib.mixInLength2(Hasher, tmp, items.len, out); }, else => { @@ -199,7 +203,7 @@ pub fn List(T: type, comptime N: usize) type { try lib.hashTreeRoot(Hasher, Item, item, &tmp, allocator); try chunks.append(allocator, tmp); } - // Always use N (max capacity) for merkleization, even when empty + // Always use N (max capacity) for merkleization, even when empty, // This ensures proper tree depth according to SSZ specification try lib.merkleize(Hasher, chunks.items, N, &tmp); lib.mixInLength2(Hasher, tmp, items.len, out); @@ -207,6 +211,45 @@ pub fn List(T: type, comptime N: usize) type { } } + // Leaf protocol consumed by TreeHasher (cached hashing) + + /// Number of data chunks currently occupied. + pub fn numDataChunks(self: *const Self) usize { + const n = self.inner.items.len; + return switch (@typeInfo(Item)) { + .int => (n * @sizeOf(Item) + BYTES_PER_CHUNK - 1) / BYTES_PER_CHUNK, + else => n, + }; + } + + /// SSZ chunk-count limit (used to derive the merkleization depth). + pub fn chunkCountLimit() usize { + return switch (@typeInfo(Item)) { + .int => (N * @sizeOf(Item) + BYTES_PER_CHUNK - 1) / BYTES_PER_CHUNK, + else => N, + }; + } + + /// Derive the bytes of data chunk `idx` (idx < numDataChunks()). + pub fn getLeafBytes(self: *const Self, idx: usize, out: *chunk, comptime Hasher: type, allocator: Allocator) !void { + switch (@typeInfo(Item)) { + .int => { + const bytes_per_item = @sizeOf(Item); + const items_per_chunk = BYTES_PER_CHUNK / bytes_per_item; + out.* = zero_chunk; + const start = idx * items_per_chunk; + const end = @min(start + items_per_chunk, self.inner.items.len); + for (start..end) |item_i| { + const pos = (item_i % items_per_chunk) * bytes_per_item; + std.mem.writeInt(Item, out[pos..][0..bytes_per_item], self.inner.items[item_i], .little); + } + }, + else => { + try lib.hashTreeRoot(Hasher, Item, self.inner.items[idx], out, allocator); + }, + } + } + /// Decodes and validates the length from dynamic input pub fn decodeDynamicLength(buf: []const u8) !u32 { if (buf.len == 0) { @@ -231,10 +274,11 @@ pub fn List(T: type, comptime N: usize) type { }; } -/// Implements the SSZ `Bitlist[N]` container +/// Implements the SSZ `Bitlist[N]` container. pub fn Bitlist(comptime N: usize) type { return struct { const Self = @This(); + pub const Item = bool; // stores list without sentinel const Inner = std.ArrayList(u8); @@ -365,7 +409,7 @@ pub fn Bitlist(comptime N: usize) type { return (self.length + 7 + 1) / 8; } - pub fn hashTreeRoot(self: *const Self, Hasher: type, out: *[32]u8, allocator: Allocator) !void { + pub fn hashTreeRoot(self: *const Self, Hasher: type, out: *[Hasher.digest_length]u8, allocator: Allocator) !void { const bit_length = self.length; var bitfield_bytes: ArrayList(u8) = .empty; @@ -377,7 +421,7 @@ pub fn Bitlist(comptime N: usize) type { try bitfield_bytes.appendSlice(allocator, sl[0..sl.len]); // Remove trailing zeros but keep at least one byte - // This avoids the wasteful pattern of removing all zeros then adding back a chunk + // This avoids the wasteful pattern of removing all zeros and then adding back a chunk while (bitfield_bytes.items.len > 1 and bitfield_bytes.items[bitfield_bytes.items.len - 1] == 0) { _ = bitfield_bytes.pop(); } @@ -390,12 +434,33 @@ pub fn Bitlist(comptime N: usize) type { const chunks = std.mem.bytesAsSlice(chunk, bitfield_bytes.items); var tmp: chunk = undefined; - // Use chunk_count limit as per SSZ specification - const chunk_count_limit = (N + 255) / 256; - try lib.merkleize(Hasher, chunks, chunk_count_limit, &tmp); + try lib.merkleize(Hasher, chunks, chunkCountLimit(), &tmp); lib.mixInLength2(Hasher, tmp, bit_length, out); } + // Leaf protocol consumed by TreeHasher (cached hashing) + + pub fn numDataChunks(self: *const Self) usize { + const n = self.inner.items.len; // bytes + return (n + BYTES_PER_CHUNK - 1) / BYTES_PER_CHUNK; + } + + pub fn chunkCountLimit() usize { + return (N + 255) / 256; + } + + pub fn getLeafBytes(self: *const Self, idx: usize, out: *chunk, comptime Hasher: type, allocator: Allocator) !void { + _ = Hasher; + _ = allocator; + out.* = zero_chunk; + const sl = self.inner.items; + const start = idx * BYTES_PER_CHUNK; + if (start < sl.len) { + const end = @min(start + BYTES_PER_CHUNK, sl.len); + @memcpy(out[0 .. end - start], sl[start..end]); + } + } + /// Validates that the bitlist is correctly formed pub fn validateBitlist(buf: []const u8) !void { const byte_len = buf.len; @@ -430,3 +495,80 @@ pub fn Bitlist(comptime N: usize) type { } }; } + +/// Hasher-agnostic Merkle-caching wrapper around a regular List/Bitlist. +/// +/// Caching is opt-in: use the raw `Inner` for no cache, +/// or `TreeHasher(Inner, Hasher)` for a persistent cache. +/// +/// The cache is owned via a heap pointer that survives by-value copies of the +/// wrapper (as happens when a containing struct is hashed), so a cached field +/// keeps its tree across calls. Leaves are refreshed content-addressed on every +/// call (re-derived from the backing store and compared), so the root never goes +/// stale even if the backing store is mutated directly. +pub fn TreeHasher(comptime Inner: type, comptime Hasher: type) type { + return struct { + const Self = @This(); + const Cache = merkle_cache.MerkleCache(Hasher); + const target_depth = merkle_cache.targetDepth(Inner.chunkCountLimit()); + + inner: Inner, + cache: *Cache, + allocator: Allocator, + + pub fn init(allocator: Allocator) !Self { + const c = try allocator.create(Cache); + errdefer allocator.destroy(c); + c.* = Cache.init(); + return .{ .inner = try Inner.init(allocator), .cache = c, .allocator = allocator }; + } + + pub fn deinit(self: *Self) void { + self.cache.deinit(self.allocator); + self.allocator.destroy(self.cache); + self.inner.deinit(); + } + + pub fn hashTreeRoot(self: *const Self, H: type, out: *[H.digest_length]u8, allocator: Allocator) !void { + // The cache is keyed to `Hasher`; a different hasher falls back to + // the regular uncached object. + if (H == Hasher) return self.hashTreeRootCached(out, allocator); + return self.inner.hashTreeRoot(H, out, allocator); + } + + fn hashTreeRootCached(self: *const Self, out: *[Hasher.digest_length]u8, allocator: Allocator) !void { + const cache = self.cache; + const data_chunks = self.inner.numDataChunks(); + try cache.ensureCapacity(self.allocator, data_chunks); + + // Content-addressed leaf refresh: re-derive each leaf from the + // backing store and mark dirty only those that actually changed. + var leaf: chunk = undefined; + for (0..cache.capacity) |i| { + if (i < data_chunks) { + try self.inner.getLeafBytes(i, &leaf, Hasher, allocator); + } else { + leaf = zero_chunk; + } + if (!std.mem.eql(u8, &leaf, cache.leafPtr(i))) { + cache.leafPtr(i).* = leaf; + cache.markDirty(i); + } + } + + out.* = cache.recomputeWithLength(self.inner.len(), target_depth); + } + + pub fn len(self: *const Self) usize { + return self.inner.len(); + } + + pub fn set(self: *Self, i: usize, item: Inner.Item) !void { + return self.inner.set(i, item); + } + + pub fn append(self: *Self, item: Inner.Item) !void { + return self.inner.append(item); + } + }; +}