feat(v2): EpochStakes parsing & zero-copy SnapshotMetadata

v2/build.zig

@@ -333,6 +333,13 @@ const Sig = struct {
                     .{ .name = "services", .module = services_mod },
                 },
             });
+
+            // accounts_db creates a lot of large globals.
+            // convert globals from [rip + 32-bit offset] to 64bit offsets
+            if (std.mem.eql(u8, service.name, "accounts_db")) {
+                service_mod.code_model = .medium;
+            }
+
             unit_tests.add(service.name, service_mod);
 
             const service_lib = b.addLibrary(.{

v2/config/example.zon

@@ -23,5 +23,6 @@
         .file = "./validator/accounts.db",
         .rooted = .{ .gb = 8 }, // mainnet needs 64, testnet only 8 or so
         .unrooted = .{ .gb = 8 }, // TODO
+        .metadata = .{ .gb = 1 },
     },
 }

v2/init/main.zig

@@ -66,6 +66,7 @@ const Config = struct {
         file: []const u8,
         rooted: MemorySize,
         unrooted: MemorySize,
+        metadata: MemorySize,
 
         // For nicer initialization of constants (instead of x * 1024 * 1024 * 1024)
         const MemorySize = union(enum) {
@@ -258,8 +259,10 @@ pub fn main() !void {
     var snapshot_ready_to_accounts_db: Region(lib.snapshot.SnapshotData) = try .simple();
     snapshot_ready_to_accounts_db.ptr().init();
 
-    var snapshot_metadata: Region(lib.accounts_db.RuntimeMetadata) = try .simple();
-    snapshot_metadata.ptr().init();
+    var snapshot_metadata: Region(lib.snapshot.SnapshotMetadata) = try .sized(
+        @sizeOf(lib.snapshot.SnapshotMetadata) + config.accounts_db.metadata.toBytes(),
+    );
+    snapshot_metadata.ptr().init(config.accounts_db.metadata.toBytes());
 
     const unrooted_memory = config.accounts_db.unrooted.toBytes();
     var account_pool: Region(lib.accounts_db.AccountPool) =

v2/init/services.zig

@@ -15,7 +15,7 @@ pub const accounts_db = struct {
     pub const ReadWrite = struct {
         config: *lib.accounts_db.RootedConfig,
         ready_snapshot_in: *lib.snapshot.SnapshotData,
-        snapshot_metadata_out: *lib.accounts_db.RuntimeMetadata,
+        snapshot_metadata_out: *lib.snapshot.SnapshotMetadata,
         account_pool: *lib.accounts_db.AccountPool,
         replay_lookups: *lib.accounts_db.AccountLookups,
         tel: *lib.telemetry.Region,
@@ -60,7 +60,7 @@ pub const replay = struct {
 
     pub const ReadWrite = struct {
         scratch_memory: *[lib.replay.scratch_buffer_size]u8,
-        snapshot_metadata_in: *lib.accounts_db.RuntimeMetadata,
+        snapshot_metadata_in: *lib.snapshot.SnapshotMetadata,
         deshredded_in: *lib.shred.DeshredRing,
         replay_transaction_pool: *lib.replay.TransactionPool,
         block_pool: *lib.replay.BlockPool,
@@ -78,7 +78,7 @@ pub const shred_receiver = struct {
 
     pub const ReadWrite = struct {
         /// Gets slot (& soon leader-schedule info) from replay / runtime init.
-        snapshot_metadata: *lib.accounts_db.RuntimeMetadata,
+        snapshot_metadata: *lib.snapshot.SnapshotMetadata,
 
         /// Transaction Validation Unit (TVU) UDP socket, i.e. where we receive
         /// shreds. This is typically port 8002. While we've obtained a net

v2/lib/accounts_db.zig

@@ -10,8 +10,6 @@ comptime {
 }
 
 const Pubkey = lib.solana.Pubkey;
-const Hash = lib.solana.Hash;
-const Slot = lib.solana.Slot;
 
 pub const AccountPool = @import("accounts_db/pool.zig").AccountPool;
 pub const Rooted = @import("accounts_db/rooted.zig").Rooted;
@@ -40,52 +38,3 @@ pub const AccountLookups = extern struct {
         self.out.init();
     }
 };
-
-/// How to consume this struct:
-/// 1. read all ring buffers in the correct order (specify the correct order
-///    here if more are added, currently there is only one: the blockhash queue)
-/// 2. call getSlotBlocking
-/// 3. read other fields
-pub const RuntimeMetadata = extern struct {
-    slot: std.atomic.Value(u64),
-    /// The merkle root of the last fec set (tower) or the root of roots (alpenglow)
-    block_id: Hash,
-    blockhash_queue: extern struct {
-        /// read after consuming all of hashes
-        max_age: u64,
-        /// Accountsdb blocks until enough hashes are read from here to make
-        /// room for accountsdb to write all of its block hashes here.
-        hashes: lib.ipc.Ring(256, Hash),
-    },
-
-    // 0 may be a valid slot, so use something that will never be reached.
-    const invalid_slot = std.math.maxInt(Slot);
-
-    pub fn init(self: *RuntimeMetadata) void {
-        self.slot = .init(invalid_slot);
-
-        self.blockhash_queue.max_age = 0;
-        self.blockhash_queue.hashes.init();
-    }
-
-    /// Unblocks all getSlotBlocking() callers with the given slot value.
-    /// Can be called only once.
-    /// Should also only call after all other RuntimeMetadata fields are populated.
-    pub fn populateSlot(self: *RuntimeMetadata, slot: Slot) void {
-        std.debug.assert(slot != invalid_slot);
-        std.debug.assert(self.slot.swap(slot, .release) == invalid_slot);
-    }
-
-    /// Accountsdb writes the slot last, so you need to empty all the ring
-    /// buffers before calling this.
-    pub fn getSlotBlocking(self: *RuntimeMetadata, runner: lib.runner.Connection) !Slot {
-        while (true) {
-            const slot = self.slot.load(.acquire);
-            if (slot != invalid_slot) {
-                try runner.activity.signalActive();
-                return slot;
-            }
-            try runner.activity.signalIdleSpinning();
-        }
-    }
-};