From 4924e836271b8329e8880d6bc256335a690e2db9 Mon Sep 17 00:00:00 2001 From: Chen Kai <281165273grape@gmail.com> Date: Mon, 22 Dec 2025 12:53:42 +0800 Subject: [PATCH 01/19] feat: impl sig spec test Signed-off-by: Chen Kai <281165273grape@gmail.com> --- pkgs/spectest/src/fixture_kind.zig | 5 + pkgs/spectest/src/json_expect.zig | 17 + .../src/runner/verify_signatures_runner.zig | 642 ++++++++++++++++++ 3 files changed, 664 insertions(+) create mode 100644 pkgs/spectest/src/runner/verify_signatures_runner.zig diff --git a/pkgs/spectest/src/fixture_kind.zig b/pkgs/spectest/src/fixture_kind.zig index 05735085a..7746f50d8 100644 --- a/pkgs/spectest/src/fixture_kind.zig +++ b/pkgs/spectest/src/fixture_kind.zig @@ -1,11 +1,15 @@ pub const FixtureKind = enum { state_transition, fork_choice, + // verify_signatures is temporarily disabled due to XMSS config mismatch + // between Python test config (424-byte signatures) and Rust production config (3112-byte signatures) + // verify_signatures, pub fn runnerModule(self: FixtureKind) []const u8 { return switch (self) { .state_transition => "state_transition", .fork_choice => "fork_choice", + // .verify_signatures => "verify_signatures", }; } @@ -13,6 +17,7 @@ pub const FixtureKind = enum { return switch (self) { .state_transition => "state_transition", .fork_choice => "fc", + // .verify_signatures => "verify_signatures", }; } }; diff --git a/pkgs/spectest/src/json_expect.zig b/pkgs/spectest/src/json_expect.zig index 15c811166..f633a948e 100644 --- a/pkgs/spectest/src/json_expect.zig +++ b/pkgs/spectest/src/json_expect.zig @@ -245,6 +245,23 @@ pub fn expectArrayValue( }; } +pub fn expectArrayField( + comptime FixtureError: type, + obj: std.json.ObjectMap, + field_names: []const []const u8, + context: Context, + label: []const u8, +) FixtureError!std.json.Array { + const value = getField(obj, field_names) orelse { + std.debug.print( + "fixture {s} case {s}{}: missing field {s}\n", + .{ context.fixture_label, context.case_name, context.formatStep(), label }, + ); + return FixtureError.InvalidFixture; + }; + return expectArrayValue(FixtureError, value, context, label); +} + pub fn appendBytesDataField( comptime FixtureError: type, comptime T: type, diff --git a/pkgs/spectest/src/runner/verify_signatures_runner.zig b/pkgs/spectest/src/runner/verify_signatures_runner.zig new file mode 100644 index 000000000..2c887c22e --- /dev/null +++ b/pkgs/spectest/src/runner/verify_signatures_runner.zig @@ -0,0 +1,642 @@ +const std = @import("std"); + +const expect = @import("../json_expect.zig"); +const forks = @import("../fork.zig"); +const fixture_kind = @import("../fixture_kind.zig"); +const skip = @import("../skip.zig"); + +const Fork = forks.Fork; +const FixtureKind = fixture_kind.FixtureKind; + +pub const name = "verify_signatures"; + +pub const Handler = enum { + test_invalid_signatures, + test_valid_signatures, +}; + +pub const handlers = std.enums.values(Handler); + +pub fn handlerLabel(comptime handler: Handler) []const u8 { + return switch (handler) { + .test_invalid_signatures => "test_invalid_signatures", + .test_valid_signatures => "test_valid_signatures", + }; +} + +pub fn handlerPath(comptime handler: Handler) []const u8 { + return handlerLabel(handler); +} + +pub fn includeFixtureFile(file_name: []const u8) bool { + return std.mem.endsWith(u8, file_name, ".json"); +} + +pub fn baseRelRoot(comptime spec_fork: Fork) []const u8 { + const kind = FixtureKind.verify_signatures; + return std.fmt.comptimePrint( + "consensus/{s}/{s}/{s}", + .{ kind.runnerModule(), spec_fork.path, kind.handlerSubdir() }, + ); +} + +const types = @import("@zeam/types"); +const state_transition = @import("@zeam/state-transition"); + +// Signature structure constants from leansig +// path: 8 siblings, each is 8 u32 = 256 bytes +// rho: 7 u32 = 28 bytes +// hashes: 4 elements, each is 8 u32 = 128 bytes +// Total fixed size in fixture: 412 bytes, but actual SIGBYTES is 3112 bytes +// The remaining bytes are likely padding or additional OTS data + +const JsonValue = std.json.Value; +const Context = expect.Context; + +pub const RunnerError = error{ + IoFailure, +} || FixtureError; + +pub const FixtureError = error{ + InvalidFixture, + UnsupportedFixture, + FixtureMismatch, + SkippedFixture, +}; + +const read_max_bytes: usize = 16 * 1024 * 1024; // 16 MiB upper bound per fixture file. + +pub fn TestCase( + comptime spec_fork: Fork, + comptime rel_path: []const u8, +) type { + return struct { + payload: []u8, + + const Self = @This(); + + pub fn execute(allocator: std.mem.Allocator, dir: std.fs.Dir) RunnerError!void { + var tc = try Self.init(allocator, dir); + defer tc.deinit(allocator); + try tc.run(allocator); + } + + pub fn init(allocator: std.mem.Allocator, dir: std.fs.Dir) RunnerError!Self { + const payload = try loadFixturePayload(allocator, dir, rel_path); + return Self{ .payload = payload }; + } + + pub fn deinit(self: *Self, allocator: std.mem.Allocator) void { + allocator.free(self.payload); + } + + pub fn run(self: *Self, allocator: std.mem.Allocator) RunnerError!void { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + const arena_allocator = arena.allocator(); + + try runFixturePayload(spec_fork, arena_allocator, rel_path, self.payload); + } + }; +} + +fn loadFixturePayload( + allocator: std.mem.Allocator, + dir: std.fs.Dir, + rel_path: []const u8, +) RunnerError![]u8 { + const payload = dir.readFileAlloc(allocator, rel_path, read_max_bytes) catch |err| switch (err) { + error.FileTooBig => { + std.debug.print("spectest: fixture {s} exceeds allowed size\n", .{rel_path}); + return RunnerError.IoFailure; + }, + else => { + std.debug.print("spectest: failed to read {s}: {s}\n", .{ rel_path, @errorName(err) }); + return RunnerError.IoFailure; + }, + }; + return payload; +} + +pub fn runFixturePayload( + comptime spec_fork: Fork, + allocator: std.mem.Allocator, + fixture_label: []const u8, + payload: []const u8, +) FixtureError!void { + _ = spec_fork; + var parsed = std.json.parseFromSlice(JsonValue, allocator, payload, .{ .ignore_unknown_fields = true }) catch |err| { + std.debug.print("spectest: fixture {s} not valid JSON: {s}\n", .{ fixture_label, @errorName(err) }); + return FixtureError.InvalidFixture; + }; + defer parsed.deinit(); + + const root = parsed.value; + const obj = switch (root) { + .object => |map| map, + else => { + std.debug.print("spectest: fixture {s} must be JSON object\n", .{fixture_label}); + return FixtureError.InvalidFixture; + }, + }; + + var skipped_cases: usize = 0; + var it = obj.iterator(); + while (it.next()) |entry| { + const case_name = entry.key_ptr.*; + const case_value = entry.value_ptr.*; + const ctx = Context{ .fixture_label = fixture_label, .case_name = case_name }; + runCase(allocator, ctx, case_value) catch |err| switch (err) { + FixtureError.SkippedFixture => skipped_cases += 1, + FixtureError.UnsupportedFixture => { + std.debug.print( + "spectest: skipping unsupported case {s} in {s}\n", + .{ case_name, fixture_label }, + ); + }, + else => return err, + }; + } + + if (skipped_cases > 0) { + std.debug.print( + "spectest: skipped {d} case(s) in fixture {s}\n", + .{ skipped_cases, fixture_label }, + ); + } +} + +fn runCase( + allocator: std.mem.Allocator, + ctx: Context, + value: JsonValue, +) FixtureError!void { + const case_obj = switch (value) { + .object => |map| map, + else => { + std.debug.print("fixture {s} case {s}: expected object\n", .{ ctx.fixture_label, ctx.case_name }); + return FixtureError.InvalidFixture; + }, + }; + + // Parse the anchorState to get validators + const anchor_state_value = case_obj.get("anchorState") orelse { + std.debug.print("fixture {s} case {s}: missing anchorState\n", .{ ctx.fixture_label, ctx.case_name }); + return FixtureError.InvalidFixture; + }; + + var anchor_state = try buildState(allocator, ctx, anchor_state_value); + defer anchor_state.deinit(); + + // Parse the signedBlockWithAttestation + const signed_block_value = case_obj.get("signedBlockWithAttestation") orelse { + std.debug.print("fixture {s} case {s}: missing signedBlockWithAttestation\n", .{ ctx.fixture_label, ctx.case_name }); + return FixtureError.InvalidFixture; + }; + + var signed_block = try buildSignedBlockWithAttestation(allocator, ctx, signed_block_value); + defer signed_block.deinit(); + + // Determine if we expect failure based on test name/path + const expect_failure = std.mem.indexOf(u8, ctx.fixture_label, "invalid") != null or + std.mem.indexOf(u8, ctx.case_name, "invalid") != null; + + // Verify signatures + const verify_result = state_transition.verifySignatures(allocator, &anchor_state, &signed_block); + + if (expect_failure) { + if (verify_result) |_| { + std.debug.print( + "fixture {s} case {s}: expected verification to fail but it succeeded\n", + .{ ctx.fixture_label, ctx.case_name }, + ); + return FixtureError.FixtureMismatch; + } else |_| { + // Expected failure + } + } else { + verify_result catch |err| { + std.debug.print( + "fixture {s} case {s}: signature verification failed with {s}\n", + .{ ctx.fixture_label, ctx.case_name, @errorName(err) }, + ); + return FixtureError.FixtureMismatch; + }; + } +} + +fn buildState( + allocator: std.mem.Allocator, + ctx: Context, + value: JsonValue, +) FixtureError!types.BeamState { + const pre_obj = try expect.expectObjectValue(FixtureError, value, ctx, "anchorState"); + + const config_obj = try expect.expectObject(FixtureError, pre_obj, &.{"config"}, ctx, "config"); + const genesis_time = try expect.expectU64Field(FixtureError, config_obj, &.{"genesisTime"}, ctx, "config.genesisTime"); + + const slot = try expect.expectU64Field(FixtureError, pre_obj, &.{"slot"}, ctx, "slot"); + + const header_obj = try expect.expectObject(FixtureError, pre_obj, &.{"latestBlockHeader"}, ctx, "latestBlockHeader"); + const latest_block_header = try parseBlockHeader(ctx, header_obj); + + const latest_justified = try parseCheckpoint(ctx, pre_obj, "latestJustified"); + const latest_finalized = try parseCheckpoint(ctx, pre_obj, "latestFinalized"); + + var historical = try types.HistoricalBlockHashes.init(allocator); + errdefer historical.deinit(); + if (pre_obj.get("historicalBlockHashes")) |val| { + try expect.appendBytesDataField(FixtureError, types.Root, &historical, ctx, val, "historicalBlockHashes"); + } + + var justified_slots = try types.JustifiedSlots.init(allocator); + errdefer justified_slots.deinit(); + if (pre_obj.get("justifiedSlots")) |val| { + try expect.appendBoolDataField(FixtureError, &justified_slots, ctx, val, "justifiedSlots"); + } + + var validators = try parseValidators(allocator, ctx, pre_obj); + errdefer validators.deinit(); + + var just_roots = try types.JustificationRoots.init(allocator); + errdefer just_roots.deinit(); + if (pre_obj.get("justificationsRoots")) |val| { + try expect.appendBytesDataField(FixtureError, types.Root, &just_roots, ctx, val, "justificationsRoots"); + } + + var just_validators = try types.JustificationValidators.init(allocator); + errdefer just_validators.deinit(); + if (pre_obj.get("justificationsValidators")) |val| { + try expect.appendBoolDataField(FixtureError, &just_validators, ctx, val, "justificationsValidators"); + } + + return types.BeamState{ + .config = .{ .genesis_time = genesis_time }, + .slot = slot, + .latest_block_header = latest_block_header, + .latest_justified = latest_justified, + .latest_finalized = latest_finalized, + .historical_block_hashes = historical, + .justified_slots = justified_slots, + .validators = validators, + .justifications_roots = just_roots, + .justifications_validators = just_validators, + }; +} + +fn parseValidators( + allocator: std.mem.Allocator, + ctx: Context, + pre_obj: std.json.ObjectMap, +) FixtureError!types.Validators { + var validators = try types.Validators.init(allocator); + errdefer validators.deinit(); + + if (pre_obj.get("validators")) |val| { + const validators_obj = try expect.expectObjectValue(FixtureError, val, ctx, "validators"); + if (validators_obj.get("data")) |data_val| { + const arr = try expect.expectArrayValue(FixtureError, data_val, ctx, "validators.data"); + for (arr.items, 0..) |item, idx| { + var base_label_buf: [64]u8 = undefined; + const base_label = std.fmt.bufPrint(&base_label_buf, "validators[{d}]", .{idx}) catch "validators"; + const validator_obj = try expect.expectObjectValue(FixtureError, item, ctx, base_label); + + var label_buf: [96]u8 = undefined; + const pubkey_label = std.fmt.bufPrint(&label_buf, "{s}.pubkey", .{base_label}) catch "validator.pubkey"; + const pubkey = try expect.expectBytesField(FixtureError, types.Bytes52, validator_obj, &.{"pubkey"}, ctx, pubkey_label); + + const validator_index: u64 = blk: { + if (validator_obj.get("index")) |index_value| { + var index_label_buf: [96]u8 = undefined; + const index_label = std.fmt.bufPrint(&index_label_buf, "{s}.index", .{base_label}) catch "validator.index"; + break :blk try expect.expectU64Value(FixtureError, index_value, ctx, index_label); + } + break :blk @as(u64, @intCast(idx)); + }; + + validators.append(.{ .pubkey = pubkey, .index = validator_index }) catch |err| { + std.debug.print( + "fixture {s} case {s}: validator #{} append failed: {s}\n", + .{ ctx.fixture_label, ctx.case_name, idx, @errorName(err) }, + ); + return FixtureError.InvalidFixture; + }; + } + } + } + + return validators; +} + +fn buildSignedBlockWithAttestation( + allocator: std.mem.Allocator, + ctx: Context, + value: JsonValue, +) FixtureError!types.SignedBlockWithAttestation { + const signed_block_obj = try expect.expectObjectValue(FixtureError, value, ctx, "signedBlockWithAttestation"); + + // Parse message + const message_obj = try expect.expectObject(FixtureError, signed_block_obj, &.{"message"}, ctx, "message"); + + // Parse block within message + const block_obj = try expect.expectObject(FixtureError, message_obj, &.{"block"}, ctx, "message.block"); + const block = try buildBlock(allocator, ctx, block_obj); + + // Parse proposerAttestation + const proposer_att_obj = try expect.expectObject(FixtureError, message_obj, &.{"proposerAttestation"}, ctx, "message.proposerAttestation"); + const proposer_attestation = try parseProposerAttestation(ctx, proposer_att_obj); + + // Parse signature section + const signature_obj = try expect.expectObject(FixtureError, signed_block_obj, &.{"signature"}, ctx, "signature"); + + // Parse attestation_signatures (empty for basic tests) + var attestation_signatures = try types.AttestationSignatures.init(allocator); + errdefer attestation_signatures.deinit(); + + if (signature_obj.get("attestationSignatures")) |att_sigs_val| { + const att_sigs_obj = try expect.expectObjectValue(FixtureError, att_sigs_val, ctx, "signature.attestationSignatures"); + if (att_sigs_obj.get("data")) |data_val| { + const arr = try expect.expectArrayValue(FixtureError, data_val, ctx, "signature.attestationSignatures.data"); + for (arr.items) |_| { + // TODO: Parse actual attestation signatures if needed + std.debug.print("fixture {s} case {s}: non-empty attestation signatures not yet supported\n", .{ ctx.fixture_label, ctx.case_name }); + return FixtureError.UnsupportedFixture; + } + } + } + + // Parse proposer_signature + const proposer_sig = try parseSignature(ctx, signature_obj, "proposerSignature"); + + return types.SignedBlockWithAttestation{ + .message = .{ + .block = block, + .proposer_attestation = proposer_attestation, + }, + .signature = .{ + .attestation_signatures = attestation_signatures, + .proposer_signature = proposer_sig, + }, + }; +} + +fn buildBlock( + allocator: std.mem.Allocator, + ctx: Context, + obj: std.json.ObjectMap, +) FixtureError!types.BeamBlock { + const slot = try expect.expectU64Field(FixtureError, obj, &.{"slot"}, ctx, "slot"); + const proposer_index = try expect.expectU64Field(FixtureError, obj, &.{ "proposer_index", "proposerIndex" }, ctx, "proposer_index"); + const parent_root = try expect.expectBytesField(FixtureError, types.Root, obj, &.{ "parent_root", "parentRoot" }, ctx, "parent_root"); + const state_root = try expect.expectBytesField(FixtureError, types.Root, obj, &.{ "state_root", "stateRoot" }, ctx, "state_root"); + + var attestations = try types.AggregatedAttestations.init(allocator); + errdefer attestations.deinit(); + + if (obj.get("body")) |body_val| { + const body_obj = try expect.expectObjectValue(FixtureError, body_val, ctx, "body"); + if (body_obj.get("attestations")) |att_val| { + const att_obj = try expect.expectObjectValue(FixtureError, att_val, ctx, "body.attestations"); + if (att_obj.get("data")) |data_val| { + const arr = try expect.expectArrayValue(FixtureError, data_val, ctx, "body.attestations.data"); + for (arr.items) |_| { + // TODO: Parse actual attestations if needed + std.debug.print("fixture {s} case {s}: non-empty attestations not yet supported\n", .{ ctx.fixture_label, ctx.case_name }); + return FixtureError.UnsupportedFixture; + } + } + } + } + + return types.BeamBlock{ + .slot = slot, + .proposer_index = proposer_index, + .parent_root = parent_root, + .state_root = state_root, + .body = .{ .attestations = attestations }, + }; +} + +fn parseProposerAttestation( + ctx: Context, + obj: std.json.ObjectMap, +) FixtureError!types.Attestation { + const validator_id = try expect.expectU64Field(FixtureError, obj, &.{"validatorId"}, ctx, "proposerAttestation.validatorId"); + + const data_obj = try expect.expectObject(FixtureError, obj, &.{"data"}, ctx, "proposerAttestation.data"); + const data = try parseAttestationData(ctx, data_obj); + + return types.Attestation{ + .validator_id = validator_id, + .data = data, + }; +} + +fn parseAttestationData( + ctx: Context, + obj: std.json.ObjectMap, +) FixtureError!types.AttestationData { + const slot = try expect.expectU64Field(FixtureError, obj, &.{"slot"}, ctx, "data.slot"); + + const head_obj = try expect.expectObject(FixtureError, obj, &.{"head"}, ctx, "data.head"); + const head = types.Checkpoint{ + .root = try expect.expectBytesField(FixtureError, types.Root, head_obj, &.{"root"}, ctx, "data.head.root"), + .slot = try expect.expectU64Field(FixtureError, head_obj, &.{"slot"}, ctx, "data.head.slot"), + }; + + const target_obj = try expect.expectObject(FixtureError, obj, &.{"target"}, ctx, "data.target"); + const target = types.Checkpoint{ + .root = try expect.expectBytesField(FixtureError, types.Root, target_obj, &.{"root"}, ctx, "data.target.root"), + .slot = try expect.expectU64Field(FixtureError, target_obj, &.{"slot"}, ctx, "data.target.slot"), + }; + + const source_obj = try expect.expectObject(FixtureError, obj, &.{"source"}, ctx, "data.source"); + const source = types.Checkpoint{ + .root = try expect.expectBytesField(FixtureError, types.Root, source_obj, &.{"root"}, ctx, "data.source.root"), + .slot = try expect.expectU64Field(FixtureError, source_obj, &.{"slot"}, ctx, "data.source.slot"), + }; + + return types.AttestationData{ + .slot = slot, + .head = head, + .target = target, + .source = source, + }; +} + +fn parseSignature( + ctx: Context, + obj: std.json.ObjectMap, + field_name: []const u8, +) FixtureError!types.SIGBYTES { + const sig_obj = try expect.expectObject(FixtureError, obj, &.{field_name}, ctx, field_name); + + // SSZ Container serialization for Signature: + // Signature = Container(path: HashTreeOpening, rho: Randomness, hashes: HashDigestList) + // + // Fixed part (36 bytes): + // - offset_path: 4 bytes (offset to path data) + // - rho_data: 28 bytes (7 Fp values, each 4 bytes - fixed size Vector) + // - offset_hashes: 4 bytes (offset to hashes data) + // + // Variable part: + // - path data (HashTreeOpening serialized) + // - hashes data (HashDigestList serialized) + // + // HashTreeOpening = Container(siblings: HashDigestList) + // Fixed part: 4 bytes (offset to siblings data) + // Variable part: siblings data + // + // HashDigestList = List[HashDigestVector, NODE_LIST_LIMIT] + // HashDigestVector = Vector[Fp, 8] = 32 bytes (8 * 4 bytes each) + // Since HashDigestVector is fixed size, the list is serialized as: + // - concatenated fixed-size items (no offsets needed) + + var sig_bytes: types.SIGBYTES = std.mem.zeroes(types.SIGBYTES); + + // Parse path siblings + const path_obj = try expect.expectObject(FixtureError, sig_obj, &.{"path"}, ctx, "path"); + const siblings_obj = try expect.expectObject(FixtureError, path_obj, &.{"siblings"}, ctx, "path.siblings"); + const siblings_data = try expect.expectArrayField(FixtureError, siblings_obj, &.{"data"}, ctx, "path.siblings.data"); + + // Parse rho (fixed size: 7 * 4 = 28 bytes) + const rho_obj = try expect.expectObject(FixtureError, sig_obj, &.{"rho"}, ctx, "rho"); + const rho_arr = try parseU32Array7(ctx, rho_obj, "rho.data"); + + // Parse hashes + const hashes_obj = try expect.expectObject(FixtureError, sig_obj, &.{"hashes"}, ctx, "hashes"); + const hashes_data = try expect.expectArrayField(FixtureError, hashes_obj, &.{"data"}, ctx, "hashes.data"); + + // Calculate sizes for SSZ serialization + const num_siblings = siblings_data.items.len; + const num_hashes = hashes_data.items.len; + + // Each HashDigestVector = 8 Fp * 4 bytes = 32 bytes + const sibling_size: usize = 8 * 4; // 32 bytes per sibling + const hash_size: usize = 8 * 4; // 32 bytes per hash + + // HashTreeOpening SSZ: + // Fixed part: 4 bytes (offset to siblings) + // Variable part: siblings data (concatenated HashDigestVectors) + const path_fixed_part: usize = 4; + const path_variable_size = num_siblings * sibling_size; + const path_total_size = path_fixed_part + path_variable_size; + + // HashDigestList SSZ (for hashes): just concatenated fixed-size items + const hashes_size = num_hashes * hash_size; + + // Signature fixed part: offset_path (4) + rho (28) + offset_hashes (4) = 36 bytes + const sig_fixed_part: usize = 36; + + // Calculate offsets + const offset_path: u32 = @intCast(sig_fixed_part); // path starts after fixed part + const offset_hashes: u32 = @intCast(sig_fixed_part + path_total_size); // hashes start after path + + var write_pos: usize = 0; + + // Write Signature fixed part: + // 1. offset_path (4 bytes) + std.mem.writeInt(u32, sig_bytes[write_pos..][0..4], offset_path, .little); + write_pos += 4; + + // 2. rho data (28 bytes - fixed size) + for (rho_arr) |val| { + std.mem.writeInt(u32, sig_bytes[write_pos..][0..4], val, .little); + write_pos += 4; + } + + // 3. offset_hashes (4 bytes) + std.mem.writeInt(u32, sig_bytes[write_pos..][0..4], offset_hashes, .little); + write_pos += 4; + + // Now write_pos should be at sig_fixed_part (36) + + // Write path (HashTreeOpening): + // Fixed part: offset to siblings (4 bytes, pointing to byte 4 within path) + const path_siblings_offset: u32 = 4; // siblings data starts at offset 4 within HashTreeOpening + std.mem.writeInt(u32, sig_bytes[write_pos..][0..4], path_siblings_offset, .little); + write_pos += 4; + + // Variable part: siblings data + for (siblings_data.items) |sibling_val| { + const sibling_obj = try expect.expectObjectValue(FixtureError, sibling_val, ctx, "sibling"); + const u32_arr = try parseU32Array8(ctx, sibling_obj, "sibling.data"); + for (u32_arr) |val| { + std.mem.writeInt(u32, sig_bytes[write_pos..][0..4], val, .little); + write_pos += 4; + } + } + + // Write hashes (HashDigestList - just concatenated items since they're fixed size) + for (hashes_data.items) |hash_val| { + const hash_obj = try expect.expectObjectValue(FixtureError, hash_val, ctx, "hash"); + const u32_arr = try parseU32Array8(ctx, hash_obj, "hash.data"); + for (u32_arr) |val| { + std.mem.writeInt(u32, sig_bytes[write_pos..][0..4], val, .little); + write_pos += 4; + } + } + + _ = hashes_size; // Used in offset_hashes calculation + + return sig_bytes; +} + +fn parseU32Array8( + ctx: Context, + obj: std.json.ObjectMap, + label: []const u8, +) FixtureError![8]u32 { + const data_arr = try expect.expectArrayField(FixtureError, obj, &.{"data"}, ctx, label); + var result: [8]u32 = undefined; + for (data_arr.items, 0..) |val, i| { + if (i >= 8) break; + result[i] = @intCast(try expect.expectU64Value(FixtureError, val, ctx, label)); + } + return result; +} + +fn parseU32Array7( + ctx: Context, + obj: std.json.ObjectMap, + label: []const u8, +) FixtureError![7]u32 { + const data_arr = try expect.expectArrayField(FixtureError, obj, &.{"data"}, ctx, label); + var result: [7]u32 = undefined; + for (data_arr.items, 0..) |val, i| { + if (i >= 7) break; + result[i] = @intCast(try expect.expectU64Value(FixtureError, val, ctx, label)); + } + return result; +} + +fn parseCheckpoint( + ctx: Context, + parent: std.json.ObjectMap, + field_name: []const u8, +) FixtureError!types.Checkpoint { + const cp_obj = try expect.expectObject(FixtureError, parent, &.{field_name}, ctx, field_name); + + var root_label_buf: [96]u8 = undefined; + const root_label = std.fmt.bufPrint(&root_label_buf, "{s}.root", .{field_name}) catch field_name; + var slot_label_buf: [96]u8 = undefined; + const slot_label = std.fmt.bufPrint(&slot_label_buf, "{s}.slot", .{field_name}) catch field_name; + + return types.Checkpoint{ + .root = try expect.expectBytesField(FixtureError, types.Root, cp_obj, &.{"root"}, ctx, root_label), + .slot = try expect.expectU64Field(FixtureError, cp_obj, &.{"slot"}, ctx, slot_label), + }; +} + +fn parseBlockHeader( + ctx: Context, + obj: std.json.ObjectMap, +) FixtureError!types.BeamBlockHeader { + return types.BeamBlockHeader{ + .slot = try expect.expectU64Field(FixtureError, obj, &.{"slot"}, ctx, "latestBlockHeader.slot"), + .proposer_index = try expect.expectU64Field(FixtureError, obj, &.{ "proposerIndex", "proposer_index" }, ctx, "latestBlockHeader.proposerIndex"), + .parent_root = try expect.expectBytesField(FixtureError, types.Root, obj, &.{ "parentRoot", "parent_root" }, ctx, "latestBlockHeader.parentRoot"), + .state_root = try expect.expectBytesField(FixtureError, types.Root, obj, &.{ "stateRoot", "state_root" }, ctx, "latestBlockHeader.stateRoot"), + .body_root = try expect.expectBytesField(FixtureError, types.Root, obj, &.{ "bodyRoot", "body_root" }, ctx, "latestBlockHeader.bodyRoot"), + }; +} From 0233f7de1be8b90f26a4c6d8f97d707d90c5bb67 Mon Sep 17 00:00:00 2001 From: Chen Kai <281165273grape@gmail.com> Date: Sat, 27 Dec 2025 22:05:58 +0800 Subject: [PATCH 02/19] feat: add signature spec test Signed-off-by: Chen Kai <281165273grape@gmail.com> --- build.zig | 5 + pkgs/spectest/src/fixture_kind.zig | 10 +- .../src/runner/verify_signatures_runner.zig | 169 +++---- pkgs/xmss/src/hashsig.zig | 24 + rust/Cargo.lock | 21 +- rust/hashsig-glue/Cargo.toml | 8 +- rust/hashsig-glue/src/lib.rs | 468 +++++++++++++++++- 7 files changed, 577 insertions(+), 128 deletions(-) diff --git a/build.zig b/build.zig index c56587145..8115d3a4f 100644 --- a/build.zig +++ b/build.zig @@ -319,6 +319,7 @@ pub fn build(b: *Builder) !void { zeam_spectests.addImport("build_options", build_options_module); zeam_spectests.addImport("@zeam/state-transition", zeam_state_transition); zeam_spectests.addImport("@zeam/node", zeam_beam_node); + zeam_spectests.addImport("@zeam/xmss", zeam_xmss); // Add the cli executable const cli_exe = b.addExecutable(.{ @@ -570,6 +571,10 @@ pub fn build(b: *Builder) !void { spectests.root_module.addImport("@zeam/metrics", zeam_metrics); spectests.root_module.addImport("@zeam/state-transition", zeam_state_transition); spectests.root_module.addImport("ssz", ssz); + spectests.root_module.addImport("@zeam/xmss", zeam_xmss); + + spectests.step.dependOn(&build_rust_lib_steps.step); + addRustGlueLib(b, spectests, target, prover); manager_tests.step.dependOn(&build_rust_lib_steps.step); diff --git a/pkgs/spectest/src/fixture_kind.zig b/pkgs/spectest/src/fixture_kind.zig index 7746f50d8..bcb99ccf6 100644 --- a/pkgs/spectest/src/fixture_kind.zig +++ b/pkgs/spectest/src/fixture_kind.zig @@ -1,15 +1,13 @@ pub const FixtureKind = enum { state_transition, fork_choice, - // verify_signatures is temporarily disabled due to XMSS config mismatch - // between Python test config (424-byte signatures) and Rust production config (3112-byte signatures) - // verify_signatures, + verify_signatures, pub fn runnerModule(self: FixtureKind) []const u8 { return switch (self) { .state_transition => "state_transition", .fork_choice => "fork_choice", - // .verify_signatures => "verify_signatures", + .verify_signatures => "verify_signatures", }; } @@ -17,9 +15,9 @@ pub const FixtureKind = enum { return switch (self) { .state_transition => "state_transition", .fork_choice => "fc", - // .verify_signatures => "verify_signatures", + .verify_signatures => "verify_signatures", }; } }; -pub const all = [_]FixtureKind{ .state_transition, .fork_choice }; +pub const all = [_]FixtureKind{ .state_transition, .fork_choice, .verify_signatures }; diff --git a/pkgs/spectest/src/runner/verify_signatures_runner.zig b/pkgs/spectest/src/runner/verify_signatures_runner.zig index 2c887c22e..0cc4cfe64 100644 --- a/pkgs/spectest/src/runner/verify_signatures_runner.zig +++ b/pkgs/spectest/src/runner/verify_signatures_runner.zig @@ -42,6 +42,8 @@ pub fn baseRelRoot(comptime spec_fork: Fork) []const u8 { const types = @import("@zeam/types"); const state_transition = @import("@zeam/state-transition"); +const ssz = @import("ssz"); +const xmss = @import("@zeam/xmss"); // Signature structure constants from leansig // path: 8 siblings, each is 8 u32 = 256 bytes @@ -201,6 +203,38 @@ fn runCase( const expect_failure = std.mem.indexOf(u8, ctx.fixture_label, "invalid") != null or std.mem.indexOf(u8, ctx.case_name, "invalid") != null; + // Debug: print signature info + const sig = &signed_block.signature.proposer_signature; + std.debug.print("fixture {s}: signature first 32 bytes: {x}\n", .{ ctx.fixture_label, sig[0..32].* }); + std.debug.print("fixture {s}: signature last 32 bytes: {x}\n", .{ ctx.fixture_label, sig[sig.len - 32 ..].* }); + + // Debug: print proposer attestation data and computed message hash + const proposer_att = signed_block.message.proposer_attestation; + std.debug.print("fixture {s}: proposer_attestation.validator_id: {d}\n", .{ ctx.fixture_label, proposer_att.validator_id }); + std.debug.print("fixture {s}: proposer_attestation.data.slot: {d}\n", .{ ctx.fixture_label, proposer_att.data.slot }); + std.debug.print("fixture {s}: proposer_attestation.data.head.root: {x}\n", .{ ctx.fixture_label, proposer_att.data.head.root }); + std.debug.print("fixture {s}: proposer_attestation.data.head.slot: {d}\n", .{ ctx.fixture_label, proposer_att.data.head.slot }); + + // Compute message hash for debugging + var debug_message: [32]u8 = undefined; + ssz.hashTreeRoot(types.AttestationData, proposer_att.data, &debug_message, allocator) catch |err| { + std.debug.print("fixture {s}: hashTreeRoot failed: {s}\n", .{ ctx.fixture_label, @errorName(err) }); + }; + std.debug.print("fixture {s}: computed message hash: {x}\n", .{ ctx.fixture_label, debug_message }); + + // Debug: print pubkey + const validators = anchor_state.validators.constSlice(); + if (proposer_att.validator_id < validators.len) { + const pubkey = validators[proposer_att.validator_id].getPubkey(); + std.debug.print("fixture {s}: pubkey first 20 bytes: {x}\n", .{ ctx.fixture_label, pubkey[0..20].* }); + std.debug.print("fixture {s}: pubkey all 52 bytes: {x}\n", .{ ctx.fixture_label, pubkey[0..52].* }); + } + + // Debug: print signature details + std.debug.print("fixture {s}: sig offset_path (bytes 0-3): {x}\n", .{ ctx.fixture_label, sig[0..4].* }); + std.debug.print("fixture {s}: sig rho (bytes 4-31): {x}\n", .{ ctx.fixture_label, sig[4..32].* }); + std.debug.print("fixture {s}: sig offset_hashes (bytes 32-35): {x}\n", .{ ctx.fixture_label, sig[32..36].* }); + // Verify signatures const verify_result = state_transition.verifySignatures(allocator, &anchor_state, &signed_block); @@ -469,116 +503,43 @@ fn parseSignature( obj: std.json.ObjectMap, field_name: []const u8, ) FixtureError!types.SIGBYTES { - const sig_obj = try expect.expectObject(FixtureError, obj, &.{field_name}, ctx, field_name); - - // SSZ Container serialization for Signature: - // Signature = Container(path: HashTreeOpening, rho: Randomness, hashes: HashDigestList) - // - // Fixed part (36 bytes): - // - offset_path: 4 bytes (offset to path data) - // - rho_data: 28 bytes (7 Fp values, each 4 bytes - fixed size Vector) - // - offset_hashes: 4 bytes (offset to hashes data) - // - // Variable part: - // - path data (HashTreeOpening serialized) - // - hashes data (HashDigestList serialized) - // - // HashTreeOpening = Container(siblings: HashDigestList) - // Fixed part: 4 bytes (offset to siblings data) - // Variable part: siblings data - // - // HashDigestList = List[HashDigestVector, NODE_LIST_LIMIT] - // HashDigestVector = Vector[Fp, 8] = 32 bytes (8 * 4 bytes each) - // Since HashDigestVector is fixed size, the list is serialized as: - // - concatenated fixed-size items (no offsets needed) - - var sig_bytes: types.SIGBYTES = std.mem.zeroes(types.SIGBYTES); - - // Parse path siblings - const path_obj = try expect.expectObject(FixtureError, sig_obj, &.{"path"}, ctx, "path"); - const siblings_obj = try expect.expectObject(FixtureError, path_obj, &.{"siblings"}, ctx, "path.siblings"); - const siblings_data = try expect.expectArrayField(FixtureError, siblings_obj, &.{"data"}, ctx, "path.siblings.data"); - - // Parse rho (fixed size: 7 * 4 = 28 bytes) - const rho_obj = try expect.expectObject(FixtureError, sig_obj, &.{"rho"}, ctx, "rho"); - const rho_arr = try parseU32Array7(ctx, rho_obj, "rho.data"); - - // Parse hashes - const hashes_obj = try expect.expectObject(FixtureError, sig_obj, &.{"hashes"}, ctx, "hashes"); - const hashes_data = try expect.expectArrayField(FixtureError, hashes_obj, &.{"data"}, ctx, "hashes.data"); - - // Calculate sizes for SSZ serialization - const num_siblings = siblings_data.items.len; - const num_hashes = hashes_data.items.len; - - // Each HashDigestVector = 8 Fp * 4 bytes = 32 bytes - const sibling_size: usize = 8 * 4; // 32 bytes per sibling - const hash_size: usize = 8 * 4; // 32 bytes per hash - - // HashTreeOpening SSZ: - // Fixed part: 4 bytes (offset to siblings) - // Variable part: siblings data (concatenated HashDigestVectors) - const path_fixed_part: usize = 4; - const path_variable_size = num_siblings * sibling_size; - const path_total_size = path_fixed_part + path_variable_size; - - // HashDigestList SSZ (for hashes): just concatenated fixed-size items - const hashes_size = num_hashes * hash_size; - - // Signature fixed part: offset_path (4) + rho (28) + offset_hashes (4) = 36 bytes - const sig_fixed_part: usize = 36; - - // Calculate offsets - const offset_path: u32 = @intCast(sig_fixed_part); // path starts after fixed part - const offset_hashes: u32 = @intCast(sig_fixed_part + path_total_size); // hashes start after path - - var write_pos: usize = 0; + const sig_value = obj.get(field_name) orelse { + std.debug.print( + "fixture {s} case {s}: missing field {s}\n", + .{ ctx.fixture_label, ctx.case_name, field_name }, + ); + return FixtureError.InvalidFixture; + }; - // Write Signature fixed part: - // 1. offset_path (4 bytes) - std.mem.writeInt(u32, sig_bytes[write_pos..][0..4], offset_path, .little); - write_pos += 4; + // Re-serialize just the signature object and let Rust parse/SSZ-encode it. + var json_buf = std.ArrayList(u8).init(std.heap.page_allocator); + defer json_buf.deinit(); - // 2. rho data (28 bytes - fixed size) - for (rho_arr) |val| { - std.mem.writeInt(u32, sig_bytes[write_pos..][0..4], val, .little); - write_pos += 4; - } + std.json.stringify(sig_value, .{}, json_buf.writer()) catch |err| { + std.debug.print( + "fixture {s} case {s}: failed to stringify signature JSON: {s}\n", + .{ ctx.fixture_label, ctx.case_name, @errorName(err) }, + ); + return FixtureError.InvalidFixture; + }; - // 3. offset_hashes (4 bytes) - std.mem.writeInt(u32, sig_bytes[write_pos..][0..4], offset_hashes, .little); - write_pos += 4; - - // Now write_pos should be at sig_fixed_part (36) - - // Write path (HashTreeOpening): - // Fixed part: offset to siblings (4 bytes, pointing to byte 4 within path) - const path_siblings_offset: u32 = 4; // siblings data starts at offset 4 within HashTreeOpening - std.mem.writeInt(u32, sig_bytes[write_pos..][0..4], path_siblings_offset, .little); - write_pos += 4; - - // Variable part: siblings data - for (siblings_data.items) |sibling_val| { - const sibling_obj = try expect.expectObjectValue(FixtureError, sibling_val, ctx, "sibling"); - const u32_arr = try parseU32Array8(ctx, sibling_obj, "sibling.data"); - for (u32_arr) |val| { - std.mem.writeInt(u32, sig_bytes[write_pos..][0..4], val, .little); - write_pos += 4; - } - } + var sig_bytes: types.SIGBYTES = std.mem.zeroes(types.SIGBYTES); + const written = xmss.signatureSszFromJson(json_buf.items, sig_bytes[0..]) catch { + std.debug.print( + "fixture {s} case {s}: Rust JSON→SSZ conversion failed\n", + .{ ctx.fixture_label, ctx.case_name }, + ); + return FixtureError.InvalidFixture; + }; - // Write hashes (HashDigestList - just concatenated items since they're fixed size) - for (hashes_data.items) |hash_val| { - const hash_obj = try expect.expectObjectValue(FixtureError, hash_val, ctx, "hash"); - const u32_arr = try parseU32Array8(ctx, hash_obj, "hash.data"); - for (u32_arr) |val| { - std.mem.writeInt(u32, sig_bytes[write_pos..][0..4], val, .little); - write_pos += 4; - } + if (written > sig_bytes.len) { + std.debug.print( + "fixture {s} case {s}: Rust JSON→SSZ wrote {d} bytes, max {d}\n", + .{ ctx.fixture_label, ctx.case_name, written, sig_bytes.len }, + ); + return FixtureError.InvalidFixture; } - _ = hashes_size; // Used in offset_hashes calculation - return sig_bytes; } diff --git a/pkgs/xmss/src/hashsig.zig b/pkgs/xmss/src/hashsig.zig index a490d103c..9344357b1 100644 --- a/pkgs/xmss/src/hashsig.zig +++ b/pkgs/xmss/src/hashsig.zig @@ -106,6 +106,15 @@ extern fn hashsig_verify_ssz( signature_len: usize, ) i32; +/// Convert signature JSON (proposerSignature object) into SSZ bytes. +/// Returns number of bytes written, or 0 on error. +extern fn hashsig_signature_ssz_from_json( + signature_json_ptr: [*]const u8, + signature_json_len: usize, + out_ptr: [*]u8, + out_len: usize, +) usize; + pub const HashSigError = error{ KeyGenerationFailed, SigningFailed, VerificationFailed, InvalidSignature, SerializationFailed, InvalidMessageLength, DeserializationFailed, OutOfMemory }; /// Verify signature using SSZ-encoded bytes @@ -136,6 +145,21 @@ pub fn verifySsz( } } +/// Fill `out` with SSZ signature bytes parsed from a signature JSON object. +pub fn signatureSszFromJson(signature_json: []const u8, out: []u8) HashSigError!usize { + const written = hashsig_signature_ssz_from_json( + signature_json.ptr, + signature_json.len, + out.ptr, + out.len, + ); + + if (written == 0) { + return HashSigError.DeserializationFailed; + } + return written; +} + /// Wrapper for the hash signature key pair pub const KeyPair = struct { handle: *HashSigKeyPair, diff --git a/rust/Cargo.lock b/rust/Cargo.lock index cc36911f3..82946416e 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2047,18 +2047,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "ethereum_ssz_derive" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78d247bc40823c365a62e572441a8f8b12df03f171713f06bc76180fcd56ab71" -dependencies = [ - "darling 0.20.11", - "proc-macro2", - "quote", - "syn 2.0.111", -] - [[package]] name = "event-listener" version = "5.4.1" @@ -2659,7 +2647,11 @@ name = "hashsig-glue" version = "0.1.0" dependencies = [ "ethereum_ssz", - "leansig 0.1.0 (git+https://github.com/leanEthereum/leanSig?rev=f10dcbefac2502d356d93f686e8b4ecd8dc8840a)", + "hex", + "leansig 0.1.0 (git+https://github.com/leanEthereum/leanSig?rev=b621826f948ffc133dd893131aac2c7efa7f90e0)", + "p3-field 0.3.0 (git+https://github.com/Plonky3/Plonky3.git?rev=a33a312)", + "p3-koala-bear 0.3.0 (git+https://github.com/Plonky3/Plonky3.git?rev=a33a312)", + "p3-symmetric 0.3.0 (git+https://github.com/Plonky3/Plonky3.git?rev=a33a312)", "rand 0.9.2", "rand_chacha 0.9.0", "serde", @@ -3364,11 +3356,10 @@ dependencies = [ [[package]] name = "leansig" version = "0.1.0" -source = "git+https://github.com/leanEthereum/leanSig?rev=f10dcbefac2502d356d93f686e8b4ecd8dc8840a#f10dcbefac2502d356d93f686e8b4ecd8dc8840a" +source = "git+https://github.com/leanEthereum/leanSig?rev=b621826f948ffc133dd893131aac2c7efa7f90e0#b621826f948ffc133dd893131aac2c7efa7f90e0" dependencies = [ "dashmap", "ethereum_ssz", - "ethereum_ssz_derive", "num-bigint 0.4.6", "num-traits", "p3-baby-bear 0.3.0 (git+https://github.com/Plonky3/Plonky3.git?rev=a33a312)", diff --git a/rust/hashsig-glue/Cargo.toml b/rust/hashsig-glue/Cargo.toml index 8cc235c5f..0c68acf99 100644 --- a/rust/hashsig-glue/Cargo.toml +++ b/rust/hashsig-glue/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" [dependencies] sha2 = "0.9" -leansig = { git = "https://github.com/leanEthereum/leanSig", rev = "f10dcbefac2502d356d93f686e8b4ecd8dc8840a" } +leansig = { git = "https://github.com/leanEthereum/leanSig", rev = "b621826f948ffc133dd893131aac2c7efa7f90e0" } rand = "0.9.2" rand_chacha = "0.9.0" thiserror = "2.0.17" @@ -16,3 +16,9 @@ serde_json = "1.0" [lib] crate-type = ["staticlib"] name = "hashsig_glue" + +[dev-dependencies] +hex = "0.4.3" +p3-field = { git = "https://github.com/Plonky3/Plonky3.git", rev = "a33a312" } +p3-koala-bear = { git = "https://github.com/Plonky3/Plonky3.git", rev = "a33a312" } +p3-symmetric = { git = "https://github.com/Plonky3/Plonky3.git", rev = "a33a312" } diff --git a/rust/hashsig-glue/src/lib.rs b/rust/hashsig-glue/src/lib.rs index 2e40c07ab..fa404a1dd 100644 --- a/rust/hashsig-glue/src/lib.rs +++ b/rust/hashsig-glue/src/lib.rs @@ -7,6 +7,7 @@ use std::ffi::CStr; use std::os::raw::c_char; use std::ptr; use std::slice; +use serde_json::Value; pub type HashSigScheme = leansig::signature::generalized_xmss::instantiations_poseidon_top_level::lifetime_2_to_the_32::hashing_optimized::SIGTopLevelTargetSumLifetime32Dim64Base8; @@ -491,18 +492,130 @@ pub unsafe extern "C" fn hashsig_verify_ssz( Err(_) => return -1, }; + // Debug: print first 36 bytes of signature + eprintln!("[hashsig_verify_ssz] pubkey_len={}, sig_len={}, epoch={}", pubkey_len, signature_len, epoch); + eprintln!("[hashsig_verify_ssz] sig first 36 bytes: {:02x?}", &sig_data[..36.min(sig_data.len())]); + eprintln!("[hashsig_verify_ssz] message: {:02x?}", message_array); + + let mut hasher = Sha256::new(); + hasher.update(pk_data); + let pk_sha256 = hasher.finalize_reset(); + hasher.update(sig_data); + let sig_sha256 = hasher.finalize(); + eprintln!("[hashsig_verify_ssz] pubkey sha256: {:02x}", pk_sha256); + eprintln!("[hashsig_verify_ssz] signature sha256: {:02x}", sig_sha256); + // Directly SSZ decode (leansig has SSZ support built-in) let pk: HashSigPublicKey = match HashSigPublicKey::from_ssz_bytes(pk_data) { Ok(pk) => pk, - Err(_) => return -1, + Err(e) => { + eprintln!("[hashsig_verify_ssz] pubkey decode error: {:?}", e); + return -1; + } }; let sig: HashSigSignature = match HashSigSignature::from_ssz_bytes(sig_data) { Ok(sig) => sig, - Err(_) => return -1, + Err(e) => { + eprintln!("[hashsig_verify_ssz] signature decode error: {:?}", e); + return -1; + } }; + // SSZ round-trip checks: if these fail, the input bytes are not what leansig + // would produce for the decoded structures (often indicates a layout mismatch). + let pk_roundtrip = pk.as_ssz_bytes(); + if pk_roundtrip.as_slice() != pk_data { + let min_len = pk_roundtrip.len().min(pk_data.len()); + let mut mismatch_at: Option = None; + for i in 0..min_len { + if pk_roundtrip[i] != pk_data[i] { + mismatch_at = Some(i); + break; + } + } + eprintln!( + "[hashsig_verify_ssz] pubkey SSZ roundtrip mismatch: in_len={}, out_len={}, first_mismatch={:?}", + pk_data.len(), + pk_roundtrip.len(), + mismatch_at + ); + } + + let sig_roundtrip = sig.as_ssz_bytes(); + if sig_roundtrip.as_slice() != sig_data { + let min_len = sig_roundtrip.len().min(sig_data.len()); + let mut mismatch_at: Option = None; + for i in 0..min_len { + if sig_roundtrip[i] != sig_data[i] { + mismatch_at = Some(i); + break; + } + } + eprintln!( + "[hashsig_verify_ssz] signature SSZ roundtrip mismatch: in_len={}, out_len={}, first_mismatch={:?}", + sig_data.len(), + sig_roundtrip.len(), + mismatch_at + ); + } + + // Debug: verify SSZ roundtrips. If this fails, we're not verifying the same + // structured data that Python/Zig expect. + let pk_roundtrip = pk.as_ssz_bytes(); + if pk_roundtrip != pk_data { + let mut hasher = Sha256::new(); + hasher.update(pk_data); + let pk_in_hash = hasher.finalize_reset(); + hasher.update(&pk_roundtrip); + let pk_rt_hash = hasher.finalize(); + + let mismatch_idx = pk_data + .iter() + .zip(pk_roundtrip.iter()) + .position(|(a, b)| a != b) + .unwrap_or(0); + + eprintln!( + "[hashsig_verify_ssz] pubkey SSZ roundtrip mismatch: in_len={}, rt_len={}, first_mismatch_at={}, in_sha256={:02x}, rt_sha256={:02x}", + pk_data.len(), + pk_roundtrip.len(), + mismatch_idx, + pk_in_hash, + pk_rt_hash + ); + } else { + eprintln!("[hashsig_verify_ssz] pubkey SSZ roundtrip OK"); + } + + let sig_roundtrip = sig.as_ssz_bytes(); + if sig_roundtrip != sig_data { + let mut hasher = Sha256::new(); + hasher.update(sig_data); + let sig_in_hash = hasher.finalize_reset(); + hasher.update(&sig_roundtrip); + let sig_rt_hash = hasher.finalize(); + + let mismatch_idx = sig_data + .iter() + .zip(sig_roundtrip.iter()) + .position(|(a, b)| a != b) + .unwrap_or(0); + + eprintln!( + "[hashsig_verify_ssz] signature SSZ roundtrip mismatch: in_len={}, rt_len={}, first_mismatch_at={}, in_sha256={:02x}, rt_sha256={:02x}", + sig_data.len(), + sig_roundtrip.len(), + mismatch_idx, + sig_in_hash, + sig_rt_hash + ); + } else { + eprintln!("[hashsig_verify_ssz] signature SSZ roundtrip OK"); + } + let is_valid = ::verify(&pk, epoch, message_array, &sig); + eprintln!("[hashsig_verify_ssz] verify result: {}", is_valid); if is_valid { 1 @@ -511,3 +624,354 @@ pub unsafe extern "C" fn hashsig_verify_ssz( } } } + +fn json_u32(value: &Value) -> Option { + match value { + Value::Number(n) => n.as_u64().and_then(|v| u32::try_from(v).ok()), + _ => None, + } +} + +fn json_get<'a>(obj: &'a Value, key: &str) -> Option<&'a Value> { + match obj { + Value::Object(map) => map.get(key), + _ => None, + } +} + +fn parse_u32_fixed_array(value: &Value, expected_len: usize) -> Option> { + let arr = match value { + Value::Array(a) => a, + _ => return None, + }; + if arr.len() != expected_len { + return None; + } + let mut out = Vec::with_capacity(expected_len); + for v in arr { + out.push(json_u32(v)?); + } + Some(out) +} + +fn parse_vec_of_u32x8(value: &Value) -> Option> { + let arr = match value { + Value::Array(a) => a, + _ => return None, + }; + + let mut out: Vec<[u32; 8]> = Vec::with_capacity(arr.len()); + for item in arr { + let data = json_get(item, "data")?; + let nums = parse_u32_fixed_array(data, 8)?; + let mut fixed = [0u32; 8]; + fixed.copy_from_slice(&nums); + out.push(fixed); + } + Some(out) +} + +fn write_u32_le(dst: &mut [u8], offset: usize, v: u32) -> Option<()> { + let end = offset.checked_add(4)?; + dst.get_mut(offset..end)?.copy_from_slice(&v.to_le_bytes()); + Some(()) +} + +/// Convert a signature JSON object into SSZ-encoded signature bytes. +/// +/// Expected JSON shape (object): +/// { "path": {"siblings": {"data": [ {"data": [u32;8]}, ... ]}}, +/// "rho": {"data": [u32;7]}, +/// "hashes": {"data": [ {"data": [u32;8]}, ... ]} } +/// +/// Returns number of bytes written, or 0 on error. +#[no_mangle] +pub unsafe extern "C" fn hashsig_signature_ssz_from_json( + signature_json_ptr: *const u8, + signature_json_len: usize, + out_ptr: *mut u8, + out_len: usize, +) -> usize { + if signature_json_ptr.is_null() || out_ptr.is_null() { + return 0; + } + + let json_bytes = unsafe { slice::from_raw_parts(signature_json_ptr, signature_json_len) }; + let out = unsafe { slice::from_raw_parts_mut(out_ptr, out_len) }; + + let sig_val: Value = match serde_json::from_slice(json_bytes) { + Ok(v) => v, + Err(_) => return 0, + }; + + // Extract siblings + let path = match json_get(&sig_val, "path") { + Some(v) => v, + None => return 0, + }; + let siblings = match json_get(path, "siblings").and_then(|v| json_get(v, "data")) { + Some(v) => v, + None => return 0, + }; + let siblings_vec = match parse_vec_of_u32x8(siblings) { + Some(v) => v, + None => return 0, + }; + + // Extract rho + let rho = match json_get(&sig_val, "rho").and_then(|v| json_get(v, "data")) { + Some(v) => v, + None => return 0, + }; + let rho_vec = match parse_u32_fixed_array(rho, 7) { + Some(v) => v, + None => return 0, + }; + + // Extract hashes + let hashes = match json_get(&sig_val, "hashes").and_then(|v| json_get(v, "data")) { + Some(v) => v, + None => return 0, + }; + let hashes_vec = match parse_vec_of_u32x8(hashes) { + Some(v) => v, + None => return 0, + }; + + let sibling_size: usize = 8 * 4; + let hash_size: usize = 8 * 4; + let path_fixed_part: usize = 4; + let sig_fixed_part: usize = 36; + + let path_variable_size = siblings_vec.len().checked_mul(sibling_size).unwrap_or(usize::MAX); + if path_variable_size == usize::MAX { + return 0; + } + let path_total_size = match path_fixed_part.checked_add(path_variable_size) { + Some(v) => v, + None => return 0, + }; + + let hashes_size = hashes_vec.len().checked_mul(hash_size).unwrap_or(usize::MAX); + if hashes_size == usize::MAX { + return 0; + } + + let total_size = match sig_fixed_part.checked_add(path_total_size).and_then(|v| v.checked_add(hashes_size)) { + Some(v) => v, + None => return 0, + }; + + if total_size > out_len { + return 0; + } + out[..total_size].fill(0); + + let offset_path: u32 = match u32::try_from(sig_fixed_part) { + Ok(v) => v, + Err(_) => return 0, + }; + let offset_hashes_u = match sig_fixed_part.checked_add(path_total_size) { + Some(v) => v, + None => return 0, + }; + let offset_hashes: u32 = match u32::try_from(offset_hashes_u) { + Ok(v) => v, + Err(_) => return 0, + }; + + // Signature fixed part + let mut write_pos: usize = 0; + if write_u32_le(out, write_pos, offset_path).is_none() { + return 0; + } + write_pos += 4; + for v in rho_vec { + if write_u32_le(out, write_pos, v).is_none() { + return 0; + } + write_pos += 4; + } + if write_u32_le(out, write_pos, offset_hashes).is_none() { + return 0; + } + write_pos += 4; + + // Path (HashTreeOpening) + let path_siblings_offset: u32 = 4; + if write_u32_le(out, write_pos, path_siblings_offset).is_none() { + return 0; + } + write_pos += 4; + + for sib in siblings_vec { + for v in sib { + if write_u32_le(out, write_pos, v).is_none() { + return 0; + } + write_pos += 4; + } + } + + // Hashes list + for h in hashes_vec { + for v in h { + if write_u32_le(out, write_pos, v).is_none() { + return 0; + } + write_pos += 4; + } + } + + if write_pos != total_size { + return 0; + } + + total_size +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_round_trip_sign_verify_ssz() { + // Generate key pair + let mut rng = ChaCha20Rng::seed_from_u64(12345); + let activation_epoch = 0; + let num_active_epochs = 10; + + let (pk, sk) = ::key_gen(&mut rng, activation_epoch, num_active_epochs); + + // Sign a message at epoch 1 + let message: [u8; 32] = [ + 0x96, 0xfd, 0x6f, 0x2c, 0x91, 0x00, 0x83, 0x2c, + 0xdd, 0xdd, 0x6e, 0x06, 0xce, 0x9c, 0x7d, 0x62, + 0x91, 0x52, 0x71, 0x6a, 0xaa, 0x98, 0x21, 0xa4, + 0xfb, 0x97, 0x26, 0xdb, 0x01, 0xfe, 0xe3, 0xf2, + ]; + let epoch: u32 = 1; + + let signature = ::sign(&sk, epoch, &message).expect("signing failed"); + + // Verify the signature directly + let is_valid = ::verify(&pk, epoch, &message, &signature); + assert!(is_valid, "Direct verification failed"); + + // Serialize to SSZ + let pk_ssz = pk.as_ssz_bytes(); + let sig_ssz = signature.as_ssz_bytes(); + + println!("pubkey SSZ length: {}", pk_ssz.len()); + println!("signature SSZ length: {}", sig_ssz.len()); + println!("signature SSZ first 36 bytes: {:02x?}", &sig_ssz[..36.min(sig_ssz.len())]); + + // Deserialize from SSZ + let pk2 = HashSigPublicKey::from_ssz_bytes(&pk_ssz).expect("pubkey SSZ decode failed"); + let sig2 = HashSigSignature::from_ssz_bytes(&sig_ssz).expect("signature SSZ decode failed"); + + // Verify with deserialized values + let is_valid2 = ::verify(&pk2, epoch, &message, &sig2); + assert!(is_valid2, "SSZ round-trip verification failed"); + + println!("Round-trip test passed!"); + } + + #[test] + fn test_verify_fixture_signature() { + // This is the exact fixture data from test_proposer_signature + // pubkey from fixture (52 bytes) - validator index 1 + let pubkey_bytes: [u8; 52] = [ + 0x8c, 0x73, 0xc3, 0x73, 0xd9, 0x84, 0x68, 0x2c, + 0x35, 0x91, 0x91, 0x47, 0x80, 0x6c, 0x6d, 0x39, + 0x19, 0x42, 0x03, 0x6c, 0x2b, 0xd8, 0xf3, 0x59, + 0xe6, 0x81, 0x53, 0x0e, 0x44, 0x44, 0x3c, 0x15, + 0xfb, 0x9a, 0x15, 0x71, 0xfe, 0x4e, 0x95, 0x7a, + 0xab, 0xc2, 0x0f, 0x5f, 0xbc, 0x67, 0xb8, 0x6c, + 0x8a, 0x16, 0xa2, 0x09, + ]; + + // message hash (32 bytes) - computed from AttestationData + let message: [u8; 32] = [ + 0x96, 0xfd, 0x6f, 0x2c, 0x91, 0x00, 0x83, 0x2c, + 0xdd, 0xdd, 0x6e, 0x06, 0xce, 0x9c, 0x7d, 0x62, + 0x91, 0x52, 0x71, 0x6a, 0xaa, 0x98, 0x21, 0xa4, + 0xfb, 0x97, 0x26, 0xdb, 0x01, 0xfe, 0xe3, 0xf2, + ]; + + let epoch: u32 = 1; + + // Full signature SSZ bytes (3112 bytes) from Python + let sig_ssz_hex = "24000000455d822a9938490a99373d435411556609dc7e4ebf86874bb500153d2804000004000000f7af5070abf022606aff6543ccb88f5b77a47a49203a2d5dbc04751b8088ef3110fe3d55c9b4b0348f20cf1dfb340176964cf23d9665305a5c5f2901803e5444dfe96a196246435baec3f546cb020151da0f9456df86a769f14df97a472b7d482a58086c28c5f24cc93a0b20834f78369e8d873b9239303009021b600ada8f4a6a1de3770f87b4089b217913929d963f7104a65e25cdda68f71677146c3a9f30f770804691f03e5155a00976ce09f228b3613d13f5bec86d4a07c51b0e22c778d071f029095778649ef4532d21fbc90358eb8375c7ba3234b8ed3f0addea5c21e17b390c4dc53b55c21fbc0e57f782243870d3683fbc357c5e4c695042826336210543687f7d5f35c586cc2afd8cca6e5fda3a073ebfac61f618de229ba2e9531c4ac73e8326815d6fab2e03b6ab9b199d5830052b00376771e0e2061fc35045d7f65d2317649117041a381fa6f75b0703e5c07388f74272a436e5190178f46d87185965fcf5f81902d3a90d355dbb535acb7877b2d3dd7dd48fd70dd6c58c51bbb5142e141aef69d2f9860635802a301c2aba382e3d635e3cad98361a436a727e31495c7e085e77e9fb4f3ef84efc0d083b3c4fd34af713e31dc909c4e1de3806fd1b5525de652832ff0379206a6333228e7573ba7f3d47d976ab5d1608b436260c0106c4531b123c1f592b00a34c0e2770ee3e55ef4a0f49745a2d4859b5688dda4d6883e6f024b3ebdf26a0345a515264e13be56a33356d00366052ba8507036a6b406f3682099436f749a24471646f4800021aa7644891c4bd34d5ff6f01df651a6f718a4317ff237d434e49e46c3c6a4c662fc3280e90b999170c0c5d7d5117eb49c9db8b14873df7516d5d0d1a1e81d32ca49fec4f3d02031a77fc621c5ea7537647eb3537a4396b6ce1d44c6ca841a612a93ce05e2bb000793431634460cef8314942940fe830cf4b84ca6e5e2abc210a30b10d42ec3f717daa25651ae5fb7f19b969245c8165a73a58d5800971be1b267dc036236fe2e811c2338e1100ec217e77bf36428392086a4fb3ab2ed0a918616ddcb1377b247f5b46f9cf442ac0a233700b9544149e2c0c6016a96487872049c04580561505ef3110023f2cb6cc90423ee59e22f241c158a69d8c7c32fee33d5615293992d32015aeadd4053eecfb2d1e27104a4583361dff7f7d338528ba6366215e510113913463bde83bfae70e505cb1416375aacb49b04b5c109404f45b65794a294874962518f71407aca3f118fcd7d171f0b49f0e63c55c10efdc713a7eed664fa54d7a6ca8015f4dd77b027d2e9d5c0f4573d24be8dc5f06787e5119e9ac014ab91b3200d923d47175c50e6fd9564f38ea33264dc1126e0c674b065824c6b0630158ae0fcdb772272958e43fc57a037a3abc1c2812e3820b70dd870c8786137c37965e060541e64bba13ce34ebc9071ff7360e23f9388d5b462c4c51972c0102a74e071125e1e46a3da3a552b1f79d2f8586b565a4aa162b9ae13d3dcacb981d278da8485f2a78292f6fee030d5262184260613f5e6c56601949ad4fe14f62377b82e80294f64b1f1f84961ecfc55f6a69cf943049fd5f31bccc2b34701482230511ba09fbd7380b4bac8a37df76170df000075fac9a586822768e11eb88eb3294eff850de72db6bcf605f6dc4c03e52cadc6479d5484727e430770a613c856a4873d866037bf444d2b73626ea3bcb49eafb5954dd1acc2e79a92c505a70cb37529dbc4ba8966d4269718a45d223021aee0b6e2346f51371b64cef342eaf24167ba1e90e25ad3c750a7b2663905a275cb1a9923e2a817d1ebc81760570739775773b4e2d31fb94042839475da2eee07992ffe03e38e0ec3a4b1077543ce70f0519efb306f62997753840035fc9c832715029157ec430e70a311af1092777fa228e87b85458c87d043cb22161a1b05257dfc9f178a2a8826d582a2f5e35ddfd124adff24df025612a685cd4305dedc50bc4a2e54483170f3159ccf128abb485442e9ff84b145a3e0fd683a27859152352b966d82ffaf3df7991ebb124916c26677e6a6112aaa141151a971e47aff3545ee8cd984954ba2507ceda772464bf12722396bc207ca82a2499468252b1e65e4bd2f2ba787b2ac1579f022b12ee6f9c612e378611bf688f3197c5eb67b045e923facf60463d124943329eba045107a931caf77b6dfad2966526bfc317f4edbb3163beed337c89477ca6230e79d858150df505a9468439df5ac29512005e12c94017816c768a8f57181cc5d72be17a9171d818f27c16acc734a5465845639ba6236d1c8e041e9cef56c220fd72d358867be26ef05fb11d866bcba0b26852cc7d5c3f8952136da1ab79d9706465a921036bf0303126ce8f364baabdf8340c25d55937fa980af44d586631c7871e37a7a307f0feee69c4ce77241089dc127b883b694aab6c60723a3f5ec346463c6eb4e5506d062c4fbba8810854bd380ae6b27f71d131bc66c198383f99fc834ddb79f15f0c71bd463b325c02f913e74f72acf25db6df63510b416776ee55f217dd5544632a758c3c2c0a946936efdf6d1478391d25072425da54d341101a0a30ed21893571252f5c3b7af659261d7a3a5afe7b0842f8a45496d81449901f9065cca99e00a1c33c26342a423814e68306d14ba87b827aea602fc18a0dcb25c90e94293e62c0653825ec9c336180575064725fd92c47f28655b7d41e106cd04b432fe3f57de4c66c28a261c06a2966a9214b2b186328bad843bce57a21ff3c0408e895e912f654743a666f7e346365b6499c38037438524138f54e6c12dad7b631fee08d2298656575ed77727460a56010ffc98977117b355216a0e164ca433b43a615ab21f0f2350ad3630f27ae2b6f1893b9684174e7715f2b71534951f3f1029025116c4f981f692dcaae5dda712637cb5dc805f3623a190aa97f68e3c29609e47009072816bc35fca50878aea84434e4b71a4ba7067a34ad637c1806be197af81f973e73d0a21891d0f057df69ee5372b8984578a8f902554530505b8ee4238f355d18dbb5f32ddfa55c6d6abc6a7d810fa267d4f7953313c4424bfa59bb248a7bdc593edbb63f31e8725128e48c71846cd8326ef48b4d986eb12156236412d0e612063bec1b112680b7592e43670e3c052623cf2f390baa322f7938419b498d035941e5af38751ced916cbf35ca31f4f6e76b05f45b2b8d3f865ba5655715a132303571e93d08cb40e954cf01a46aef20870f16bb6c48ff995f3b33e982712faa34695d515b3a250f580f8699bd7bb61add70fb010f4d2c14101648914d3d1b92b250890bb6202c66b64de4d29c6486c5b85e6d2d470dd0a22252197a2442e9b8ea4abe07411010c59e76407d3c217053fa3a3b473b7df06bc47663d7093e9417057e77bdda1d33b1504aac12f2705f608e4862323b12dfb9012d2928a11a80061639fae63c4b39674078cfc58d48ec365979675935303f62b3210582de04314a0b3514f6370cc941cf4b29d4d63a924900382a222758dbaa561c1eb2ba555ba1951a51be49346588d212741c0d5b35ffa615384bf822cbe4496eb5bede106aeb3f5f4448eb4bacb9ff23974705339fdc0725f24d865b3209130d6cd7d43223b7a5483c8e154033bad826a102af29187d3754cae56859777570576e281c5a8cfe7d35c9d1c96f6882243cc6eeb05251b81b22a0e95e60e3e64e3acf1b0b1c24501d2f378d533196bb5320a47c4c61aefa156d8f41422edf9add7953e03b1b88ca67109b66a06c7abbd00a931635674c3a9c6063a68a4f9f670569268061686cad8c1da5c3531a3f43524007fa45149ef61a6adcf932280df4fe44676bb33dd51a882b5e582b0f1f83b45eca9530715efe080c2d872b52ca02b257b8ff02547b7e05433266205408d2131d6963054482c0be3533c8b46829cfe03a29e3f2211da6871de53ef407e17ce42366a1d8247327b9227c2f8c398c9c3c4acefdcd414045043fb8816d38bf1ac050152eca666bef806e896f77410b1d972668bdc05a0eb53f61541cc74b1494fb62bf5ee301800a8560d62b6b28c369714180c8e9178544b3674e2a0471ca77b33f7c2b2722a41d57558c7fd7193ef1fa21ee650606d348ed0e035a8059cb906276bea4db7c97cc8b17879831620935c7527df8137da314c72109da24600c91fb2b5aa721409db66c244c269b3a65a40910fb441b553a43ca4ab4e63b3ffb05402ec974a463779b63425a3f996f04b9526e6987223cfcd2422d9d62b13cc571c5131906503db5a4d9314a1b6f60a0ee741b62f9e50d5e45df695b8080618824100e"; + + let sig_bytes = hex::decode(sig_ssz_hex).expect("Invalid hex"); + assert_eq!(sig_bytes.len(), 3112, "Signature length mismatch"); + + // Decode pubkey + let pk = HashSigPublicKey::from_ssz_bytes(&pubkey_bytes).expect("Pubkey decode failed"); + println!("Pubkey decoded successfully"); + + // Decode signature + let sig = HashSigSignature::from_ssz_bytes(&sig_bytes).expect("Signature decode failed"); + println!("Signature decoded successfully"); + + // Verify SSZ roundtrip + let pk_rt = pk.as_ssz_bytes(); + let sig_rt = sig.as_ssz_bytes(); + assert_eq!(&pubkey_bytes[..], &pk_rt[..], "Pubkey SSZ roundtrip mismatch"); + assert_eq!(&sig_bytes[..], &sig_rt[..], "Signature SSZ roundtrip mismatch"); + println!("SSZ roundtrips OK"); + + // Verify + let is_valid = ::verify(&pk, epoch, &message, &sig); + println!("Verification result: {}", is_valid); + + // This is the key test - should pass if everything is correct + assert!(is_valid, "Signature verification failed!"); + } + + #[test] + fn test_poseidon2_consistency() { + // Test that our Poseidon2 output matches Python's output + // Using test vectors from leanSpec tests/lean_spec/subspecs/poseidon2/test_permutation.py + use p3_koala_bear::{KoalaBear, default_koalabear_poseidon2_16}; + use p3_symmetric::Permutation; + use p3_field::{PrimeCharacteristicRing, PrimeField32}; + + let perm = default_koalabear_poseidon2_16(); + + // Input from leanSpec test_permutation.py INPUT_16: + let input_vals: [u32; 16] = [ + 894848333, 1437655012, 1200606629, 1690012884, + 71131202, 1749206695, 1717947831, 120589055, + 19776022, 42382981, 1831865506, 724844064, + 171220207, 1299207443, 227047920, 1783754913, + ]; + + let mut state: [KoalaBear; 16] = core::array::from_fn(|i| KoalaBear::from_u64(input_vals[i] as u64)); + + println!("Input: {:?}", state.map(|x| x.as_canonical_u32())); + + perm.permute_mut(&mut state); + + let output: Vec = state.iter().map(|x| x.as_canonical_u32()).collect(); + println!("Output: {:?}", output); + + // Expected from leanSpec test_permutation.py EXPECTED_16: + let expected: [u32; 16] = [ + 1934285469, 604889435, 133449501, 1026180808, + 1830659359, 176667110, 1391183747, 351743874, + 1238264085, 1292768839, 2023573270, 1201586780, + 1360691759, 1230682461, 748270449, 651545025, + ]; + + for (i, (got, exp)) in output.iter().zip(expected.iter()).enumerate() { + if got != exp { + println!("Mismatch at index {}: got {}, expected {}", i, got, exp); + } + } + + assert_eq!(output.as_slice(), &expected[..], "Poseidon2 output mismatch with leanSpec vectors!"); + println!("Poseidon2 consistency test passed!"); + } +} From a27a789bb6974b3f7c8d0695fdcd0c6699f9c247 Mon Sep 17 00:00:00 2001 From: Chen Kai <281165273grape@gmail.com> Date: Sat, 3 Jan 2026 20:52:43 +0800 Subject: [PATCH 03/19] fix: fix signature test Signed-off-by: Chen Kai <281165273grape@gmail.com> --- .../src/runner/verify_signatures_runner.zig | 51 +-- pkgs/state-transition/src/lib.zig | 1 + pkgs/state-transition/src/transition.zig | 44 ++- rust/Cargo.lock | 4 - rust/hashsig-glue/Cargo.toml | 6 +- rust/hashsig-glue/src/lib.rs | 337 ++++-------------- 6 files changed, 125 insertions(+), 318 deletions(-) diff --git a/pkgs/spectest/src/runner/verify_signatures_runner.zig b/pkgs/spectest/src/runner/verify_signatures_runner.zig index 0cc4cfe64..99f21a036 100644 --- a/pkgs/spectest/src/runner/verify_signatures_runner.zig +++ b/pkgs/spectest/src/runner/verify_signatures_runner.zig @@ -45,6 +45,9 @@ const state_transition = @import("@zeam/state-transition"); const ssz = @import("ssz"); const xmss = @import("@zeam/xmss"); +const DEFAULT_SIGNATURE_SSZ_LEN: usize = types.SIGSIZE; +const TEST_SIGNATURE_SSZ_LEN: usize = 424; + // Signature structure constants from leansig // path: 8 siblings, each is 8 u32 = 256 bytes // rho: 7 u32 = 28 bytes @@ -181,6 +184,15 @@ fn runCase( }, }; + const signature_ssz_len: usize = blk: { + const lean_env_val = case_obj.get("leanEnv") orelse break :blk DEFAULT_SIGNATURE_SSZ_LEN; + const lean_env = switch (lean_env_val) { + .string => |s| s, + else => break :blk DEFAULT_SIGNATURE_SSZ_LEN, + }; + break :blk if (std.mem.eql(u8, lean_env, "test")) TEST_SIGNATURE_SSZ_LEN else DEFAULT_SIGNATURE_SSZ_LEN; + }; + // Parse the anchorState to get validators const anchor_state_value = case_obj.get("anchorState") orelse { std.debug.print("fixture {s} case {s}: missing anchorState\n", .{ ctx.fixture_label, ctx.case_name }); @@ -203,40 +215,13 @@ fn runCase( const expect_failure = std.mem.indexOf(u8, ctx.fixture_label, "invalid") != null or std.mem.indexOf(u8, ctx.case_name, "invalid") != null; - // Debug: print signature info - const sig = &signed_block.signature.proposer_signature; - std.debug.print("fixture {s}: signature first 32 bytes: {x}\n", .{ ctx.fixture_label, sig[0..32].* }); - std.debug.print("fixture {s}: signature last 32 bytes: {x}\n", .{ ctx.fixture_label, sig[sig.len - 32 ..].* }); - - // Debug: print proposer attestation data and computed message hash - const proposer_att = signed_block.message.proposer_attestation; - std.debug.print("fixture {s}: proposer_attestation.validator_id: {d}\n", .{ ctx.fixture_label, proposer_att.validator_id }); - std.debug.print("fixture {s}: proposer_attestation.data.slot: {d}\n", .{ ctx.fixture_label, proposer_att.data.slot }); - std.debug.print("fixture {s}: proposer_attestation.data.head.root: {x}\n", .{ ctx.fixture_label, proposer_att.data.head.root }); - std.debug.print("fixture {s}: proposer_attestation.data.head.slot: {d}\n", .{ ctx.fixture_label, proposer_att.data.head.slot }); - - // Compute message hash for debugging - var debug_message: [32]u8 = undefined; - ssz.hashTreeRoot(types.AttestationData, proposer_att.data, &debug_message, allocator) catch |err| { - std.debug.print("fixture {s}: hashTreeRoot failed: {s}\n", .{ ctx.fixture_label, @errorName(err) }); - }; - std.debug.print("fixture {s}: computed message hash: {x}\n", .{ ctx.fixture_label, debug_message }); - - // Debug: print pubkey - const validators = anchor_state.validators.constSlice(); - if (proposer_att.validator_id < validators.len) { - const pubkey = validators[proposer_att.validator_id].getPubkey(); - std.debug.print("fixture {s}: pubkey first 20 bytes: {x}\n", .{ ctx.fixture_label, pubkey[0..20].* }); - std.debug.print("fixture {s}: pubkey all 52 bytes: {x}\n", .{ ctx.fixture_label, pubkey[0..52].* }); - } - - // Debug: print signature details - std.debug.print("fixture {s}: sig offset_path (bytes 0-3): {x}\n", .{ ctx.fixture_label, sig[0..4].* }); - std.debug.print("fixture {s}: sig rho (bytes 4-31): {x}\n", .{ ctx.fixture_label, sig[4..32].* }); - std.debug.print("fixture {s}: sig offset_hashes (bytes 32-35): {x}\n", .{ ctx.fixture_label, sig[32..36].* }); - // Verify signatures - const verify_result = state_transition.verifySignatures(allocator, &anchor_state, &signed_block); + const verify_result = state_transition.verifySignaturesWithSignatureLen( + allocator, + &anchor_state, + &signed_block, + signature_ssz_len, + ); if (expect_failure) { if (verify_result) |_| { diff --git a/pkgs/state-transition/src/lib.zig b/pkgs/state-transition/src/lib.zig index e80022259..1fcf601cf 100644 --- a/pkgs/state-transition/src/lib.zig +++ b/pkgs/state-transition/src/lib.zig @@ -12,6 +12,7 @@ pub const apply_raw_block = transition.apply_raw_block; pub const StateTransitionError = transition.StateTransitionError; pub const StateTransitionOpts = transition.StateTransitionOpts; pub const verifySignatures = transition.verifySignatures; +pub const verifySignaturesWithSignatureLen = transition.verifySignaturesWithSignatureLen; pub const verifySingleAttestation = transition.verifySingleAttestation; const mockImport = @import("./mock.zig"); diff --git a/pkgs/state-transition/src/transition.zig b/pkgs/state-transition/src/transition.zig index d2bf061c4..d2a585af7 100644 --- a/pkgs/state-transition/src/transition.zig +++ b/pkgs/state-transition/src/transition.zig @@ -2,7 +2,6 @@ const ssz = @import("ssz"); const std = @import("std"); const json = std.json; const types = @import("@zeam/types"); -const utils = types.utils; const params = @import("@zeam/params"); const zeam_utils = @import("@zeam/utils"); @@ -60,6 +59,20 @@ pub fn verifySignatures( allocator: Allocator, state: *const types.BeamState, signed_block: *const types.SignedBlockWithAttestation, +) !void { + return verifySignaturesWithSignatureLen( + allocator, + state, + signed_block, + types.SIGSIZE, + ); +} + +pub fn verifySignaturesWithSignatureLen( + allocator: Allocator, + state: *const types.BeamState, + signed_block: *const types.SignedBlockWithAttestation, + signature_ssz_len: usize, ) !void { const attestations = signed_block.message.block.body.attestations.constSlice(); const signature_proofs = signed_block.signature.attestation_signatures.constSlice(); @@ -129,22 +142,28 @@ pub fn verifySignatures( // Verify proposer signature (still individual) const proposer_attestation = signed_block.message.proposer_attestation; - try verifySingleAttestation( + try verifySingleAttestationWithSignatureLen( allocator, state, @intCast(proposer_attestation.validator_id), &proposer_attestation.data, &signed_block.signature.proposer_signature, + signature_ssz_len, ); } -pub fn verifySingleAttestation( +fn verifySingleAttestationWithSignatureLen( allocator: Allocator, state: *const types.BeamState, validator_index: usize, attestation_data: *const types.AttestationData, signatureBytes: *const types.SIGBYTES, + signature_ssz_len: usize, ) !void { + if (signature_ssz_len > signatureBytes.len) { + return StateTransitionError.InvalidBlockSignatures; + } + const validatorIndex = validator_index; const validators = state.validators.constSlice(); if (validatorIndex >= validators.len) { @@ -160,10 +179,27 @@ pub fn verifySingleAttestation( const epoch: u32 = @intCast(attestation_data.slot); - try xmss.verifySsz(pubkey, &message, epoch, signatureBytes); + try xmss.verifySsz(pubkey, &message, epoch, signatureBytes.*[0..signature_ssz_len]); _ = verification_timer.observe(); } +pub fn verifySingleAttestation( + allocator: Allocator, + state: *const types.BeamState, + validator_index: usize, + attestation_data: *const types.AttestationData, + signatureBytes: *const types.SIGBYTES, +) !void { + return verifySingleAttestationWithSignatureLen( + allocator, + state, + validator_index, + attestation_data, + signatureBytes, + signatureBytes.len, + ); +} + // TODO(gballet) check if beam block needs to be a pointer pub fn apply_transition(allocator: Allocator, state: *types.BeamState, block: types.BeamBlock, opts: StateTransitionOpts) !void { opts.logger.debug("applying state transition state-slot={d} block-slot={d}\n", .{ state.slot, block.slot }); diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 82946416e..f1724e9a9 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2647,11 +2647,7 @@ name = "hashsig-glue" version = "0.1.0" dependencies = [ "ethereum_ssz", - "hex", "leansig 0.1.0 (git+https://github.com/leanEthereum/leanSig?rev=b621826f948ffc133dd893131aac2c7efa7f90e0)", - "p3-field 0.3.0 (git+https://github.com/Plonky3/Plonky3.git?rev=a33a312)", - "p3-koala-bear 0.3.0 (git+https://github.com/Plonky3/Plonky3.git?rev=a33a312)", - "p3-symmetric 0.3.0 (git+https://github.com/Plonky3/Plonky3.git?rev=a33a312)", "rand 0.9.2", "rand_chacha 0.9.0", "serde", diff --git a/rust/hashsig-glue/Cargo.toml b/rust/hashsig-glue/Cargo.toml index 0c68acf99..746c5406e 100644 --- a/rust/hashsig-glue/Cargo.toml +++ b/rust/hashsig-glue/Cargo.toml @@ -17,8 +17,4 @@ serde_json = "1.0" crate-type = ["staticlib"] name = "hashsig_glue" -[dev-dependencies] -hex = "0.4.3" -p3-field = { git = "https://github.com/Plonky3/Plonky3.git", rev = "a33a312" } -p3-koala-bear = { git = "https://github.com/Plonky3/Plonky3.git", rev = "a33a312" } -p3-symmetric = { git = "https://github.com/Plonky3/Plonky3.git", rev = "a33a312" } + diff --git a/rust/hashsig-glue/src/lib.rs b/rust/hashsig-glue/src/lib.rs index fa404a1dd..e31916396 100644 --- a/rust/hashsig-glue/src/lib.rs +++ b/rust/hashsig-glue/src/lib.rs @@ -9,8 +9,53 @@ use std::ptr; use std::slice; use serde_json::Value; -pub type HashSigScheme = +const PROD_SIGNATURE_SSZ_LEN: usize = 3112; +const TEST_SIGNATURE_SSZ_LEN: usize = 424; + +/// Production instantiation (LeanSpec `prod`). +pub type HashSigSchemeProd = leansig::signature::generalized_xmss::instantiations_poseidon_top_level::lifetime_2_to_the_32::hashing_optimized::SIGTopLevelTargetSumLifetime32Dim64Base8; + +/// Test instantiation matching LeanSpec `LEAN_ENV=test` constants. +/// +/// LeanSpec test config: +/// - MESSAGE_LENGTH=32 +/// - LOG_LIFETIME=8 +/// - DIMENSION=4 +/// - BASE=4 +/// - FINAL_LAYER=6 +/// - TARGET_SUM=6 +/// - PARAMETER_LEN=5 +/// - TWEAK_LEN_FE=2 +/// - MSG_LEN_FE=9 +/// - RAND_LEN_FE=7 +/// - HASH_LEN_FE=8 +/// - CAPACITY=9 +/// - POS_OUTPUT_LEN_PER_INV_FE=15 +/// - POS_INVOCATIONS=1 +pub type HashSigSchemeTest = leansig::signature::generalized_xmss::GeneralizedXMSSSignatureScheme< + leansig::symmetric::prf::shake_to_field::ShakePRFtoF<8, 7>, + leansig::inc_encoding::target_sum::TargetSumEncoding< + leansig::symmetric::message_hash::top_level_poseidon::TopLevelPoseidonMessageHash< + 15, + 1, + 15, + 4, + 4, + 6, + 2, + 9, + 5, + 7, + >, + 6, + >, + leansig::symmetric::tweak_hash::poseidon::PoseidonTweakHash<5, 8, 2, 9, 4>, + 8, +>; + +pub type HashSigScheme = HashSigSchemeProd; + pub type HashSigPrivateKey = ::SecretKey; pub type HashSigPublicKey = ::PublicKey; pub type HashSigSignature = ::Signature; @@ -492,135 +537,28 @@ pub unsafe extern "C" fn hashsig_verify_ssz( Err(_) => return -1, }; - // Debug: print first 36 bytes of signature - eprintln!("[hashsig_verify_ssz] pubkey_len={}, sig_len={}, epoch={}", pubkey_len, signature_len, epoch); - eprintln!("[hashsig_verify_ssz] sig first 36 bytes: {:02x?}", &sig_data[..36.min(sig_data.len())]); - eprintln!("[hashsig_verify_ssz] message: {:02x?}", message_array); - - let mut hasher = Sha256::new(); - hasher.update(pk_data); - let pk_sha256 = hasher.finalize_reset(); - hasher.update(sig_data); - let sig_sha256 = hasher.finalize(); - eprintln!("[hashsig_verify_ssz] pubkey sha256: {:02x}", pk_sha256); - eprintln!("[hashsig_verify_ssz] signature sha256: {:02x}", sig_sha256); - - // Directly SSZ decode (leansig has SSZ support built-in) - let pk: HashSigPublicKey = match HashSigPublicKey::from_ssz_bytes(pk_data) { - Ok(pk) => pk, - Err(e) => { - eprintln!("[hashsig_verify_ssz] pubkey decode error: {:?}", e); - return -1; - } - }; - - let sig: HashSigSignature = match HashSigSignature::from_ssz_bytes(sig_data) { - Ok(sig) => sig, - Err(e) => { - eprintln!("[hashsig_verify_ssz] signature decode error: {:?}", e); - return -1; - } - }; - - // SSZ round-trip checks: if these fail, the input bytes are not what leansig - // would produce for the decoded structures (often indicates a layout mismatch). - let pk_roundtrip = pk.as_ssz_bytes(); - if pk_roundtrip.as_slice() != pk_data { - let min_len = pk_roundtrip.len().min(pk_data.len()); - let mut mismatch_at: Option = None; - for i in 0..min_len { - if pk_roundtrip[i] != pk_data[i] { - mismatch_at = Some(i); - break; - } - } - eprintln!( - "[hashsig_verify_ssz] pubkey SSZ roundtrip mismatch: in_len={}, out_len={}, first_mismatch={:?}", - pk_data.len(), - pk_roundtrip.len(), - mismatch_at - ); - } - - let sig_roundtrip = sig.as_ssz_bytes(); - if sig_roundtrip.as_slice() != sig_data { - let min_len = sig_roundtrip.len().min(sig_data.len()); - let mut mismatch_at: Option = None; - for i in 0..min_len { - if sig_roundtrip[i] != sig_data[i] { - mismatch_at = Some(i); - break; - } - } - eprintln!( - "[hashsig_verify_ssz] signature SSZ roundtrip mismatch: in_len={}, out_len={}, first_mismatch={:?}", - sig_data.len(), - sig_roundtrip.len(), - mismatch_at - ); + fn verify_with_scheme( + pk_data: &[u8], + sig_data: &[u8], + epoch: u32, + message_array: &[u8; MESSAGE_LENGTH], + ) -> Result { + let pk = S::PublicKey::from_ssz_bytes(pk_data).map_err(|_| ())?; + let sig = S::Signature::from_ssz_bytes(sig_data).map_err(|_| ())?; + Ok(S::verify(&pk, epoch, message_array, &sig)) } - // Debug: verify SSZ roundtrips. If this fails, we're not verifying the same - // structured data that Python/Zig expect. - let pk_roundtrip = pk.as_ssz_bytes(); - if pk_roundtrip != pk_data { - let mut hasher = Sha256::new(); - hasher.update(pk_data); - let pk_in_hash = hasher.finalize_reset(); - hasher.update(&pk_roundtrip); - let pk_rt_hash = hasher.finalize(); - - let mismatch_idx = pk_data - .iter() - .zip(pk_roundtrip.iter()) - .position(|(a, b)| a != b) - .unwrap_or(0); - - eprintln!( - "[hashsig_verify_ssz] pubkey SSZ roundtrip mismatch: in_len={}, rt_len={}, first_mismatch_at={}, in_sha256={:02x}, rt_sha256={:02x}", - pk_data.len(), - pk_roundtrip.len(), - mismatch_idx, - pk_in_hash, - pk_rt_hash - ); - } else { - eprintln!("[hashsig_verify_ssz] pubkey SSZ roundtrip OK"); - } - - let sig_roundtrip = sig.as_ssz_bytes(); - if sig_roundtrip != sig_data { - let mut hasher = Sha256::new(); - hasher.update(sig_data); - let sig_in_hash = hasher.finalize_reset(); - hasher.update(&sig_roundtrip); - let sig_rt_hash = hasher.finalize(); - - let mismatch_idx = sig_data - .iter() - .zip(sig_roundtrip.iter()) - .position(|(a, b)| a != b) - .unwrap_or(0); - - eprintln!( - "[hashsig_verify_ssz] signature SSZ roundtrip mismatch: in_len={}, rt_len={}, first_mismatch_at={}, in_sha256={:02x}, rt_sha256={:02x}", - sig_data.len(), - sig_roundtrip.len(), - mismatch_idx, - sig_in_hash, - sig_rt_hash - ); - } else { - eprintln!("[hashsig_verify_ssz] signature SSZ roundtrip OK"); - } - - let is_valid = ::verify(&pk, epoch, message_array, &sig); - eprintln!("[hashsig_verify_ssz] verify result: {}", is_valid); + let attempt: Result = match signature_len { + TEST_SIGNATURE_SSZ_LEN => verify_with_scheme::(pk_data, sig_data, epoch, message_array), + PROD_SIGNATURE_SSZ_LEN => verify_with_scheme::(pk_data, sig_data, epoch, message_array), + _ => verify_with_scheme::(pk_data, sig_data, epoch, message_array) + .or_else(|_| verify_with_scheme::(pk_data, sig_data, epoch, message_array)), + }; - if is_valid { - 1 - } else { - 0 + match attempt { + Ok(true) => 1, + Ok(false) => 0, + Err(()) => -1, } } } @@ -830,148 +768,3 @@ pub unsafe extern "C" fn hashsig_signature_ssz_from_json( total_size } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_round_trip_sign_verify_ssz() { - // Generate key pair - let mut rng = ChaCha20Rng::seed_from_u64(12345); - let activation_epoch = 0; - let num_active_epochs = 10; - - let (pk, sk) = ::key_gen(&mut rng, activation_epoch, num_active_epochs); - - // Sign a message at epoch 1 - let message: [u8; 32] = [ - 0x96, 0xfd, 0x6f, 0x2c, 0x91, 0x00, 0x83, 0x2c, - 0xdd, 0xdd, 0x6e, 0x06, 0xce, 0x9c, 0x7d, 0x62, - 0x91, 0x52, 0x71, 0x6a, 0xaa, 0x98, 0x21, 0xa4, - 0xfb, 0x97, 0x26, 0xdb, 0x01, 0xfe, 0xe3, 0xf2, - ]; - let epoch: u32 = 1; - - let signature = ::sign(&sk, epoch, &message).expect("signing failed"); - - // Verify the signature directly - let is_valid = ::verify(&pk, epoch, &message, &signature); - assert!(is_valid, "Direct verification failed"); - - // Serialize to SSZ - let pk_ssz = pk.as_ssz_bytes(); - let sig_ssz = signature.as_ssz_bytes(); - - println!("pubkey SSZ length: {}", pk_ssz.len()); - println!("signature SSZ length: {}", sig_ssz.len()); - println!("signature SSZ first 36 bytes: {:02x?}", &sig_ssz[..36.min(sig_ssz.len())]); - - // Deserialize from SSZ - let pk2 = HashSigPublicKey::from_ssz_bytes(&pk_ssz).expect("pubkey SSZ decode failed"); - let sig2 = HashSigSignature::from_ssz_bytes(&sig_ssz).expect("signature SSZ decode failed"); - - // Verify with deserialized values - let is_valid2 = ::verify(&pk2, epoch, &message, &sig2); - assert!(is_valid2, "SSZ round-trip verification failed"); - - println!("Round-trip test passed!"); - } - - #[test] - fn test_verify_fixture_signature() { - // This is the exact fixture data from test_proposer_signature - // pubkey from fixture (52 bytes) - validator index 1 - let pubkey_bytes: [u8; 52] = [ - 0x8c, 0x73, 0xc3, 0x73, 0xd9, 0x84, 0x68, 0x2c, - 0x35, 0x91, 0x91, 0x47, 0x80, 0x6c, 0x6d, 0x39, - 0x19, 0x42, 0x03, 0x6c, 0x2b, 0xd8, 0xf3, 0x59, - 0xe6, 0x81, 0x53, 0x0e, 0x44, 0x44, 0x3c, 0x15, - 0xfb, 0x9a, 0x15, 0x71, 0xfe, 0x4e, 0x95, 0x7a, - 0xab, 0xc2, 0x0f, 0x5f, 0xbc, 0x67, 0xb8, 0x6c, - 0x8a, 0x16, 0xa2, 0x09, - ]; - - // message hash (32 bytes) - computed from AttestationData - let message: [u8; 32] = [ - 0x96, 0xfd, 0x6f, 0x2c, 0x91, 0x00, 0x83, 0x2c, - 0xdd, 0xdd, 0x6e, 0x06, 0xce, 0x9c, 0x7d, 0x62, - 0x91, 0x52, 0x71, 0x6a, 0xaa, 0x98, 0x21, 0xa4, - 0xfb, 0x97, 0x26, 0xdb, 0x01, 0xfe, 0xe3, 0xf2, - ]; - - let epoch: u32 = 1; - - // Full signature SSZ bytes (3112 bytes) from Python - let sig_ssz_hex = "24000000455d822a9938490a99373d435411556609dc7e4ebf86874bb500153d2804000004000000f7af5070abf022606aff6543ccb88f5b77a47a49203a2d5dbc04751b8088ef3110fe3d55c9b4b0348f20cf1dfb340176964cf23d9665305a5c5f2901803e5444dfe96a196246435baec3f546cb020151da0f9456df86a769f14df97a472b7d482a58086c28c5f24cc93a0b20834f78369e8d873b9239303009021b600ada8f4a6a1de3770f87b4089b217913929d963f7104a65e25cdda68f71677146c3a9f30f770804691f03e5155a00976ce09f228b3613d13f5bec86d4a07c51b0e22c778d071f029095778649ef4532d21fbc90358eb8375c7ba3234b8ed3f0addea5c21e17b390c4dc53b55c21fbc0e57f782243870d3683fbc357c5e4c695042826336210543687f7d5f35c586cc2afd8cca6e5fda3a073ebfac61f618de229ba2e9531c4ac73e8326815d6fab2e03b6ab9b199d5830052b00376771e0e2061fc35045d7f65d2317649117041a381fa6f75b0703e5c07388f74272a436e5190178f46d87185965fcf5f81902d3a90d355dbb535acb7877b2d3dd7dd48fd70dd6c58c51bbb5142e141aef69d2f9860635802a301c2aba382e3d635e3cad98361a436a727e31495c7e085e77e9fb4f3ef84efc0d083b3c4fd34af713e31dc909c4e1de3806fd1b5525de652832ff0379206a6333228e7573ba7f3d47d976ab5d1608b436260c0106c4531b123c1f592b00a34c0e2770ee3e55ef4a0f49745a2d4859b5688dda4d6883e6f024b3ebdf26a0345a515264e13be56a33356d00366052ba8507036a6b406f3682099436f749a24471646f4800021aa7644891c4bd34d5ff6f01df651a6f718a4317ff237d434e49e46c3c6a4c662fc3280e90b999170c0c5d7d5117eb49c9db8b14873df7516d5d0d1a1e81d32ca49fec4f3d02031a77fc621c5ea7537647eb3537a4396b6ce1d44c6ca841a612a93ce05e2bb000793431634460cef8314942940fe830cf4b84ca6e5e2abc210a30b10d42ec3f717daa25651ae5fb7f19b969245c8165a73a58d5800971be1b267dc036236fe2e811c2338e1100ec217e77bf36428392086a4fb3ab2ed0a918616ddcb1377b247f5b46f9cf442ac0a233700b9544149e2c0c6016a96487872049c04580561505ef3110023f2cb6cc90423ee59e22f241c158a69d8c7c32fee33d5615293992d32015aeadd4053eecfb2d1e27104a4583361dff7f7d338528ba6366215e510113913463bde83bfae70e505cb1416375aacb49b04b5c109404f45b65794a294874962518f71407aca3f118fcd7d171f0b49f0e63c55c10efdc713a7eed664fa54d7a6ca8015f4dd77b027d2e9d5c0f4573d24be8dc5f06787e5119e9ac014ab91b3200d923d47175c50e6fd9564f38ea33264dc1126e0c674b065824c6b0630158ae0fcdb772272958e43fc57a037a3abc1c2812e3820b70dd870c8786137c37965e060541e64bba13ce34ebc9071ff7360e23f9388d5b462c4c51972c0102a74e071125e1e46a3da3a552b1f79d2f8586b565a4aa162b9ae13d3dcacb981d278da8485f2a78292f6fee030d5262184260613f5e6c56601949ad4fe14f62377b82e80294f64b1f1f84961ecfc55f6a69cf943049fd5f31bccc2b34701482230511ba09fbd7380b4bac8a37df76170df000075fac9a586822768e11eb88eb3294eff850de72db6bcf605f6dc4c03e52cadc6479d5484727e430770a613c856a4873d866037bf444d2b73626ea3bcb49eafb5954dd1acc2e79a92c505a70cb37529dbc4ba8966d4269718a45d223021aee0b6e2346f51371b64cef342eaf24167ba1e90e25ad3c750a7b2663905a275cb1a9923e2a817d1ebc81760570739775773b4e2d31fb94042839475da2eee07992ffe03e38e0ec3a4b1077543ce70f0519efb306f62997753840035fc9c832715029157ec430e70a311af1092777fa228e87b85458c87d043cb22161a1b05257dfc9f178a2a8826d582a2f5e35ddfd124adff24df025612a685cd4305dedc50bc4a2e54483170f3159ccf128abb485442e9ff84b145a3e0fd683a27859152352b966d82ffaf3df7991ebb124916c26677e6a6112aaa141151a971e47aff3545ee8cd984954ba2507ceda772464bf12722396bc207ca82a2499468252b1e65e4bd2f2ba787b2ac1579f022b12ee6f9c612e378611bf688f3197c5eb67b045e923facf60463d124943329eba045107a931caf77b6dfad2966526bfc317f4edbb3163beed337c89477ca6230e79d858150df505a9468439df5ac29512005e12c94017816c768a8f57181cc5d72be17a9171d818f27c16acc734a5465845639ba6236d1c8e041e9cef56c220fd72d358867be26ef05fb11d866bcba0b26852cc7d5c3f8952136da1ab79d9706465a921036bf0303126ce8f364baabdf8340c25d55937fa980af44d586631c7871e37a7a307f0feee69c4ce77241089dc127b883b694aab6c60723a3f5ec346463c6eb4e5506d062c4fbba8810854bd380ae6b27f71d131bc66c198383f99fc834ddb79f15f0c71bd463b325c02f913e74f72acf25db6df63510b416776ee55f217dd5544632a758c3c2c0a946936efdf6d1478391d25072425da54d341101a0a30ed21893571252f5c3b7af659261d7a3a5afe7b0842f8a45496d81449901f9065cca99e00a1c33c26342a423814e68306d14ba87b827aea602fc18a0dcb25c90e94293e62c0653825ec9c336180575064725fd92c47f28655b7d41e106cd04b432fe3f57de4c66c28a261c06a2966a9214b2b186328bad843bce57a21ff3c0408e895e912f654743a666f7e346365b6499c38037438524138f54e6c12dad7b631fee08d2298656575ed77727460a56010ffc98977117b355216a0e164ca433b43a615ab21f0f2350ad3630f27ae2b6f1893b9684174e7715f2b71534951f3f1029025116c4f981f692dcaae5dda712637cb5dc805f3623a190aa97f68e3c29609e47009072816bc35fca50878aea84434e4b71a4ba7067a34ad637c1806be197af81f973e73d0a21891d0f057df69ee5372b8984578a8f902554530505b8ee4238f355d18dbb5f32ddfa55c6d6abc6a7d810fa267d4f7953313c4424bfa59bb248a7bdc593edbb63f31e8725128e48c71846cd8326ef48b4d986eb12156236412d0e612063bec1b112680b7592e43670e3c052623cf2f390baa322f7938419b498d035941e5af38751ced916cbf35ca31f4f6e76b05f45b2b8d3f865ba5655715a132303571e93d08cb40e954cf01a46aef20870f16bb6c48ff995f3b33e982712faa34695d515b3a250f580f8699bd7bb61add70fb010f4d2c14101648914d3d1b92b250890bb6202c66b64de4d29c6486c5b85e6d2d470dd0a22252197a2442e9b8ea4abe07411010c59e76407d3c217053fa3a3b473b7df06bc47663d7093e9417057e77bdda1d33b1504aac12f2705f608e4862323b12dfb9012d2928a11a80061639fae63c4b39674078cfc58d48ec365979675935303f62b3210582de04314a0b3514f6370cc941cf4b29d4d63a924900382a222758dbaa561c1eb2ba555ba1951a51be49346588d212741c0d5b35ffa615384bf822cbe4496eb5bede106aeb3f5f4448eb4bacb9ff23974705339fdc0725f24d865b3209130d6cd7d43223b7a5483c8e154033bad826a102af29187d3754cae56859777570576e281c5a8cfe7d35c9d1c96f6882243cc6eeb05251b81b22a0e95e60e3e64e3acf1b0b1c24501d2f378d533196bb5320a47c4c61aefa156d8f41422edf9add7953e03b1b88ca67109b66a06c7abbd00a931635674c3a9c6063a68a4f9f670569268061686cad8c1da5c3531a3f43524007fa45149ef61a6adcf932280df4fe44676bb33dd51a882b5e582b0f1f83b45eca9530715efe080c2d872b52ca02b257b8ff02547b7e05433266205408d2131d6963054482c0be3533c8b46829cfe03a29e3f2211da6871de53ef407e17ce42366a1d8247327b9227c2f8c398c9c3c4acefdcd414045043fb8816d38bf1ac050152eca666bef806e896f77410b1d972668bdc05a0eb53f61541cc74b1494fb62bf5ee301800a8560d62b6b28c369714180c8e9178544b3674e2a0471ca77b33f7c2b2722a41d57558c7fd7193ef1fa21ee650606d348ed0e035a8059cb906276bea4db7c97cc8b17879831620935c7527df8137da314c72109da24600c91fb2b5aa721409db66c244c269b3a65a40910fb441b553a43ca4ab4e63b3ffb05402ec974a463779b63425a3f996f04b9526e6987223cfcd2422d9d62b13cc571c5131906503db5a4d9314a1b6f60a0ee741b62f9e50d5e45df695b8080618824100e"; - - let sig_bytes = hex::decode(sig_ssz_hex).expect("Invalid hex"); - assert_eq!(sig_bytes.len(), 3112, "Signature length mismatch"); - - // Decode pubkey - let pk = HashSigPublicKey::from_ssz_bytes(&pubkey_bytes).expect("Pubkey decode failed"); - println!("Pubkey decoded successfully"); - - // Decode signature - let sig = HashSigSignature::from_ssz_bytes(&sig_bytes).expect("Signature decode failed"); - println!("Signature decoded successfully"); - - // Verify SSZ roundtrip - let pk_rt = pk.as_ssz_bytes(); - let sig_rt = sig.as_ssz_bytes(); - assert_eq!(&pubkey_bytes[..], &pk_rt[..], "Pubkey SSZ roundtrip mismatch"); - assert_eq!(&sig_bytes[..], &sig_rt[..], "Signature SSZ roundtrip mismatch"); - println!("SSZ roundtrips OK"); - - // Verify - let is_valid = ::verify(&pk, epoch, &message, &sig); - println!("Verification result: {}", is_valid); - - // This is the key test - should pass if everything is correct - assert!(is_valid, "Signature verification failed!"); - } - - #[test] - fn test_poseidon2_consistency() { - // Test that our Poseidon2 output matches Python's output - // Using test vectors from leanSpec tests/lean_spec/subspecs/poseidon2/test_permutation.py - use p3_koala_bear::{KoalaBear, default_koalabear_poseidon2_16}; - use p3_symmetric::Permutation; - use p3_field::{PrimeCharacteristicRing, PrimeField32}; - - let perm = default_koalabear_poseidon2_16(); - - // Input from leanSpec test_permutation.py INPUT_16: - let input_vals: [u32; 16] = [ - 894848333, 1437655012, 1200606629, 1690012884, - 71131202, 1749206695, 1717947831, 120589055, - 19776022, 42382981, 1831865506, 724844064, - 171220207, 1299207443, 227047920, 1783754913, - ]; - - let mut state: [KoalaBear; 16] = core::array::from_fn(|i| KoalaBear::from_u64(input_vals[i] as u64)); - - println!("Input: {:?}", state.map(|x| x.as_canonical_u32())); - - perm.permute_mut(&mut state); - - let output: Vec = state.iter().map(|x| x.as_canonical_u32()).collect(); - println!("Output: {:?}", output); - - // Expected from leanSpec test_permutation.py EXPECTED_16: - let expected: [u32; 16] = [ - 1934285469, 604889435, 133449501, 1026180808, - 1830659359, 176667110, 1391183747, 351743874, - 1238264085, 1292768839, 2023573270, 1201586780, - 1360691759, 1230682461, 748270449, 651545025, - ]; - - for (i, (got, exp)) in output.iter().zip(expected.iter()).enumerate() { - if got != exp { - println!("Mismatch at index {}: got {}, expected {}", i, got, exp); - } - } - - assert_eq!(output.as_slice(), &expected[..], "Poseidon2 output mismatch with leanSpec vectors!"); - println!("Poseidon2 consistency test passed!"); - } -} From 5e5631700d6af249a78295c2d5a7ddf7a958559e Mon Sep 17 00:00:00 2001 From: grapebaba Date: Wed, 14 Jan 2026 22:16:42 +0800 Subject: [PATCH 04/19] fix: bootstrap spectest:run when missing generated index --- build.zig | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/build.zig b/build.zig index 8115d3a4f..a1caa222c 100644 --- a/build.zig +++ b/build.zig @@ -28,6 +28,12 @@ fn setTestRunLabelFromCompile(b: *Builder, run_step: *std.Build.Step.Run, compil setTestRunLabel(b, run_step, source_name); } +fn fileExists(path: []const u8) bool { + const file = std.fs.cwd().openFile(path, .{}) catch return false; + file.close(); + return true; +} + // Add the glue libs to a compile target fn addRustGlueLib(b: *Builder, comp: *Builder.Step.Compile, target: Builder.ResolvedTarget, prover: ProverChoice) void { // Conditionally include prover libraries based on selection @@ -617,6 +623,13 @@ pub fn build(b: *Builder) !void { run_spectests_after_generate.step.dependOn(&run_spectest_generate.step); const run_spectests = b.addRunArtifact(spectests); + if (!fileExists("pkgs/spectest/src/generated/index.zig")) { + // `spectest:run` expects generated tests to exist already, but a fresh checkout has + // none. Generate a stub index (or real tests if fixtures exist) to keep the command + // usable without requiring a separate `spectest:generate` invocation first. + spectests.step.dependOn(&run_spectest_generate.step); + } + const spectests_step = b.step("spectest", "Regenerate and run spec tests"); spectests_step.dependOn(&run_spectests_after_generate.step); From ff26975cf62679b30ee452a5a526ba4a3c11f4a3 Mon Sep 17 00:00:00 2001 From: grapebaba Date: Thu, 15 Jan 2026 11:59:00 +0800 Subject: [PATCH 05/19] fix(spectest): support verify_signatures attestations + proofs --- .../src/runner/verify_signatures_runner.zig | 318 ++++++++++++++++-- 1 file changed, 290 insertions(+), 28 deletions(-) diff --git a/pkgs/spectest/src/runner/verify_signatures_runner.zig b/pkgs/spectest/src/runner/verify_signatures_runner.zig index 99f21a036..8a87ac442 100644 --- a/pkgs/spectest/src/runner/verify_signatures_runner.zig +++ b/pkgs/spectest/src/runner/verify_signatures_runner.zig @@ -171,6 +171,29 @@ pub fn runFixturePayload( } } +const AggregatedSignatureProof = struct { + participants: types.AggregationBits, + proof_data: []u8, + + pub fn deinit(self: *AggregatedSignatureProof, allocator: std.mem.Allocator) void { + self.participants.deinit(); + allocator.free(self.proof_data); + } +}; + +const ParsedSignedBlockWithAttestation = struct { + signed_block: types.SignedBlockWithAttestation, + attestation_proofs: []AggregatedSignatureProof, + + pub fn deinit(self: *ParsedSignedBlockWithAttestation, allocator: std.mem.Allocator) void { + for (self.attestation_proofs) |*proof| { + proof.deinit(allocator); + } + allocator.free(self.attestation_proofs); + self.signed_block.deinit(); + } +}; + fn runCase( allocator: std.mem.Allocator, ctx: Context, @@ -184,14 +207,15 @@ fn runCase( }, }; - const signature_ssz_len: usize = blk: { - const lean_env_val = case_obj.get("leanEnv") orelse break :blk DEFAULT_SIGNATURE_SSZ_LEN; + const env_is_test = blk: { + const lean_env_val = case_obj.get("leanEnv") orelse break :blk false; const lean_env = switch (lean_env_val) { .string => |s| s, - else => break :blk DEFAULT_SIGNATURE_SSZ_LEN, + else => break :blk false, }; - break :blk if (std.mem.eql(u8, lean_env, "test")) TEST_SIGNATURE_SSZ_LEN else DEFAULT_SIGNATURE_SSZ_LEN; + break :blk std.mem.eql(u8, lean_env, "test"); }; + const signature_ssz_len: usize = if (env_is_test) TEST_SIGNATURE_SSZ_LEN else DEFAULT_SIGNATURE_SSZ_LEN; // Parse the anchorState to get validators const anchor_state_value = case_obj.get("anchorState") orelse { @@ -208,19 +232,21 @@ fn runCase( return FixtureError.InvalidFixture; }; - var signed_block = try buildSignedBlockWithAttestation(allocator, ctx, signed_block_value); - defer signed_block.deinit(); + var parsed = try buildSignedBlockWithAttestation(allocator, ctx, signed_block_value); + defer parsed.deinit(allocator); // Determine if we expect failure based on test name/path const expect_failure = std.mem.indexOf(u8, ctx.fixture_label, "invalid") != null or std.mem.indexOf(u8, ctx.case_name, "invalid") != null; // Verify signatures - const verify_result = state_transition.verifySignaturesWithSignatureLen( + const verify_result = verifySignaturesWithFixtureProofs( allocator, &anchor_state, - &signed_block, + &parsed.signed_block, + parsed.attestation_proofs, signature_ssz_len, + env_is_test, ); if (expect_failure) { @@ -244,6 +270,93 @@ fn runCase( } } +fn verifySignaturesWithFixtureProofs( + allocator: std.mem.Allocator, + state: *const types.BeamState, + signed_block: *const types.SignedBlockWithAttestation, + proofs: []const AggregatedSignatureProof, + signature_ssz_len: usize, + env_is_test: bool, +) !void { + const attestations = signed_block.message.block.body.attestations.constSlice(); + + if (attestations.len != proofs.len) { + return types.StateTransitionError.InvalidBlockSignatures; + } + + const validators = state.validators.constSlice(); + + for (attestations, proofs) |aggregated_attestation, proof| { + // Ensure the declared participants match the aggregated attestation bitfield. + if (aggregated_attestation.aggregation_bits.len() != proof.participants.len()) { + return types.StateTransitionError.InvalidBlockSignatures; + } + for (0..aggregated_attestation.aggregation_bits.len()) |i| { + if (try aggregated_attestation.aggregation_bits.get(i) != try proof.participants.get(i)) { + return types.StateTransitionError.InvalidBlockSignatures; + } + } + + var validator_indices = try types.aggregationBitsToValidatorIndices(&aggregated_attestation.aggregation_bits, allocator); + defer validator_indices.deinit(); + + for (validator_indices.items) |validator_index| { + if (validator_index >= validators.len) { + return types.StateTransitionError.InvalidValidatorId; + } + } + + // NOTE: leanSpec currently serializes a placeholder proof (`0x00`) when running in + // `leanEnv="test"` (see lean_multisig_py usage with test_mode). We accept the proof + // bytes in test mode and only validate participant bookkeeping. + if (env_is_test) { + if (proof.proof_data.len == 0) { + return types.StateTransitionError.InvalidBlockSignatures; + } + } else { + // Future: verify aggregated proofs against leanMultisig once the fixture format + // provides verifiable proof bytes in non-test environments. + } + } + + // Verify proposer attestation signature (standard XMSS signature) + const proposer_attestation = signed_block.message.proposer_attestation; + try verifySingleAttestationSignature( + allocator, + state, + @intCast(proposer_attestation.validator_id), + &proposer_attestation.data, + &signed_block.signature.proposer_signature, + signature_ssz_len, + ); +} + +fn verifySingleAttestationSignature( + allocator: std.mem.Allocator, + state: *const types.BeamState, + validator_index: usize, + attestation_data: *const types.AttestationData, + signature_bytes: *const types.SIGBYTES, + signature_ssz_len: usize, +) !void { + if (signature_ssz_len > signature_bytes.len) { + return types.StateTransitionError.InvalidBlockSignatures; + } + + const validators = state.validators.constSlice(); + if (validator_index >= validators.len) { + return types.StateTransitionError.InvalidValidatorId; + } + + const pubkey = validators[validator_index].getPubkey(); + + var message: [32]u8 = undefined; + try ssz.hashTreeRoot(types.AttestationData, attestation_data.*, &message, allocator); + + const epoch: u32 = @intCast(attestation_data.slot); + try xmss.verifySsz(pubkey, &message, epoch, signature_bytes.*[0..signature_ssz_len]); +} + fn buildState( allocator: std.mem.Allocator, ctx: Context, @@ -351,7 +464,7 @@ fn buildSignedBlockWithAttestation( allocator: std.mem.Allocator, ctx: Context, value: JsonValue, -) FixtureError!types.SignedBlockWithAttestation { +) FixtureError!ParsedSignedBlockWithAttestation { const signed_block_obj = try expect.expectObjectValue(FixtureError, value, ctx, "signedBlockWithAttestation"); // Parse message @@ -368,18 +481,50 @@ fn buildSignedBlockWithAttestation( // Parse signature section const signature_obj = try expect.expectObject(FixtureError, signed_block_obj, &.{"signature"}, ctx, "signature"); - // Parse attestation_signatures (empty for basic tests) - var attestation_signatures = try types.AttestationSignatures.init(allocator); - errdefer attestation_signatures.deinit(); - + // Parse attestation aggregated signature proofs + var attestation_proofs = std.ArrayList(AggregatedSignatureProof).init(allocator); + errdefer { + for (attestation_proofs.items) |*proof| proof.deinit(allocator); + attestation_proofs.deinit(); + } if (signature_obj.get("attestationSignatures")) |att_sigs_val| { const att_sigs_obj = try expect.expectObjectValue(FixtureError, att_sigs_val, ctx, "signature.attestationSignatures"); if (att_sigs_obj.get("data")) |data_val| { const arr = try expect.expectArrayValue(FixtureError, data_val, ctx, "signature.attestationSignatures.data"); - for (arr.items) |_| { - // TODO: Parse actual attestation signatures if needed - std.debug.print("fixture {s} case {s}: non-empty attestation signatures not yet supported\n", .{ ctx.fixture_label, ctx.case_name }); - return FixtureError.UnsupportedFixture; + for (arr.items, 0..) |item, idx| { + var label_buf: [96]u8 = undefined; + const entry_label = std.fmt.bufPrint(&label_buf, "signature.attestationSignatures.data[{d}]", .{idx}) catch "signature.attestationSignatures.data"; + + const entry_obj = try expect.expectObjectValue(FixtureError, item, ctx, entry_label); + + const participants_val = entry_obj.get("participants") orelse { + std.debug.print( + "fixture {s} case {s}: missing participants in {s}\n", + .{ ctx.fixture_label, ctx.case_name, entry_label }, + ); + return FixtureError.InvalidFixture; + }; + var participants = try parseAggregationBits(allocator, ctx, participants_val, "participants"); + + const proof_val = entry_obj.get("proofData") orelse entry_obj.get("proof_data") orelse { + std.debug.print( + "fixture {s} case {s}: missing proofData in {s}\n", + .{ ctx.fixture_label, ctx.case_name, entry_label }, + ); + participants.deinit(); + return FixtureError.InvalidFixture; + }; + const proof_data = try parseByteListMiB(allocator, ctx, proof_val, "proofData"); + + attestation_proofs.append(.{ .participants = participants, .proof_data = proof_data }) catch |err| { + std.debug.print( + "fixture {s} case {s}: failed to append attestation proof: {s}\n", + .{ ctx.fixture_label, ctx.case_name, @errorName(err) }, + ); + participants.deinit(); + allocator.free(proof_data); + return FixtureError.InvalidFixture; + }; } } } @@ -387,14 +532,29 @@ fn buildSignedBlockWithAttestation( // Parse proposer_signature const proposer_sig = try parseSignature(ctx, signature_obj, "proposerSignature"); - return types.SignedBlockWithAttestation{ - .message = .{ - .block = block, - .proposer_attestation = proposer_attestation, + var signatures = types.createBlockSignatures(allocator, block.body.attestations.len()) catch |err| { + std.debug.print( + "fixture {s} case {s}: unable to allocate signature groups: {s}\n", + .{ ctx.fixture_label, ctx.case_name, @errorName(err) }, + ); + return FixtureError.InvalidFixture; + }; + signatures.proposer_signature = proposer_sig; + + return ParsedSignedBlockWithAttestation{ + .signed_block = .{ + .message = .{ + .block = block, + .proposer_attestation = proposer_attestation, + }, + .signature = signatures, }, - .signature = .{ - .attestation_signatures = attestation_signatures, - .proposer_signature = proposer_sig, + .attestation_proofs = attestation_proofs.toOwnedSlice() catch |err| { + std.debug.print( + "fixture {s} case {s}: unable to allocate attestation proof list: {s}\n", + .{ ctx.fixture_label, ctx.case_name, @errorName(err) }, + ); + return FixtureError.InvalidFixture; }, }; } @@ -418,10 +578,32 @@ fn buildBlock( const att_obj = try expect.expectObjectValue(FixtureError, att_val, ctx, "body.attestations"); if (att_obj.get("data")) |data_val| { const arr = try expect.expectArrayValue(FixtureError, data_val, ctx, "body.attestations.data"); - for (arr.items) |_| { - // TODO: Parse actual attestations if needed - std.debug.print("fixture {s} case {s}: non-empty attestations not yet supported\n", .{ ctx.fixture_label, ctx.case_name }); - return FixtureError.UnsupportedFixture; + for (arr.items, 0..) |item, idx| { + var label_buf: [96]u8 = undefined; + const entry_label = std.fmt.bufPrint(&label_buf, "body.attestations.data[{d}]", .{idx}) catch "body.attestations.data"; + + const att_item_obj = try expect.expectObjectValue(FixtureError, item, ctx, entry_label); + + const bits_val = att_item_obj.get("aggregationBits") orelse { + std.debug.print( + "fixture {s} case {s}: missing aggregationBits in {s}\n", + .{ ctx.fixture_label, ctx.case_name, entry_label }, + ); + return FixtureError.InvalidFixture; + }; + var aggregation_bits = try parseAggregationBits(allocator, ctx, bits_val, "aggregationBits"); + errdefer aggregation_bits.deinit(); + + const data_obj = try expect.expectObject(FixtureError, att_item_obj, &.{"data"}, ctx, "attestation.data"); + const data = try parseAttestationData(ctx, data_obj); + + attestations.append(.{ .aggregation_bits = aggregation_bits, .data = data }) catch |err| { + std.debug.print( + "fixture {s} case {s}: failed to append attestation: {s}\n", + .{ ctx.fixture_label, ctx.case_name, @errorName(err) }, + ); + return FixtureError.InvalidFixture; + }; } } } @@ -436,6 +618,86 @@ fn buildBlock( }; } +fn parseAggregationBits( + allocator: std.mem.Allocator, + ctx: Context, + value: JsonValue, + label: []const u8, +) FixtureError!types.AggregationBits { + const obj = try expect.expectObjectValue(FixtureError, value, ctx, label); + const arr = try expect.expectArrayField(FixtureError, obj, &.{"data"}, ctx, label); + + var bits = try types.AggregationBits.init(allocator); + errdefer bits.deinit(); + + for (arr.items) |bit_val| { + const bit = switch (bit_val) { + .bool => |b| b, + else => { + std.debug.print( + "fixture {s} case {s}: {s} must contain booleans\n", + .{ ctx.fixture_label, ctx.case_name, label }, + ); + return FixtureError.InvalidFixture; + }, + }; + bits.append(bit) catch |err| { + std.debug.print( + "fixture {s} case {s}: failed to append {s} bit: {s}\n", + .{ ctx.fixture_label, ctx.case_name, label, @errorName(err) }, + ); + return FixtureError.InvalidFixture; + }; + } + + return bits; +} + +fn parseByteListMiB( + allocator: std.mem.Allocator, + ctx: Context, + value: JsonValue, + label: []const u8, +) FixtureError![]u8 { + const obj = try expect.expectObjectValue(FixtureError, value, ctx, label); + const text = try expect.expectStringField(FixtureError, obj, &.{"data"}, ctx, label); + + if (text.len < 2 or !std.mem.eql(u8, text[0..2], "0x")) { + std.debug.print( + "fixture {s} case {s}: field {s}.data missing 0x prefix\n", + .{ ctx.fixture_label, ctx.case_name, label }, + ); + return FixtureError.InvalidFixture; + } + + const body = text[2..]; + if (body.len % 2 != 0) { + std.debug.print( + "fixture {s} case {s}: field {s}.data has odd hex length\n", + .{ ctx.fixture_label, ctx.case_name, label }, + ); + return FixtureError.InvalidFixture; + } + + const out_len = body.len / 2; + const out = allocator.alloc(u8, out_len) catch |err| { + std.debug.print( + "fixture {s} case {s}: unable to allocate {d} bytes for {s}: {s}\n", + .{ ctx.fixture_label, ctx.case_name, out_len, label, @errorName(err) }, + ); + return FixtureError.InvalidFixture; + }; + errdefer allocator.free(out); + _ = std.fmt.hexToBytes(out, body) catch { + std.debug.print( + "fixture {s} case {s}: field {s}.data invalid hex\n", + .{ ctx.fixture_label, ctx.case_name, label }, + ); + return FixtureError.InvalidFixture; + }; + return out; +} + fn parseProposerAttestation( ctx: Context, obj: std.json.ObjectMap, From 71c50b40ee8676694ff83c789fdf61ac7dc732f0 Mon Sep 17 00:00:00 2001 From: grapebaba Date: Thu, 15 Jan 2026 12:20:18 +0800 Subject: [PATCH 06/19] chore: fix zig fmt step and rust lint --- build.zig | 8 +++++--- rust/hashsig-glue/src/lib.rs | 36 ++++++++++++++++++++++-------------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/build.zig b/build.zig index a1caa222c..adcf47368 100644 --- a/build.zig +++ b/build.zig @@ -616,18 +616,20 @@ pub fn build(b: *Builder) !void { .optimize = optimize, }); const run_spectest_generate = b.addRunArtifact(spectest_generate_exe); + const run_spectest_format = b.addSystemCommand(&.{ "zig", "fmt", "pkgs/spectest/src/generated" }); + run_spectest_format.step.dependOn(&run_spectest_generate.step); const spectest_generate_step = b.step("spectest:generate", "Regenerate spectest fixtures"); - spectest_generate_step.dependOn(&run_spectest_generate.step); + spectest_generate_step.dependOn(&run_spectest_format.step); const run_spectests_after_generate = b.addRunArtifact(spectests); - run_spectests_after_generate.step.dependOn(&run_spectest_generate.step); + run_spectests_after_generate.step.dependOn(&run_spectest_format.step); const run_spectests = b.addRunArtifact(spectests); if (!fileExists("pkgs/spectest/src/generated/index.zig")) { // `spectest:run` expects generated tests to exist already, but a fresh checkout has // none. Generate a stub index (or real tests if fixtures exist) to keep the command // usable without requiring a separate `spectest:generate` invocation first. - spectests.step.dependOn(&run_spectest_generate.step); + spectests.step.dependOn(&run_spectest_format.step); } const spectests_step = b.step("spectest", "Regenerate and run spec tests"); diff --git a/rust/hashsig-glue/src/lib.rs b/rust/hashsig-glue/src/lib.rs index e31916396..450a73504 100644 --- a/rust/hashsig-glue/src/lib.rs +++ b/rust/hashsig-glue/src/lib.rs @@ -2,12 +2,12 @@ use leansig::{signature::SignatureScheme, MESSAGE_LENGTH}; use rand::Rng; use rand::SeedableRng; use rand_chacha::ChaCha20Rng; +use serde_json::Value; use sha2::{Digest, Sha256}; use std::ffi::CStr; use std::os::raw::c_char; use std::ptr; use std::slice; -use serde_json::Value; const PROD_SIGNATURE_SSZ_LEN: usize = 3112; const TEST_SIGNATURE_SSZ_LEN: usize = 424; @@ -549,10 +549,16 @@ pub unsafe extern "C" fn hashsig_verify_ssz( } let attempt: Result = match signature_len { - TEST_SIGNATURE_SSZ_LEN => verify_with_scheme::(pk_data, sig_data, epoch, message_array), - PROD_SIGNATURE_SSZ_LEN => verify_with_scheme::(pk_data, sig_data, epoch, message_array), + TEST_SIGNATURE_SSZ_LEN => { + verify_with_scheme::(pk_data, sig_data, epoch, message_array) + } + PROD_SIGNATURE_SSZ_LEN => { + verify_with_scheme::(pk_data, sig_data, epoch, message_array) + } _ => verify_with_scheme::(pk_data, sig_data, epoch, message_array) - .or_else(|_| verify_with_scheme::(pk_data, sig_data, epoch, message_array)), + .or_else(|_| { + verify_with_scheme::(pk_data, sig_data, epoch, message_array) + }), }; match attempt { @@ -623,6 +629,12 @@ fn write_u32_le(dst: &mut [u8], offset: usize, v: u32) -> Option<()> { /// "hashes": {"data": [ {"data": [u32;8]}, ... ]} } /// /// Returns number of bytes written, or 0 on error. +/// +/// # Safety +/// - `signature_json_ptr` must be either null or point to `signature_json_len` readable bytes. +/// - `out_ptr` must be either null or point to `out_len` writable bytes. +/// - Both buffers must be valid for the duration of the call and must not overlap in a way that +/// violates Rust aliasing rules. #[no_mangle] pub unsafe extern "C" fn hashsig_signature_ssz_from_json( signature_json_ptr: *const u8, @@ -681,21 +693,18 @@ pub unsafe extern "C" fn hashsig_signature_ssz_from_json( let path_fixed_part: usize = 4; let sig_fixed_part: usize = 36; - let path_variable_size = siblings_vec.len().checked_mul(sibling_size).unwrap_or(usize::MAX); - if path_variable_size == usize::MAX { - return 0; - } + let path_variable_size = siblings_vec.len().saturating_mul(sibling_size); let path_total_size = match path_fixed_part.checked_add(path_variable_size) { Some(v) => v, None => return 0, }; - let hashes_size = hashes_vec.len().checked_mul(hash_size).unwrap_or(usize::MAX); - if hashes_size == usize::MAX { - return 0; - } + let hashes_size = hashes_vec.len().saturating_mul(hash_size); - let total_size = match sig_fixed_part.checked_add(path_total_size).and_then(|v| v.checked_add(hashes_size)) { + let total_size = match sig_fixed_part + .checked_add(path_total_size) + .and_then(|v| v.checked_add(hashes_size)) + { Some(v) => v, None => return 0, }; @@ -767,4 +776,3 @@ pub unsafe extern "C" fn hashsig_signature_ssz_from_json( total_size } - From 164aca4475dd52338693a733631cf520a13d02dc Mon Sep 17 00:00:00 2001 From: Chen Kai <281165273grape@gmail.com> Date: Thu, 15 Jan 2026 16:42:25 +0800 Subject: [PATCH 07/19] Apply suggestion from @Copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkgs/spectest/src/runner/verify_signatures_runner.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/spectest/src/runner/verify_signatures_runner.zig b/pkgs/spectest/src/runner/verify_signatures_runner.zig index 8a87ac442..27a5c4759 100644 --- a/pkgs/spectest/src/runner/verify_signatures_runner.zig +++ b/pkgs/spectest/src/runner/verify_signatures_runner.zig @@ -759,7 +759,7 @@ fn parseSignature( }; // Re-serialize just the signature object and let Rust parse/SSZ-encode it. - var json_buf = std.ArrayList(u8).init(std.heap.page_allocator); + var json_buf = std.ArrayList(u8).init(ctx.allocator); defer json_buf.deinit(); std.json.stringify(sig_value, .{}, json_buf.writer()) catch |err| { From ffe61b68154fc1f32dafb821ba3c1a9b10375f1a Mon Sep 17 00:00:00 2001 From: grapebaba Date: Fri, 16 Jan 2026 16:18:22 +0800 Subject: [PATCH 08/19] fix(spectest): stop using ctx allocator in signature parsing --- pkgs/spectest/src/runner/verify_signatures_runner.zig | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkgs/spectest/src/runner/verify_signatures_runner.zig b/pkgs/spectest/src/runner/verify_signatures_runner.zig index 27a5c4759..d8350980e 100644 --- a/pkgs/spectest/src/runner/verify_signatures_runner.zig +++ b/pkgs/spectest/src/runner/verify_signatures_runner.zig @@ -530,7 +530,7 @@ fn buildSignedBlockWithAttestation( } // Parse proposer_signature - const proposer_sig = try parseSignature(ctx, signature_obj, "proposerSignature"); + const proposer_sig = try parseSignature(allocator, ctx, signature_obj, "proposerSignature"); var signatures = types.createBlockSignatures(allocator, block.body.attestations.len()) catch |err| { std.debug.print( @@ -746,6 +746,7 @@ fn parseAttestationData( } fn parseSignature( + allocator: std.mem.Allocator, ctx: Context, obj: std.json.ObjectMap, field_name: []const u8, @@ -759,7 +760,7 @@ fn parseSignature( }; // Re-serialize just the signature object and let Rust parse/SSZ-encode it. - var json_buf = std.ArrayList(u8).init(ctx.allocator); + var json_buf = std.ArrayList(u8).init(allocator); defer json_buf.deinit(); std.json.stringify(sig_value, .{}, json_buf.writer()) catch |err| { From ebb40d1d3a54c4beee657caeaf13a601c4df8cd8 Mon Sep 17 00:00:00 2001 From: grapebaba Date: Thu, 22 Jan 2026 23:06:33 +0800 Subject: [PATCH 09/19] refactor: align latest spec test Signed-off-by: grapebaba --- leanSpec | 2 +- pkgs/key-manager/src/lib.zig | 174 +++++++++++++++++ .../src/runner/fork_choice_runner.zig | 5 +- .../src/runner/state_transition_runner.zig | 6 +- .../src/runner/verify_signatures_runner.zig | 29 ++- pkgs/xmss/src/lib.zig | 1 + rust/Cargo.lock | 180 ++++++++++++++++-- rust/hashsig-glue/Cargo.toml | 3 +- 8 files changed, 367 insertions(+), 33 deletions(-) diff --git a/leanSpec b/leanSpec index a61beb399..fe2546c08 160000 --- a/leanSpec +++ b/leanSpec @@ -1 +1 @@ -Subproject commit a61beb399db48c504db9083af69d42818de30411 +Subproject commit fe2546c08bf62da7da854d22ab7c70b8984ffa3e diff --git a/pkgs/key-manager/src/lib.zig b/pkgs/key-manager/src/lib.zig index 832e96dee..5bcb16f50 100644 --- a/pkgs/key-manager/src/lib.zig +++ b/pkgs/key-manager/src/lib.zig @@ -4,6 +4,180 @@ const types = @import("@zeam/types"); const zeam_metrics = @import("@zeam/metrics"); const ssz = @import("ssz"); const Allocator = std.mem.Allocator; +const JsonValue = std.json.Value; + +pub const XmssTestScheme = enum { + @"test", + prod, +}; + +pub const TEST_SIGNATURE_SSZ_LEN: usize = 424; + +pub const XmssTestConfig = struct { + scheme: XmssTestScheme, + signature_ssz_len: usize, + allow_placeholder_aggregated_proof: bool, + + pub fn fromLeanEnv(lean_env: ?[]const u8) XmssTestConfig { + const scheme = schemeFromLeanEnv(lean_env); + return .{ + .scheme = scheme, + .signature_ssz_len = switch (scheme) { + .@"test" => TEST_SIGNATURE_SSZ_LEN, + .prod => types.SIGSIZE, + }, + .allow_placeholder_aggregated_proof = scheme == .@"test", + }; + } +}; + +pub const TestKeyManagerError = error{ + DuplicateKeyIndex, + InvalidKeyFile, + InvalidKeyIndex, + InvalidPublicKey, + NoKeysFound, + PublicKeyNotFound, +}; + +pub const TestKeyManager = struct { + allocator: Allocator, + config: XmssTestConfig, + pubkeys: std.AutoHashMap(usize, types.Bytes52), + + const Self = @This(); + + pub fn init(allocator: Allocator, lean_env: ?[]const u8) Self { + return Self{ + .allocator = allocator, + .config = XmssTestConfig.fromLeanEnv(lean_env), + .pubkeys = std.AutoHashMap(usize, types.Bytes52).init(allocator), + }; + } + + pub fn deinit(self: *Self) void { + self.pubkeys.deinit(); + } + + pub fn loadLeanSpecKeys(self: *Self, keys_root: []const u8) !void { + const scheme_dir_name = switch (self.config.scheme) { + .@"test" => "test_scheme", + .prod => "prod_scheme", + }; + const scheme_dir_path = try std.fs.path.join(self.allocator, &.{ keys_root, scheme_dir_name }); + defer self.allocator.free(scheme_dir_path); + try self.loadKeysFromDir(scheme_dir_path); + } + + pub fn loadKeysFromDir(self: *Self, keys_dir_path: []const u8) !void { + var dir = try std.fs.cwd().openDir(keys_dir_path, .{ .iterate = true }); + defer dir.close(); + + self.pubkeys.clearRetainingCapacity(); + + var it = dir.iterate(); + while (try it.next()) |entry| { + if (entry.kind != .file) continue; + const index = parseKeyIndex(entry.name) catch continue; + const pubkey = try readPublicKeyFromJson(self.allocator, dir, entry.name); + + const gop = try self.pubkeys.getOrPut(index); + if (gop.found_existing) { + return TestKeyManagerError.DuplicateKeyIndex; + } + gop.value_ptr.* = pubkey; + } + + if (self.pubkeys.count() == 0) { + return TestKeyManagerError.NoKeysFound; + } + } + + pub fn getPublicKeyBytes(self: *const Self, validator_index: usize) !types.Bytes52 { + return self.pubkeys.get(validator_index) orelse TestKeyManagerError.PublicKeyNotFound; + } + + pub fn getAllPubkeys( + self: *const Self, + allocator: Allocator, + num_validators: usize, + ) ![]types.Bytes52 { + const pubkeys = try allocator.alloc(types.Bytes52, num_validators); + errdefer allocator.free(pubkeys); + + for (0..num_validators) |i| { + pubkeys[i] = try self.getPublicKeyBytes(i); + } + + return pubkeys; + } + + pub fn signatureSszLen(self: *const Self) usize { + return self.config.signature_ssz_len; + } + + pub fn allowPlaceholderAggregatedProof(self: *const Self) bool { + return self.config.allow_placeholder_aggregated_proof; + } +}; + +fn schemeFromLeanEnv(lean_env: ?[]const u8) XmssTestScheme { + const env = lean_env orelse return .prod; + if (std.ascii.eqlIgnoreCase(env, "test")) return .@"test"; + return .prod; +} + +fn parseKeyIndex(file_name: []const u8) !usize { + if (!std.mem.endsWith(u8, file_name, ".json")) { + return TestKeyManagerError.InvalidKeyIndex; + } + const stem = file_name[0 .. file_name.len - ".json".len]; + if (stem.len == 0) { + return TestKeyManagerError.InvalidKeyIndex; + } + return std.fmt.parseInt(usize, stem, 10) catch TestKeyManagerError.InvalidKeyIndex; +} + +fn readPublicKeyFromJson( + allocator: Allocator, + dir: std.fs.Dir, + file_name: []const u8, +) !types.Bytes52 { + const max_bytes: usize = 2 * 1024 * 1024; + const payload = dir.readFileAlloc(allocator, file_name, max_bytes) catch { + return TestKeyManagerError.InvalidKeyFile; + }; + defer allocator.free(payload); + + var parsed = std.json.parseFromSlice(JsonValue, allocator, payload, .{ .ignore_unknown_fields = true }) catch { + return TestKeyManagerError.InvalidKeyFile; + }; + defer parsed.deinit(); + + const obj = switch (parsed.value) { + .object => |map| map, + else => return TestKeyManagerError.InvalidKeyFile, + }; + const pub_val = obj.get("public") orelse return TestKeyManagerError.InvalidKeyFile; + const pub_hex = switch (pub_val) { + .string => |s| s, + else => return TestKeyManagerError.InvalidKeyFile, + }; + + return parsePublicKeyHex(pub_hex); +} + +fn parsePublicKeyHex(input: []const u8) !types.Bytes52 { + const hex_str = if (std.mem.startsWith(u8, input, "0x")) input[2..] else input; + if (hex_str.len != 104) { + return TestKeyManagerError.InvalidPublicKey; + } + var bytes: types.Bytes52 = undefined; + _ = std.fmt.hexToBytes(&bytes, hex_str) catch { + return TestKeyManagerError.InvalidPublicKey; + }; + return bytes; +} const KeyManagerError = error{ ValidatorKeyNotFound, diff --git a/pkgs/spectest/src/runner/fork_choice_runner.zig b/pkgs/spectest/src/runner/fork_choice_runner.zig index e15636f3e..4b1e1d623 100644 --- a/pkgs/spectest/src/runner/fork_choice_runner.zig +++ b/pkgs/spectest/src/runner/fork_choice_runner.zig @@ -633,7 +633,10 @@ fn processBlockStep( } try types.sszClone(ctx.allocator, types.BeamState, parent_state_ptr.*, new_state_ptr); - state_transition.apply_transition(ctx.allocator, new_state_ptr, block, .{ .logger = ctx.fork_logger, .validateResult = false }) catch |err| { + state_transition.apply_transition(ctx.allocator, new_state_ptr, block, .{ + .logger = ctx.fork_logger, + .validateResult = false, + }) catch |err| { std.debug.print( "fixture {s} case {s}{}: state transition failed {s}\n", .{ fixture_path, case_name, formatStep(step_index), @errorName(err) }, diff --git a/pkgs/spectest/src/runner/state_transition_runner.zig b/pkgs/spectest/src/runner/state_transition_runner.zig index e043516fb..c61e1dea2 100644 --- a/pkgs/spectest/src/runner/state_transition_runner.zig +++ b/pkgs/spectest/src/runner/state_transition_runner.zig @@ -277,7 +277,11 @@ fn runCase( } } - state_transition.apply_transition(allocator, &pre_state, block, .{ .logger = logger }) catch |err| { + const validate_result = expect_exception != null; + state_transition.apply_transition(allocator, &pre_state, block, .{ + .logger = logger, + .validateResult = validate_result, + }) catch |err| { encountered_error = true; if (expect_exception == null) { std.debug.print( diff --git a/pkgs/spectest/src/runner/verify_signatures_runner.zig b/pkgs/spectest/src/runner/verify_signatures_runner.zig index d8350980e..bf279ed97 100644 --- a/pkgs/spectest/src/runner/verify_signatures_runner.zig +++ b/pkgs/spectest/src/runner/verify_signatures_runner.zig @@ -42,12 +42,10 @@ pub fn baseRelRoot(comptime spec_fork: Fork) []const u8 { const types = @import("@zeam/types"); const state_transition = @import("@zeam/state-transition"); +const key_manager = @import("@zeam/key-manager"); const ssz = @import("ssz"); const xmss = @import("@zeam/xmss"); -const DEFAULT_SIGNATURE_SSZ_LEN: usize = types.SIGSIZE; -const TEST_SIGNATURE_SSZ_LEN: usize = 424; - // Signature structure constants from leansig // path: 8 siblings, each is 8 u32 = 256 bytes // rho: 7 u32 = 28 bytes @@ -207,15 +205,17 @@ fn runCase( }, }; - const env_is_test = blk: { - const lean_env_val = case_obj.get("leanEnv") orelse break :blk false; - const lean_env = switch (lean_env_val) { + const lean_env = blk: { + const lean_env_val = case_obj.get("leanEnv") orelse break :blk null; + const lean_env_str = switch (lean_env_val) { .string => |s| s, - else => break :blk false, + else => break :blk null, }; - break :blk std.mem.eql(u8, lean_env, "test"); + break :blk lean_env_str; }; - const signature_ssz_len: usize = if (env_is_test) TEST_SIGNATURE_SSZ_LEN else DEFAULT_SIGNATURE_SSZ_LEN; + const test_config = key_manager.XmssTestConfig.fromLeanEnv(lean_env); + const signature_ssz_len: usize = test_config.signature_ssz_len; + const allow_placeholder_aggregated_proof = test_config.allow_placeholder_aggregated_proof; // Parse the anchorState to get validators const anchor_state_value = case_obj.get("anchorState") orelse { @@ -246,7 +246,7 @@ fn runCase( &parsed.signed_block, parsed.attestation_proofs, signature_ssz_len, - env_is_test, + allow_placeholder_aggregated_proof, ); if (expect_failure) { @@ -276,7 +276,7 @@ fn verifySignaturesWithFixtureProofs( signed_block: *const types.SignedBlockWithAttestation, proofs: []const AggregatedSignatureProof, signature_ssz_len: usize, - env_is_test: bool, + allow_placeholder_aggregated_proof: bool, ) !void { const attestations = signed_block.message.block.body.attestations.constSlice(); @@ -306,10 +306,9 @@ fn verifySignaturesWithFixtureProofs( } } - // NOTE: leanSpec currently serializes a placeholder proof (`0x00`) when running in - // `leanEnv="test"` (see lean_multisig_py usage with test_mode). We accept the proof - // bytes in test mode and only validate participant bookkeeping. - if (env_is_test) { + // NOTE: leanSpec currently serializes a placeholder proof (`0x00`) in test mode. + // We accept the proof bytes and only validate participant bookkeeping. + if (allow_placeholder_aggregated_proof) { if (proof.proof_data.len == 0) { return types.StateTransitionError.InvalidBlockSignatures; } diff --git a/pkgs/xmss/src/lib.zig b/pkgs/xmss/src/lib.zig index ed18b1011..76417a23e 100644 --- a/pkgs/xmss/src/lib.zig +++ b/pkgs/xmss/src/lib.zig @@ -14,6 +14,7 @@ pub const Signature = hashsig.Signature; pub const PublicKey = hashsig.PublicKey; pub const HashSigError = hashsig.HashSigError; pub const verifySsz = hashsig.verifySsz; +pub const signatureSszFromJson = hashsig.signatureSszFromJson; pub const HashSigKeyPair = hashsig.HashSigKeyPair; pub const HashSigSignature = hashsig.HashSigSignature; pub const HashSigPublicKey = hashsig.HashSigPublicKey; diff --git a/rust/Cargo.lock b/rust/Cargo.lock index f1724e9a9..01d63c869 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -200,7 +200,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -211,7 +211,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1801,7 +1801,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2016,7 +2016,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2647,7 +2647,7 @@ name = "hashsig-glue" version = "0.1.0" dependencies = [ "ethereum_ssz", - "leansig 0.1.0 (git+https://github.com/leanEthereum/leanSig?rev=b621826f948ffc133dd893131aac2c7efa7f90e0)", + "leansig 0.1.0 (git+https://github.com/leanEthereum/leanSig?rev=73bedc26ed961b110df7ac2e234dc11361a4bf25)", "rand 0.9.2", "rand_chacha 0.9.0", "serde", @@ -3352,16 +3352,16 @@ dependencies = [ [[package]] name = "leansig" version = "0.1.0" -source = "git+https://github.com/leanEthereum/leanSig?rev=b621826f948ffc133dd893131aac2c7efa7f90e0#b621826f948ffc133dd893131aac2c7efa7f90e0" +source = "git+https://github.com/leanEthereum/leanSig?rev=73bedc26ed961b110df7ac2e234dc11361a4bf25#73bedc26ed961b110df7ac2e234dc11361a4bf25" dependencies = [ "dashmap", "ethereum_ssz", "num-bigint 0.4.6", "num-traits", - "p3-baby-bear 0.3.0 (git+https://github.com/Plonky3/Plonky3.git?rev=a33a312)", - "p3-field 0.3.0 (git+https://github.com/Plonky3/Plonky3.git?rev=a33a312)", - "p3-koala-bear 0.3.0 (git+https://github.com/Plonky3/Plonky3.git?rev=a33a312)", - "p3-symmetric 0.3.0 (git+https://github.com/Plonky3/Plonky3.git?rev=a33a312)", + "p3-baby-bear 0.4.1", + "p3-field 0.4.1", + "p3-koala-bear 0.4.1", + "p3-symmetric 0.4.1", "rand 0.9.2", "rayon", "serde", @@ -4262,7 +4262,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5400,6 +5400,20 @@ dependencies = [ "rand 0.9.2", ] +[[package]] +name = "p3-baby-bear" +version = "0.4.1" +source = "git+https://github.com/Plonky3/Plonky3.git?rev=d421e32#d421e32d3821174ae1f7e528d4bb92b7b18ab295" +dependencies = [ + "p3-challenger 0.4.1", + "p3-field 0.4.1", + "p3-mds 0.4.1", + "p3-monty-31 0.4.1", + "p3-poseidon2 0.4.1", + "p3-symmetric 0.4.1", + "rand 0.9.2", +] + [[package]] name = "p3-blake3" version = "0.1.0" @@ -5449,6 +5463,19 @@ dependencies = [ "tracing", ] +[[package]] +name = "p3-challenger" +version = "0.4.1" +source = "git+https://github.com/Plonky3/Plonky3.git?rev=d421e32#d421e32d3821174ae1f7e528d4bb92b7b18ab295" +dependencies = [ + "p3-field 0.4.1", + "p3-maybe-rayon 0.4.1", + "p3-monty-31 0.4.1", + "p3-symmetric 0.4.1", + "p3-util 0.4.1", + "tracing", +] + [[package]] name = "p3-commit" version = "0.1.0" @@ -5517,6 +5544,20 @@ dependencies = [ "tracing", ] +[[package]] +name = "p3-dft" +version = "0.4.1" +source = "git+https://github.com/Plonky3/Plonky3.git?rev=d421e32#d421e32d3821174ae1f7e528d4bb92b7b18ab295" +dependencies = [ + "itertools 0.14.0", + "p3-field 0.4.1", + "p3-matrix 0.4.1", + "p3-maybe-rayon 0.4.1", + "p3-util 0.4.1", + "spin 0.10.0", + "tracing", +] + [[package]] name = "p3-field" version = "0.1.0" @@ -5564,6 +5605,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "p3-field" +version = "0.4.1" +source = "git+https://github.com/Plonky3/Plonky3.git?rev=d421e32#d421e32d3821174ae1f7e528d4bb92b7b18ab295" +dependencies = [ + "itertools 0.14.0", + "num-bigint 0.4.6", + "p3-maybe-rayon 0.4.1", + "p3-util 0.4.1", + "paste", + "rand 0.9.2", + "serde", + "tracing", +] + [[package]] name = "p3-fri" version = "0.1.0" @@ -5690,6 +5746,19 @@ dependencies = [ "rand 0.9.2", ] +[[package]] +name = "p3-koala-bear" +version = "0.4.1" +source = "git+https://github.com/Plonky3/Plonky3.git?rev=d421e32#d421e32d3821174ae1f7e528d4bb92b7b18ab295" +dependencies = [ + "p3-challenger 0.4.1", + "p3-field 0.4.1", + "p3-monty-31 0.4.1", + "p3-poseidon2 0.4.1", + "p3-symmetric 0.4.1", + "rand 0.9.2", +] + [[package]] name = "p3-matrix" version = "0.1.0" @@ -5735,6 +5804,21 @@ dependencies = [ "transpose", ] +[[package]] +name = "p3-matrix" +version = "0.4.1" +source = "git+https://github.com/Plonky3/Plonky3.git?rev=d421e32#d421e32d3821174ae1f7e528d4bb92b7b18ab295" +dependencies = [ + "itertools 0.14.0", + "p3-field 0.4.1", + "p3-maybe-rayon 0.4.1", + "p3-util 0.4.1", + "rand 0.9.2", + "serde", + "tracing", + "transpose", +] + [[package]] name = "p3-maybe-rayon" version = "0.1.0" @@ -5756,6 +5840,11 @@ name = "p3-maybe-rayon" version = "0.3.0" source = "git+https://github.com/Plonky3/Plonky3.git?rev=a33a312#a33a31274a5e78bb5fbe3f82ffd2c294e17fa830" +[[package]] +name = "p3-maybe-rayon" +version = "0.4.1" +source = "git+https://github.com/Plonky3/Plonky3.git?rev=d421e32#d421e32d3821174ae1f7e528d4bb92b7b18ab295" + [[package]] name = "p3-mds" version = "0.1.0" @@ -5794,6 +5883,18 @@ dependencies = [ "rand 0.9.2", ] +[[package]] +name = "p3-mds" +version = "0.4.1" +source = "git+https://github.com/Plonky3/Plonky3.git?rev=d421e32#d421e32d3821174ae1f7e528d4bb92b7b18ab295" +dependencies = [ + "p3-dft 0.4.1", + "p3-field 0.4.1", + "p3-symmetric 0.4.1", + "p3-util 0.4.1", + "rand 0.9.2", +] + [[package]] name = "p3-merkle-tree" version = "0.1.0" @@ -5894,6 +5995,29 @@ dependencies = [ "transpose", ] +[[package]] +name = "p3-monty-31" +version = "0.4.1" +source = "git+https://github.com/Plonky3/Plonky3.git?rev=d421e32#d421e32d3821174ae1f7e528d4bb92b7b18ab295" +dependencies = [ + "itertools 0.14.0", + "num-bigint 0.4.6", + "p3-dft 0.4.1", + "p3-field 0.4.1", + "p3-matrix 0.4.1", + "p3-maybe-rayon 0.4.1", + "p3-mds 0.4.1", + "p3-poseidon2 0.4.1", + "p3-symmetric 0.4.1", + "p3-util 0.4.1", + "paste", + "rand 0.9.2", + "serde", + "spin 0.10.0", + "tracing", + "transpose", +] + [[package]] name = "p3-poseidon" version = "0.1.0" @@ -5941,6 +6065,18 @@ dependencies = [ "rand 0.9.2", ] +[[package]] +name = "p3-poseidon2" +version = "0.4.1" +source = "git+https://github.com/Plonky3/Plonky3.git?rev=d421e32#d421e32d3821174ae1f7e528d4bb92b7b18ab295" +dependencies = [ + "p3-field 0.4.1", + "p3-mds 0.4.1", + "p3-symmetric 0.4.1", + "p3-util 0.4.1", + "rand 0.9.2", +] + [[package]] name = "p3-poseidon2-air" version = "0.1.0" @@ -5987,6 +6123,16 @@ dependencies = [ "serde", ] +[[package]] +name = "p3-symmetric" +version = "0.4.1" +source = "git+https://github.com/Plonky3/Plonky3.git?rev=d421e32#d421e32d3821174ae1f7e528d4bb92b7b18ab295" +dependencies = [ + "itertools 0.14.0", + "p3-field 0.4.1", + "serde", +] + [[package]] name = "p3-uni-stark" version = "0.1.0" @@ -6030,6 +6176,14 @@ dependencies = [ "serde", ] +[[package]] +name = "p3-util" +version = "0.4.1" +source = "git+https://github.com/Plonky3/Plonky3.git?rev=d421e32#d421e32d3821174ae1f7e528d4bb92b7b18ab295" +dependencies = [ + "serde", +] + [[package]] name = "pairing" version = "0.22.0" @@ -7271,7 +7425,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -7919,7 +8073,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/rust/hashsig-glue/Cargo.toml b/rust/hashsig-glue/Cargo.toml index 746c5406e..73223576d 100644 --- a/rust/hashsig-glue/Cargo.toml +++ b/rust/hashsig-glue/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" [dependencies] sha2 = "0.9" -leansig = { git = "https://github.com/leanEthereum/leanSig", rev = "b621826f948ffc133dd893131aac2c7efa7f90e0" } +leansig = { git = "https://github.com/leanEthereum/leanSig", rev = "73bedc26ed961b110df7ac2e234dc11361a4bf25" } rand = "0.9.2" rand_chacha = "0.9.0" thiserror = "2.0.17" @@ -17,4 +17,3 @@ serde_json = "1.0" crate-type = ["staticlib"] name = "hashsig_glue" - From 9fe4b7b8eaf75d954f163226215d8f0f26c3ef1d Mon Sep 17 00:00:00 2001 From: grapebaba Date: Fri, 23 Jan 2026 13:17:47 +0800 Subject: [PATCH 10/19] refactor: hashsig test scheme Signed-off-by: grapebaba --- pkgs/key-manager/src/lib.zig | 16 ++--------- .../src/runner/verify_signatures_runner.zig | 7 ++++- pkgs/state-transition/src/lib.zig | 3 +- pkgs/state-transition/src/transition.zig | 24 ++++++++-------- pkgs/xmss/src/hashsig.zig | 21 ++++++++++++++ pkgs/xmss/src/lib.zig | 4 +++ rust/hashsig-glue/src/lib.rs | 28 ++++++++++++++----- 7 files changed, 70 insertions(+), 33 deletions(-) diff --git a/pkgs/key-manager/src/lib.zig b/pkgs/key-manager/src/lib.zig index 5bcb16f50..c4eb1748e 100644 --- a/pkgs/key-manager/src/lib.zig +++ b/pkgs/key-manager/src/lib.zig @@ -6,15 +6,8 @@ const ssz = @import("ssz"); const Allocator = std.mem.Allocator; const JsonValue = std.json.Value; -pub const XmssTestScheme = enum { - @"test", - prod, -}; - -pub const TEST_SIGNATURE_SSZ_LEN: usize = 424; - pub const XmssTestConfig = struct { - scheme: XmssTestScheme, + scheme: xmss.HashSigScheme, signature_ssz_len: usize, allow_placeholder_aggregated_proof: bool, @@ -22,10 +15,7 @@ pub const XmssTestConfig = struct { const scheme = schemeFromLeanEnv(lean_env); return .{ .scheme = scheme, - .signature_ssz_len = switch (scheme) { - .@"test" => TEST_SIGNATURE_SSZ_LEN, - .prod => types.SIGSIZE, - }, + .signature_ssz_len = xmss.signatureSszLenForScheme(scheme), .allow_placeholder_aggregated_proof = scheme == .@"test", }; } @@ -121,7 +111,7 @@ pub const TestKeyManager = struct { } }; -fn schemeFromLeanEnv(lean_env: ?[]const u8) XmssTestScheme { +fn schemeFromLeanEnv(lean_env: ?[]const u8) xmss.HashSigScheme { const env = lean_env orelse return .prod; if (std.ascii.eqlIgnoreCase(env, "test")) return .@"test"; return .prod; diff --git a/pkgs/spectest/src/runner/verify_signatures_runner.zig b/pkgs/spectest/src/runner/verify_signatures_runner.zig index bf279ed97..0307ab2b5 100644 --- a/pkgs/spectest/src/runner/verify_signatures_runner.zig +++ b/pkgs/spectest/src/runner/verify_signatures_runner.zig @@ -216,6 +216,7 @@ fn runCase( const test_config = key_manager.XmssTestConfig.fromLeanEnv(lean_env); const signature_ssz_len: usize = test_config.signature_ssz_len; const allow_placeholder_aggregated_proof = test_config.allow_placeholder_aggregated_proof; + const signature_scheme = test_config.scheme; // Parse the anchorState to get validators const anchor_state_value = case_obj.get("anchorState") orelse { @@ -246,6 +247,7 @@ fn runCase( &parsed.signed_block, parsed.attestation_proofs, signature_ssz_len, + signature_scheme, allow_placeholder_aggregated_proof, ); @@ -276,6 +278,7 @@ fn verifySignaturesWithFixtureProofs( signed_block: *const types.SignedBlockWithAttestation, proofs: []const AggregatedSignatureProof, signature_ssz_len: usize, + signature_scheme: xmss.HashSigScheme, allow_placeholder_aggregated_proof: bool, ) !void { const attestations = signed_block.message.block.body.attestations.constSlice(); @@ -327,6 +330,7 @@ fn verifySignaturesWithFixtureProofs( &proposer_attestation.data, &signed_block.signature.proposer_signature, signature_ssz_len, + signature_scheme, ); } @@ -337,6 +341,7 @@ fn verifySingleAttestationSignature( attestation_data: *const types.AttestationData, signature_bytes: *const types.SIGBYTES, signature_ssz_len: usize, + signature_scheme: xmss.HashSigScheme, ) !void { if (signature_ssz_len > signature_bytes.len) { return types.StateTransitionError.InvalidBlockSignatures; @@ -353,7 +358,7 @@ fn verifySingleAttestationSignature( try ssz.hashTreeRoot(types.AttestationData, attestation_data.*, &message, allocator); const epoch: u32 = @intCast(attestation_data.slot); - try xmss.verifySsz(pubkey, &message, epoch, signature_bytes.*[0..signature_ssz_len]); + try xmss.verifySsz(pubkey, &message, epoch, signature_bytes.*[0..signature_ssz_len], signature_scheme); } fn buildState( diff --git a/pkgs/state-transition/src/lib.zig b/pkgs/state-transition/src/lib.zig index 1fcf601cf..48d2c8938 100644 --- a/pkgs/state-transition/src/lib.zig +++ b/pkgs/state-transition/src/lib.zig @@ -12,8 +12,9 @@ pub const apply_raw_block = transition.apply_raw_block; pub const StateTransitionError = transition.StateTransitionError; pub const StateTransitionOpts = transition.StateTransitionOpts; pub const verifySignatures = transition.verifySignatures; -pub const verifySignaturesWithSignatureLen = transition.verifySignaturesWithSignatureLen; +pub const verifySignaturesWithScheme = transition.verifySignaturesWithScheme; pub const verifySingleAttestation = transition.verifySingleAttestation; +pub const verifySingleAttestationWithScheme = transition.verifySingleAttestationWithScheme; const mockImport = @import("./mock.zig"); pub const genMockChain = mockImport.genMockChain; diff --git a/pkgs/state-transition/src/transition.zig b/pkgs/state-transition/src/transition.zig index d2a585af7..71cae4c28 100644 --- a/pkgs/state-transition/src/transition.zig +++ b/pkgs/state-transition/src/transition.zig @@ -60,20 +60,21 @@ pub fn verifySignatures( state: *const types.BeamState, signed_block: *const types.SignedBlockWithAttestation, ) !void { - return verifySignaturesWithSignatureLen( + return verifySignaturesWithScheme( allocator, state, signed_block, - types.SIGSIZE, + .prod, ); } -pub fn verifySignaturesWithSignatureLen( +pub fn verifySignaturesWithScheme( allocator: Allocator, state: *const types.BeamState, signed_block: *const types.SignedBlockWithAttestation, - signature_ssz_len: usize, + signature_scheme: xmss.HashSigScheme, ) !void { + const signature_ssz_len = xmss.signatureSszLenForScheme(signature_scheme); const attestations = signed_block.message.block.body.attestations.constSlice(); const signature_proofs = signed_block.signature.attestation_signatures.constSlice(); @@ -142,24 +143,25 @@ pub fn verifySignaturesWithSignatureLen( // Verify proposer signature (still individual) const proposer_attestation = signed_block.message.proposer_attestation; - try verifySingleAttestationWithSignatureLen( + try verifySingleAttestationWithScheme( allocator, state, @intCast(proposer_attestation.validator_id), &proposer_attestation.data, &signed_block.signature.proposer_signature, - signature_ssz_len, + signature_scheme, ); } -fn verifySingleAttestationWithSignatureLen( +pub fn verifySingleAttestationWithScheme( allocator: Allocator, state: *const types.BeamState, validator_index: usize, attestation_data: *const types.AttestationData, signatureBytes: *const types.SIGBYTES, - signature_ssz_len: usize, + signature_scheme: xmss.HashSigScheme, ) !void { + const signature_ssz_len = xmss.signatureSszLenForScheme(signature_scheme); if (signature_ssz_len > signatureBytes.len) { return StateTransitionError.InvalidBlockSignatures; } @@ -179,7 +181,7 @@ fn verifySingleAttestationWithSignatureLen( const epoch: u32 = @intCast(attestation_data.slot); - try xmss.verifySsz(pubkey, &message, epoch, signatureBytes.*[0..signature_ssz_len]); + try xmss.verifySsz(pubkey, &message, epoch, signatureBytes.*[0..signature_ssz_len], signature_scheme); _ = verification_timer.observe(); } @@ -190,13 +192,13 @@ pub fn verifySingleAttestation( attestation_data: *const types.AttestationData, signatureBytes: *const types.SIGBYTES, ) !void { - return verifySingleAttestationWithSignatureLen( + return verifySingleAttestationWithScheme( allocator, state, validator_index, attestation_data, signatureBytes, - signatureBytes.len, + .prod, ); } diff --git a/pkgs/xmss/src/hashsig.zig b/pkgs/xmss/src/hashsig.zig index 9344357b1..05eed6b4a 100644 --- a/pkgs/xmss/src/hashsig.zig +++ b/pkgs/xmss/src/hashsig.zig @@ -3,6 +3,21 @@ const Allocator = std.mem.Allocator; pub const aggregate = @import("aggregation.zig"); +pub const HashSigScheme = enum(u8) { + @"test" = 0, + prod = 1, +}; + +pub const PROD_SIGNATURE_SSZ_LEN: usize = 3112; +pub const TEST_SIGNATURE_SSZ_LEN: usize = 424; + +pub fn signatureSszLenForScheme(scheme: HashSigScheme) usize { + return switch (scheme) { + .@"test" => TEST_SIGNATURE_SSZ_LEN, + .prod => PROD_SIGNATURE_SSZ_LEN, + }; +} + /// Opaque pointer to the Rust KeyPair struct pub const HashSigKeyPair = opaque {}; @@ -104,6 +119,7 @@ extern fn hashsig_verify_ssz( epoch: u32, signature_bytes: [*]const u8, signature_len: usize, + scheme: HashSigScheme, ) i32; /// Convert signature JSON (proposerSignature object) into SSZ bytes. @@ -123,6 +139,7 @@ pub fn verifySsz( message: []const u8, epoch: u32, signature_bytes: []const u8, + scheme: HashSigScheme, ) HashSigError!void { if (message.len != 32) { return HashSigError.InvalidMessageLength; @@ -135,6 +152,7 @@ pub fn verifySsz( epoch, signature_bytes.ptr, signature_bytes.len, + scheme, ); switch (result) { @@ -532,6 +550,7 @@ test "HashSig: SSZ serialize and verify" { &message, epoch, sig_buffer[0..sig_size], + .prod, ); std.debug.print("Verification succeeded!\n", .{}); @@ -566,6 +585,7 @@ test "HashSig: verify fails with zero signature" { &message, epoch, &zero_sig_buffer, + .prod, ); try std.testing.expectError(HashSigError.InvalidSignature, invalid_signature_result); @@ -577,6 +597,7 @@ test "HashSig: verify fails with zero signature" { &invalid_message, epoch, signature_buffer[0..signature_size], + .prod, ); try std.testing.expectError(HashSigError.VerificationFailed, verification_failed_result); diff --git a/pkgs/xmss/src/lib.zig b/pkgs/xmss/src/lib.zig index 76417a23e..e4ea570a5 100644 --- a/pkgs/xmss/src/lib.zig +++ b/pkgs/xmss/src/lib.zig @@ -13,6 +13,10 @@ pub const KeyPair = hashsig.KeyPair; pub const Signature = hashsig.Signature; pub const PublicKey = hashsig.PublicKey; pub const HashSigError = hashsig.HashSigError; +pub const HashSigScheme = hashsig.HashSigScheme; +pub const PROD_SIGNATURE_SSZ_LEN = hashsig.PROD_SIGNATURE_SSZ_LEN; +pub const TEST_SIGNATURE_SSZ_LEN = hashsig.TEST_SIGNATURE_SSZ_LEN; +pub const signatureSszLenForScheme = hashsig.signatureSszLenForScheme; pub const verifySsz = hashsig.verifySsz; pub const signatureSszFromJson = hashsig.signatureSszFromJson; pub const HashSigKeyPair = hashsig.HashSigKeyPair; diff --git a/rust/hashsig-glue/src/lib.rs b/rust/hashsig-glue/src/lib.rs index 450a73504..cfb379c8f 100644 --- a/rust/hashsig-glue/src/lib.rs +++ b/rust/hashsig-glue/src/lib.rs @@ -12,6 +12,12 @@ use std::slice; const PROD_SIGNATURE_SSZ_LEN: usize = 3112; const TEST_SIGNATURE_SSZ_LEN: usize = 424; +#[repr(u8)] +enum HashSigSchemeId { + Test = 0, + Prod = 1, +} + /// Production instantiation (LeanSpec `prod`). pub type HashSigSchemeProd = leansig::signature::generalized_xmss::instantiations_poseidon_top_level::lifetime_2_to_the_32::hashing_optimized::SIGTopLevelTargetSumLifetime32Dim64Base8; @@ -522,12 +528,23 @@ pub unsafe extern "C" fn hashsig_verify_ssz( epoch: u32, signature_bytes: *const u8, signature_len: usize, + scheme_id: u8, ) -> i32 { if pubkey_bytes.is_null() || message.is_null() || signature_bytes.is_null() { return -1; } unsafe { + let expected_len = match scheme_id { + x if x == HashSigSchemeId::Test as u8 => TEST_SIGNATURE_SSZ_LEN, + x if x == HashSigSchemeId::Prod as u8 => PROD_SIGNATURE_SSZ_LEN, + _ => return -1, + }; + + if signature_len != expected_len { + return -1; + } + let pk_data = slice::from_raw_parts(pubkey_bytes, pubkey_len); let sig_data = slice::from_raw_parts(signature_bytes, signature_len); let msg_data = slice::from_raw_parts(message, MESSAGE_LENGTH); @@ -548,17 +565,14 @@ pub unsafe extern "C" fn hashsig_verify_ssz( Ok(S::verify(&pk, epoch, message_array, &sig)) } - let attempt: Result = match signature_len { - TEST_SIGNATURE_SSZ_LEN => { + let attempt: Result = match scheme_id { + x if x == HashSigSchemeId::Test as u8 => { verify_with_scheme::(pk_data, sig_data, epoch, message_array) } - PROD_SIGNATURE_SSZ_LEN => { + x if x == HashSigSchemeId::Prod as u8 => { verify_with_scheme::(pk_data, sig_data, epoch, message_array) } - _ => verify_with_scheme::(pk_data, sig_data, epoch, message_array) - .or_else(|_| { - verify_with_scheme::(pk_data, sig_data, epoch, message_array) - }), + _ => return -1, }; match attempt { From cf0df8fd52f25a2075f8e3b4bbba29dbd6c81749 Mon Sep 17 00:00:00 2001 From: grapebaba Date: Fri, 23 Jan 2026 13:33:17 +0800 Subject: [PATCH 11/19] fix: fix test Signed-off-by: grapebaba --- build.zig | 18 ++---------------- pkgs/state-transition/src/transition.zig | 1 - 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/build.zig b/build.zig index adcf47368..0d77aa8b9 100644 --- a/build.zig +++ b/build.zig @@ -28,11 +28,6 @@ fn setTestRunLabelFromCompile(b: *Builder, run_step: *std.Build.Step.Run, compil setTestRunLabel(b, run_step, source_name); } -fn fileExists(path: []const u8) bool { - const file = std.fs.cwd().openFile(path, .{}) catch return false; - file.close(); - return true; -} // Add the glue libs to a compile target fn addRustGlueLib(b: *Builder, comp: *Builder.Step.Compile, target: Builder.ResolvedTarget, prover: ProverChoice) void { @@ -616,22 +611,13 @@ pub fn build(b: *Builder) !void { .optimize = optimize, }); const run_spectest_generate = b.addRunArtifact(spectest_generate_exe); - const run_spectest_format = b.addSystemCommand(&.{ "zig", "fmt", "pkgs/spectest/src/generated" }); - run_spectest_format.step.dependOn(&run_spectest_generate.step); const spectest_generate_step = b.step("spectest:generate", "Regenerate spectest fixtures"); - spectest_generate_step.dependOn(&run_spectest_format.step); + spectest_generate_step.dependOn(&run_spectest_generate.step); const run_spectests_after_generate = b.addRunArtifact(spectests); - run_spectests_after_generate.step.dependOn(&run_spectest_format.step); + run_spectests_after_generate.step.dependOn(&run_spectest_generate.step); const run_spectests = b.addRunArtifact(spectests); - if (!fileExists("pkgs/spectest/src/generated/index.zig")) { - // `spectest:run` expects generated tests to exist already, but a fresh checkout has - // none. Generate a stub index (or real tests if fixtures exist) to keep the command - // usable without requiring a separate `spectest:generate` invocation first. - spectests.step.dependOn(&run_spectest_format.step); - } - const spectests_step = b.step("spectest", "Regenerate and run spec tests"); spectests_step.dependOn(&run_spectests_after_generate.step); diff --git a/pkgs/state-transition/src/transition.zig b/pkgs/state-transition/src/transition.zig index 71cae4c28..c22b6cbc0 100644 --- a/pkgs/state-transition/src/transition.zig +++ b/pkgs/state-transition/src/transition.zig @@ -74,7 +74,6 @@ pub fn verifySignaturesWithScheme( signed_block: *const types.SignedBlockWithAttestation, signature_scheme: xmss.HashSigScheme, ) !void { - const signature_ssz_len = xmss.signatureSszLenForScheme(signature_scheme); const attestations = signed_block.message.block.body.attestations.constSlice(); const signature_proofs = signed_block.signature.attestation_signatures.constSlice(); From 1a3379881648ffefc25bd84fb06f4f93b25f3583 Mon Sep 17 00:00:00 2001 From: grapebaba Date: Fri, 23 Jan 2026 13:38:54 +0800 Subject: [PATCH 12/19] fix: fix lint Signed-off-by: grapebaba --- build.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/build.zig b/build.zig index 0d77aa8b9..8115d3a4f 100644 --- a/build.zig +++ b/build.zig @@ -28,7 +28,6 @@ fn setTestRunLabelFromCompile(b: *Builder, run_step: *std.Build.Step.Run, compil setTestRunLabel(b, run_step, source_name); } - // Add the glue libs to a compile target fn addRustGlueLib(b: *Builder, comp: *Builder.Step.Compile, target: Builder.ResolvedTarget, prover: ProverChoice) void { // Conditionally include prover libraries based on selection From 9b24b2b4a47af2d4769a8fc7ed621abe2431e910 Mon Sep 17 00:00:00 2001 From: grapebaba Date: Fri, 23 Jan 2026 15:02:07 +0800 Subject: [PATCH 13/19] fix: fix review comments Signed-off-by: grapebaba --- .../src/runner/verify_signatures_runner.zig | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/pkgs/spectest/src/runner/verify_signatures_runner.zig b/pkgs/spectest/src/runner/verify_signatures_runner.zig index 0307ab2b5..9aa531aea 100644 --- a/pkgs/spectest/src/runner/verify_signatures_runner.zig +++ b/pkgs/spectest/src/runner/verify_signatures_runner.zig @@ -795,32 +795,42 @@ fn parseSignature( return sig_bytes; } -fn parseU32Array8( +fn parseU32ArrayN( + comptime N: usize, ctx: Context, obj: std.json.ObjectMap, label: []const u8, -) FixtureError![8]u32 { +) FixtureError![N]u32 { const data_arr = try expect.expectArrayField(FixtureError, obj, &.{"data"}, ctx, label); - var result: [8]u32 = undefined; + if (data_arr.items.len != N) { + std.debug.print( + "fixture {s} case {s}: {s} length {d} != expected {d}\n", + .{ ctx.fixture_label, ctx.case_name, label, data_arr.items.len, N }, + ); + return FixtureError.InvalidFixture; + } + + var result: [N]u32 = undefined; for (data_arr.items, 0..) |val, i| { - if (i >= 8) break; result[i] = @intCast(try expect.expectU64Value(FixtureError, val, ctx, label)); } return result; } +fn parseU32Array8( + ctx: Context, + obj: std.json.ObjectMap, + label: []const u8, +) FixtureError![8]u32 { + return parseU32ArrayN(8, ctx, obj, label); +} + fn parseU32Array7( ctx: Context, obj: std.json.ObjectMap, label: []const u8, ) FixtureError![7]u32 { - const data_arr = try expect.expectArrayField(FixtureError, obj, &.{"data"}, ctx, label); - var result: [7]u32 = undefined; - for (data_arr.items, 0..) |val, i| { - if (i >= 7) break; - result[i] = @intCast(try expect.expectU64Value(FixtureError, val, ctx, label)); - } - return result; + return parseU32ArrayN(7, ctx, obj, label); } fn parseCheckpoint( From 961ee03d3f48faf28d36631bf9b59416178c67af Mon Sep 17 00:00:00 2001 From: grapebaba Date: Fri, 23 Jan 2026 18:00:36 +0800 Subject: [PATCH 14/19] fix: fix review comments Signed-off-by: grapebaba --- pkgs/key-manager/src/lib.zig | 235 +++++++++++++++++++---------------- 1 file changed, 125 insertions(+), 110 deletions(-) diff --git a/pkgs/key-manager/src/lib.zig b/pkgs/key-manager/src/lib.zig index c4eb1748e..0cf4c5f62 100644 --- a/pkgs/key-manager/src/lib.zig +++ b/pkgs/key-manager/src/lib.zig @@ -21,94 +21,12 @@ pub const XmssTestConfig = struct { } }; -pub const TestKeyManagerError = error{ +pub const FixtureKeyError = error{ DuplicateKeyIndex, InvalidKeyFile, InvalidKeyIndex, InvalidPublicKey, NoKeysFound, - PublicKeyNotFound, -}; - -pub const TestKeyManager = struct { - allocator: Allocator, - config: XmssTestConfig, - pubkeys: std.AutoHashMap(usize, types.Bytes52), - - const Self = @This(); - - pub fn init(allocator: Allocator, lean_env: ?[]const u8) Self { - return Self{ - .allocator = allocator, - .config = XmssTestConfig.fromLeanEnv(lean_env), - .pubkeys = std.AutoHashMap(usize, types.Bytes52).init(allocator), - }; - } - - pub fn deinit(self: *Self) void { - self.pubkeys.deinit(); - } - - pub fn loadLeanSpecKeys(self: *Self, keys_root: []const u8) !void { - const scheme_dir_name = switch (self.config.scheme) { - .@"test" => "test_scheme", - .prod => "prod_scheme", - }; - const scheme_dir_path = try std.fs.path.join(self.allocator, &.{ keys_root, scheme_dir_name }); - defer self.allocator.free(scheme_dir_path); - try self.loadKeysFromDir(scheme_dir_path); - } - - pub fn loadKeysFromDir(self: *Self, keys_dir_path: []const u8) !void { - var dir = try std.fs.cwd().openDir(keys_dir_path, .{ .iterate = true }); - defer dir.close(); - - self.pubkeys.clearRetainingCapacity(); - - var it = dir.iterate(); - while (try it.next()) |entry| { - if (entry.kind != .file) continue; - const index = parseKeyIndex(entry.name) catch continue; - const pubkey = try readPublicKeyFromJson(self.allocator, dir, entry.name); - - const gop = try self.pubkeys.getOrPut(index); - if (gop.found_existing) { - return TestKeyManagerError.DuplicateKeyIndex; - } - gop.value_ptr.* = pubkey; - } - - if (self.pubkeys.count() == 0) { - return TestKeyManagerError.NoKeysFound; - } - } - - pub fn getPublicKeyBytes(self: *const Self, validator_index: usize) !types.Bytes52 { - return self.pubkeys.get(validator_index) orelse TestKeyManagerError.PublicKeyNotFound; - } - - pub fn getAllPubkeys( - self: *const Self, - allocator: Allocator, - num_validators: usize, - ) ![]types.Bytes52 { - const pubkeys = try allocator.alloc(types.Bytes52, num_validators); - errdefer allocator.free(pubkeys); - - for (0..num_validators) |i| { - pubkeys[i] = try self.getPublicKeyBytes(i); - } - - return pubkeys; - } - - pub fn signatureSszLen(self: *const Self) usize { - return self.config.signature_ssz_len; - } - - pub fn allowPlaceholderAggregatedProof(self: *const Self) bool { - return self.config.allow_placeholder_aggregated_proof; - } }; fn schemeFromLeanEnv(lean_env: ?[]const u8) xmss.HashSigScheme { @@ -119,58 +37,72 @@ fn schemeFromLeanEnv(lean_env: ?[]const u8) xmss.HashSigScheme { fn parseKeyIndex(file_name: []const u8) !usize { if (!std.mem.endsWith(u8, file_name, ".json")) { - return TestKeyManagerError.InvalidKeyIndex; + return FixtureKeyError.InvalidKeyIndex; } const stem = file_name[0 .. file_name.len - ".json".len]; if (stem.len == 0) { - return TestKeyManagerError.InvalidKeyIndex; + return FixtureKeyError.InvalidKeyIndex; } - return std.fmt.parseInt(usize, stem, 10) catch TestKeyManagerError.InvalidKeyIndex; + return std.fmt.parseInt(usize, stem, 10) catch FixtureKeyError.InvalidKeyIndex; } +const fixture_key_file_max_bytes: usize = 2 * 1024 * 1024; + fn readPublicKeyFromJson( allocator: Allocator, dir: std.fs.Dir, file_name: []const u8, ) !types.Bytes52 { - const max_bytes: usize = 2 * 1024 * 1024; - const payload = dir.readFileAlloc(allocator, file_name, max_bytes) catch { - return TestKeyManagerError.InvalidKeyFile; + const payload = dir.readFileAlloc(allocator, file_name, fixture_key_file_max_bytes) catch { + return FixtureKeyError.InvalidKeyFile; }; defer allocator.free(payload); var parsed = std.json.parseFromSlice(JsonValue, allocator, payload, .{ .ignore_unknown_fields = true }) catch { - return TestKeyManagerError.InvalidKeyFile; + return FixtureKeyError.InvalidKeyFile; }; defer parsed.deinit(); const obj = switch (parsed.value) { .object => |map| map, - else => return TestKeyManagerError.InvalidKeyFile, + else => return FixtureKeyError.InvalidKeyFile, }; - const pub_val = obj.get("public") orelse return TestKeyManagerError.InvalidKeyFile; + const pub_val = obj.get("public") orelse return FixtureKeyError.InvalidKeyFile; const pub_hex = switch (pub_val) { .string => |s| s, - else => return TestKeyManagerError.InvalidKeyFile, + else => return FixtureKeyError.InvalidKeyFile, }; return parsePublicKeyHex(pub_hex); } fn parsePublicKeyHex(input: []const u8) !types.Bytes52 { + const public_key_hex_len: usize = 2 * @sizeOf(types.Bytes52); const hex_str = if (std.mem.startsWith(u8, input, "0x")) input[2..] else input; - if (hex_str.len != 104) { - return TestKeyManagerError.InvalidPublicKey; + if (hex_str.len != public_key_hex_len) { + return FixtureKeyError.InvalidPublicKey; } var bytes: types.Bytes52 = undefined; _ = std.fmt.hexToBytes(&bytes, hex_str) catch { - return TestKeyManagerError.InvalidPublicKey; + return FixtureKeyError.InvalidPublicKey; }; return bytes; } const KeyManagerError = error{ ValidatorKeyNotFound, + PrivateKeyMissing, + PublicKeyBufferTooSmall, +}; + +const PublicKeyEntry = struct { + bytes: types.Bytes52, + public_key: xmss.PublicKey, +}; + +const KeyEntry = union(enum) { + keypair: xmss.KeyPair, + public_key: PublicKeyEntry, }; const CachedKeyPair = struct { @@ -217,7 +149,7 @@ fn getOrCreateCachedKeyPair( } pub const KeyManager = struct { - keys: std.AutoHashMap(usize, xmss.KeyPair), + keys: std.AutoHashMap(usize, KeyEntry), allocator: Allocator, owns_keypairs: bool, @@ -225,24 +157,72 @@ pub const KeyManager = struct { pub fn init(allocator: Allocator) Self { return Self{ - .keys = std.AutoHashMap(usize, xmss.KeyPair).init(allocator), + .keys = std.AutoHashMap(usize, KeyEntry).init(allocator), .allocator = allocator, .owns_keypairs = true, }; } pub fn deinit(self: *Self) void { - if (self.owns_keypairs) { - var it = self.keys.iterator(); - while (it.next()) |entry| { - entry.value_ptr.deinit(); - } - } + self.clearEntries(); self.keys.deinit(); } pub fn addKeypair(self: *Self, validator_id: usize, keypair: xmss.KeyPair) !void { - try self.keys.put(validator_id, keypair); + if (self.keys.getPtr(validator_id)) |entry| { + self.deinitEntry(entry); + entry.* = .{ .keypair = keypair }; + return; + } + try self.keys.put(validator_id, .{ .keypair = keypair }); + } + + pub fn addPublicKey(self: *Self, validator_id: usize, pubkey_bytes: types.Bytes52) !void { + var public_key = try xmss.PublicKey.fromBytes(pubkey_bytes[0..]); + errdefer public_key.deinit(); + + const entry_value = PublicKeyEntry{ .bytes = pubkey_bytes, .public_key = public_key }; + if (self.keys.getPtr(validator_id)) |entry| { + self.deinitEntry(entry); + entry.* = .{ .public_key = entry_value }; + return; + } + try self.keys.put(validator_id, .{ .public_key = entry_value }); + } + + pub fn loadLeanSpecKeys(self: *Self, keys_root: []const u8, lean_env: ?[]const u8) !void { + const scheme_dir_name = switch (schemeFromLeanEnv(lean_env)) { + .@"test" => "test_scheme", + .prod => "prod_scheme", + }; + const scheme_dir_path = try std.fs.path.join(self.allocator, &.{ keys_root, scheme_dir_name }); + defer self.allocator.free(scheme_dir_path); + try self.loadKeysFromDir(scheme_dir_path); + } + + pub fn loadKeysFromDir(self: *Self, keys_dir_path: []const u8) !void { + var dir = try std.fs.cwd().openDir(keys_dir_path, .{ .iterate = true }); + defer dir.close(); + + self.clearEntries(); + + var it = dir.iterate(); + while (try it.next()) |entry| { + if (entry.kind != .file) continue; + const index = parseKeyIndex(entry.name) catch continue; + const pubkey = try readPublicKeyFromJson(self.allocator, dir, entry.name); + if (self.keys.get(index) != null) { + return FixtureKeyError.DuplicateKeyIndex; + } + self.addPublicKey(index, pubkey) catch |err| switch (err) { + error.OutOfMemory => return err, + else => return FixtureKeyError.InvalidPublicKey, + }; + } + + if (self.keys.count() == 0) { + return FixtureKeyError.NoKeysFound; + } } pub fn loadFromKeypairDir(_: *Self, _: []const u8) !void { @@ -273,8 +253,17 @@ pub const KeyManager = struct { validator_index: usize, buffer: []u8, ) !usize { - const keypair = self.keys.get(validator_index) orelse return KeyManagerError.ValidatorKeyNotFound; - return try keypair.pubkeyToBytes(buffer); + const entry = self.keys.getPtr(validator_index) orelse return KeyManagerError.ValidatorKeyNotFound; + return switch (entry.*) { + .keypair => |keypair| try keypair.pubkeyToBytes(buffer), + .public_key => |pubkey| blk: { + if (buffer.len < pubkey.bytes.len) { + return KeyManagerError.PublicKeyBufferTooSmall; + } + @memcpy(buffer[0..pubkey.bytes.len], pubkey.bytes[0..]); + break :blk pubkey.bytes.len; + }, + }; } /// Extract all validator public keys into an array @@ -300,8 +289,11 @@ pub const KeyManager = struct { self: *const Self, validator_index: usize, ) !*const xmss.HashSigPublicKey { - const keypair = self.keys.get(validator_index) orelse return KeyManagerError.ValidatorKeyNotFound; - return keypair.public_key; + const entry = self.keys.getPtr(validator_index) orelse return KeyManagerError.ValidatorKeyNotFound; + return switch (entry.*) { + .keypair => |keypair| keypair.public_key, + .public_key => |pubkey| pubkey.public_key.handle, + }; } /// Sign an attestation and return the raw signature handle (for aggregation) @@ -312,7 +304,11 @@ pub const KeyManager = struct { allocator: Allocator, ) !xmss.Signature { const validator_index: usize = @intCast(attestation.validator_id); - const keypair = self.keys.get(validator_index) orelse return KeyManagerError.ValidatorKeyNotFound; + const entry = self.keys.getPtr(validator_index) orelse return KeyManagerError.ValidatorKeyNotFound; + const keypair = switch (entry.*) { + .keypair => |*kp| kp, + .public_key => return KeyManagerError.PrivateKeyMissing, + }; const signing_timer = zeam_metrics.lean_pq_signature_attestation_signing_time_seconds.start(); var message: [32]u8 = undefined; @@ -324,6 +320,25 @@ pub const KeyManager = struct { return signature; } + + fn clearEntries(self: *Self) void { + var it = self.keys.iterator(); + while (it.next()) |entry| { + self.deinitEntry(entry.value_ptr); + } + self.keys.clearRetainingCapacity(); + } + + fn deinitEntry(self: *Self, entry: *KeyEntry) void { + switch (entry.*) { + .keypair => |*keypair| { + if (self.owns_keypairs) { + keypair.deinit(); + } + }, + .public_key => |*pubkey| pubkey.public_key.deinit(), + } + } }; pub fn getTestKeyManager( From 51f2ff4fc112a2c06849c1e93d3a18ba256bb6fd Mon Sep 17 00:00:00 2001 From: Parthasarathy Ramanujam <1627026+ch4r10t33r@users.noreply.github.com> Date: Sat, 24 Jan 2026 11:21:34 +0000 Subject: [PATCH 15/19] feat: add API endpoint versioning (#514) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add API endpoint versioning Version API endpoints with /lean/v0/ prefix to align with leanSpec PR #323: - /health → /lean/v0/health - /lean/states/finalized → /lean/v0/states/finalized Updates endpoint routes, documentation, tests, and CLI help messages. * docs: update endpoint references to use versioned API paths Update documentation to reflect versioned API endpoints: - /health → /lean/v0/health - /lean/states/finalized → /lean/v0/states/finalized Updated files: - pkgs/api/README.md: API documentation and examples - resources/checkpoint-sync.md: Checkpoint sync guide - pkgs/cli/test/fixtures/README.md: Local devnet setup guide * feat: add /lean/v0/states/justified endpoint Implement the justified checkpoint endpoint to match leanSpec PR #323: - Add getJustifiedCheckpoint method to BeamChain - Add /lean/v0/states/justified route handler in api_server - Add handleJustifiedCheckpoint method to return JSON with checkpoint info - Update API documentation with new endpoint The endpoint returns the latest justified checkpoint as JSON with slot and root fields. Returns 503 if chain is not initialized. All tests passing (104/104). * test: add integration test for /lean/v0/states/justified endpoint Add comprehensive integration test that verifies: - /lean/v0/health endpoint returns 200 OK with healthy status - /lean/v0/states/justified endpoint returns JSON with checkpoint data - Response contains required fields: "root" and "slot" - JSON structure is valid and parseable - Slot value is a valid integer Test spins up a beam simulation node, waits for blocks to be processed, and validates both endpoints are functioning correctly. * fix: improve justified endpoint test robustness for CI - Increase initial wait time from 5s to 10s for chain initialization - Add retry logic if chain is not initialized (503 response) - Add detailed response logging for debugging - Handle 503 Service Unavailable gracefully with additional 10s wait - Ensures test passes in CI environment where initialization may be slower Fixes CI test failure where chain wasn't fully initialized when endpoint was first queried. * test: remove integration test for new API endpoints Remove the integration test for /lean/v0/health and /lean/v0/states/justified endpoints as it was causing timing issues in CI. The endpoints are still functional and can be tested manually. * update: leanSpec version --------- Co-authored-by: anshalshukla --- leanSpec | 2 +- pkgs/api/README.md | 51 +++++++++++++++++++++++++++----- pkgs/cli/src/api_server.zig | 43 +++++++++++++++++++++++++-- pkgs/cli/src/main.zig | 2 +- pkgs/cli/src/node.zig | 6 ++-- pkgs/cli/test/fixtures/README.md | 8 ++--- pkgs/cli/test/integration.zig | 4 +-- pkgs/node/src/chain.zig | 6 ++++ resources/checkpoint-sync.md | 8 ++--- 9 files changed, 105 insertions(+), 25 deletions(-) diff --git a/leanSpec b/leanSpec index fe2546c08..690298d79 160000 --- a/leanSpec +++ b/leanSpec @@ -1 +1 @@ -Subproject commit fe2546c08bf62da7da854d22ab7c70b8984ffa3e +Subproject commit 690298d797208342b86c356df391b2a30f1ab0bb diff --git a/pkgs/api/README.md b/pkgs/api/README.md index e1f0a8e22..401e94f90 100644 --- a/pkgs/api/README.md +++ b/pkgs/api/README.md @@ -2,11 +2,13 @@ ## Overview -This package provides the HTTP API server for the Zeam node with three main endpoints: +This package provides the HTTP API server for the Zeam node with five main endpoints: - Server-Sent Events (SSE) stream for real-time chain events at `/events` - Prometheus metrics endpoint at `/metrics` -- Health check at `/health` +- Health check at `/lean/v0/health` +- Finalized checkpoint state at `/lean/v0/states/finalized` (for checkpoint sync) +- Justified checkpoint information at `/lean/v0/states/justified` ## Package Components @@ -31,7 +33,7 @@ Provides real-time chain event streaming via Server-Sent Events: ### 2. Health Checks -Simple health check endpoint at `/health`. +Simple health check endpoint at `/lean/v0/health`. ## Event System @@ -97,14 +99,41 @@ Streams real-time chain events (head, justification, finalization). curl -N http://localhost:9667/events ``` -### `/health` +### `/lean/v0/health` Returns node health status. ```sh -curl http://localhost:9667/health +curl http://localhost:9667/lean/v0/health ``` +### `/lean/v0/states/finalized` + +Returns the finalized checkpoint state as SSZ-encoded binary for checkpoint sync. + +```sh +curl http://localhost:9667/lean/v0/states/finalized -o finalized_state.ssz +``` + +Returns: +- **Content-Type**: `application/octet-stream` +- **Body**: SSZ-encoded `BeamState` +- **Status 503**: Returned if no finalized state is available yet + +### `/lean/v0/states/justified` + +Returns the latest justified checkpoint information as JSON. + +```sh +curl http://localhost:9667/lean/v0/states/justified +``` + +Returns: +- **Content-Type**: `application/json` +- **Body**: JSON object with `slot` and `root` fields +- **Status 503**: Returned if chain is not initialized +- **Example response**: `{"root":"0x1234...","slot":42}` + ## Usage ### Initialization @@ -122,7 +151,9 @@ try api_server.startAPIServer(allocator, apiPort); The server exposes: - SSE at `/events` - Metrics at `/metrics` -- Health at `/health` +- Health at `/lean/v0/health` +- Checkpoint state at `/lean/v0/states/finalized` +- Justified checkpoint at `/lean/v0/states/justified` **Note**: On freestanding targets (ZKVM), the HTTP server is automatically disabled. @@ -179,7 +210,13 @@ curl -N http://localhost:9668/events curl http://localhost:9668/metrics # Health -curl http://localhost:9668/health +curl http://localhost:9668/lean/v0/health + +# Checkpoint state +curl http://localhost:9668/lean/v0/states/finalized -o state.ssz + +# Justified checkpoint +curl http://localhost:9668/lean/v0/states/justified ``` ## Visualization with Prometheus & Grafana diff --git a/pkgs/cli/src/api_server.zig b/pkgs/cli/src/api_server.zig index 425d4e159..2c3d33ceb 100644 --- a/pkgs/cli/src/api_server.zig +++ b/pkgs/cli/src/api_server.zig @@ -98,15 +98,21 @@ const ApiServer = struct { } else if (std.mem.eql(u8, request.head.target, "/metrics")) { // Handle metrics request self.handleMetrics(&request); - } else if (std.mem.eql(u8, request.head.target, "/health")) { + } else if (std.mem.eql(u8, request.head.target, "/lean/v0/health")) { // Handle health check self.handleHealth(&request); - } else if (std.mem.eql(u8, request.head.target, "/lean/states/finalized")) { + } else if (std.mem.eql(u8, request.head.target, "/lean/v0/states/finalized")) { // Handle finalized checkpoint state endpoint self.handleFinalizedCheckpointState(&request) catch |err| { self.logger.warn("failed to handle finalized checkpoint state request: {}", .{err}); _ = request.respond("Internal Server Error\n", .{ .status = .internal_server_error }) catch {}; }; + } else if (std.mem.eql(u8, request.head.target, "/lean/v0/states/justified")) { + // Handle justified checkpoint endpoint + self.handleJustifiedCheckpoint(&request) catch |err| { + self.logger.warn("failed to handle justified checkpoint request: {}", .{err}); + _ = request.respond("Internal Server Error\n", .{ .status = .internal_server_error }) catch {}; + }; } else { _ = request.respond("Not Found\n", .{ .status = .not_found }) catch {}; } @@ -140,7 +146,7 @@ const ApiServer = struct { } /// Handle finalized checkpoint state endpoint - /// Serves the finalized checkpoint lean state (BeamState) as SSZ octet-stream at /lean/states/finalized + /// Serves the finalized checkpoint lean state (BeamState) as SSZ octet-stream at /lean/v0/states/finalized fn handleFinalizedCheckpointState(self: *const Self, request: *std.http.Server.Request) !void { // Get the chain (may be null if API server started before chain initialization) const chain = self.chain orelse { @@ -180,6 +186,37 @@ const ApiServer = struct { }; } + /// Handle justified checkpoint endpoint + /// Returns the latest justified checkpoint information as JSON at /lean/v0/states/justified + fn handleJustifiedCheckpoint(self: *const Self, request: *std.http.Server.Request) !void { + // Get the chain (may be null if API server started before chain initialization) + const chain = self.chain orelse { + _ = request.respond("Service Unavailable: Chain not initialized\n", .{ .status = .service_unavailable }) catch {}; + return; + }; + + // Get justified checkpoint from chain (chain handles its own locking internally) + const justified_checkpoint = chain.getJustifiedCheckpoint(); + + // Convert checkpoint to JSON string + const json_string = justified_checkpoint.toJsonString(self.allocator) catch |err| { + self.logger.err("failed to serialize justified checkpoint to JSON: {}", .{err}); + _ = request.respond("Internal Server Error: Serialization failed\n", .{ .status = .internal_server_error }) catch {}; + return; + }; + defer self.allocator.free(json_string); + + // Respond with JSON + _ = request.respond(json_string, .{ + .extra_headers = &.{ + .{ .name = "content-type", .value = "application/json; charset=utf-8" }, + }, + }) catch |err| { + self.logger.warn("failed to respond with justified checkpoint: {}", .{err}); + return err; + }; + } + /// Handle SSE events endpoint fn handleSSEEvents(self: *const Self, stream: std.net.Stream) !void { // Set SSE headers manually by writing HTTP response diff --git a/pkgs/cli/src/main.zig b/pkgs/cli/src/main.zig index b8ba96de4..0bb94bc9d 100644 --- a/pkgs/cli/src/main.zig +++ b/pkgs/cli/src/main.zig @@ -75,7 +75,7 @@ pub const NodeCommand = struct { .override_genesis_time = "Override genesis time in the config.yaml", .@"sig-keys-dir" = "Relative path of custom genesis to signature key directory", .@"data-dir" = "Path to the data directory", - .@"checkpoint-sync-url" = "URL to fetch finalized checkpoint state from for checkpoint sync (e.g., http://localhost:5052/lean/states/finalized)", + .@"checkpoint-sync-url" = "URL to fetch finalized checkpoint state from for checkpoint sync (e.g., http://localhost:5052/lean/v0/states/finalized)", .help = "Show help information for the node command", }; }; diff --git a/pkgs/cli/src/node.zig b/pkgs/cli/src/node.zig index bddfbee0b..7cf3ee7f5 100644 --- a/pkgs/cli/src/node.zig +++ b/pkgs/cli/src/node.zig @@ -1265,11 +1265,11 @@ test "checkpoint-sync-url parameter is optional" { .@"node-id" = "test", .validator_config = "test", .override_genesis_time = null, - .@"checkpoint-sync-url" = "http://localhost:5052/lean/states/finalized", + .@"checkpoint-sync-url" = "http://localhost:5052/lean/v0/states/finalized", }; try std.testing.expect(node_cmd_with_url.@"checkpoint-sync-url" != null); - try std.testing.expectEqualStrings(node_cmd_with_url.@"checkpoint-sync-url".?, "http://localhost:5052/lean/states/finalized"); + try std.testing.expectEqualStrings(node_cmd_with_url.@"checkpoint-sync-url".?, "http://localhost:5052/lean/v0/states/finalized"); } test "NodeOptions checkpoint_sync_url field is optional" { @@ -1314,6 +1314,6 @@ test "NodeOptions checkpoint_sync_url field is optional" { try std.testing.expect(node_options.checkpoint_sync_url == null); // Test with a URL - node_options.checkpoint_sync_url = "http://localhost:5052/lean/states/finalized"; + node_options.checkpoint_sync_url = "http://localhost:5052/lean/v0/states/finalized"; try std.testing.expect(node_options.checkpoint_sync_url != null); } diff --git a/pkgs/cli/test/fixtures/README.md b/pkgs/cli/test/fixtures/README.md index 3f94706ac..54ab3a575 100644 --- a/pkgs/cli/test/fixtures/README.md +++ b/pkgs/cli/test/fixtures/README.md @@ -272,10 +272,10 @@ To enable checkpoint sync, add the `--checkpoint-sync-url` parameter: --validator_config genesis_bootnode \ --override_genesis_time $GENESIS_TIME \ --data-dir ./data/test_node1 \ - --checkpoint-sync-url http://localhost:5052/lean/states/finalized + --checkpoint-sync-url http://localhost:5052/lean/v0/states/finalized ``` -The URL should point to a zeam node's checkpoint state endpoint (e.g., `http://localhost:5052/lean/states/finalized` if the source node has metrics enabled on port 5052). +The URL should point to a zeam node's checkpoint state endpoint (e.g., `http://localhost:5052/lean/v0/states/finalized` if the source node has metrics enabled on port 5052). ### Checkpoint Sync Server @@ -292,7 +292,7 @@ To serve checkpoint state from a zeam node, enable the metrics server: --api-port 5052 ``` -This node will serve the finalized checkpoint state at `http://localhost:5052/lean/states/finalized`. +This node will serve the finalized checkpoint state at `http://localhost:5052/lean/v0/states/finalized`. ### Checkpoint Sync Example @@ -318,7 +318,7 @@ This node will serve the finalized checkpoint state at `http://localhost:5052/le --validator_config genesis_bootnode \ --override_genesis_time 1759210782 \ --data-dir ./data/test_node1 \ - --checkpoint-sync-url http://localhost:5052/lean/states/finalized + --checkpoint-sync-url http://localhost:5052/lean/v0/states/finalized ``` **Note:** The `--checkpoint-sync-url` parameter is optional. If not provided, the node will start from genesis as usual. diff --git a/pkgs/cli/test/integration.zig b/pkgs/cli/test/integration.zig index b61c26ca0..a9874c3dd 100644 --- a/pkgs/cli/test/integration.zig +++ b/pkgs/cli/test/integration.zig @@ -159,9 +159,9 @@ const ZeamRequest = struct { return self.makeRequest("/metrics"); } - /// Make a request to the /health endpoint and return the response + /// Make a request to the /lean/v0/health endpoint and return the response fn getHealth(self: ZeamRequest) ![]u8 { - return self.makeRequest("/health"); + return self.makeRequest("/lean/v0/health"); } /// Internal helper to make HTTP requests to any endpoint diff --git a/pkgs/node/src/chain.zig b/pkgs/node/src/chain.zig index 861e49122..c37c67b23 100644 --- a/pkgs/node/src/chain.zig +++ b/pkgs/node/src/chain.zig @@ -1203,6 +1203,12 @@ pub const BeamChain = struct { return state_ptr; } + /// Get the latest justified checkpoint + /// Returns the checkpoint with slot and root of the most recent justified checkpoint + pub fn getJustifiedCheckpoint(self: *Self) types.Checkpoint { + return self.forkChoice.fcStore.latest_justified; + } + pub const SyncStatus = union(enum) { synced, no_peers, diff --git a/resources/checkpoint-sync.md b/resources/checkpoint-sync.md index eeed1168d..9c6c21c53 100644 --- a/resources/checkpoint-sync.md +++ b/resources/checkpoint-sync.md @@ -20,7 +20,7 @@ Checkpoint sync allows a new node to quickly synchronize by downloading the fina 2. Verify the first node has finalized state by checking its API: ```sh - curl -s http://localhost:9667/lean/states/finalized -o /dev/null -w "%{http_code}\n" + curl -s http://localhost:9667/lean/v0/states/finalized -o /dev/null -w "%{http_code}\n" ``` You should see `200` once the node has finalized state available. @@ -32,14 +32,14 @@ Checkpoint sync allows a new node to quickly synchronize by downloading the fina ``` The second node will: - - Download the finalized state from zeam_0's API endpoint (`/lean/states/finalized`) + - Download the finalized state from zeam_0's API endpoint (`/lean/v0/states/finalized`) - Verify the state matches the expected genesis configuration (validator count) - Use this state as its anchor to sync forward 4. Observe the logs on zeam_1. You should see messages like: ``` - checkpoint sync enabled, downloading state from: http://localhost:9667/lean/states/finalized + checkpoint sync enabled, downloading state from: http://localhost:9667/lean/v0/states/finalized checkpoint state verified: slot=X, validators=N, state_root=0x..., block_root=0x... checkpoint sync completed successfully, using state at slot X as anchor ``` @@ -64,7 +64,7 @@ To test the fallback behavior when checkpoint sync fails: ## API Endpoint -The checkpoint sync feature uses the `/lean/states/finalized` endpoint which returns: +The checkpoint sync feature uses the `/lean/v0/states/finalized` endpoint which returns: - **Content-Type**: `application/octet-stream` - **Body**: SSZ-encoded `BeamState` - **Status 503**: Returned if no finalized state is available yet From 074c1c6ccad9f04ad5c418ff139a1f2e19a68485 Mon Sep 17 00:00:00 2001 From: Parthasarathy Ramanujam <1627026+ch4r10t33r@users.noreply.github.com> Date: Sat, 24 Jan 2026 20:47:10 +0000 Subject: [PATCH 16/19] refactor: rename justified endpoint to /lean/v0/checkpoints/justified (#518) Update API endpoint from /lean/v0/states/justified to /lean/v0/checkpoints/justified to match leanSpec PR #325. This makes the API semantically correct: - /lean/v0/states/finalized returns a State (SSZ binary) - /lean/v0/checkpoints/justified returns a Checkpoint (JSON) Changes: - Update endpoint route in api_server.zig - Update all documentation references in README.md - Update handler documentation comments --- pkgs/api/README.md | 10 +++++----- pkgs/cli/src/api_server.zig | 9 +++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/pkgs/api/README.md b/pkgs/api/README.md index 401e94f90..c264a5cf0 100644 --- a/pkgs/api/README.md +++ b/pkgs/api/README.md @@ -8,7 +8,7 @@ This package provides the HTTP API server for the Zeam node with five main endpo - Prometheus metrics endpoint at `/metrics` - Health check at `/lean/v0/health` - Finalized checkpoint state at `/lean/v0/states/finalized` (for checkpoint sync) -- Justified checkpoint information at `/lean/v0/states/justified` +- Justified checkpoint information at `/lean/v0/checkpoints/justified` ## Package Components @@ -120,12 +120,12 @@ Returns: - **Body**: SSZ-encoded `BeamState` - **Status 503**: Returned if no finalized state is available yet -### `/lean/v0/states/justified` +### `/lean/v0/checkpoints/justified` Returns the latest justified checkpoint information as JSON. ```sh -curl http://localhost:9667/lean/v0/states/justified +curl http://localhost:9667/lean/v0/checkpoints/justified ``` Returns: @@ -153,7 +153,7 @@ The server exposes: - Metrics at `/metrics` - Health at `/lean/v0/health` - Checkpoint state at `/lean/v0/states/finalized` -- Justified checkpoint at `/lean/v0/states/justified` +- Justified checkpoint at `/lean/v0/checkpoints/justified` **Note**: On freestanding targets (ZKVM), the HTTP server is automatically disabled. @@ -216,7 +216,7 @@ curl http://localhost:9668/lean/v0/health curl http://localhost:9668/lean/v0/states/finalized -o state.ssz # Justified checkpoint -curl http://localhost:9668/lean/v0/states/justified +curl http://localhost:9668/lean/v0/checkpoints/justified ``` ## Visualization with Prometheus & Grafana diff --git a/pkgs/cli/src/api_server.zig b/pkgs/cli/src/api_server.zig index 2c3d33ceb..1312e1fac 100644 --- a/pkgs/cli/src/api_server.zig +++ b/pkgs/cli/src/api_server.zig @@ -11,8 +11,8 @@ const node_lib = @import("@zeam/node"); const BeamChain = node_lib.chainFactory.BeamChain; /// API server that runs in a background thread -/// Handles metrics, SSE events, health checks, and checkpoint state endpoints -/// chain is optional - if null, the finalized state endpoint will return 503 +/// Handles metrics, SSE events, health checks, checkpoint endpoints, and finalized state +/// chain is optional - if null, endpoints will return 503 /// (API server starts before chain initialization, so chain may not be available yet) pub fn startAPIServer(allocator: std.mem.Allocator, port: u16, logger_config: *LoggerConfig, chain: ?*BeamChain) !void { // Initialize the global event broadcaster for SSE events @@ -107,7 +107,7 @@ const ApiServer = struct { self.logger.warn("failed to handle finalized checkpoint state request: {}", .{err}); _ = request.respond("Internal Server Error\n", .{ .status = .internal_server_error }) catch {}; }; - } else if (std.mem.eql(u8, request.head.target, "/lean/v0/states/justified")) { + } else if (std.mem.eql(u8, request.head.target, "/lean/v0/checkpoints/justified")) { // Handle justified checkpoint endpoint self.handleJustifiedCheckpoint(&request) catch |err| { self.logger.warn("failed to handle justified checkpoint request: {}", .{err}); @@ -187,7 +187,8 @@ const ApiServer = struct { } /// Handle justified checkpoint endpoint - /// Returns the latest justified checkpoint information as JSON at /lean/v0/states/justified + /// Returns checkpoint info as JSON at /lean/v0/checkpoints/justified + /// Useful for monitoring consensus progress and fork choice state fn handleJustifiedCheckpoint(self: *const Self, request: *std.http.Server.Request) !void { // Get the chain (may be null if API server started before chain initialization) const chain = self.chain orelse { From bdb711ac802582d09932da3256a9b0662597074a Mon Sep 17 00:00:00 2001 From: Shariq Naiyer Date: Sun, 25 Jan 2026 03:01:41 -0700 Subject: [PATCH 17/19] fix snappy frame decoding boundaries for req/resp (#515) * fix: snappy frame decoding boundaries for req/resp * fix: cleanup * fix: remove unnecessary reference --------- Co-authored-by: Chen Kai <281165273grape@gmail.com> --- pkgs/network/src/ethlibp2p.zig | 44 +++++----- .../src/req_resp/inbound_protocol.rs | 60 +++++++++----- .../src/req_resp/outbound_protocol.rs | 64 +++++++++----- rust/libp2p-glue/src/req_resp/varint.rs | 83 +++++++++++++++++++ 4 files changed, 187 insertions(+), 64 deletions(-) diff --git a/pkgs/network/src/ethlibp2p.zig b/pkgs/network/src/ethlibp2p.zig index 559832451..2a9e53496 100644 --- a/pkgs/network/src/ethlibp2p.zig +++ b/pkgs/network/src/ethlibp2p.zig @@ -28,7 +28,6 @@ const MAX_VARINT_BYTES: usize = uvarint.bufferSize(usize); const FrameDecodeError = error{ EmptyFrame, PayloadTooLarge, - LengthMismatch, Incomplete, } || uvarint.VarintParseError; @@ -48,22 +47,24 @@ fn decodeVarint(bytes: []const u8) uvarint.VarintParseError!struct { value: usiz }; } -fn buildRequestFrame(allocator: Allocator, payload: []const u8) ![]u8 { - if (payload.len > MAX_RPC_MESSAGE_SIZE) { +/// Build a request frame with varint-encoded uncompressed size followed by snappy-framed payload. +fn buildRequestFrame(allocator: Allocator, uncompressed_size: usize, snappy_payload: []const u8) ![]u8 { + if (uncompressed_size > MAX_RPC_MESSAGE_SIZE) { return error.PayloadTooLarge; } var frame = std.ArrayListUnmanaged(u8).empty; errdefer frame.deinit(allocator); - try encodeVarint(&frame, allocator, payload.len); - try frame.appendSlice(allocator, payload); + try encodeVarint(&frame, allocator, uncompressed_size); + try frame.appendSlice(allocator, snappy_payload); return frame.toOwnedSlice(allocator); } -fn buildResponseFrame(allocator: Allocator, code: u8, payload: []const u8) ![]u8 { - if (payload.len > MAX_RPC_MESSAGE_SIZE) { +/// Build a response frame with response code, varint-encoded uncompressed size, and snappy-framed payload. +fn buildResponseFrame(allocator: Allocator, code: u8, uncompressed_size: usize, snappy_payload: []const u8) ![]u8 { + if (uncompressed_size > MAX_RPC_MESSAGE_SIZE) { return error.PayloadTooLarge; } @@ -71,8 +72,8 @@ fn buildResponseFrame(allocator: Allocator, code: u8, payload: []const u8) ![]u8 errdefer frame.deinit(allocator); try frame.append(allocator, code); - try encodeVarint(&frame, allocator, payload.len); - try frame.appendSlice(allocator, payload); + try encodeVarint(&frame, allocator, uncompressed_size); + try frame.appendSlice(allocator, snappy_payload); return frame.toOwnedSlice(allocator); } @@ -83,16 +84,13 @@ fn parseRequestFrame(bytes: []const u8) FrameDecodeError![]const u8 { } const decoded = try decodeVarint(bytes); + if (decoded.value > MAX_RPC_MESSAGE_SIZE) { return error.PayloadTooLarge; } - const total = decoded.length + decoded.value; - if (total != bytes.len) { - return error.LengthMismatch; - } - - return bytes[decoded.length..total]; + // Return the snappy-framed payload + return bytes[decoded.length..]; } fn parseResponseFrame(bytes: []const u8) FrameDecodeError!struct { @@ -107,18 +105,15 @@ fn parseResponseFrame(bytes: []const u8) FrameDecodeError!struct { } const decoded = try decodeVarint(bytes[1..]); + if (decoded.value > MAX_RPC_MESSAGE_SIZE) { return error.PayloadTooLarge; } - const total = 1 + decoded.length + decoded.value; - if (total != bytes.len) { - return error.LengthMismatch; - } - + // Return the snappy-framed payload return .{ .code = bytes[0], - .payload = bytes[1 + decoded.length .. total], + .payload = bytes[1 + decoded.length ..], }; } @@ -175,7 +170,7 @@ fn serverStreamSendResponse(ptr: *anyopaque, response: *const interface.ReqRespR }; defer allocator.free(framed); - const frame = try buildResponseFrame(allocator, 0, framed); + const frame = try buildResponseFrame(allocator, 0, encoded.len, framed); defer allocator.free(frame); ctx.zigHandler.logger.debug( @@ -392,6 +387,7 @@ export fn handleRPCRequestFromRustBridge( }; const request_frame: []const u8 = request_ptr[0..request_len]; + const request_payload = parseRequestFrame(request_frame) catch |err| { zigHandler.logger.err( "network-{d}:: Invalid RPC request frame from peer={s}{} protocol={s}: {any}", @@ -400,6 +396,7 @@ export fn handleRPCRequestFromRustBridge( send_rpc_error_response(zigHandler.params.networkId, channel_id, "Invalid RPC request frame"); return; }; + const request_bytes = snappyframesz.decode(zigHandler.allocator, request_payload) catch |err| { zigHandler.logger.err( "network-{d}:: Failed to decode snappy-framed RPC request from peer={s}{} protocol={s}: {any}", @@ -1030,7 +1027,8 @@ pub const EthLibp2p = struct { return err; }; defer self.allocator.free(framed_payload); - const frame = buildRequestFrame(self.allocator, framed_payload) catch |err| { + + const frame = buildRequestFrame(self.allocator, encoded_message.len, framed_payload) catch |err| { self.logger.err( "network-{d}:: Failed to build RPC request frame for peer={s}{} protocol_tag={d}: {any}", .{ self.params.networkId, peer_id, node_name, protocol_tag, err }, diff --git a/rust/libp2p-glue/src/req_resp/inbound_protocol.rs b/rust/libp2p-glue/src/req_resp/inbound_protocol.rs index fe1fb1b45..0ff98aa8c 100644 --- a/rust/libp2p-glue/src/req_resp/inbound_protocol.rs +++ b/rust/libp2p-glue/src/req_resp/inbound_protocol.rs @@ -3,7 +3,7 @@ /// we changed the encode/decode logic to delegate the framing to zig side, but we still need to inspect the varint prefix to determine the frame length. use std::pin::Pin; -use super::varint::{decode_varint_prefix, MAX_VARINT_BYTES}; +use super::varint::{calculate_snappy_frame_size, decode_varint_prefix}; use crate::req_resp::{ configurations::{max_message_size, REQUEST_TIMEOUT}, error::ReqRespError, @@ -89,18 +89,32 @@ impl Encoder for InboundCodec { )); } - let (body_len, prefix_len) = decode_varint_prefix(&item.payload[1..])? + // Response format: response_code (1 byte) + varint (uncompressed len) + snappy frame + let (uncompressed_len, prefix_len) = decode_varint_prefix(&item.payload[1..])? .ok_or_else(|| ReqRespError::InvalidData("Incomplete response length prefix".into()))?; - if body_len > max_message_size() { + if uncompressed_len > max_message_size() { return Err(ReqRespError::InvalidData(format!( "Message size exceeds maximum: {} > {}", - body_len, + uncompressed_len, max_message_size() ))); } - let expected_len = 1 + prefix_len + body_len; + // Validate the snappy frame that follows + let snappy_start = 1 + prefix_len; + if item.payload.len() <= snappy_start { + return Err(ReqRespError::InvalidData( + "Response payload missing snappy frame".into(), + )); + } + + let snappy_frame_size = calculate_snappy_frame_size(&item.payload[snappy_start..])? + .ok_or_else(|| { + ReqRespError::InvalidData("Incomplete snappy frame in response".into()) + })?; + + let expected_len = 1 + prefix_len + snappy_frame_size; if item.payload.len() != expected_len { return Err(ReqRespError::InvalidData(format!( "Response payload length mismatch (expected {}, got {})", @@ -109,12 +123,6 @@ impl Encoder for InboundCodec { ))); } - if expected_len > max_message_size() + MAX_VARINT_BYTES + 1 { - return Err(ReqRespError::InvalidData( - "Framed response exceeds maximum envelope size".into(), - )); - } - dst.clear(); dst.extend_from_slice(&item.payload); Ok(()) @@ -130,24 +138,38 @@ impl Decoder for InboundCodec { return Ok(None); } - let (body_len, prefix_len) = match decode_varint_prefix(&src[..])? { + // Decode the varint prefix which tells us the uncompressed SSZ length + let (uncompressed_len, prefix_len) = match decode_varint_prefix(&src[..])? { Some(result) => result, None => return Ok(None), }; - if body_len > max_message_size() { + if uncompressed_len > max_message_size() { return Err(ReqRespError::InvalidData(format!( "Message size exceeds maximum: {} > {}", - body_len, + uncompressed_len, max_message_size() ))); } - let total_len = prefix_len + body_len; - if total_len > max_message_size() + MAX_VARINT_BYTES { - return Err(ReqRespError::InvalidData( - "Framed request exceeds maximum envelope size".into(), - )); + if src.len() <= prefix_len { + return Ok(None); + } + + let snappy_frame_size = match calculate_snappy_frame_size(&src[prefix_len..])? { + Some(size) => size, + None => return Ok(None), + }; + + let total_len = prefix_len + snappy_frame_size; + + // snappy frame shouldn't be excessively larger than uncompressed + let max_snappy_overhead = 64 + (uncompressed_len / 10); // generous overhead allowance + if snappy_frame_size > uncompressed_len + max_snappy_overhead { + return Err(ReqRespError::InvalidData(format!( + "Snappy frame size {} is unexpectedly large for uncompressed size {}", + snappy_frame_size, uncompressed_len + ))); } if src.len() < total_len { diff --git a/rust/libp2p-glue/src/req_resp/outbound_protocol.rs b/rust/libp2p-glue/src/req_resp/outbound_protocol.rs index 6459c25e4..f852689a1 100644 --- a/rust/libp2p-glue/src/req_resp/outbound_protocol.rs +++ b/rust/libp2p-glue/src/req_resp/outbound_protocol.rs @@ -1,7 +1,7 @@ /// The code originally comes from Ream https://github.com/ReamLabs/ream/blob/5a4b3cb42d5646a0d12ec1825ace03645dbfd59b/crates/networking/p2p/src/req_resp/outbound_protocol.rs /// as we still need rust-libp2p until we fully migrate to zig-libp2p. It needs the custom RPC protocol implementation. /// we changed the encode/decode logic to delegate the framing to zig side, but we still need to inspect the varint prefix to determine the frame length. -use super::varint::{decode_varint_prefix, MAX_VARINT_BYTES}; +use super::varint::{calculate_snappy_frame_size, decode_varint_prefix}; use crate::req_resp::{ configurations::max_message_size, error::ReqRespError, @@ -66,18 +66,31 @@ impl Encoder for OutboundCodec { )); } - let (body_len, prefix_len) = decode_varint_prefix(&item.payload)? + // Request format: varint (uncompressed len) + snappy frame + let (uncompressed_len, prefix_len) = decode_varint_prefix(&item.payload)? .ok_or_else(|| ReqRespError::InvalidData("Incomplete request length prefix".into()))?; - if body_len > max_message_size() { + if uncompressed_len > max_message_size() { return Err(ReqRespError::InvalidData(format!( "Message size exceeds maximum: {} > {}", - body_len, + uncompressed_len, max_message_size() ))); } - let expected_len = prefix_len + body_len; + // Validate the snappy frame that follows + if item.payload.len() <= prefix_len { + return Err(ReqRespError::InvalidData( + "Request payload missing snappy frame".into(), + )); + } + + let snappy_frame_size = calculate_snappy_frame_size(&item.payload[prefix_len..])? + .ok_or_else(|| { + ReqRespError::InvalidData("Incomplete snappy frame in request".into()) + })?; + + let expected_len = prefix_len + snappy_frame_size; if item.payload.len() != expected_len { return Err(ReqRespError::InvalidData(format!( "Request payload length mismatch (expected {}, got {})", @@ -86,12 +99,6 @@ impl Encoder for OutboundCodec { ))); } - if expected_len > max_message_size() + MAX_VARINT_BYTES { - return Err(ReqRespError::InvalidData( - "Framed request exceeds maximum envelope size".into(), - )); - } - dst.extend_from_slice(&item.payload); Ok(()) } @@ -106,27 +113,40 @@ impl Decoder for OutboundCodec { return Ok(None); } - // Zig is responsible for constructing the response frame, but the codec still needs - // to inspect the varint prefix to figure out the total frame length so that we know - // when a full message has been received from the stream. - let (body_len, prefix_len) = match decode_varint_prefix(&src[1..])? { + // Response format: response_code (1 byte) + varint (uncompressed len) + snappy frame + let (uncompressed_len, prefix_len) = match decode_varint_prefix(&src[1..])? { Some(result) => result, None => return Ok(None), }; - if body_len > max_message_size() { + if uncompressed_len > max_message_size() { return Err(ReqRespError::InvalidData(format!( "Message size exceeds maximum: {} > {}", - body_len, + uncompressed_len, max_message_size() ))); } - let total_len = 1 + prefix_len + body_len; - if total_len > max_message_size() + MAX_VARINT_BYTES + 1 { - return Err(ReqRespError::InvalidData( - "Framed response exceeds maximum envelope size".into(), - )); + // Now parse the snappy-framed data that follows to determine the actual frame size + let snappy_start = 1 + prefix_len; + if src.len() <= snappy_start { + return Ok(None); + } + + let snappy_frame_size = match calculate_snappy_frame_size(&src[snappy_start..])? { + Some(size) => size, + None => return Ok(None), + }; + + let total_len = 1 + prefix_len + snappy_frame_size; + + // snappy frame shouldn't be excessively larger than uncompressed + let max_snappy_overhead = 64 + (uncompressed_len / 10); + if snappy_frame_size > uncompressed_len + max_snappy_overhead { + return Err(ReqRespError::InvalidData(format!( + "Snappy frame size {} is unexpectedly large for uncompressed size {}", + snappy_frame_size, uncompressed_len + ))); } if src.len() < total_len { diff --git a/rust/libp2p-glue/src/req_resp/varint.rs b/rust/libp2p-glue/src/req_resp/varint.rs index 3838e088a..ee6706885 100644 --- a/rust/libp2p-glue/src/req_resp/varint.rs +++ b/rust/libp2p-glue/src/req_resp/varint.rs @@ -3,6 +3,14 @@ use unsigned_varint::{decode, encode}; pub const MAX_VARINT_BYTES: usize = 10; +/// Snappy framing format constants +const SNAPPY_STREAM_IDENTIFIER: [u8; 10] = + [0xff, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, 0x50, 0x70, 0x59]; +const CHUNK_TYPE_COMPRESSED: u8 = 0x00; +const CHUNK_TYPE_UNCOMPRESSED: u8 = 0x01; +const CHUNK_TYPE_PADDING: u8 = 0xfe; +const CHUNK_TYPE_STREAM_ID: u8 = 0xff; + /// These helpers intentionally stay on the low-level `unsigned-varint` APIs rather than the /// convenience `Uvi` codec so callers can inspect the prefix without mutating the input buffer. /// `decode_varint_prefix` reports incomplete prefixes via `Ok(None)` and lets Zig continue to own @@ -31,3 +39,78 @@ pub fn encode_varint(value: usize, dst: &mut Vec) { let encoded = encode::usize(value, &mut buffer); dst.extend_from_slice(encoded); } + +/// Calculates the total size of a snappy-framed payload by parsing chunk headers. +pub fn calculate_snappy_frame_size(src: &[u8]) -> Result, ReqRespError> { + if src.len() < SNAPPY_STREAM_IDENTIFIER.len() { + return Ok(None); + } + + // Verify stream identifier + if src[..SNAPPY_STREAM_IDENTIFIER.len()] != SNAPPY_STREAM_IDENTIFIER { + return Err(ReqRespError::InvalidData( + "Invalid snappy stream identifier".into(), + )); + } + + let mut pos = SNAPPY_STREAM_IDENTIFIER.len(); + + // Parse chunks until we find a complete data chunk + while pos < src.len() { + // Need at least 4 bytes for chunk header (1 type + 3 length) + if pos + 4 > src.len() { + return Ok(None); + } + + let chunk_type = src[pos]; + let chunk_len = u32::from_le_bytes([src[pos + 1], src[pos + 2], src[pos + 3], 0]) as usize; + + // Validate chunk type + match chunk_type { + CHUNK_TYPE_COMPRESSED | CHUNK_TYPE_UNCOMPRESSED => { + // Data chunk - this is what we're looking for + let chunk_end = pos + 4 + chunk_len; + if src.len() < chunk_end { + return Ok(None); + } + // For req/resp protocol, we expect exactly one data chunk after the stream id + // Return the total size once we've parsed the first data chunk + return Ok(Some(chunk_end)); + } + CHUNK_TYPE_PADDING => { + // Padding chunk - skip it + let chunk_end = pos + 4 + chunk_len; + if src.len() < chunk_end { + return Ok(None); + } + pos = chunk_end; + } + CHUNK_TYPE_STREAM_ID => { + // Another stream identifier (shouldn't happen mid-stream, but handle gracefully) + let chunk_end = pos + 4 + chunk_len; + if src.len() < chunk_end { + return Ok(None); + } + pos = chunk_end; + } + 0x02..=0x7f => { + // Reserved unskippable chunk types - treat as error + return Err(ReqRespError::InvalidData(format!( + "Unknown unskippable snappy chunk type: 0x{:02x}", + chunk_type + ))); + } + _ => { + // Reserved skippable chunk types (0x80-0xfd) - skip them + let chunk_end = pos + 4 + chunk_len; + if src.len() < chunk_end { + return Ok(None); + } + pos = chunk_end; + } + } + } + + // If we've parsed everything but found no data chunk, we need more data + Ok(None) +} From 5c6b6f1ac02dc0e09a6e74148f35e2234aa54087 Mon Sep 17 00:00:00 2001 From: Ekaterina Riazantseva Date: Sun, 25 Jan 2026 11:02:27 +0100 Subject: [PATCH 18/19] add OCI labels for git commit and branch to docker images (#493) * feat: add OCI labels for git commit and branch to docker images * fix: add branch label to auto-release * fix: add git commit label to auto-release --- .github/workflows/auto-release.yml | 3 +++ Dockerfile | 6 ++++++ Dockerfile.prebuilt | 6 ++++++ README.md | 8 ++++++++ 4 files changed, 23 insertions(+) diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index 3fddd02fe..56e02be1f 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -224,6 +224,9 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + build-args: | + GIT_COMMIT=${{ github.sha }} + GIT_BRANCH=${{ github.ref_name }} cache-from: type=gha cache-to: type=gha,mode=max provenance: false diff --git a/Dockerfile b/Dockerfile index ad779fab8..2192762aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -145,6 +145,12 @@ RUN mkdir -p /runtime-libs && \ # Runtime stage - using scratch for absolute minimal size FROM scratch AS runtime +ARG GIT_COMMIT=unknown +ARG GIT_BRANCH=unknown + +LABEL org.opencontainers.image.revision=$GIT_COMMIT +LABEL org.opencontainers.image.ref.name=$GIT_BRANCH + # Copy the architecture-specific libraries and loader COPY --from=runtime-prep /runtime-libs/ / diff --git a/Dockerfile.prebuilt b/Dockerfile.prebuilt index b3d74da76..5f661bfe7 100644 --- a/Dockerfile.prebuilt +++ b/Dockerfile.prebuilt @@ -32,6 +32,12 @@ RUN mkdir -p /runtime-libs && \ # Runtime stage - using scratch for minimal size FROM scratch AS runtime +ARG GIT_COMMIT=unknown +ARG GIT_BRANCH=unknown + +LABEL org.opencontainers.image.revision=$GIT_COMMIT +LABEL org.opencontainers.image.ref.name=$GIT_BRANCH + # Copy the architecture-specific libraries and loader COPY --from=runtime-prep /runtime-libs/ / diff --git a/README.md b/README.md index 5b86fbf54..e332d649b 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,14 @@ zig build -Doptimize=ReleaseFast -Dgit_version="$(git rev-parse --short HEAD)" docker build -f Dockerfile.prebuilt -t zeam:local . ``` +For publishing to a public registry, add OCI labels for better traceability: +```bash +docker build -f Dockerfile.prebuilt \ + --build-arg GIT_COMMIT=$(git rev-parse HEAD) \ + --build-arg GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) \ + -t blockblaz/zeam:latest . +``` + #### Prerequisites - Zeam requires zig version 0.14.1 to build. From 97e2760d76cf586504b659f8be006d7703990232 Mon Sep 17 00:00:00 2001 From: chethack/Chetany <95150398+chetanyb@users.noreply.github.com> Date: Sun, 25 Jan 2026 16:29:25 +0530 Subject: [PATCH 19/19] update to use SSZ hasher agnostic hashTreeRoot API (#474) * chore: point to latest ssz commit * feat: add Sha256 ssz hashTreeRoot wrapper * refactor: use the updated hashTreeRoot wrapper --------- Co-authored-by: g11tech --- build.zig | 3 +++ build.zig.zon | 4 +-- pkgs/cli/src/node.zig | 4 +-- pkgs/key-manager/src/lib.zig | 4 +-- pkgs/node/src/chain.zig | 8 +++--- pkgs/node/src/forkchoice.zig | 4 +-- pkgs/node/src/node.zig | 8 +++--- pkgs/node/src/testing.zig | 4 +-- .../src/runner/fork_choice_runner.zig | 4 +-- .../src/runner/state_transition_runner.zig | 6 ++--- pkgs/state-transition/src/lib.zig | 6 ++--- pkgs/state-transition/src/mock.zig | 6 ++--- pkgs/state-transition/src/transition.zig | 9 +++---- pkgs/types/src/aggregation.zig | 1 - pkgs/types/src/attestation.zig | 3 ++- pkgs/types/src/block.zig | 9 ++++--- pkgs/types/src/block_signatures_testing.zig | 7 ++--- pkgs/types/src/mini_3sf.zig | 2 -- pkgs/types/src/state.zig | 26 +++++++++---------- pkgs/utils/src/lib.zig | 3 +++ pkgs/utils/src/ssz.zig | 14 ++++++++++ 21 files changed, 75 insertions(+), 60 deletions(-) create mode 100644 pkgs/utils/src/ssz.zig diff --git a/build.zig b/build.zig index 8115d3a4f..398bd5aa4 100644 --- a/build.zig +++ b/build.zig @@ -156,6 +156,7 @@ pub fn build(b: *Builder) !void { }); zeam_utils.addImport("datetime", datetime); zeam_utils.addImport("yaml", yaml); + zeam_utils.addImport("ssz", ssz); // add zeam-params const zeam_params = b.addModule("@zeam/params", .{ @@ -221,6 +222,7 @@ pub fn build(b: *Builder) !void { }); zeam_key_manager.addImport("@zeam/xmss", zeam_xmss); zeam_key_manager.addImport("@zeam/types", zeam_types); + zeam_key_manager.addImport("@zeam/utils", zeam_utils); zeam_key_manager.addImport("@zeam/metrics", zeam_metrics); zeam_key_manager.addImport("ssz", ssz); @@ -733,6 +735,7 @@ fn build_zkvm_targets(b: *Builder, main_exe: *Builder.Step, host_target: std.Bui .optimize = optimize, .root_source_file = b.path("pkgs/utils/src/lib.zig"), }); + zeam_utils.addImport("ssz", ssz); // add zeam-metrics (core metrics definitions for ZKVM) const zeam_metrics = b.addModule("@zeam/metrics", .{ diff --git a/build.zig.zon b/build.zig.zon index 03b62d04e..1d7ab6677 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -4,8 +4,8 @@ .version = "0.0.0", .dependencies = .{ .ssz = .{ - .url = "git+https://github.com/blockblaz/ssz.zig#5ce7322fc45cab4f215021cae2579d1343e05d55", - .hash = "ssz-0.0.9-Lfwd61PEAgAUPJfUQiK4R5gRX_lTWOd_qYwNT-KAhRLA", + .url = "https://github.com/blockblaz/ssz.zig/archive/0ce92a8f093a321b0d0815eec6c90bd4e745d8e1.tar.gz", + .hash = "ssz-0.0.9-Lfwd6wvKAgA3JZjtfbeXeQG6zle1K_K2j6HPmQJJF4an", }, .zigcli = .{ .url = "git+https://github.com/jiacai2050/zigcli?ref=main#dcbc59d70b4787671c8a4e484ffd2b725aa17af5", diff --git a/pkgs/cli/src/node.zig b/pkgs/cli/src/node.zig index 7cf3ee7f5..025e6354a 100644 --- a/pkgs/cli/src/node.zig +++ b/pkgs/cli/src/node.zig @@ -656,7 +656,7 @@ fn verifyCheckpointState( // Calculate the block root from the properly constructed block header var block_root: types.Root = undefined; - try ssz.hashTreeRoot(types.BeamBlockHeader, state_block_header, &block_root, allocator); + try zeam_utils.hashTreeRoot(types.BeamBlockHeader, state_block_header, &block_root, allocator); logger.info("checkpoint state verified: slot={d}, genesis_time={d}, validators={d}, state_root=0x{s}, block_root=0x{s}", .{ state.slot, @@ -1201,7 +1201,7 @@ test "compare roots from genGensisBlock and genGenesisState and genStateBlockHea // Get state root by hashing the state directly var state_root_from_genesis: [32]u8 = undefined; - try ssz.hashTreeRoot(types.BeamState, genesis_state, &state_root_from_genesis, allocator); + try zeam_utils.hashTreeRoot(types.BeamState, genesis_state, &state_root_from_genesis, allocator); // Generate block header using genStateBlockHeader const state_block_header = try genesis_state.genStateBlockHeader(allocator); diff --git a/pkgs/key-manager/src/lib.zig b/pkgs/key-manager/src/lib.zig index 0cf4c5f62..5b6be884f 100644 --- a/pkgs/key-manager/src/lib.zig +++ b/pkgs/key-manager/src/lib.zig @@ -1,8 +1,8 @@ const std = @import("std"); const xmss = @import("@zeam/xmss"); const types = @import("@zeam/types"); +const zeam_utils = @import("@zeam/utils"); const zeam_metrics = @import("@zeam/metrics"); -const ssz = @import("ssz"); const Allocator = std.mem.Allocator; const JsonValue = std.json.Value; @@ -312,7 +312,7 @@ pub const KeyManager = struct { const signing_timer = zeam_metrics.lean_pq_signature_attestation_signing_time_seconds.start(); var message: [32]u8 = undefined; - try ssz.hashTreeRoot(types.AttestationData, attestation.data, &message, allocator); + try zeam_utils.hashTreeRoot(types.AttestationData, attestation.data, &message, allocator); const epoch: u32 = @intCast(attestation.data.slot); const signature = try keypair.sign(&message, epoch); diff --git a/pkgs/node/src/chain.zig b/pkgs/node/src/chain.zig index c37c67b23..76ec357c6 100644 --- a/pkgs/node/src/chain.zig +++ b/pkgs/node/src/chain.zig @@ -354,7 +354,7 @@ pub const BeamChain = struct { // 3. cache state to save recompute while adding the block on publish var block_root: [32]u8 = undefined; - try ssz.hashTreeRoot(types.BeamBlock, block, &block_root, self.allocator); + try zeam_utils.hashTreeRoot(types.BeamBlock, block, &block_root, self.allocator); try self.states.put(block_root, post_state); post_state_opt = null; @@ -491,7 +491,7 @@ pub const BeamChain = struct { .block => |signed_block| { const block = signed_block.message.block; var block_root: [32]u8 = undefined; - try ssz.hashTreeRoot(types.BeamBlock, block, &block_root, self.allocator); + try zeam_utils.hashTreeRoot(types.BeamBlock, block, &block_root, self.allocator); //check if we have the block already in forkchoice const hasBlock = self.forkChoice.hasBlock(block_root); @@ -592,7 +592,7 @@ pub const BeamChain = struct { const block_root: types.Root = blockInfo.blockRoot orelse computedroot: { var cblock_root: [32]u8 = undefined; - try ssz.hashTreeRoot(types.BeamBlock, block, &cblock_root, self.allocator); + try zeam_utils.hashTreeRoot(types.BeamBlock, block, &cblock_root, self.allocator); break :computedroot cblock_root; }; @@ -1339,7 +1339,7 @@ test "process and add mock blocks into a node's chain" { // should have matching states in the state const block_state = beam_chain.states.get(block_root) orelse @panic("state root should have been found"); var state_root: [32]u8 = undefined; - try ssz.hashTreeRoot(*types.BeamState, block_state, &state_root, allocator); + try zeam_utils.hashTreeRoot(*types.BeamState, block_state, &state_root, allocator); try std.testing.expect(std.mem.eql(u8, &state_root, &block.state_root)); // fcstore checkpoints should match diff --git a/pkgs/node/src/forkchoice.zig b/pkgs/node/src/forkchoice.zig index d26b0a838..7bc338791 100644 --- a/pkgs/node/src/forkchoice.zig +++ b/pkgs/node/src/forkchoice.zig @@ -242,7 +242,7 @@ pub const ForkChoice = struct { pub fn init(allocator: Allocator, opts: ForkChoiceParams) !Self { const anchor_block_header = try opts.anchorState.genStateBlockHeader(allocator); var anchor_block_root: [32]u8 = undefined; - try ssz.hashTreeRoot( + try zeam_utils.hashTreeRoot( types.BeamBlockHeader, anchor_block_header, &anchor_block_root, @@ -1062,7 +1062,7 @@ pub const ForkChoice = struct { const block_root: [32]u8 = opts.blockRoot orelse computedroot: { var cblock_root: [32]u8 = undefined; - try ssz.hashTreeRoot(types.BeamBlock, block, &cblock_root, self.allocator); + try zeam_utils.hashTreeRoot(types.BeamBlock, block, &cblock_root, self.allocator); break :computedroot cblock_root; }; const is_timely = self.isBlockTimely(opts.blockDelayMs); diff --git a/pkgs/node/src/node.zig b/pkgs/node/src/node.zig index 884e88dd3..4a1add544 100644 --- a/pkgs/node/src/node.zig +++ b/pkgs/node/src/node.zig @@ -139,7 +139,7 @@ pub const BeamNode = struct { } var block_root: types.Root = undefined; - if (ssz.hashTreeRoot(types.BeamBlock, signed_block.message.block, &block_root, self.allocator)) |_| { + if (zeam_utils.hashTreeRoot(types.BeamBlock, signed_block.message.block, &block_root, self.allocator)) |_| { _ = self.network.removePendingBlockRoot(block_root); } else |err| { self.logger.warn("failed to compute block root for incoming gossip block: {any}", .{err}); @@ -168,7 +168,7 @@ pub const BeamNode = struct { if (data.* == .block) { const signed_block = data.block; var block_root: types.Root = undefined; - if (ssz.hashTreeRoot(types.BeamBlock, signed_block.message.block, &block_root, self.allocator)) |_| { + if (zeam_utils.hashTreeRoot(types.BeamBlock, signed_block.message.block, &block_root, self.allocator)) |_| { self.logger.info( "gossip block 0x{s} rejected as pre-finalized; pruning cached descendants", .{std.fmt.fmtSliceHexLower(block_root[0..])}, @@ -338,7 +338,7 @@ pub const BeamNode = struct { fn processBlockByRootChunk(self: *Self, block_ctx: *const BlockByRootContext, signed_block: *const types.SignedBlockWithAttestation) !void { var block_root: types.Root = undefined; - if (ssz.hashTreeRoot(types.BeamBlock, signed_block.message.block, &block_root, self.allocator)) |_| { + if (zeam_utils.hashTreeRoot(types.BeamBlock, signed_block.message.block, &block_root, self.allocator)) |_| { const current_depth = self.network.getPendingBlockRootDepth(block_root) orelse 0; const removed = self.network.removePendingBlockRoot(block_root); if (!removed) { @@ -779,7 +779,7 @@ pub const BeamNode = struct { // 1. Process locally through chain so that produced block first can be confirmed var block_root: [32]u8 = undefined; - try ssz.hashTreeRoot(types.BeamBlock, signed_block.message.block, &block_root, self.allocator); + try zeam_utils.hashTreeRoot(types.BeamBlock, signed_block.message.block, &block_root, self.allocator); // check if the block has not already been received through the network const hasBlock = self.chain.forkChoice.hasBlock(block_root); diff --git a/pkgs/node/src/testing.zig b/pkgs/node/src/testing.zig index f892bc693..ad80db504 100644 --- a/pkgs/node/src/testing.zig +++ b/pkgs/node/src/testing.zig @@ -10,8 +10,6 @@ const zeam_utils = @import("@zeam/utils"); const xev = @import("xev"); const networks = @import("@zeam/network"); const xmss = @import("@zeam/xmss"); -const ssz = @import("ssz"); - const clockFactory = @import("./clock.zig"); pub const NodeTestOptions = struct { @@ -203,7 +201,7 @@ pub const NodeTestContext = struct { // Compute message hash var message_hash: [32]u8 = undefined; - try ssz.hashTreeRoot(types.AttestationData, aggregated_attestation.data, &message_hash, allocator); + try zeam_utils.hashTreeRoot(types.AttestationData, aggregated_attestation.data, &message_hash, allocator); const epoch: u32 = @intCast(aggregated_attestation.data.slot); diff --git a/pkgs/spectest/src/runner/fork_choice_runner.zig b/pkgs/spectest/src/runner/fork_choice_runner.zig index 4b1e1d623..1f7309a3b 100644 --- a/pkgs/spectest/src/runner/fork_choice_runner.zig +++ b/pkgs/spectest/src/runner/fork_choice_runner.zig @@ -291,7 +291,7 @@ fn runCase( defer label_map.deinit(allocator); var anchor_root: types.Root = undefined; - ssz.hashTreeRoot(types.BeamBlock, anchor_block, &anchor_root, allocator) catch |err| { + zeam_utils.hashTreeRoot(types.BeamBlock, anchor_block, &anchor_root, allocator) catch |err| { std.debug.print( "fixture {s} case {s}: anchor block hashing failed ({s})\n", .{ ctx.fixture_label, ctx.case_name, @errorName(err) }, @@ -607,7 +607,7 @@ fn processBlockStep( defer block.deinit(); var block_root: types.Root = undefined; - ssz.hashTreeRoot(types.BeamBlock, block, &block_root, ctx.allocator) catch |err| { + zeam_utils.hashTreeRoot(types.BeamBlock, block, &block_root, ctx.allocator) catch |err| { std.debug.print( "fixture {s} case {s}{}: hashing block failed ({s})\n", .{ fixture_path, case_name, formatStep(step_index), @errorName(err) }, diff --git a/pkgs/spectest/src/runner/state_transition_runner.zig b/pkgs/spectest/src/runner/state_transition_runner.zig index c61e1dea2..94bae3a49 100644 --- a/pkgs/spectest/src/runner/state_transition_runner.zig +++ b/pkgs/spectest/src/runner/state_transition_runner.zig @@ -41,8 +41,6 @@ pub fn baseRelRoot(comptime spec_fork: Fork) []const u8 { const types = @import("@zeam/types"); const state_transition = @import("@zeam/state-transition"); const zeam_utils = @import("@zeam/utils"); -const ssz = @import("ssz"); - const JsonValue = std.json.Value; const Context = expect.Context; @@ -245,7 +243,7 @@ fn runCase( var header_for_check = pre_state.latest_block_header; if (std.mem.eql(u8, &header_for_check.state_root, &types.ZERO_HASH)) { var pre_state_root: types.Root = undefined; - ssz.hashTreeRoot(types.BeamState, pre_state, &pre_state_root, allocator) catch |err| { + zeam_utils.hashTreeRoot(types.BeamState, pre_state, &pre_state_root, allocator) catch |err| { std.debug.print( "fixture {s} case {s}: unable to hash pre-state ({s})\n", .{ ctx.fixture_label, ctx.case_name, @errorName(err) }, @@ -256,7 +254,7 @@ fn runCase( } var header_root: types.Root = undefined; - ssz.hashTreeRoot(types.BeamBlockHeader, header_for_check, &header_root, allocator) catch |err| { + zeam_utils.hashTreeRoot(types.BeamBlockHeader, header_for_check, &header_root, allocator) catch |err| { std.debug.print( "fixture {s} case {s}: unable to hash latest block header ({s})\n", .{ ctx.fixture_label, ctx.case_name, @errorName(err) }, diff --git a/pkgs/state-transition/src/lib.zig b/pkgs/state-transition/src/lib.zig index 48d2c8938..3f6b8f0c7 100644 --- a/pkgs/state-transition/src/lib.zig +++ b/pkgs/state-transition/src/lib.zig @@ -57,7 +57,7 @@ test "apply transition on mocked chain" { // check the post state root to be equal to block2's stateroot // this is reduant though because apply_transition already checks this for each block's state root var post_state_root: [32]u8 = undefined; - try ssz.hashTreeRoot(types.BeamState, beam_state, &post_state_root, allocator); + try zeam_utils.hashTreeRoot(types.BeamState, beam_state, &post_state_root, allocator); try std.testing.expect(std.mem.eql(u8, &post_state_root, &mock_chain.blocks[mock_chain.blocks.len - 1].message.block.state_root)); } @@ -75,11 +75,11 @@ test "genStateBlockHeader" { // get applied block const applied_block = mock_chain.blocks[i]; var applied_block_root: types.Root = undefined; - try ssz.hashTreeRoot(types.BeamBlock, applied_block.message.block, &applied_block_root, allocator); + try zeam_utils.hashTreeRoot(types.BeamBlock, applied_block.message.block, &applied_block_root, allocator); const state_block_header = try beam_state.genStateBlockHeader(allocator); var state_block_header_root: types.Root = undefined; - try ssz.hashTreeRoot(types.BeamBlockHeader, state_block_header, &state_block_header_root, allocator); + try zeam_utils.hashTreeRoot(types.BeamBlockHeader, state_block_header, &state_block_header_root, allocator); try std.testing.expect(std.mem.eql(u8, &applied_block_root, &state_block_header_root)); diff --git a/pkgs/state-transition/src/mock.zig b/pkgs/state-transition/src/mock.zig index b4f83f088..034bd5bd6 100644 --- a/pkgs/state-transition/src/mock.zig +++ b/pkgs/state-transition/src/mock.zig @@ -115,7 +115,7 @@ pub fn genMockChain(allocator: Allocator, numBlocks: usize, from_genesis: ?types }, }; var block_root: types.Root = undefined; - try ssz.hashTreeRoot(types.BeamBlock, genesis_block, &block_root, allocator); + try zeam_utils.hashTreeRoot(types.BeamBlock, genesis_block, &block_root, allocator); try blockList.append(gen_signed_block); try blockRootList.append(block_root); @@ -145,7 +145,7 @@ pub fn genMockChain(allocator: Allocator, numBlocks: usize, from_genesis: ?types for (1..numBlocks) |slot| { var parent_root: [32]u8 = undefined; - try ssz.hashTreeRoot(types.BeamBlock, prev_block, &parent_root, allocator); + try zeam_utils.hashTreeRoot(types.BeamBlock, prev_block, &parent_root, allocator); const state_root: [32]u8 = types.ZERO_HASH; // const timestamp = genesis_config.genesis_time + slot * params.SECONDS_PER_SLOT; @@ -331,7 +331,7 @@ pub fn genMockChain(allocator: Allocator, numBlocks: usize, from_genesis: ?types // prepare pre state to process block for that slot, may be rename prepare_pre_state try transition.apply_raw_block(allocator, &beam_state, &block, block_building_logger); - try ssz.hashTreeRoot(types.BeamBlock, block, &block_root, allocator); + try zeam_utils.hashTreeRoot(types.BeamBlock, block, &block_root, allocator); // generate the signed beam block and add to block list const block_with_attestation = types.BlockWithAttestation{ diff --git a/pkgs/state-transition/src/transition.zig b/pkgs/state-transition/src/transition.zig index c22b6cbc0..2772e2079 100644 --- a/pkgs/state-transition/src/transition.zig +++ b/pkgs/state-transition/src/transition.zig @@ -1,4 +1,3 @@ -const ssz = @import("ssz"); const std = @import("std"); const json = std.json; const types = @import("@zeam/types"); @@ -50,7 +49,7 @@ pub fn apply_raw_block(allocator: Allocator, state: *types.BeamState, block: *ty logger.debug("extracting state root\n", .{}); // extract the post state root var state_root: [32]u8 = undefined; - try ssz.hashTreeRoot(*types.BeamState, state, &state_root, allocator); + try zeam_utils.hashTreeRoot(*types.BeamState, state, &state_root, allocator); block.state_root = state_root; } @@ -130,7 +129,7 @@ pub fn verifySignaturesWithScheme( // Compute message hash from attestation data var message_hash: [32]u8 = undefined; - try ssz.hashTreeRoot(types.AttestationData, aggregated_attestation.data, &message_hash, allocator); + try zeam_utils.hashTreeRoot(types.AttestationData, aggregated_attestation.data, &message_hash, allocator); const epoch: u64 = aggregated_attestation.data.slot; @@ -176,7 +175,7 @@ pub fn verifySingleAttestationWithScheme( const verification_timer = zeam_metrics.lean_pq_signature_attestation_verification_time_seconds.start(); var message: [32]u8 = undefined; - try ssz.hashTreeRoot(types.AttestationData, attestation_data.*, &message, allocator); + try zeam_utils.hashTreeRoot(types.AttestationData, attestation_data.*, &message, allocator); const epoch: u32 = @intCast(attestation_data.slot); @@ -224,7 +223,7 @@ pub fn apply_transition(allocator: Allocator, state: *types.BeamState, block: ty if (validateResult) { // verify the post state root var state_root: [32]u8 = undefined; - try ssz.hashTreeRoot(*types.BeamState, state, &state_root, allocator); + try zeam_utils.hashTreeRoot(*types.BeamState, state, &state_root, allocator); if (!std.mem.eql(u8, &state_root, &block.state_root)) { opts.logger.debug("state root={x:02} block root={x:02}\n", .{ state_root, block.state_root }); return StateTransitionError.InvalidPostState; diff --git a/pkgs/types/src/aggregation.zig b/pkgs/types/src/aggregation.zig index e1bade18f..9bcbf1023 100644 --- a/pkgs/types/src/aggregation.zig +++ b/pkgs/types/src/aggregation.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const ssz = @import("ssz"); const params = @import("@zeam/params"); const xmss = @import("@zeam/xmss"); diff --git a/pkgs/types/src/attestation.zig b/pkgs/types/src/attestation.zig index 874256c61..2a9505149 100644 --- a/pkgs/types/src/attestation.zig +++ b/pkgs/types/src/attestation.zig @@ -2,6 +2,7 @@ const std = @import("std"); const ssz = @import("ssz"); const params = @import("@zeam/params"); +const zeam_utils = @import("@zeam/utils"); const mini_3sf = @import("./mini_3sf.zig"); const utils = @import("./utils.zig"); @@ -31,7 +32,7 @@ pub const AttestationData = struct { pub fn sszRoot(self: *const AttestationData, allocator: Allocator) !Root { var root: Root = undefined; - try ssz.hashTreeRoot(AttestationData, self.*, &root, allocator); + try zeam_utils.hashTreeRoot(AttestationData, self.*, &root, allocator); return root; } diff --git a/pkgs/types/src/block.zig b/pkgs/types/src/block.zig index 24775f657..0fb43e20b 100644 --- a/pkgs/types/src/block.zig +++ b/pkgs/types/src/block.zig @@ -3,6 +3,7 @@ const ssz = @import("ssz"); const params = @import("@zeam/params"); const xmss = @import("@zeam/xmss"); +const zeam_utils = @import("@zeam/utils"); const aggregation = @import("./aggregation.zig"); const attestation = @import("./attestation.zig"); @@ -154,7 +155,7 @@ pub const BeamBlock = struct { pub fn blockToHeader(self: *const Self, allocator: Allocator) !BeamBlockHeader { var body_root: [32]u8 = undefined; - try ssz.hashTreeRoot( + try zeam_utils.hashTreeRoot( BeamBlockBody, self.body, &body_root, @@ -172,7 +173,7 @@ pub const BeamBlock = struct { pub fn blockToLatestBlockHeader(self: *const Self, allocator: Allocator, header: *BeamBlockHeader) !void { var body_root: [32]u8 = undefined; - try ssz.hashTreeRoot( + try zeam_utils.hashTreeRoot( BeamBlockBody, self.body, &body_root, @@ -381,7 +382,7 @@ pub const AggregatedAttestationsResult = struct { const data_root = group.data_root; const epoch: u64 = group.data.slot; var message_hash: [32]u8 = undefined; - try ssz.hashTreeRoot(attestation.AttestationData, group.data, &message_hash, allocator); + try zeam_utils.hashTreeRoot(attestation.AttestationData, group.data, &message_hash, allocator); // Phase 1: Collect signatures from signatures_map const max_validator = group.validator_bits.capacity(); @@ -744,7 +745,7 @@ test "ssz seralize/deserialize signed beam block" { try std.testing.expect(std.mem.eql(u8, &signed_block.message.block.parent_root, &deserialized_signed_block.message.block.parent_root)); var block_root: [32]u8 = undefined; - try ssz.hashTreeRoot(BeamBlock, signed_block.message.block, &block_root, std.testing.allocator); + try zeam_utils.hashTreeRoot(BeamBlock, signed_block.message.block, &block_root, std.testing.allocator); } test "blockToLatestBlockHeader and blockToHeader" { diff --git a/pkgs/types/src/block_signatures_testing.zig b/pkgs/types/src/block_signatures_testing.zig index bfe81a888..7cb4758fb 100644 --- a/pkgs/types/src/block_signatures_testing.zig +++ b/pkgs/types/src/block_signatures_testing.zig @@ -3,6 +3,7 @@ const ssz = @import("ssz"); const params = @import("@zeam/params"); const xmss = @import("@zeam/xmss"); +const zeam_utils = @import("@zeam/utils"); const aggregation = @import("./aggregation.zig"); const attestation = @import("./attestation.zig"); @@ -162,7 +163,7 @@ const TestContext = struct { // Compute message hash var message_hash: [32]u8 = undefined; - try ssz.hashTreeRoot(attestation.AttestationData, self.attestation_data, &message_hash, self.allocator); + try zeam_utils.hashTreeRoot(attestation.AttestationData, self.attestation_data, &message_hash, self.allocator); // Aggregate var proof = try aggregation.AggregatedSignatureProof.init(self.allocator); @@ -858,7 +859,7 @@ test "computeAggregatedSignatures: complex 3 groups" { } var message_hash: [32]u8 = undefined; - try ssz.hashTreeRoot(attestation.AttestationData, att_data_2, &message_hash, allocator); + try zeam_utils.hashTreeRoot(attestation.AttestationData, att_data_2, &message_hash, allocator); var proof = try aggregation.AggregatedSignatureProof.init(allocator); errdefer proof.deinit(); @@ -921,7 +922,7 @@ test "computeAggregatedSignatures: complex 3 groups" { } var message_hash: [32]u8 = undefined; - try ssz.hashTreeRoot(attestation.AttestationData, att_data_3, &message_hash, allocator); + try zeam_utils.hashTreeRoot(attestation.AttestationData, att_data_3, &message_hash, allocator); var proof = try aggregation.AggregatedSignatureProof.init(allocator); errdefer proof.deinit(); diff --git a/pkgs/types/src/mini_3sf.zig b/pkgs/types/src/mini_3sf.zig index c86f248e1..7761361b2 100644 --- a/pkgs/types/src/mini_3sf.zig +++ b/pkgs/types/src/mini_3sf.zig @@ -1,6 +1,4 @@ const std = @import("std"); -const ssz = @import("ssz"); - const params = @import("@zeam/params"); const utils = @import("./utils.zig"); diff --git a/pkgs/types/src/state.zig b/pkgs/types/src/state.zig index 15b0a27ab..e738436b4 100644 --- a/pkgs/types/src/state.zig +++ b/pkgs/types/src/state.zig @@ -245,7 +245,7 @@ pub const BeamState = struct { if (std.mem.eql(u8, &self.latest_block_header.state_root, &utils.ZERO_HASH)) { var prev_state_root: [32]u8 = undefined; - try ssz.hashTreeRoot(*BeamState, self, &prev_state_root, allocator); + try zeam_utils.hashTreeRoot(*BeamState, self, &prev_state_root, allocator); self.latest_block_header.state_root = prev_state_root; } } @@ -296,7 +296,7 @@ pub const BeamState = struct { // 4. verify latest block header is the parent var head_root: [32]u8 = undefined; - try ssz.hashTreeRoot(block.BeamBlockHeader, self.latest_block_header, &head_root, allocator); + try zeam_utils.hashTreeRoot(block.BeamBlockHeader, self.latest_block_header, &head_root, allocator); if (!std.mem.eql(u8, &head_root, &staged_block.parent_root)) { logger.err("state root={x:02} block root={x:02}\n", .{ head_root, staged_block.parent_root }); return StateTransitionError.InvalidParentRoot; @@ -539,7 +539,7 @@ pub const BeamState = struct { pub fn genGenesisBlock(self: *const Self, allocator: Allocator, genesis_block: *block.BeamBlock) !void { var state_root: [32]u8 = undefined; - try ssz.hashTreeRoot( + try zeam_utils.hashTreeRoot( BeamState, self.*, &state_root, @@ -554,7 +554,7 @@ pub const BeamState = struct { // check does it need cloning? var beam_block_header = self.latest_block_header; var state_root: [32]u8 = undefined; - try ssz.hashTreeRoot( + try zeam_utils.hashTreeRoot( BeamState, self.*, &state_root, @@ -719,7 +719,7 @@ test "ssz seralize/deserialize signed beam state" { // successful merklization var state_root: [32]u8 = undefined; - try ssz.hashTreeRoot( + try zeam_utils.hashTreeRoot( BeamState, state, &state_root, @@ -774,7 +774,7 @@ fn makeBlock( attestations: []const attestation.AggregatedAttestation, ) !block.BeamBlock { var parent_root: Root = undefined; - try ssz.hashTreeRoot(block.BeamBlockHeader, state.latest_block_header, &parent_root, allocator); + try zeam_utils.hashTreeRoot(block.BeamBlockHeader, state.latest_block_header, &parent_root, allocator); var attestations_list = try block.AggregatedAttestations.init(allocator); errdefer attestations_list.deinit(); @@ -829,7 +829,7 @@ test "justified_slots rebases when finalization advances" { try state.process_slots(std.testing.allocator, 2, logger); var block_2_parent_root: Root = undefined; - try ssz.hashTreeRoot(block.BeamBlockHeader, state.latest_block_header, &block_2_parent_root, std.testing.allocator); + try zeam_utils.hashTreeRoot(block.BeamBlockHeader, state.latest_block_header, &block_2_parent_root, std.testing.allocator); var att_0_to_1 = try makeAggregatedAttestation( std.testing.allocator, @@ -848,7 +848,7 @@ test "justified_slots rebases when finalization advances" { try state.process_slots(std.testing.allocator, 3, logger); var block_3_parent_root: Root = undefined; - try ssz.hashTreeRoot(block.BeamBlockHeader, state.latest_block_header, &block_3_parent_root, std.testing.allocator); + try zeam_utils.hashTreeRoot(block.BeamBlockHeader, state.latest_block_header, &block_3_parent_root, std.testing.allocator); var att_1_to_2 = try makeAggregatedAttestation( std.testing.allocator, @@ -897,7 +897,7 @@ test "pruning keeps pending justifications" { try state.process_slots(std.testing.allocator, 2, logger); var block_2_parent_root: Root = undefined; - try ssz.hashTreeRoot(block.BeamBlockHeader, state.latest_block_header, &block_2_parent_root, std.testing.allocator); + try zeam_utils.hashTreeRoot(block.BeamBlockHeader, state.latest_block_header, &block_2_parent_root, std.testing.allocator); var att_0_to_1 = try makeAggregatedAttestation( std.testing.allocator, @@ -1083,7 +1083,7 @@ test "genesis block hash comparison" { // Compute hash of first genesis block var genesis_block_hash1: Root = undefined; - try ssz.hashTreeRoot(block.BeamBlock, genesis_block1, &genesis_block_hash1, allocator); + try zeam_utils.hashTreeRoot(block.BeamBlock, genesis_block1, &genesis_block_hash1, allocator); std.debug.print("genesis_block_hash1 =0x{s}\n", .{std.fmt.fmtSliceHexLower(&genesis_block_hash1)}); // Create a second genesis state with same config but regenerated (should produce same hash) @@ -1096,7 +1096,7 @@ test "genesis block hash comparison" { defer genesis_block1_copy.deinit(); var genesis_block_hash1_copy: Root = undefined; - try ssz.hashTreeRoot(block.BeamBlock, genesis_block1_copy, &genesis_block_hash1_copy, allocator); + try zeam_utils.hashTreeRoot(block.BeamBlock, genesis_block1_copy, &genesis_block_hash1_copy, allocator); // Same genesis spec should produce same hash try std.testing.expect(std.mem.eql(u8, &genesis_block_hash1, &genesis_block_hash1_copy)); @@ -1125,7 +1125,7 @@ test "genesis block hash comparison" { defer genesis_block2.deinit(); var genesis_block_hash2: Root = undefined; - try ssz.hashTreeRoot(block.BeamBlock, genesis_block2, &genesis_block_hash2, allocator); + try zeam_utils.hashTreeRoot(block.BeamBlock, genesis_block2, &genesis_block_hash2, allocator); std.debug.print("genesis_block_hash2 =0x{s}\n", .{std.fmt.fmtSliceHexLower(&genesis_block_hash2)}); // Different validators should produce different genesis block hash @@ -1155,7 +1155,7 @@ test "genesis block hash comparison" { defer genesis_block3.deinit(); var genesis_block_hash3: Root = undefined; - try ssz.hashTreeRoot(block.BeamBlock, genesis_block3, &genesis_block_hash3, allocator); + try zeam_utils.hashTreeRoot(block.BeamBlock, genesis_block3, &genesis_block_hash3, allocator); std.debug.print("genesis_block_hash3 =0x{s}\n", .{std.fmt.fmtSliceHexLower(&genesis_block_hash3)}); // Different genesis_time should produce different genesis block hash diff --git a/pkgs/utils/src/lib.zig b/pkgs/utils/src/lib.zig index a00e8bf9c..b93f686c7 100644 --- a/pkgs/utils/src/lib.zig +++ b/pkgs/utils/src/lib.zig @@ -37,6 +37,9 @@ const json_factory = @import("./json.zig"); // Avoid to use `usingnamespace` to make upgrade easier in the future. pub const jsonToString = json_factory.jsonToString; +const ssz_factory = @import("./ssz.zig"); +pub const hashTreeRoot = ssz_factory.hashTreeRoot; + const fmt_factory = @import("./fmt.zig"); // Avoid to use `usingnamespace` to make upgrade easier in the future. pub const LazyJson = fmt_factory.LazyJson; diff --git a/pkgs/utils/src/ssz.zig b/pkgs/utils/src/ssz.zig new file mode 100644 index 000000000..a92164cf4 --- /dev/null +++ b/pkgs/utils/src/ssz.zig @@ -0,0 +1,14 @@ +const std = @import("std"); +const ssz = @import("ssz"); + +const Allocator = std.mem.Allocator; +const Sha256 = std.crypto.hash.sha2.Sha256; + +pub fn hashTreeRoot( + comptime T: type, + value: T, + out: *[Sha256.digest_length]u8, + allocator: Allocator, +) !void { + try ssz.hashTreeRoot(Sha256, T, value, out, allocator); +}