diff --git a/contracts/sysio.msgch/include/sysio.msgch/sysio.msgch.hpp b/contracts/sysio.msgch/include/sysio.msgch/sysio.msgch.hpp index 7a3e8b9aab..0784913cd7 100644 --- a/contracts/sysio.msgch/include/sysio.msgch/sysio.msgch.hpp +++ b/contracts/sysio.msgch/include/sysio.msgch/sysio.msgch.hpp @@ -198,9 +198,16 @@ namespace sysio { sysio::const_mem_fun> >; - /// Outbound envelope table. One-deep per outpost: `buildenv` erases every older row for the - /// destination `chain_code` after inserting the new emit, so the surviving row doubles as the - /// per-outpost chain tip. + /// Outbound envelope table. After inserting a new emit, `buildenv` erases the older rows for + /// the destination `chain_code` that the outpost has CONSUMED -- those whose `epoch_index` is + /// covered by `outpcons.epoch_index` -- and retains any it has not. The epoch-advance + /// interlock (`chkcons` releases `advance`, and therefore `buildenv`, only once every active + /// outpost has reached consensus at the current epoch) makes that the whole table in the + /// healthy path, so it is normally one-deep; a retained row means the interlock did not hold + /// and its `raw_envelope` is the only remaining copy of an envelope the outpost still needs. + /// + /// The per-outpost chain tip is therefore the NEWEST row, selected through `byoutepoch` -- + /// NOT `byoutpost`, whose entries sort by ascending primary key and so yield the oldest. /// /// `envelope_hash` is the canonical epoch digest: keccak256 over the canonical /// field-complete encoding with the in-envelope `envelope_hash` field blanked (equal to diff --git a/contracts/sysio.msgch/src/sysio.msgch.cpp b/contracts/sysio.msgch/src/sysio.msgch.cpp index 38ef1d82e4..91a12d202b 100644 --- a/contracts/sysio.msgch/src/sysio.msgch.cpp +++ b/contracts/sysio.msgch/src/sysio.msgch.cpp @@ -1870,21 +1870,31 @@ void msgch::buildenv(uint64_t chain_code) { candidate_ids.begin(), candidate_ids.begin() + included_count); - // Chain links: the previous envelope emitted for this outpost. `outenvelopes` is one-deep per - // outpost (see the cleanup below), so the single surviving row is the previous emit; its - // `envelope_hash` is that envelope's epoch digest and its `last_message_id` is this outpost's - // message-stream tip. The first emit for an outpost has no row and chains both links from - // empty (genesis), matching the outpost contracts' zero genesis tip. + // Chain links: the previous envelope emitted for this outpost -- the NEWEST row for + // `chain_code`, its `envelope_hash` that envelope's epoch digest and its `last_message_id` + // this outpost's message-stream tip. The first emit for an outpost has no row and chains both + // links from empty (genesis), matching the outpost contracts' zero genesis tip. + // + // Walked over `byoutepoch` rather than `byoutpost`: KV secondary entries sort by ascending + // PRIMARY key within one secondary key, so a `byoutpost` lower_bound yields the OLDEST row for + // the outpost. That was the tip only while exactly one row survived; the ack-gated cleanup + // below legitimately retains an emit the outpost has not consumed, so the tip must be selected + // by epoch instead of by "the only row left". + // + // `byoutepoch` packs `(chain_code, epoch_index)`, so this outpost's rows are contiguous and + // ascending by epoch -- the LAST match in the walk is the tip. Forward-only (`lower_bound` + + // `++` + the chain_code guard), the same shape as the sweep below; the walk is one row in the + // healthy path and bounded by the retained emits otherwise. outenvelopes_t envelopes(get_self()); std::vector prev_envelope_digest; std::vector prev_message_id; { - auto by_outpost = envelopes.get_index<"byoutpost"_n>(); - auto prev_it = by_outpost.lower_bound(chain_code); - if (prev_it != by_outpost.end() && prev_it->chain_code == chain_code) { - const auto digest_bytes = prev_it->envelope_hash.extract_as_byte_array(); + auto by_epoch = envelopes.get_index<"byoutepoch"_n>(); + for (auto it = by_epoch.lower_bound(opp::outpost_epoch_key(chain_code, 0)); + it != by_epoch.end() && it->chain_code == chain_code; ++it) { + const auto digest_bytes = it->envelope_hash.extract_as_byte_array(); prev_envelope_digest.assign(digest_bytes.begin(), digest_bytes.end()); - const auto tip_bytes = prev_it->last_message_id.extract_as_byte_array(); + const auto tip_bytes = it->last_message_id.extract_as_byte_array(); prev_message_id.assign(tip_bytes.begin(), tip_bytes.end()); } } @@ -2003,20 +2013,56 @@ void msgch::buildenv(uint64_t chain_code) { // === AUDIT LOG + INLINE CLEANUP OF WORKING STATE === // // Audit-log row mirrors the outbound emit (WIRE → outpost). Followed - // by inline drains of the previous-epoch outenvelopes row (one-deep - // retention; the batch op only ever reads the most-recent emit) and - // the just-PROCESSED attestations for this outpost (their bytes are - // now baked into `packed` above). + // by inline drains of the outenvelopes rows this outpost has already + // CONSUMED (see the ack gate below) and the just-PROCESSED + // attestations for this outpost (their bytes are now baked into + // `packed` above). { // Same endpoints the wire envelope carries (derived above from the // destination's `sysio.chains` row). write_envelope_log(get_self(), route_endpoints, epoch, envelope_digest); - // Drop previous outpost emits — keep only the row we just inserted. + // Drop previous outpost emits the destination has CONSUMED, and only those. + // + // `outpcons.epoch_index` is the last epoch whose INBOUND envelope from this outpost + // reached consensus, and an outpost can only emit its epoch-N envelope after accepting + // the depot's epoch-N emit -- Solana asserts it directly + // (`emit.rs`: `require!(wire_epoch_index < config.next_epoch_index, EmitBeforeEpochAccepted)`, + // which binds the admin recovery instruction too), and Ethereum reaches + // `OPP::emitOutboundEnvelope` only from the `OPPInbound` consensus tip under + // `OPP_FINALIZER_ROLE`. So `epoch_index <= acked` is PROOF the outpost consumed that emit. + // + // In the healthy path this erases exactly what the old unconditional sweep did and leaves + // the table one-deep: `chkcons` only released the `advance` that reached this `buildenv` + // because every active outpost had already reached consensus at the previous epoch, so + // `acked` covers the prior emit. A row survives only when that interlock did not hold, in + // which case its bytes are the ONLY copy of an envelope the outpost still needs -- the + // audit log keeps a checksum, not a payload, and the source attestations are drained just + // below. Retaining is what stops a chain break from becoming unrecoverable data loss. + // + // `has_ack` is load-bearing, not defensive: NO row means the outpost has acknowledged + // NOTHING, which a bare `epoch_index` default of 0 would make indistinguishable from + // having acknowledged epoch 0 -- and would erase an epoch-0 emit on that reading. + outpost_consensus_t opcons(get_self()); + const auto opc_pk = outpost_consensus_key{chain_code}; + const bool has_ack = opcons.contains(opc_pk); + const uint32_t acked = has_ack ? opcons.get(opc_pk).epoch_index : 0; + auto by_outpost = envelopes.get_index<"byoutpost"_n>(); for (auto it = by_outpost.lower_bound(chain_code); it != by_outpost.end() && it->chain_code == chain_code; ) { if (it->id == out_id) { ++it; continue; } + if (!has_ack || it->epoch_index > acked) { + // Never expected: reaching this `buildenv` required consensus past this emit. + // Logged (visible under --contracts-console) so a broken interlock is greppable + // rather than a silently growing table. + sysio::print_f("msgch::buildenv: retaining unacknowledged emit chain_code=%llu " + "epoch=%u has_ack=%d acked=%u\n", + static_cast(chain_code), it->epoch_index, + static_cast(has_ack), acked); + ++it; + continue; + } it = by_outpost.erase(std::move(it)); } diff --git a/contracts/sysio.msgch/sysio.msgch.wasm b/contracts/sysio.msgch/sysio.msgch.wasm index b581f28a4f..0ba30f0061 100755 Binary files a/contracts/sysio.msgch/sysio.msgch.wasm and b/contracts/sysio.msgch/sysio.msgch.wasm differ diff --git a/contracts/tests/sysio.msgch_chain_tests.cpp b/contracts/tests/sysio.msgch_chain_tests.cpp index 49f0144484..e7eaf1311e 100644 --- a/contracts/tests/sysio.msgch_chain_tests.cpp +++ b/contracts/tests/sysio.msgch_chain_tests.cpp @@ -492,20 +492,53 @@ class sysio_msgch_chain_tester : public tester { // -- Table readers -- - /// The single surviving outbound envelope row for `chain_code` (the table is one-deep per - /// outpost); null variant when none. + /// The NEWEST outbound envelope row for `chain_code` -- the per-outpost chain tip + /// `buildenv` chains from; null variant when none. Deliberately not "the first row + /// found": `buildenv` retains emits the outpost has not acknowledged, so the lowest + /// id can be a retained predecessor rather than the tip. fc::variant find_outbound_envelope(uint64_t chain_code, uint64_t scan_until = 32) { + fc::variant newest; for (uint64_t id = 0; id < scan_until; ++id) { auto data = get_row_by_id(MSGCH_ACCOUNT, MSGCH_ACCOUNT, "outenvelopes"_n, id); if (data.empty()) continue; auto row = msgch_abi.binary_to_variant( "outbound_envelope", data, abi_serializer::create_yield_function(abi_serializer_max_time)); - if (row["chain_code"].as_uint64() == chain_code) return row; + if (row["chain_code"].as_uint64() == chain_code) newest = row; + } + return newest; + } + + /// The outbound envelope row `chain_code` emitted for `epoch_index`; null when it is + /// absent -- either never emitted, or erased because the outpost acknowledged it. + fc::variant outbound_envelope_for_epoch(uint64_t chain_code, uint32_t epoch_index, + uint64_t scan_until = 32) { + for (uint64_t id = 0; id < scan_until; ++id) { + auto data = get_row_by_id(MSGCH_ACCOUNT, MSGCH_ACCOUNT, "outenvelopes"_n, id); + if (data.empty()) continue; + auto row = msgch_abi.binary_to_variant( + "outbound_envelope", data, + abi_serializer::create_yield_function(abi_serializer_max_time)); + if (row["chain_code"].as_uint64() == chain_code && + row["epoch_index"].as() == epoch_index) return row; } return fc::variant{}; } + /// Live outbound envelope rows for `chain_code`. + uint32_t outbound_envelope_count(uint64_t chain_code, uint64_t scan_until = 32) { + uint32_t n = 0; + for (uint64_t id = 0; id < scan_until; ++id) { + auto data = get_row_by_id(MSGCH_ACCOUNT, MSGCH_ACCOUNT, "outenvelopes"_n, id); + if (data.empty()) continue; + auto row = msgch_abi.binary_to_variant( + "outbound_envelope", data, + abi_serializer::create_yield_function(abi_serializer_max_time)); + if (row["chain_code"].as_uint64() == chain_code) ++n; + } + return n; + } + /// Per-outpost consensus row (`outpcons`, primary key = chain_code); null variant when absent. fc::variant get_outpcons(uint64_t chain_code) { auto data = get_row_by_id(MSGCH_ACCOUNT, MSGCH_ACCOUNT, "outpcons"_n, chain_code); @@ -1063,6 +1096,56 @@ BOOST_FIXTURE_TEST_CASE(buildenv_first_emit_chains_from_empty, sysio_msgch_chain fc::to_hex(header.message_id().data(), header.message_id().size())); } FC_LOG_AND_RETHROW() } +// --------------------------------------------------------------------------- +// Outbound retention: an emit is erased only once the outpost acknowledges it. +// --------------------------------------------------------------------------- + +/// WNS-18 / WIRE-348, the acknowledged half. `buildenv` erases prior emits for an outpost +/// only up to `outpcons.epoch_index` -- the last epoch whose INBOUND envelope from that +/// outpost reached consensus, which the outpost can only have produced after consuming the +/// depot's emit for that epoch. So: +/// +/// * while nothing is acknowledged, successive emits ACCUMULATE (their `raw_envelope` is +/// the only surviving copy of an envelope the outpost still needs), and +/// * one acknowledgement drains every emit at or below it, leaving the table one-deep -- +/// which is what the healthy path always looks like, because `chkcons` releases the +/// `advance` that reaches `buildenv` only once every active outpost has reached +/// consensus at the current epoch. +BOOST_FIXTURE_TEST_CASE(buildenv_erases_acknowledged_outenvelope, sysio_msgch_chain_tester) { try { + bootstrap(); + + // bootstrap()'s genesis `advance` fanned a buildenv to every registered outpost. + const uint32_t epoch_a = current_epoch(); + BOOST_REQUIRE(!outbound_envelope_for_epoch(ETH_OUTPOST_ID, epoch_a).is_null()); + BOOST_REQUIRE(get_outpcons(ETH_OUTPOST_ID).is_null()); // nothing acknowledged yet + + // Unacknowledged: the next epoch's emit does NOT displace epoch A's. + const uint32_t epoch_b = advance_one_epoch(); + BOOST_REQUIRE_NE(epoch_a, epoch_b); + BOOST_REQUIRE(!outbound_envelope_for_epoch(ETH_OUTPOST_ID, epoch_a).is_null()); + BOOST_REQUIRE(!outbound_envelope_for_epoch(ETH_OUTPOST_ID, epoch_b).is_null()); + BOOST_REQUIRE_EQUAL(2u, outbound_envelope_count(ETH_OUTPOST_ID)); + + // The tip is the NEWEST row, not the retained predecessor. + BOOST_REQUIRE_EQUAL(find_outbound_envelope(ETH_OUTPOST_ID)["epoch_index"].as(), + epoch_b); + + // An inbound envelope accepted for epoch B is the acknowledgement: the outpost could not + // have emitted it without first consuming the depot's epoch-B envelope. + BOOST_REQUIRE_EQUAL(success(), deliver(ETH_OUTPOST_ID, encode_delivery(epoch_b, "alpha"))); + produce_blocks(); + auto opc = get_outpcons(ETH_OUTPOST_ID); + BOOST_REQUIRE(!opc.is_null()); + BOOST_REQUIRE_EQUAL(opc["epoch_index"].as(), epoch_b); + + // The next emit now drains everything at or below the acknowledged epoch -- both A and B. + const uint32_t epoch_c = advance_one_epoch(); + BOOST_REQUIRE(outbound_envelope_for_epoch(ETH_OUTPOST_ID, epoch_a).is_null()); + BOOST_REQUIRE(outbound_envelope_for_epoch(ETH_OUTPOST_ID, epoch_b).is_null()); + BOOST_REQUIRE(!outbound_envelope_for_epoch(ETH_OUTPOST_ID, epoch_c).is_null()); + BOOST_REQUIRE_EQUAL(1u, outbound_envelope_count(ETH_OUTPOST_ID)); +} FC_LOG_AND_RETHROW() } + // --------------------------------------------------------------------------- // Inbound: apply_consensus records and verifies the per-outpost chain tip. // --------------------------------------------------------------------------- @@ -1306,7 +1389,7 @@ BOOST_FIXTURE_TEST_CASE(inbound_rejects_cross_stream_link_for_svm_outposts, // Next epoch: emit a depot outbound envelope, then deliver an inbound envelope whose prev is // that emit's digest (NOT the inbound tip). Building the outbound envelope in the delivery - // epoch pins the one-deep outenvelopes row so no epoch-advance emission replaces it. + // epoch pins the outenvelopes tip so no epoch-advance emission supersedes it. epoch = advance_one_epoch(); BOOST_REQUIRE_EQUAL(success(), queueout(SOL_OUTPOST_ID, sysio::opp::types::ATTESTATION_TYPE_OPERATORS, std::vector{0x0a})); diff --git a/contracts/tests/sysio.msgch_tests.cpp b/contracts/tests/sysio.msgch_tests.cpp index e237de42bd..00e4d84af6 100644 --- a/contracts/tests/sysio.msgch_tests.cpp +++ b/contracts/tests/sysio.msgch_tests.cpp @@ -323,16 +323,29 @@ class sysio_msgch_envlog_tester : public tester { ); } - /// Return the first outbound envelope row found in a small id scan. + /// Return the NEWEST outbound envelope row in a small id scan -- the per-outpost + /// chain tip, which is what `buildenv` chains from. Deliberately not "the first + /// row found": `buildenv` retains emits the outpost has not acknowledged, so the + /// lowest id can be a retained predecessor rather than the tip. fc::variant find_outbound_envelope(uint64_t scan_until = 16) { + fc::variant newest; for (uint64_t id = 0; id < scan_until; ++id) { auto data = get_row_by_id(MSGCH_ACCOUNT, MSGCH_ACCOUNT, "outenvelopes"_n, id); if (data.empty()) continue; - return msgch_abi.binary_to_variant( + newest = msgch_abi.binary_to_variant( "outbound_envelope", data, abi_serializer::create_yield_function(abi_serializer_max_time)); } - return fc::variant{}; + return newest; + } + + /// Number of live `outenvelopes` rows in the id range `[0, scan_until)`. + uint32_t outbound_envelope_count(uint64_t scan_until = 16) { + uint32_t n = 0; + for (uint64_t id = 0; id < scan_until; ++id) { + if (!get_row_by_id(MSGCH_ACCOUNT, MSGCH_ACCOUNT, "outenvelopes"_n, id).empty()) ++n; + } + return n; } /// Count populated `envlog` rows in the id range `[0, max_id_exclusive)`. @@ -475,10 +488,18 @@ BOOST_FIXTURE_TEST_CASE(envlog_cap_tracks_outpost_count, sysio_msgch_envlog_test BOOST_REQUIRE_EQUAL(7u, alive); } FC_LOG_AND_RETHROW() } -/// Existing `outenvelopes` row gets dropped on the next `buildenv` for -/// the same outpost — one-deep retention (the batch op only ever reads -/// the most-recent emit). -BOOST_FIXTURE_TEST_CASE(buildenv_drops_previous_outenvelopes, sysio_msgch_envlog_tester) { try { +/// WNS-18 / WIRE-348. `buildenv` erases a previous emit only once the destination +/// outpost has CONSUMED it -- i.e. `outpcons.epoch_index` reaches that row's epoch. +/// This fixture pushes `buildenv` directly and never drives inbound consensus, so no +/// `outpcons` row exists (acked = 0) and the earlier emit MUST survive: its +/// `raw_envelope` is the only remaining copy of an envelope the outpost still needs +/// (the audit log keeps a checksum, not a payload, and the source attestations are +/// drained by the same `buildenv`). +/// +/// The acknowledged half of the contract -- the erase actually firing, and the table +/// staying one-deep in the healthy path -- needs real consensus and lives in +/// `sysio.msgch_chain_tests.cpp::buildenv_erases_acknowledged_outenvelope`. +BOOST_FIXTURE_TEST_CASE(buildenv_retains_unacknowledged_outenvelopes, sysio_msgch_envlog_tester) { try { bootstrap_epoch_config(/*retention=*/200); register_outpost(opp::types::CHAIN_KIND_EVM, 31337); produce_blocks(); @@ -490,16 +511,25 @@ BOOST_FIXTURE_TEST_CASE(buildenv_drops_previous_outenvelopes, sysio_msgch_envlog produce_blocks(); auto first = get_row_by_id(MSGCH_ACCOUNT, MSGCH_ACCOUNT, "outenvelopes"_n, 1); BOOST_REQUIRE(!first.empty()); + BOOST_REQUIRE_EQUAL(1u, outbound_envelope_count()); BOOST_REQUIRE_EQUAL(success(), queueout(ETH_OUTPOST_ID, EVM_TEST_ATTESTATION_TYPE)); BOOST_REQUIRE_EQUAL(success(), buildenv(ETH_OUTPOST_ID)); produce_blocks(); - // First row is now gone (replaced by the second emit). + // Both rows live: the outpost acknowledged neither. first = get_row_by_id(MSGCH_ACCOUNT, MSGCH_ACCOUNT, "outenvelopes"_n, 1); - BOOST_REQUIRE(first.empty()); + BOOST_REQUIRE(!first.empty()); auto second = get_row_by_id(MSGCH_ACCOUNT, MSGCH_ACCOUNT, "outenvelopes"_n, 2); BOOST_REQUIRE(!second.empty()); + BOOST_REQUIRE_EQUAL(2u, outbound_envelope_count()); + + // With a predecessor retained, the tip must still be the NEWEST row -- the + // `byoutepoch` read in `buildenv`, not a `byoutpost` lower_bound (which orders by + // ascending primary key and would hand back the retained row 1). + auto tip = find_outbound_envelope(); + BOOST_REQUIRE(!tip.is_null()); + BOOST_REQUIRE_EQUAL(2u, tip["id"].as_uint64()); } FC_LOG_AND_RETHROW() } /// `attestations` PROCESSED rows for a given outpost are dropped at the @@ -575,20 +605,12 @@ BOOST_FIXTURE_TEST_CASE(buildenv_packs_until_cap_then_leaves_remainder, BOOST_REQUIRE_EQUAL(success(), buildenv(/*chain_code=*/ETH_OUTPOST_ID)); produce_blocks(); - // The most recent emit lives at one of the early ids; the one-deep - // retention sweep means at most one row exists per outpost. Find it. - fc::variant emitted_row; - uint64_t emitted_id = std::numeric_limits::max(); - for (uint64_t id = 0; id < 16; ++id) { - auto data = get_row_by_id(MSGCH_ACCOUNT, MSGCH_ACCOUNT, "outenvelopes"_n, id); - if (data.empty()) continue; - emitted_row = msgch_abi.binary_to_variant( - "outbound_envelope", data, - abi_serializer::create_yield_function(abi_serializer_max_time)); - emitted_id = id; - break; - } + // Take the NEWEST emit. This fixture never drives inbound consensus, so nothing is + // acknowledged and `buildenv` retains each predecessor -- the lowest id is not the + // row under test. + fc::variant emitted_row = find_outbound_envelope(); BOOST_REQUIRE(!emitted_row.is_null()); + const uint64_t emitted_id = emitted_row["id"].as_uint64(); // ── Invariant 1: emitted envelope is at or under the cross-chain cap. const auto& raw = emitted_row["raw_envelope"].as>(); @@ -609,18 +631,11 @@ BOOST_FIXTURE_TEST_CASE(buildenv_packs_until_cap_then_leaves_remainder, BOOST_REQUIRE_EQUAL(success(), buildenv(/*chain_code=*/ETH_OUTPOST_ID)); produce_blocks(); - // Find the new emitted row (one-deep retention dropped the prior one). - fc::variant emitted_row_2; - for (uint64_t id = 0; id < 16; ++id) { - if (id == emitted_id) continue; // prior row was evicted - auto data = get_row_by_id(MSGCH_ACCOUNT, MSGCH_ACCOUNT, "outenvelopes"_n, id); - if (data.empty()) continue; - emitted_row_2 = msgch_abi.binary_to_variant( - "outbound_envelope", data, - abi_serializer::create_yield_function(abi_serializer_max_time)); - break; - } + // The new tip. emit#1's row is still present (unacknowledged), so select by + // recency rather than by "the other row". + fc::variant emitted_row_2 = find_outbound_envelope(); BOOST_REQUIRE(!emitted_row_2.is_null()); + BOOST_REQUIRE_NE(emitted_id, emitted_row_2["id"].as_uint64()); const auto& raw2 = emitted_row_2["raw_envelope"].as>(); BOOST_TEST_MESSAGE("emit#2 envelope size = " << raw2.size() << " bytes"); BOOST_REQUIRE_LE(raw2.size(), MAX_ENV_BYTES);