diff --git a/contracts/sysio.epoch/include/sysio.epoch/sysio.epoch.hpp b/contracts/sysio.epoch/include/sysio.epoch/sysio.epoch.hpp index 5a8e6a5154..59a998090b 100644 --- a/contracts/sysio.epoch/include/sysio.epoch/sysio.epoch.hpp +++ b/contracts/sysio.epoch/include/sysio.epoch/sysio.epoch.hpp @@ -31,6 +31,10 @@ namespace sysio { [[sysio::action]] void advance(); + /// Internal continuation after epoch-close operator mutations execute. + [[sysio::action]] + void finishadv(uint32_t epoch_index, int64_t emission_amount); + /// Group assignment — reads AVAILABLE batch ops from sysio.opreg. [[sysio::action]] void schbatchgps(); diff --git a/contracts/sysio.epoch/src/sysio.epoch.cpp b/contracts/sysio.epoch/src/sysio.epoch.cpp index 3f4e9dd23b..7f4ec40d15 100644 --- a/contracts/sysio.epoch/src/sysio.epoch.cpp +++ b/contracts/sysio.epoch/src/sysio.epoch.cpp @@ -19,6 +19,7 @@ #include #include #include +#include namespace sysio { @@ -45,6 +46,7 @@ namespace { constexpr name SYSTEM_ACCOUNT = "sysio"_n; constexpr name TOKEN_ACCOUNT = "sysio.token"_n; +constexpr name FINISH_ADVANCE = "finishadv"_n; /// Action identifiers owned by sysio.chalg and invoked by epoch close. namespace chalg_actions { @@ -594,9 +596,9 @@ void epoch::advance() { // // Invariant — no cross-epoch double slash: opreg::slash THROWS on an already-SLASHED operator, // which would abort advance and stall OPP epoch advancement. These inline slashes execute only - // after advance returns, so the schedule slide below can temporarily place a just-slashed - // operator in its new tail while the operator still reads ACTIVE. That member cannot create a - // later non-canonical observation: sysio.msgch::deliver requires its current sysio.opreg status + // after advance returns. The finishadv continuation waits for those mutations before + // selecting the new tail. A removed member cannot create a later non-canonical + // observation: sysio.msgch::deliver requires its current sysio.opreg status // to be ACTIVE before accepting delivery. Once the slash has executed, the scheduled SLASHED // member cannot deliver or be queued for another non-canonical-delivery slash. The collection // above also deduplicates multiple non-canonical observations for one member in this advance. @@ -642,13 +644,84 @@ void epoch::advance() { // cron tick and trips kv-index-remove on already-evicted buckets. } - const bool had_expiring_group = state.current_epoch_index > 0; - state.current_epoch_index++; state.current_epoch_start = (state.next_epoch_start.sec_since_epoch() == 0) ? now : state.next_epoch_start; state.next_epoch_start = state.current_epoch_start + microseconds(static_cast(cfg.epoch_duration_sec) * 1'000'000); + state_tbl.set(state, ram_payer); + + // Withdrawal flushing can also change eligibility. Its nested callbacks + // must complete before schedule selection and roster serialization. + // Inline siblings execute their complete subtrees in order, atomically. + action( + permission_level{get_self(), "owner"_n}, + OPREG_ACCOUNT, + "flushwtdw"_n, + std::make_tuple(state.current_epoch_index) + ).send(); + + // Keep the refund subtree at its original depth; refundwire can itself + // transfer a fee or sweep expired claims. It needs the new epoch index, + // but not the new schedule, and finishes before roster publication. + // Drain the swap-from-WIRE queue: each row queued via + // `sysio.uwrit::swapfromwire` since the last advance is re-validated + // (target reserve ACTIVE + public, variance) and either becomes a + // PENDING uwreq for the single-leg underwriter race or is refunded. + // Runs before `buildenv` so this epoch's envelopes reflect any state + // the drain produced; never throws (refund-and-drop semantics). + action( + permission_level{get_self(), "owner"_n}, + UWRIT_ACCOUNT, + "drainfwq"_n, + std::make_tuple() + ).send(); + + action( + permission_level{get_self(), "owner"_n}, + get_self(), + FINISH_ADVANCE, + std::make_tuple(state.current_epoch_index, gate.emission_amount) + ).send(); + + // Preserve payout depth: finishadv and its accrual/history descendants + // complete before this sibling executes. + if (gate.is_pay_epoch) { + action( + permission_level{get_self(), "owner"_n}, + SYSTEM_ACCOUNT, + "payepoch"_n, + std::make_tuple( + state.current_epoch_index, + std::vector>{}, + gate.period_emission + ) + ).send(); + } +} + +void epoch::finishadv(uint32_t epoch_index, int64_t emission_amount) { + require_auth(get_self()); + check(get_sender() == get_self(), "finishadv must be sent inline by sysio.epoch"); + epochcfg_t cfg_tbl(get_self()); + const auto cfg = cfg_tbl.get(); + epochstate_t state_tbl(get_self()); + auto state = state_tbl.get(); + check(state.current_epoch_index == epoch_index, "finishadv epoch mismatch"); + const bool had_expiring_group = epoch_index > 1; + + // A seated operator can have lost eligibility since the window was built. + // Preserve healthy members' order and never reuse a resident to fill a gap. + opreg::operators_t current_ops(OPREG_ACCOUNT); + for (auto& group : state.batch_op_groups) { + group.erase(std::remove_if(group.begin(), group.end(), [&](name account) { + const auto key = opreg::operator_key{account.value}; + if (!current_ops.contains(key)) return true; + const auto op = current_ops.get(key); + return op.status != OperatorStatus::OPERATOR_STATUS_ACTIVE || + op.type != OperatorType::OPERATOR_TYPE_BATCH; + }), group.end()); + } // ── Slide the schedule window ─────────────────────────────────────────── // Skip on the genesis advance (0 → 1): schbatchgps just placed @@ -665,6 +738,7 @@ void epoch::advance() { // After: window = [current, current+1, ..., current+N-1], front is // always the active group → current_batch_op_group stays at 0. if (had_expiring_group && !state.batch_op_groups.empty()) { + const auto expired = state.batch_op_groups.front(); state.batch_op_groups.erase(state.batch_op_groups.begin()); // Collect already-resident accounts so the new tail excludes them. @@ -709,6 +783,31 @@ void epoch::advance() { return a.first < b.first; }); + // Repair future seats before selecting the tail. Otherwise a removed + // operator leaves a hole that eventually becomes an empty active group, + // even when a healthy standby could have been announced one epoch ahead. + // Prefer true standbys: recycling the expired group early would shorten + // its duty interval unnecessarily. All selections consume the same pool, + // so repaired groups and the tail remain disjoint. + // Vacancy recovery is an exception to the normal N-epoch duty spacing: + // absence from this window does not prove an operator has never served + // recently, particularly with windows larger than three groups. + // Do not insert a new member into the CURRENT group here: outposts have + // not received this window yet, and their old chunk-slot assignments may + // collide with a replacement's position. That case retains the existing + // incomplete-window withholding behavior and requires roster recovery. + for (size_t g = 1; g < state.batch_op_groups.size(); ++g) { + auto& group = state.batch_op_groups[g]; + while (group.size() < cfg.operators_per_epoch) { + const auto standby = std::find_if(pool.begin(), pool.end(), [&](const auto& candidate) { + return std::find(expired.begin(), expired.end(), candidate.first) == expired.end(); + }); + if (standby == pool.end()) break; + group.push_back(standby->first); + pool.erase(standby); + } + } + std::vector new_tail; new_tail.reserve(cfg.operators_per_epoch); for (size_t i = 0; i < pool.size() && new_tail.size() < cfg.operators_per_epoch; ++i) { @@ -727,9 +826,8 @@ void epoch::advance() { // is an exact half and two competing digests can both tip. // // So the schedule is left as-is and the DECISION is pushed to the emit - // site: an empty active group is never published (see the withhold - // below). Short-but-non-empty is pre-existing behaviour and is not made - // safe here -- it is reported so the roster can be repaired off-chain. + // site: an incomplete window is never published (see the withhold + // below), and is reported so the roster can be repaired off-chain. if (new_tail.size() < cfg.operators_per_epoch) { sysio::print("sysio.epoch::advance: only ", new_tail.size(), " of ", cfg.operators_per_epoch, @@ -756,21 +854,6 @@ void epoch::advance() { state_tbl.set(state, ram_payer); - // Drain matured rows from `sysio.opreg::wtdwqueue`. Operators that queued - // a withdrawal at least WITHDRAW_WAIT_EPOCHS ago are now eligible — opreg - // subtracts from the balance and emits OPERATOR_ACTION(WITHDRAW_REMIT) to - // the matching outpost (or, for WIRE-direct withdraws, CREDITS the operator's - // `sysio.opreg::remitclaims` row, which it pulls with `claimremit` — nothing - // is transferred from this path, precisely because it runs inline from here). - // Slashed-during-the-wait rows are dropped silently inside - // opreg's flushwtdw. See CLAUDE-WIRE-OPERATOR-COLLATERAL-IMPL-PLAN.md §3.3. - action( - permission_level{get_self(), "owner"_n}, - OPREG_ACCOUNT, - "flushwtdw"_n, - std::make_tuple(state.current_epoch_index) - ).send(); - // Queue OPERATORS attestation (full roster with authex chain addresses) for each outpost. // IMPORTANT: Must come before BATCH_OPERATOR_GROUPS so that the ETH outpost's // _handleOperators populates operatorEthAddress before _handleBatchOperatorGroups @@ -900,41 +983,18 @@ void epoch::advance() { const uint32_t next_group_index = next_index < group_count ? next_index : state.current_batch_op_group; - // NEVER publish an empty "next". The index names the group the outpost - // will admit `epoch_in` against and size its quorum from, so an empty - // one is not a degraded roster -- it is an invalid attestation, and - // seating it wedges the outpost permanently (the handler that could - // replace the window runs only past the gate the empty group breaks). - // - // This is the ONE sound guarantee available here. The slide cannot buy - // non-emptiness by backfilling: with an ACTIVE pool smaller than the - // window, N groups that are both FULL and DISJOINT do not exist, and - // both escapes are unsound (see the slide's comment -- re-seating a - // resident breaks Ethereum's chunk-position disjointness; a short group - // lowers the quorum denominator it defines). So the schedule is left - // alone and the decision lands here. Withholding the attestation leaves - // the outpost on its previous window -- the same end state its own - // guards reach, without shipping an invalid payload. - // - // Cost, accepted deliberately: the withheld attestation also carries - // `epoch_duration_sec` and the whole-window resync that - // batch-operator-schedule-window.md wants on every envelope, so both are - // skipped for this epoch too. Shipping the payload with the index pinned - // to the CURRENT group instead would keep them, but it names a group the - // outpost must not treat as next, and the Solana handler refuses a window - // carrying an empty group regardless -- so it buys nothing here. - // - // Withheld by SKIPPING THE QUEUEOUT ONLY -- never by returning from - // `advance`, which still has the epoch's remaining attestations and - // actions to issue after this block. - const bool have_next_group = - next_group_index < group_count && !state.batch_op_groups[next_group_index].empty(); - if (!have_next_group) { - sysio::print("sysio.epoch::advance: no non-empty next group to publish at epoch ", + // Removing ineligible members must not lower an outpost's quorum + // denominator or publish an empty group. Withhold an incomplete window, + // while still sending OPERATORS with the authoritative removal statuses. + // Never duplicate residents to fill it: Ethereum's chunk routing assumes + // disjoint groups. Epoch accounting and envelope construction still run. + const bool have_complete_window = next_group_index < group_count && + std::all_of(state.batch_op_groups.begin(), state.batch_op_groups.end(), + [&](const auto& group) { return group.size() == cfg.operators_per_epoch; }); + if (!have_complete_window) { + sysio::print("sysio.epoch::finishadv: incomplete operator window at epoch ", state.current_epoch_index, - " (groups=", group_count, ", next_index=", next_group_index, - "); withholding BatchOperatorGroups -- outposts retain their " - "previous window\n"); + "; withholding BatchOperatorGroups until the roster is repaired\n"); } attest.active_group_index = zpp::bits::vuint32_t{next_group_index}; attest.epoch_index = zpp::bits::vuint32_t{state.current_epoch_index}; @@ -959,8 +1019,8 @@ void epoch::advance() { auto out = zpp::bits::out{encoded, zpp::bits::no_size{}}; (void)out(attest); - // `have_next_group` gates the QUEUEOUT, not `advance` -- see above. - if (have_next_group) { + // Withhold only the group attestation, never the remaining epoch work. + if (have_complete_window) { sysio::chains::chains_t chains_tbl(CHAINS_ACCOUNT); for (auto it = chains_tbl.begin(); it != chains_tbl.end(); ++it) { if (!is_active_outpost(*it)) continue; @@ -978,19 +1038,6 @@ void epoch::advance() { } } - // Drain the swap-from-WIRE queue: each row queued via - // `sysio.uwrit::swapfromwire` since the last advance is re-validated - // (target reserve ACTIVE + public, variance) and either becomes a - // PENDING uwreq for the single-leg underwriter race or is refunded. - // Runs before `buildenv` so this epoch's envelopes reflect any state - // the drain produced; never throws (refund-and-drop semantics). - action( - permission_level{get_self(), "owner"_n}, - UWRIT_ACCOUNT, - "drainfwq"_n, - std::make_tuple() - ).send(); - // Build outbound envelopes for each outpost { sysio::chains::chains_t chains_tbl(CHAINS_ACCOUNT); @@ -1005,7 +1052,7 @@ void epoch::advance() { } } - // Emissions side. Three inline actions queued in FIFO order: + // Emissions side. Accrual and history precede advance's payout sibling: // 1. accrueepoch: always queued. Records this epoch's per-epoch share // onto t5state (pending_emission_amount + batch_group_epochs[group] // + last_epoch_emission for decay continuity). @@ -1014,7 +1061,7 @@ void epoch::advance() { // 3. payepoch: queued only on pay-epochs. Reads the now-updated t5state // (which already includes this epoch's contribution from step 1), // distributes period_emission, and resets the accumulator. - // Both run after advance() returns; their FIFO ordering guarantees + // The continuation completes before that sibling; this ordering guarantees // payepoch sees the post-accrue roster history and state. std::vector active_batch_op_members; if (state.current_batch_op_group < state.batch_op_groups.size()) { @@ -1028,7 +1075,7 @@ void epoch::advance() { std::make_tuple( state.current_epoch_index, state.current_batch_op_group, - gate.emission_amount + emission_amount ) ).send(); @@ -1039,19 +1086,6 @@ void epoch::advance() { std::make_tuple(state.current_epoch_index, active_batch_op_members) ).send(); - if (gate.is_pay_epoch) { - action( - permission_level{get_self(), "owner"_n}, - SYSTEM_ACCOUNT, - "payepoch"_n, - std::make_tuple( - state.current_epoch_index, - std::vector>{}, - gate.period_emission - ) - ).send(); - } - // Working tables on `sysio.msgch` (`envelopes` / `messages` / // `attestations` / `outenvelopes`) are now drained inline by the // `evalcons` consensus-reach + `buildenv` write paths. The durable diff --git a/contracts/sysio.epoch/sysio.epoch.abi b/contracts/sysio.epoch/sysio.epoch.abi index 7d7ed6b341..2f292b432f 100644 --- a/contracts/sysio.epoch/sysio.epoch.abi +++ b/contracts/sysio.epoch/sysio.epoch.abi @@ -121,6 +121,20 @@ } ] }, + { + "name": "finishadv", + "base": "", + "fields": [ + { + "name": "epoch_index", + "type": "uint32" + }, + { + "name": "emission_amount", + "type": "int64" + } + ] + }, { "name": "pause", "base": "", @@ -169,6 +183,11 @@ "type": "advance", "ricardian_contract": "" }, + { + "name": "finishadv", + "type": "finishadv", + "ricardian_contract": "" + }, { "name": "pause", "type": "pause", diff --git a/contracts/sysio.epoch/sysio.epoch.wasm b/contracts/sysio.epoch/sysio.epoch.wasm index 23a5d0d812..3f5f0a766a 100755 Binary files a/contracts/sysio.epoch/sysio.epoch.wasm and b/contracts/sysio.epoch/sysio.epoch.wasm differ diff --git a/contracts/tests/sysio.msgch_chain_tests.cpp b/contracts/tests/sysio.msgch_chain_tests.cpp index 3cb66df44a..0d2c909d5b 100644 --- a/contracts/tests/sysio.msgch_chain_tests.cpp +++ b/contracts/tests/sysio.msgch_chain_tests.cpp @@ -25,6 +25,7 @@ /// oracle == Solidity/Rust (golden vectors) and contract == oracle (round-trip assertions) /// together give contract == outposts. #include +#include #include #include #include @@ -687,6 +688,59 @@ class sysio_msgch_chain_tester : public tester { return groups_attestation; } + /// Inspect the actual emitted envelope, after inline buildenv drained queueout. + /// The final OPERATORS snapshot must match the registry, and any published + /// schedule must follow that snapshot and contain only active batch operators. + void require_fresh_roster(uint64_t chain_code, name account, + opp::types::OperatorStatus expected_status) { + const auto row = find_outbound_envelope(chain_code); + BOOST_REQUIRE(!row.is_null()); + const auto env = decode_envelope(row["raw_envelope"].as>()); + BOOST_REQUIRE_EQUAL(env.messages_size(), 1); + bool found_operator = false; + bool have_operators = false; + for (const auto& att : env.messages(0).payload().attestations()) { + if (att.type() == opp::types::ATTESTATION_TYPE_OPERATORS) { + opp::attestations::Operators roster; + BOOST_REQUIRE(roster.ParseFromString(att.data())); + have_operators = true; + for (const auto& entry : roster.operators()) { + const auto registered = get_operator(name{entry.account().name()}); + BOOST_REQUIRE(!registered.is_null()); + BOOST_REQUIRE_EQUAL(entry.status(), + registered["status"].as()); + if (entry.account().name() == account.to_string()) { + BOOST_REQUIRE_EQUAL(entry.status(), expected_status); + found_operator = true; + } + } + } else if (att.type() == opp::types::ATTESTATION_TYPE_BATCH_OPERATOR_GROUPS) { + BOOST_REQUIRE(have_operators); + opp::attestations::BatchOperatorGroups groups; + BOOST_REQUIRE(groups.ParseFromString(att.data())); + std::set members; + for (const auto& group : groups.groups()) { + for (const auto& address : group.operators()) { + BOOST_REQUIRE(members.insert(address.address()).second); + const auto registered = get_operator(name{address.address()}); + BOOST_REQUIRE(!registered.is_null()); + BOOST_REQUIRE_EQUAL(opp::types::OPERATOR_STATUS_ACTIVE, + registered["status"].as()); + } + } + } + } + BOOST_REQUIRE(found_operator); + if (expected_status != opp::types::OPERATOR_STATUS_ACTIVE) { + const auto state = read_epoch_state(); + for (const auto& group : state["batch_op_groups"].get_array()) { + for (const auto& member : group.get_array()) { + BOOST_REQUIRE(member.as_string() != account.to_string()); + } + } + } + } + /// How many BATCH_OPERATOR_GROUPS attestations the most recent `advance` shipped to /// `chain_code` -- 0 when the depot WITHHELD it. Distinct from /// `shipped_batch_operator_groups`, which fails the test on absence: the withhold path @@ -856,8 +910,10 @@ class sysio_msgch_chain_tester : public tester { /// bootstrap() variant for a real rotation: THREE single-operator groups (so a resident op is on /// duty once per 3-epoch rotation), the SEC-28 percent rail disabled up to its accepted ceiling /// (99, so an anchored run terminates on the CONSECUTIVE rail), and `terminate_window_ms` set by - /// the caller (the exact span bound for this schedule). ETH outpost registered; genesis advance run. - void bootstrap_rotation(uint64_t terminate_window_ms) { + /// the caller (the exact span bound for this schedule). The target is non-bootstrapped + /// by default; healthy-rotation tests may opt into the bootstrap exemption. + /// ETH outpost registered; genesis advance run. + void bootstrap_rotation(uint64_t terminate_window_ms, bool batchop_is_bootstrapped = false) { BOOST_REQUIRE_EQUAL(success(), push(EPOCH_ACCOUNT, epoch_abi, EPOCH_ACCOUNT, "setconfig"_n, mvo() ("epoch_duration_sec", EPOCH_DURATION_SEC) ("operators_per_epoch", 1) @@ -883,13 +939,13 @@ class sysio_msgch_chain_tester : public tester { register_chain(opp::types::ChainKind::CHAIN_KIND_EVM, "ETH", 31337); - // BATCHOP is the termination target: NON-bootstrapped (bootstrapped operators are exempt from + // By default BATCHOP is the termination target: NON-bootstrapped (bootstrapped operators are exempt from // rolling-window termination -- see opreg::termcheck) and collateralized so it activates. // BATCHOP_B / BATCHOP_C are bootstrapped fillers for the other two groups. schbatchgps sorts // non-bootstrapped first, so BATCHOP lands in group 0 (on duty at epochs 1, 4, 7, ...). BOOST_REQUIRE_EQUAL(success(), push(OPREG_ACCOUNT, opreg_abi, OPREG_ACCOUNT, "regoperator"_n, mvo() ("account", BATCHOP.to_string())("type", opp::types::OperatorType::OPERATOR_TYPE_BATCH) - ("is_bootstrapped", false))); + ("is_bootstrapped", batchop_is_bootstrapped))); BOOST_REQUIRE_EQUAL(success(), depositinle(BATCHOP, "ETH", "ETH", 1)); for (const auto& op : {BATCHOP_B, BATCHOP_C}) { BOOST_REQUIRE_EQUAL(success(), push(OPREG_ACCOUNT, opreg_abi, OPREG_ACCOUNT, "regoperator"_n, mvo() @@ -899,7 +955,8 @@ class sysio_msgch_chain_tester : public tester { BOOST_REQUIRE(!get_operator(BATCHOP).is_null()); BOOST_REQUIRE(opp::types::OperatorStatus::OPERATOR_STATUS_ACTIVE == get_operator(BATCHOP)["status"].as()); - BOOST_REQUIRE_EQUAL(0, get_operator(BATCHOP)["is_bootstrapped"].as_uint64()); + BOOST_REQUIRE_EQUAL(static_cast(batchop_is_bootstrapped), + get_operator(BATCHOP)["is_bootstrapped"].as_uint64()); BOOST_REQUIRE_EQUAL(success(), push(EPOCH_ACCOUNT, epoch_abi, EPOCH_ACCOUNT, "schbatchgps"_n, mvo())); BOOST_REQUIRE_EQUAL(success(), push(EPOCH_ACCOUNT, epoch_abi, EPOCH_ACCOUNT, "advance"_n, mvo())); @@ -1646,6 +1703,11 @@ BOOST_FIXTURE_TEST_CASE(noncanonical_delivery_slashes_before_termination, sysio_ slash_action_count(BATCHOP, SOL_OUTPOST_ID)); BOOST_REQUIRE_EQUAL(epoch + kEpochAdvanceCount, current_epoch()); BOOST_REQUIRE_EQUAL(kExpectedDeliveredLogCount, delivered_dellog_count(BATCHOP)); + for (const auto chain : {ETH_OUTPOST_ID, SOL_OUTPOST_ID}) { + require_fresh_roster(chain, BATCHOP, opp::types::OPERATOR_STATUS_SLASHED); + // The remaining two members must not be advertised with a reduced quorum. + BOOST_REQUIRE_EQUAL(0, shipped_batch_operator_groups_count(chain)); + } } FC_LOG_AND_RETHROW() } // SEC-28 (huang review): terminate on the CONSECUTIVE-miss rail through the REAL rotation -- a @@ -1703,6 +1765,8 @@ BOOST_FIXTURE_TEST_CASE(terminate_at_duty_rotation_via_advance, sysio_msgch_chai // whereas termination + reason hold either way. BATCHOP delivered exactly once, so exactly // one delivered row must remain. BOOST_REQUIRE_EQUAL(1u, delivered_dellog_count(BATCHOP)); + require_fresh_roster(ETH_OUTPOST_ID, BATCHOP, + opp::types::OPERATOR_STATUS_TERMINATED); } else { // Still ACTIVE: BATCHOP must not terminate before its sixth miss (its 7th duty). BOOST_REQUIRE(status == opp::types::OperatorStatus::OPERATOR_STATUS_ACTIVE); @@ -1998,10 +2062,10 @@ BOOST_FIXTURE_TEST_CASE(slash_after_delivery_does_not_count_toward_consensus, sy /// rotation so every group is promised (and verified) at least once. BOOST_FIXTURE_TEST_CASE(advance_ships_lookahead_batch_operator_group, sysio_msgch_chain_tester) { try { constexpr uint32_t kGroups = 3; - // Termination rails are irrelevant here — a comfortably wide window keeps recorddel/termcheck - // quiet while the rotation is walked (same span shape the SEC-28 fixture derives). + // Use bootstrapped operators so missed deliveries cannot terminate a + // member during this test of healthy rotation and lookahead. constexpr uint64_t kRotationWindowMs = 12ULL * kGroups * EPOCH_DURATION_SEC * 1000ULL; - bootstrap_rotation(kRotationWindowMs); + bootstrap_rotation(kRotationWindowMs, /*batchop_is_bootstrapped=*/true); for (uint32_t round = 0; round < kGroups + 1; ++round) { const auto shipped = shipped_batch_operator_groups(ETH_OUTPOST_ID); @@ -2077,9 +2141,9 @@ BOOST_FIXTURE_TEST_CASE(advance_withholds_batch_operator_groups_when_next_group_ } produce_blocks(); - // The terminated pair still occupy their seats until they slide out, so the sole survivor is - // resident and the residency-excluded pool is empty: every tail from here is empty. Walk the - // window so an empty group reaches the lookahead seat, then hold there. + // Removed operators are pruned from surviving seats immediately. The pool + // cannot fill the window, so group attestations remain withheld while epoch + // accounting and authoritative operator-status publication continue. bool observed_withhold = false; for (uint32_t round = 0; round < kGroups + 1; ++round) { advance_to_next_epoch(); @@ -2096,4 +2160,85 @@ BOOST_FIXTURE_TEST_CASE(advance_withholds_batch_operator_groups_when_next_group_ "starved window never withheld BATCH_OPERATOR_GROUPS -- an empty active group was published"); } FC_LOG_AND_RETHROW() } +// WIRE-385: a removal during this advance must be visible in BOTH emitted +// attestations, with a healthy standby filling the newly selected tail. +BOOST_FIXTURE_TEST_CASE(advance_roster_excludes_same_epoch_termination, + sysio_msgch_chain_tester) { try { + bootstrap(/*n_batch_ops=*/3, /*batchop_is_bootstrapped=*/false); + BOOST_REQUIRE_EQUAL(success(), push(OPREG_ACCOUNT, opreg_abi, OPREG_ACCOUNT, + "regoperator"_n, mvo()("account", BATCHOP_D.to_string()) + ("type", opp::types::OPERATOR_TYPE_BATCH)("is_bootstrapped", true))); + produce_blocks(); + advance_to_next_epoch(); // missed delivery terminates only the non-bootstrap operator + for (const auto chain : {ETH_OUTPOST_ID, SOL_OUTPOST_ID}) { + require_fresh_roster(chain, BATCHOP, opp::types::OPERATOR_STATUS_TERMINATED); + require_fresh_roster(chain, BATCHOP_B, opp::types::OPERATOR_STATUS_ACTIVE); + const auto groups = shipped_batch_operator_groups(chain); + BOOST_REQUIRE_EQUAL(groups.groups_size(), 1); + BOOST_REQUIRE_EQUAL(groups.groups(0).operators_size(), 3); + BOOST_REQUIRE_EQUAL(groups.groups(0).operators(0).address(), BATCHOP_B.to_string()); + BOOST_REQUIRE_EQUAL(groups.groups(0).operators(1).address(), BATCHOP_C.to_string()); + BOOST_REQUIRE_EQUAL(groups.groups(0).operators(2).address(), BATCHOP_D.to_string()); + } +} FC_LOG_AND_RETHROW() } + +BOOST_FIXTURE_TEST_CASE(advance_removes_inactive_surviving_group_members, + sysio_msgch_chain_tester) { try { + constexpr uint32_t GROUP_COUNT = 3; + constexpr uint64_t WINDOW_MS = 12ULL * GROUP_COUNT * EPOCH_DURATION_SEC * 1000ULL; + bootstrap_rotation(WINDOW_MS); + for (const auto op : {BATCHOP_B, BATCHOP_C}) { + BOOST_REQUIRE_EQUAL(success(), push(OPREG_ACCOUNT, opreg_abi, CHALG_ACCOUNT, + "slash"_n, mvo()("account", op.to_string())("reason", "roster regression"))); + } + produce_blocks(); + advance_to_next_epoch(); + for (const auto op : {BATCHOP_B, BATCHOP_C}) { + require_fresh_roster(ETH_OUTPOST_ID, op, opp::types::OPERATOR_STATUS_SLASHED); + } + BOOST_REQUIRE_EQUAL(0, shipped_batch_operator_groups_count(ETH_OUTPOST_ID)); +} FC_LOG_AND_RETHROW() } + +BOOST_FIXTURE_TEST_CASE(advance_repairs_future_group_before_it_becomes_current, + sysio_msgch_chain_tester) { try { + constexpr uint64_t WINDOW_MS = 12ULL * 3 * EPOCH_DURATION_SEC * 1000ULL; + bootstrap_rotation(WINDOW_MS, /*batchop_is_bootstrapped=*/true); + // schbatchgps interleaves the sorted roster: the initial window is [A,C,B]. + // Remove the last group so its vacancy is still in the future after sliding. + const auto initial = shipped_batch_operator_groups(ETH_OUTPOST_ID); + BOOST_REQUIRE_EQUAL(initial.groups_size(), 3); + BOOST_REQUIRE_EQUAL(initial.groups(2).operators(0).address(), BATCHOP_B.to_string()); + BOOST_REQUIRE_EQUAL(success(), push(OPREG_ACCOUNT, opreg_abi, OPREG_ACCOUNT, + "regoperator"_n, mvo()("account", BATCHOP_D.to_string()) + ("type", opp::types::OPERATOR_TYPE_BATCH)("is_bootstrapped", true))); + BOOST_REQUIRE_EQUAL(success(), push(OPREG_ACCOUNT, opreg_abi, OPREG_ACCOUNT, + "terminate"_n, mvo()("account", BATCHOP_B.to_string())("reason", "future seat removed"))); + produce_blocks(); + advance_to_next_epoch(); + require_fresh_roster(ETH_OUTPOST_ID, BATCHOP_B, opp::types::OPERATOR_STATUS_TERMINATED); + const auto repaired = shipped_batch_operator_groups(ETH_OUTPOST_ID); + BOOST_REQUIRE_EQUAL(repaired.groups_size(), 3); + for (int i = 0; i < repaired.groups_size(); ++i) { + BOOST_REQUIRE_EQUAL(repaired.groups(i).operators_size(), 1); + } + BOOST_REQUIRE_EQUAL(repaired.groups(0).operators(0).address(), BATCHOP_C.to_string()); + BOOST_REQUIRE_EQUAL(repaired.groups(1).operators(0).address(), BATCHOP_D.to_string()); + BOOST_REQUIRE_EQUAL(repaired.groups(2).operators(0).address(), BATCHOP.to_string()); + advance_to_next_epoch(); + const auto next = shipped_batch_operator_groups(ETH_OUTPOST_ID); + BOOST_REQUIRE_EQUAL(next.groups_size(), 3); + BOOST_REQUIRE_EQUAL(next.groups(0).operators_size(), 1); + BOOST_REQUIRE_EQUAL(next.groups(0).operators(0).address(), BATCHOP_D.to_string()); + require_fresh_roster(ETH_OUTPOST_ID, BATCHOP_B, opp::types::OPERATOR_STATUS_TERMINATED); +} FC_LOG_AND_RETHROW() } + +BOOST_FIXTURE_TEST_CASE(finishadv_rejects_direct_calls, sysio_msgch_chain_tester) { try { + bootstrap(); + const auto args = mvo()("epoch_index", current_epoch())("emission_amount", int64_t{0}); + BOOST_REQUIRE_EQUAL(error("missing authority of sysio.epoch"), + push(EPOCH_ACCOUNT, epoch_abi, BATCHOP, "finishadv"_n, args)); + BOOST_REQUIRE_EQUAL(error("assertion failure with message: finishadv must be sent inline by sysio.epoch"), + push(EPOCH_ACCOUNT, epoch_abi, EPOCH_ACCOUNT, "finishadv"_n, args)); +} FC_LOG_AND_RETHROW() } + BOOST_AUTO_TEST_SUITE_END()