From fcbebbccf24989c043f2cd83a206ba814c1fa4cc Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 29 Aug 2026 00:32:17 +0200 Subject: [PATCH 01/34] feat(dpp): add ReducedPlatformState stored in replicated state for state sync Adds a minimal, platform-versioned subset of the Platform state that will be written into the replicated GroveDB state (Misc tree) so state-synced nodes can reconstruct the full Platform state, which is otherwise only persisted to non-replicated aux storage. Unlike the earlier prototype, fee versions of previous epochs are persisted faithfully by version number, and unknown-at-store-time block fields (app hash, block id hash, signature) are Options instead of zero-filled placeholders. Co-Authored-By: Claude Fable 5 --- packages/rs-dpp/src/lib.rs | 2 + .../rs-dpp/src/reduced_platform_state/mod.rs | 94 +++++++++++++++++++ .../src/reduced_platform_state/v0/mod.rs | 61 ++++++++++++ 3 files changed, 157 insertions(+) create mode 100644 packages/rs-dpp/src/reduced_platform_state/mod.rs create mode 100644 packages/rs-dpp/src/reduced_platform_state/v0/mod.rs diff --git a/packages/rs-dpp/src/lib.rs b/packages/rs-dpp/src/lib.rs index 7a7c90a8080..a8ebf21b25b 100644 --- a/packages/rs-dpp/src/lib.rs +++ b/packages/rs-dpp/src/lib.rs @@ -61,6 +61,8 @@ pub mod core_subsidy; pub mod fee; pub mod nft; pub mod prefunded_specialized_balance; +/// Reduced platform state stored in replicated state for state sync reconstruction +pub mod reduced_platform_state; pub mod serialization; #[cfg(any( feature = "message-signing", diff --git a/packages/rs-dpp/src/reduced_platform_state/mod.rs b/packages/rs-dpp/src/reduced_platform_state/mod.rs new file mode 100644 index 00000000000..05ab4151e77 --- /dev/null +++ b/packages/rs-dpp/src/reduced_platform_state/mod.rs @@ -0,0 +1,94 @@ +//! Reduced platform state +//! +//! A minimal subset of the Platform state that is stored inside the replicated GroveDB +//! state (under the Misc tree), allowing a node that syncs via ABCI state sync to +//! reconstruct the full Platform state. The full Platform state itself is only persisted +//! to GroveDB aux storage, which is not replicated by GroveDB state sync. + +use crate::serialization::{PlatformDeserializableFromVersionedStructure, PlatformSerializable}; +use crate::ProtocolError; +use bincode::{Decode, Encode}; +use platform_version::version::PlatformVersion; + +pub mod v0; + +use v0::ReducedPlatformStateV0; + +/// Reduced Platform State (platform-versioned wrapper) +#[derive(Clone, Debug, PartialEq, Encode, Decode, derive_more::From)] +pub enum ReducedPlatformState { + /// Version 0 + V0(ReducedPlatformStateV0), +} + +impl PlatformSerializable for ReducedPlatformState { + type Error = ProtocolError; + + fn serialize_to_bytes(&self) -> Result, Self::Error> { + let config = bincode::config::standard(); + bincode::encode_to_vec(self, config).map_err(|e| { + ProtocolError::PlatformSerializationError(format!( + "cannot serialize ReducedPlatformState: {}", + e + )) + }) + } +} + +impl PlatformDeserializableFromVersionedStructure for ReducedPlatformState { + fn versioned_deserialize( + data: &[u8], + _platform_version: &PlatformVersion, + ) -> Result + where + Self: Sized, + { + // The version of the structure is encoded in the enum discriminant, so the + // platform version is not needed to pick the variant. + let config = bincode::config::standard(); + bincode::decode_from_slice(data, config) + .map_err(|e| { + ProtocolError::PlatformDeserializationError(format!( + "cannot deserialize ReducedPlatformState: {}", + e + )) + }) + .map(|(object, _)| object) + } +} + +#[cfg(test)] +mod tests { + use super::v0::{ReducedBlockInfoV0, ReducedPlatformStateV0}; + use super::*; + use crate::block::block_info::BlockInfo; + + #[test] + fn should_roundtrip_reduced_platform_state_serialization() { + let state = ReducedPlatformState::V0(ReducedPlatformStateV0 { + last_committed_block_info: Some(ReducedBlockInfoV0 { + basic_info: BlockInfo::default_with_time(1_700_000_000_000), + app_hash: None, + quorum_hash: [1u8; 32].into(), + block_id_hash: None, + proposer_pro_tx_hash: [2u8; 32].into(), + signature: None, + round: 3, + }), + current_protocol_version_in_consensus: 15, + next_epoch_protocol_version: 15, + current_validator_set_quorum_hash: [4u8; 32].into(), + next_validator_set_quorum_hash: Some([5u8; 32].into()), + previous_fee_versions: [(0u16, 1u32)].into_iter().collect(), + quorum_positions: vec![[4u8; 32].into(), [5u8; 32].into()], + proposed_core_chain_locked_height: 1000, + }); + + let bytes = state.serialize_to_bytes().expect("should serialize"); + let restored = + ReducedPlatformState::versioned_deserialize(&bytes, PlatformVersion::latest()) + .expect("should deserialize"); + + assert_eq!(state, restored); + } +} diff --git a/packages/rs-dpp/src/reduced_platform_state/v0/mod.rs b/packages/rs-dpp/src/reduced_platform_state/v0/mod.rs new file mode 100644 index 00000000000..edda9bd2cdb --- /dev/null +++ b/packages/rs-dpp/src/reduced_platform_state/v0/mod.rs @@ -0,0 +1,61 @@ +use crate::block::block_info::BlockInfo; +use crate::fee::default_costs::EpochIndexFeeVersionsForStorage; +use crate::util::deserializer::ProtocolVersion; +use bincode::{Decode, Encode}; +use platform_value::Bytes32; + +/// Block information persisted as part of the reduced platform state. +/// +/// The reduced state is written while the block is still being executed, before it is +/// signed and before the resulting app hash is known, so `app_hash`, `block_id_hash` and +/// `signature` are `Option`s rather than zero-filled placeholders. They are `None` when +/// stored and are filled in (where possible) during state reconstruction. +#[derive(Clone, Debug, PartialEq, Encode, Decode)] +pub struct ReducedBlockInfoV0 { + /// Basic block info (height, core height, time, epoch) + pub basic_info: BlockInfo, + /// The app hash resulting from this block; unknown at store time + pub app_hash: Option, + /// The quorum that signed (or will sign) this block + pub quorum_hash: Bytes32, + /// The block id hash; unknown at store time + pub block_id_hash: Option, + /// The block proposer's pro tx hash + pub proposer_pro_tx_hash: Bytes32, + /// The block signature; unknown at store time + pub signature: Option<[u8; 96]>, + /// The consensus round that produced this block + pub round: u32, +} + +/// Reduced Platform State V0. +/// +/// This minimal version of the Platform state is written into GroveDB (under the Misc +/// tree, hence below the root hash) on every block proposal. Because it is part of the +/// replicated state, a freshly state-synced node can read it back and reconstruct the +/// full in-memory Platform state, which is otherwise only persisted to non-replicated +/// GroveDB aux storage. +#[derive(Clone, Debug, PartialEq, Encode, Decode)] +pub struct ReducedPlatformStateV0 { + /// Info about the block that was being processed when this state was written + /// (it becomes the last committed block once the block finalizes) + pub last_committed_block_info: Option, + /// Current protocol version in consensus + pub current_protocol_version_in_consensus: ProtocolVersion, + /// Upcoming protocol version + pub next_epoch_protocol_version: ProtocolVersion, + /// Current validator set quorum hash + pub current_validator_set_quorum_hash: Bytes32, + /// Next validator set quorum hash + pub next_validator_set_quorum_hash: Option, + /// Fee versions of previous epochs, stored by fee version number so they can be + /// restored faithfully on reconstruction + pub previous_fee_versions: EpochIndexFeeVersionsForStorage, + /// Ordered list of quorum hashes reflecting validator set quorum positions + // TODO: optimize this to not store the whole quorum hash, but only some index + pub quorum_positions: Vec, + /// Core chain locked height, as provided in RequestProcessProposal ABCI message; + /// note this can differ from the one in RequestPrepareProposal, as it can be + /// modified by the proposer. + pub proposed_core_chain_locked_height: u32, +} From 671cf99f32d6cdd8c3aadbbefbfca683a0c8caee Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 29 Aug 2026 00:40:11 +0200 Subject: [PATCH 02/34] feat(drive): store and fetch reduced platform state bytes in the Misc tree Persists the reduced platform state under Misc/reduced_saved_state inside the replicated grovedb state (unlike the full platform state, which lives in non-replicated aux storage). fetch returns Ok(None) when the key is absent, so callers can distinguish pre-activation snapshots. Adds the DriveError::Snapshot variant and the platform_state method version fields for the new methods. Co-Authored-By: Claude Fable 5 --- .../fetch_reduced_platform_state_bytes/mod.rs | 33 ++++++++++ .../v0/mod.rs | 24 ++++++++ .../rs-drive/src/drive/platform_state/mod.rs | 60 +++++++++++++++++++ .../store_reduced_platform_state_bytes/mod.rs | 35 +++++++++++ .../v0/mod.rs | 28 +++++++++ packages/rs-drive/src/error/drive.rs | 4 ++ .../src/version/drive_versions/mod.rs | 2 + .../src/version/drive_versions/v1.rs | 2 + .../src/version/drive_versions/v2.rs | 2 + .../src/version/drive_versions/v3.rs | 2 + .../src/version/drive_versions/v4.rs | 2 + .../src/version/drive_versions/v5.rs | 2 + .../src/version/drive_versions/v6.rs | 2 + .../src/version/drive_versions/v7.rs | 2 + .../src/version/drive_versions/v8.rs | 2 + .../src/version/drive_versions/v9.rs | 2 + .../src/version/mocks/v2_test.rs | 2 + 17 files changed, 206 insertions(+) create mode 100644 packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/mod.rs create mode 100644 packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/v0/mod.rs create mode 100644 packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/mod.rs create mode 100644 packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/v0/mod.rs diff --git a/packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/mod.rs b/packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/mod.rs new file mode 100644 index 00000000000..2045c905235 --- /dev/null +++ b/packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/mod.rs @@ -0,0 +1,33 @@ +mod v0; + +use crate::drive::Drive; +use crate::error::drive::DriveError; +use crate::error::Error; +use dpp::version::PlatformVersion; +use grovedb::TransactionArg; + +impl Drive { + /// Fetch the reduced platform state from the replicated grovedb state (Misc tree). + /// + /// Returns `Ok(None)` when the key is absent (for example before the protocol + /// version that introduced the reduced state activated). + pub fn fetch_reduced_platform_state_bytes( + &self, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result>, Error> { + match platform_version + .drive + .methods + .platform_state + .fetch_reduced_platform_state_bytes + { + 0 => self.fetch_reduced_platform_state_bytes_v0(transaction, platform_version), + version => Err(Error::Drive(DriveError::UnknownVersionMismatch { + method: "fetch_reduced_platform_state_bytes".to_string(), + known_versions: vec![0], + received: version, + })), + } + } +} diff --git a/packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/v0/mod.rs b/packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/v0/mod.rs new file mode 100644 index 00000000000..7df67f7c084 --- /dev/null +++ b/packages/rs-drive/src/drive/platform_state/fetch_reduced_platform_state_bytes/v0/mod.rs @@ -0,0 +1,24 @@ +use crate::drive::platform_state::REDUCED_PLATFORM_STATE_KEY; +use crate::drive::system::misc_path; +use crate::drive::Drive; +use crate::error::Error; +use crate::util::grove_operations::DirectQueryType; +use grovedb::TransactionArg; +use platform_version::version::PlatformVersion; + +impl Drive { + pub(super) fn fetch_reduced_platform_state_bytes_v0( + &self, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result>, Error> { + self.grove_get_raw_optional_item( + (&misc_path()).into(), + REDUCED_PLATFORM_STATE_KEY, + DirectQueryType::StatefulDirectQuery, + transaction, + &mut vec![], + &platform_version.drive, + ) + } +} diff --git a/packages/rs-drive/src/drive/platform_state/mod.rs b/packages/rs-drive/src/drive/platform_state/mod.rs index d6a0ce16c49..5a0e2fe5d9b 100644 --- a/packages/rs-drive/src/drive/platform_state/mod.rs +++ b/packages/rs-drive/src/drive/platform_state/mod.rs @@ -1,4 +1,64 @@ mod fetch_platform_state_bytes; +mod fetch_reduced_platform_state_bytes; mod store_platform_state_bytes; +mod store_reduced_platform_state_bytes; const PLATFORM_STATE_KEY: &[u8; 11] = b"saved_state"; +const REDUCED_PLATFORM_STATE_KEY: &[u8; 19] = b"reduced_saved_state"; + +#[cfg(test)] +mod tests { + use crate::util::test_helpers::setup::setup_drive_with_initial_state_structure; + use platform_version::version::PlatformVersion; + + #[test] + fn should_return_none_when_reduced_platform_state_is_absent() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + + let fetched = drive + .fetch_reduced_platform_state_bytes(None, platform_version) + .expect("fetching an absent reduced platform state should not error"); + + assert_eq!(fetched, None); + } + + #[test] + fn should_roundtrip_reduced_platform_state_bytes() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + + let state_bytes = vec![1u8, 2, 3, 4, 5]; + + drive + .store_reduced_platform_state_bytes(&state_bytes, None, platform_version) + .expect("should store reduced platform state"); + + let fetched = drive + .fetch_reduced_platform_state_bytes(None, platform_version) + .expect("should fetch reduced platform state"); + + assert_eq!(fetched, Some(state_bytes)); + } + + #[test] + fn should_overwrite_reduced_platform_state_bytes() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + + drive + .store_reduced_platform_state_bytes(&[1u8, 2, 3], None, platform_version) + .expect("should store reduced platform state"); + + let updated_bytes = vec![9u8, 8, 7]; + drive + .store_reduced_platform_state_bytes(&updated_bytes, None, platform_version) + .expect("should overwrite reduced platform state"); + + let fetched = drive + .fetch_reduced_platform_state_bytes(None, platform_version) + .expect("should fetch reduced platform state"); + + assert_eq!(fetched, Some(updated_bytes)); + } +} diff --git a/packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/mod.rs b/packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/mod.rs new file mode 100644 index 00000000000..0346e197218 --- /dev/null +++ b/packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/mod.rs @@ -0,0 +1,35 @@ +mod v0; + +use crate::drive::Drive; +use crate::error::drive::DriveError; +use crate::error::Error; +use dpp::version::PlatformVersion; +use grovedb::TransactionArg; + +impl Drive { + /// Store the reduced platform state in the replicated grovedb state (Misc tree) + pub fn store_reduced_platform_state_bytes( + &self, + state_bytes: &[u8], + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + match platform_version + .drive + .methods + .platform_state + .store_reduced_platform_state_bytes + { + 0 => self.store_reduced_platform_state_bytes_v0( + state_bytes, + transaction, + platform_version, + ), + version => Err(Error::Drive(DriveError::UnknownVersionMismatch { + method: "store_reduced_platform_state_bytes".to_string(), + known_versions: vec![0], + received: version, + })), + } + } +} diff --git a/packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/v0/mod.rs b/packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/v0/mod.rs new file mode 100644 index 00000000000..55d61b2e08a --- /dev/null +++ b/packages/rs-drive/src/drive/platform_state/store_reduced_platform_state_bytes/v0/mod.rs @@ -0,0 +1,28 @@ +use crate::drive::platform_state::REDUCED_PLATFORM_STATE_KEY; +use crate::drive::system::misc_path; +use crate::drive::Drive; +use crate::error::Error; +use grovedb::{Element, TransactionArg}; +use platform_version::version::PlatformVersion; + +impl Drive { + pub(super) fn store_reduced_platform_state_bytes_v0( + &self, + reduced_state_bytes: &[u8], + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + self.grove + .insert( + &misc_path(), + REDUCED_PLATFORM_STATE_KEY, + Element::Item(reduced_state_bytes.to_vec(), None), + None, + transaction, + &platform_version.drive.grove_version, + ) + .unwrap() + .map_err(Error::from)?; + Ok(()) + } +} diff --git a/packages/rs-drive/src/error/drive.rs b/packages/rs-drive/src/error/drive.rs index 3412c9df1aa..a64d5c94adb 100644 --- a/packages/rs-drive/src/error/drive.rs +++ b/packages/rs-drive/src/error/drive.rs @@ -218,4 +218,8 @@ pub enum DriveError { /// Checkpoint not found for specified block height #[error("checkpoint not found for block height: {0}")] CheckpointNotFound(u64), + + /// Snapshot error + #[error("snapshot error: {0}")] + Snapshot(String), } diff --git a/packages/rs-platform-version/src/version/drive_versions/mod.rs b/packages/rs-platform-version/src/version/drive_versions/mod.rs index ca7c22c6e3f..c768c5a36a1 100644 --- a/packages/rs-platform-version/src/version/drive_versions/mod.rs +++ b/packages/rs-platform-version/src/version/drive_versions/mod.rs @@ -78,6 +78,8 @@ pub struct DriveMethodVersions { pub struct DrivePlatformStateMethodVersions { pub fetch_platform_state_bytes: FeatureVersion, pub store_platform_state_bytes: FeatureVersion, + pub fetch_reduced_platform_state_bytes: FeatureVersion, + pub store_reduced_platform_state_bytes: FeatureVersion, } #[derive(Clone, Debug, Default)] diff --git a/packages/rs-platform-version/src/version/drive_versions/v1.rs b/packages/rs-platform-version/src/version/drive_versions/v1.rs index 6e87d8fe420..f7dc6e68748 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v1.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v1.rs @@ -90,6 +90,8 @@ pub const DRIVE_VERSION_V1: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v2.rs b/packages/rs-platform-version/src/version/drive_versions/v2.rs index 0fe4f8f235e..02a4ab14d14 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v2.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v2.rs @@ -90,6 +90,8 @@ pub const DRIVE_VERSION_V2: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v3.rs b/packages/rs-platform-version/src/version/drive_versions/v3.rs index a542fe99e85..13d5a29cc94 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v3.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v3.rs @@ -90,6 +90,8 @@ pub const DRIVE_VERSION_V3: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v4.rs b/packages/rs-platform-version/src/version/drive_versions/v4.rs index 4481d8b90ac..d2c09c3a2fd 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v4.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v4.rs @@ -90,6 +90,8 @@ pub const DRIVE_VERSION_V4: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v5.rs b/packages/rs-platform-version/src/version/drive_versions/v5.rs index bfbce3d74b1..6cd9ffdefe8 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v5.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v5.rs @@ -92,6 +92,8 @@ pub const DRIVE_VERSION_V5: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v6.rs b/packages/rs-platform-version/src/version/drive_versions/v6.rs index 304cbdb70c7..7b265daeafa 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v6.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v6.rs @@ -94,6 +94,8 @@ pub const DRIVE_VERSION_V6: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v7.rs b/packages/rs-platform-version/src/version/drive_versions/v7.rs index 05d8ad2d05e..3761c8fbd6d 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v7.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v7.rs @@ -92,6 +92,8 @@ pub const DRIVE_VERSION_V7: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v8.rs b/packages/rs-platform-version/src/version/drive_versions/v8.rs index 7f421173191..f48deed6d37 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v8.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v8.rs @@ -92,6 +92,8 @@ pub const DRIVE_VERSION_V8: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/v9.rs b/packages/rs-platform-version/src/version/drive_versions/v9.rs index fade08c521d..a9cd31b2da7 100644 --- a/packages/rs-platform-version/src/version/drive_versions/v9.rs +++ b/packages/rs-platform-version/src/version/drive_versions/v9.rs @@ -106,6 +106,8 @@ pub const DRIVE_VERSION_V9: DriveVersion = DriveVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { diff --git a/packages/rs-platform-version/src/version/mocks/v2_test.rs b/packages/rs-platform-version/src/version/mocks/v2_test.rs index 76579f3a0d7..5a8774f08b2 100644 --- a/packages/rs-platform-version/src/version/mocks/v2_test.rs +++ b/packages/rs-platform-version/src/version/mocks/v2_test.rs @@ -128,6 +128,8 @@ pub const TEST_PLATFORM_V2: PlatformVersion = PlatformVersion { platform_state: DrivePlatformStateMethodVersions { fetch_platform_state_bytes: 0, store_platform_state_bytes: 0, + fetch_reduced_platform_state_bytes: 0, + store_reduced_platform_state_bytes: 0, }, fetch: DriveFetchMethodVersions { fetch_elements: 0 }, prefunded_specialized_balances: DrivePrefundedSpecializedMethodVersions { From 1e24b84687d31ab9ee47520a2ccd05d96dd9e147 Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 29 Aug 2026 00:52:15 +0200 Subject: [PATCH 03/34] feat(platform-version): add protocol v15 with state sync version plumbing Adds PLATFORM_V15 (drive-abci method versions v11: run_block_proposal 1, consensus_params_update 2), the DriveAbciStateSyncVersions substructure carrying the grovedb state sync wire protocol version on every platform version, and the reduced-platform-state storage method version slots on DriveAbciPlatformStateStorageMethodVersions. Pure plumbing: no behavior changes outside version selection. Co-Authored-By: Claude Fable 5 --- .../drive_abci_method_versions/mod.rs | 3 + .../drive_abci_method_versions/v1.rs | 2 + .../drive_abci_method_versions/v10.rs | 2 + .../drive_abci_method_versions/v11.rs | 146 ++++++++++++++++++ .../drive_abci_method_versions/v2.rs | 2 + .../drive_abci_method_versions/v3.rs | 2 + .../drive_abci_method_versions/v4.rs | 2 + .../drive_abci_method_versions/v5.rs | 2 + .../drive_abci_method_versions/v6.rs | 2 + .../drive_abci_method_versions/v7.rs | 2 + .../drive_abci_method_versions/v8.rs | 2 + .../drive_abci_method_versions/v9.rs | 2 + .../drive_abci_state_sync_versions/mod.rs | 14 ++ .../drive_abci_state_sync_versions/v1.rs | 6 + .../src/version/drive_abci_versions/mod.rs | 3 + .../src/version/mocks/v2_test.rs | 2 + .../src/version/mocks/v3_test.rs | 4 + .../rs-platform-version/src/version/mod.rs | 5 +- .../src/version/protocol_version.rs | 4 +- .../rs-platform-version/src/version/v1.rs | 2 + .../rs-platform-version/src/version/v10.rs | 2 + .../rs-platform-version/src/version/v11.rs | 2 + .../rs-platform-version/src/version/v12.rs | 2 + .../rs-platform-version/src/version/v13.rs | 2 + .../rs-platform-version/src/version/v14.rs | 2 + .../rs-platform-version/src/version/v15.rs | 129 ++++++++++++++++ .../rs-platform-version/src/version/v2.rs | 2 + .../rs-platform-version/src/version/v3.rs | 2 + .../rs-platform-version/src/version/v4.rs | 2 + .../rs-platform-version/src/version/v5.rs | 2 + .../rs-platform-version/src/version/v6.rs | 2 + .../rs-platform-version/src/version/v7.rs | 2 + .../rs-platform-version/src/version/v8.rs | 2 + .../rs-platform-version/src/version/v9.rs | 2 + 34 files changed, 361 insertions(+), 3 deletions(-) create mode 100644 packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v11.rs create mode 100644 packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/mod.rs create mode 100644 packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/v1.rs create mode 100644 packages/rs-platform-version/src/version/v15.rs diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/mod.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/mod.rs index a1bf5fdd754..cfd94cf9969 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/mod.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/mod.rs @@ -2,6 +2,7 @@ use versioned_feature_core::{FeatureVersion, OptionalFeatureVersion}; pub mod v1; pub mod v10; +pub mod v11; pub mod v2; pub mod v3; pub mod v4; @@ -36,6 +37,8 @@ pub struct DriveAbciMethodVersions { pub struct DriveAbciPlatformStateStorageMethodVersions { pub fetch_platform_state: FeatureVersion, pub store_platform_state: FeatureVersion, + pub fetch_reduced_platform_state: FeatureVersion, + pub store_reduced_platform_state: FeatureVersion, } #[derive(Clone, Copy, Debug, Default)] diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v1.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v1.rs index 9798d693037..b03335c73d7 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v1.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v1.rs @@ -131,5 +131,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V1: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v10.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v10.rs index 38be834a186..a88ace56cab 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v10.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v10.rs @@ -140,5 +140,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V10: DriveAbciMethodVersions = DriveAbciMet platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v11.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v11.rs new file mode 100644 index 00000000000..8ea89d7169d --- /dev/null +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v11.rs @@ -0,0 +1,146 @@ +use crate::version::drive_abci_versions::drive_abci_method_versions::{ + DriveAbciBlockEndMethodVersions, DriveAbciBlockFeeProcessingMethodVersions, + DriveAbciBlockStartMethodVersions, DriveAbciCoreBasedUpdatesMethodVersions, + DriveAbciCoreChainLockMethodVersionsAndConstants, DriveAbciCoreInstantSendLockMethodVersions, + DriveAbciEngineMethodVersions, DriveAbciEpochMethodVersions, + DriveAbciFeePoolInwardsDistributionMethodVersions, + DriveAbciFeePoolOutwardsDistributionMethodVersions, + DriveAbciIdentityCreditWithdrawalMethodVersions, DriveAbciInitializationMethodVersions, + DriveAbciMasternodeIdentitiesUpdatesMethodVersions, DriveAbciMethodVersions, + DriveAbciPlatformStateStorageMethodVersions, DriveAbciProtocolUpgradeMethodVersions, + DriveAbciStateTransitionProcessingMethodVersions, DriveAbciTokensProcessingMethodVersions, + DriveAbciVotingMethodVersions, +}; + +/// Drive ABCI method versions 11. Introduced in protocol v15 for state sync: +/// `run_block_proposal` 0 -> 1 (the reduced platform state is written into the replicated +/// state each block, and `validator_set_update` moves above the root-hash computation so the +/// stored reduced state is sufficient to reconstruct the post-rotation state), and +/// `consensus_params_update` 1 -> 2 (emits evidence params when crossing to v15). +/// Everything else matches `DRIVE_ABCI_METHOD_VERSIONS_V10`. +pub const DRIVE_ABCI_METHOD_VERSIONS_V11: DriveAbciMethodVersions = DriveAbciMethodVersions { + engine: DriveAbciEngineMethodVersions { + init_chain: 0, + check_tx: 0, + run_block_proposal: 1, + finalize_block_proposal: 0, + consensus_params_update: 2, + }, + initialization: DriveAbciInitializationMethodVersions { + initial_core_height_and_time: 0, + create_genesis_state: 1, + }, + core_based_updates: DriveAbciCoreBasedUpdatesMethodVersions { + update_core_info: 0, + update_masternode_list: 0, + update_quorum_info: 0, + masternode_updates: DriveAbciMasternodeIdentitiesUpdatesMethodVersions { + get_voter_identity_key: 0, + get_operator_identity_keys: 0, + get_owner_identity_withdrawal_key: 0, + get_owner_identity_owner_key: 0, + get_voter_identifier_from_masternode_list_item: 0, + get_operator_identifier_from_masternode_list_item: 0, + create_operator_identity: 0, + create_owner_identity: 1, + create_voter_identity: 0, + disable_identity_keys: 0, + update_masternode_identities: 0, + update_operator_identity: 0, + update_owner_withdrawal_address: 1, + update_voter_identity: 0, + }, + }, + protocol_upgrade: DriveAbciProtocolUpgradeMethodVersions { + check_for_desired_protocol_upgrade: 1, + upgrade_protocol_version_on_epoch_change: 0, + perform_events_on_first_block_of_protocol_change: Some(1), + protocol_version_upgrade_percentage_needed: 67, + }, + block_fee_processing: DriveAbciBlockFeeProcessingMethodVersions { + add_process_epoch_change_operations: 0, + process_block_fees_and_validate_sum_trees: 1, + }, + tokens_processing: DriveAbciTokensProcessingMethodVersions { + validate_token_aggregated_balance: 0, + }, + core_chain_lock: DriveAbciCoreChainLockMethodVersionsAndConstants { + choose_quorum: 0, + verify_chain_lock: 0, + verify_chain_lock_locally: 0, + verify_chain_lock_through_core: 0, + make_sure_core_is_synced_to_chain_lock: 0, + recent_block_count_amount: 2, + }, + core_instant_send_lock: DriveAbciCoreInstantSendLockMethodVersions { + verify_recent_signature_locally: 0, + }, + fee_pool_inwards_distribution: DriveAbciFeePoolInwardsDistributionMethodVersions { + add_distribute_block_fees_into_pools_operations: 0, + add_distribute_storage_fee_to_epochs_operations: 0, + }, + fee_pool_outwards_distribution: DriveAbciFeePoolOutwardsDistributionMethodVersions { + add_distribute_fees_from_oldest_unpaid_epoch_pool_to_proposers_operations: 1, + add_epoch_pool_to_proposers_payout_operations: 0, + find_oldest_epoch_needing_payment: 0, + fetch_reward_shares_list_for_masternode: 0, + }, + withdrawals: DriveAbciIdentityCreditWithdrawalMethodVersions { + build_untied_withdrawal_transactions_from_documents: 0, + dequeue_and_build_unsigned_withdrawal_transactions: 0, + fetch_transactions_block_inclusion_status: 0, + pool_withdrawals_into_transactions_queue: 1, + update_broadcasted_withdrawal_statuses: 0, + rebroadcast_expired_withdrawal_documents: 1, + append_signatures_and_broadcast_withdrawal_transactions: 0, + cleanup_expired_locks_of_withdrawal_amounts: 1, // changed in v14: also prunes expired entries of the credit inflows sum tree + record_credit_inflows_for_withdrawals: Some(0), // new in v14: the block's credit mints recorded as an inflow for the net daily withdrawal limit + record_total_credits_history_for_withdrawals: Some(0), // changed in v14: per-block total credits history for the day-lagged daily withdrawal limit + }, + voting: DriveAbciVotingMethodVersions { + keep_record_of_finished_contested_resource_vote_poll: 0, + clean_up_after_vote_poll_end: 0, + clean_up_after_contested_resources_vote_poll_end: 1, + check_for_ended_vote_polls: 0, + tally_votes_for_contested_document_resource_vote_poll: 0, + award_document_to_winner: 0, + delay_vote_poll: 0, + run_dao_platform_events: 0, + remove_votes_for_removed_masternodes: 0, + }, + state_transition_processing: DriveAbciStateTransitionProcessingMethodVersions { + execute_event: 0, + process_raw_state_transitions: 0, + // unchanged from V9: v1 since v13 (records the balance effects of paid-INVALID / + // unsuccessful-paid transitions) + process_validation_result: 1, + decode_raw_state_transitions: 0, + validate_fees_of_event: 0, + store_address_balances_to_recent_block_storage: Some(0), + cleanup_recent_block_storage_address_balances: Some(0), + // unchanged from V9: v1 since v13 (records shielded-spend transparent credits) + record_added_balance_outputs: 1, + }, + epoch: DriveAbciEpochMethodVersions { + gather_epoch_info: 0, + get_genesis_time: 0, + }, + block_start: DriveAbciBlockStartMethodVersions { + clear_drive_block_cache: 0, + }, + block_end: DriveAbciBlockEndMethodVersions { + update_state_cache: 0, + update_drive_cache: 0, + validator_set_update: 2, + should_checkpoint: Some(0), + update_checkpoints: Some(0), + record_shielded_pool_anchor: Some(0), + prune_shielded_pool_anchors: Some(0), + }, + platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { + fetch_platform_state: 0, + store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, + }, +}; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v2.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v2.rs index c3177e006f7..e781dbca981 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v2.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v2.rs @@ -132,5 +132,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V2: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v3.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v3.rs index 06fe75413e5..e9a0fb51daa 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v3.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v3.rs @@ -131,5 +131,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V3: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v4.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v4.rs index 843e71c6d40..b5c18a3c727 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v4.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v4.rs @@ -131,5 +131,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V4: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v5.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v5.rs index bed7af26bf2..84d1c011f38 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v5.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v5.rs @@ -135,5 +135,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V5: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v6.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v6.rs index df56f534d95..f696c3d7dba 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v6.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v6.rs @@ -133,5 +133,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V6: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v7.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v7.rs index da44e9b81a9..6a10f182858 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v7.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v7.rs @@ -142,5 +142,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V7: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v8.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v8.rs index d9a461cc62f..0629a7808c7 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v8.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v8.rs @@ -142,5 +142,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V8: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v9.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v9.rs index 434f216c461..f3d4d857fd0 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v9.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_method_versions/v9.rs @@ -160,5 +160,7 @@ pub const DRIVE_ABCI_METHOD_VERSIONS_V9: DriveAbciMethodVersions = DriveAbciMeth platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/mod.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/mod.rs new file mode 100644 index 00000000000..bffcad033a1 --- /dev/null +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/mod.rs @@ -0,0 +1,14 @@ +pub mod v1; + +use versioned_feature_core::FeatureVersion; + +/// Versions for ABCI state sync (snapshot serving and consumption). +#[derive(Clone, Debug, Default)] +pub struct DriveAbciStateSyncVersions { + /// The grovedb state sync wire protocol version used for snapshots this node + /// creates and serves. Snapshots offered by peers are validated against the + /// supported set in `drive-abci`'s snapshot module; bumping to a new grovedb + /// wire version means adding a new `DriveAbciStateSyncVersions` const here and + /// extending that supported set. + pub protocol_version: FeatureVersion, +} diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/v1.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/v1.rs new file mode 100644 index 00000000000..f5a3d13e884 --- /dev/null +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/v1.rs @@ -0,0 +1,6 @@ +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::DriveAbciStateSyncVersions; + +pub const DRIVE_ABCI_STATE_SYNC_VERSIONS_V1: DriveAbciStateSyncVersions = + DriveAbciStateSyncVersions { + protocol_version: 1, + }; diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/mod.rs b/packages/rs-platform-version/src/version/drive_abci_versions/mod.rs index 6df817b3dfd..9adde49dbe5 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/mod.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/mod.rs @@ -1,6 +1,7 @@ pub mod drive_abci_checkpoint_parameters; pub mod drive_abci_method_versions; pub mod drive_abci_query_versions; +pub mod drive_abci_state_sync_versions; pub mod drive_abci_structure_versions; pub mod drive_abci_validation_versions; pub mod drive_abci_withdrawal_constants; @@ -8,6 +9,7 @@ pub mod drive_abci_withdrawal_constants; use drive_abci_checkpoint_parameters::DriveAbciCheckpointParameters; use drive_abci_method_versions::DriveAbciMethodVersions; use drive_abci_query_versions::DriveAbciQueryVersions; +use drive_abci_state_sync_versions::DriveAbciStateSyncVersions; use drive_abci_structure_versions::DriveAbciStructureVersions; use drive_abci_validation_versions::DriveAbciValidationVersions; use drive_abci_withdrawal_constants::DriveAbciWithdrawalConstants; @@ -20,4 +22,5 @@ pub struct DriveAbciVersion { pub withdrawal_constants: DriveAbciWithdrawalConstants, pub query: DriveAbciQueryVersions, pub checkpoints: DriveAbciCheckpointParameters, + pub state_sync: DriveAbciStateSyncVersions, } diff --git a/packages/rs-platform-version/src/version/mocks/v2_test.rs b/packages/rs-platform-version/src/version/mocks/v2_test.rs index 5a8774f08b2..838b110151d 100644 --- a/packages/rs-platform-version/src/version/mocks/v2_test.rs +++ b/packages/rs-platform-version/src/version/mocks/v2_test.rs @@ -23,6 +23,7 @@ use crate::version::drive_abci_versions::drive_abci_query_versions::{ DriveAbciQueryShieldedVersions, DriveAbciQuerySystemVersions, DriveAbciQueryTokenVersions, DriveAbciQueryValidatorVersions, DriveAbciQueryVersions, DriveAbciQueryVotingVersions, }; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v1::DRIVE_ABCI_VALIDATION_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v1::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1; @@ -480,6 +481,7 @@ pub const TEST_PLATFORM_V2: PlatformVersion = PlatformVersion { }, }, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/mocks/v3_test.rs b/packages/rs-platform-version/src/version/mocks/v3_test.rs index e5cfd15b1bb..d6325c633c0 100644 --- a/packages/rs-platform-version/src/version/mocks/v3_test.rs +++ b/packages/rs-platform-version/src/version/mocks/v3_test.rs @@ -29,6 +29,7 @@ use crate::version::drive_abci_versions::drive_abci_method_versions::{ DriveAbciVotingMethodVersions, }; use crate::version::drive_abci_versions::drive_abci_query_versions::v1::DRIVE_ABCI_QUERY_VERSIONS_V1; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v3::DRIVE_ABCI_VALIDATION_VERSIONS_V3; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -166,12 +167,15 @@ pub const TEST_PLATFORM_V3: PlatformVersion = PlatformVersion { platform_state_storage: DriveAbciPlatformStateStorageMethodVersions { fetch_platform_state: 0, store_platform_state: 0, + fetch_reduced_platform_state: 0, + store_reduced_platform_state: 0, }, }, validation_and_processing: DRIVE_ABCI_VALIDATION_VERSIONS_V3, withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V1, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/mod.rs b/packages/rs-platform-version/src/version/mod.rs index 1b1635efb42..ae5fd0887e1 100644 --- a/packages/rs-platform-version/src/version/mod.rs +++ b/packages/rs-platform-version/src/version/mod.rs @@ -1,6 +1,6 @@ mod protocol_version; -use crate::version::v14::PROTOCOL_VERSION_14; +use crate::version::v15::PROTOCOL_VERSION_15; pub use protocol_version::*; use std::ops::RangeInclusive; @@ -20,6 +20,7 @@ pub mod v11; pub mod v12; pub mod v13; pub mod v14; +pub mod v15; pub mod v2; pub mod v3; pub mod v4; @@ -33,5 +34,5 @@ pub type ProtocolVersion = u32; pub const ALL_VERSIONS: RangeInclusive = 1..=LATEST_VERSION; -pub const LATEST_VERSION: ProtocolVersion = PROTOCOL_VERSION_14; +pub const LATEST_VERSION: ProtocolVersion = PROTOCOL_VERSION_15; pub const INITIAL_PROTOCOL_VERSION: ProtocolVersion = 1; diff --git a/packages/rs-platform-version/src/version/protocol_version.rs b/packages/rs-platform-version/src/version/protocol_version.rs index 0eded570c10..2ba05cc8366 100644 --- a/packages/rs-platform-version/src/version/protocol_version.rs +++ b/packages/rs-platform-version/src/version/protocol_version.rs @@ -22,6 +22,7 @@ use crate::version::v11::PLATFORM_V11; use crate::version::v12::PLATFORM_V12; use crate::version::v13::PLATFORM_V13; use crate::version::v14::PLATFORM_V14; +use crate::version::v15::PLATFORM_V15; use crate::version::v2::PLATFORM_V2; use crate::version::v3::PLATFORM_V3; use crate::version::v4::PLATFORM_V4; @@ -61,6 +62,7 @@ pub const PLATFORM_VERSIONS: &[PlatformVersion] = &[ PLATFORM_V12, PLATFORM_V13, PLATFORM_V14, + PLATFORM_V15, ]; #[cfg(feature = "mock-versions")] @@ -69,7 +71,7 @@ pub static PLATFORM_TEST_VERSIONS: OnceLock> = OnceLock::ne #[cfg(feature = "mock-versions")] const DEFAULT_PLATFORM_TEST_VERSIONS: &[PlatformVersion] = &[TEST_PLATFORM_V2, TEST_PLATFORM_V3]; -pub const LATEST_PLATFORM_VERSION: &PlatformVersion = &PLATFORM_V14; +pub const LATEST_PLATFORM_VERSION: &PlatformVersion = &PLATFORM_V15; pub const DESIRED_PLATFORM_VERSION: &PlatformVersion = LATEST_PLATFORM_VERSION; diff --git a/packages/rs-platform-version/src/version/v1.rs b/packages/rs-platform-version/src/version/v1.rs index b3c54787c77..9a81628e63c 100644 --- a/packages/rs-platform-version/src/version/v1.rs +++ b/packages/rs-platform-version/src/version/v1.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v1::DRIVE_ABCI_METHOD_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v1::DRIVE_ABCI_VALIDATION_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v1::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1; @@ -40,6 +41,7 @@ pub const PLATFORM_V1: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v10.rs b/packages/rs-platform-version/src/version/v10.rs index f04b14d341a..66eabb8d84e 100644 --- a/packages/rs-platform-version/src/version/v10.rs +++ b/packages/rs-platform-version/src/version/v10.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v6::DRIVE_ABCI_METHOD_VERSIONS_V6; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v6::DRIVE_ABCI_VALIDATION_VERSIONS_V6; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V10: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v11.rs b/packages/rs-platform-version/src/version/v11.rs index eb039ad49cf..414326d77a3 100644 --- a/packages/rs-platform-version/src/version/v11.rs +++ b/packages/rs-platform-version/src/version/v11.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v7::DRIVE_ABCI_METHOD_VERSIONS_V7; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v7::DRIVE_ABCI_VALIDATION_VERSIONS_V7; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V11: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v12.rs b/packages/rs-platform-version/src/version/v12.rs index 2d334b1fd7d..cac54b6cb49 100644 --- a/packages/rs-platform-version/src/version/v12.rs +++ b/packages/rs-platform-version/src/version/v12.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v8::DRIVE_ABCI_METHOD_VERSIONS_V8; use crate::version::drive_abci_versions::drive_abci_query_versions::v1::DRIVE_ABCI_QUERY_VERSIONS_V1; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v8::DRIVE_ABCI_VALIDATION_VERSIONS_V8; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -44,6 +45,7 @@ pub const PLATFORM_V12: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V1, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v13.rs b/packages/rs-platform-version/src/version/v13.rs index b6249ba91fc..776ad91a2e1 100644 --- a/packages/rs-platform-version/src/version/v13.rs +++ b/packages/rs-platform-version/src/version/v13.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v9::DRIVE_ABCI_METHOD_VERSIONS_V9; use crate::version::drive_abci_versions::drive_abci_query_versions::v1::DRIVE_ABCI_QUERY_VERSIONS_V1; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v9::DRIVE_ABCI_VALIDATION_VERSIONS_V9; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -70,6 +71,7 @@ pub const PLATFORM_V13: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V1, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v14.rs b/packages/rs-platform-version/src/version/v14.rs index 7589c485738..ddaf8397da3 100644 --- a/packages/rs-platform-version/src/version/v14.rs +++ b/packages/rs-platform-version/src/version/v14.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v10::DRIVE_ABCI_METHOD_VERSIONS_V10; use crate::version::drive_abci_versions::drive_abci_query_versions::v3::DRIVE_ABCI_QUERY_VERSIONS_V3; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v10::DRIVE_ABCI_VALIDATION_VERSIONS_V10; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v3::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V3; @@ -204,6 +205,7 @@ pub const PLATFORM_V14: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V3, // changed: prune bound for the total credits history query: DRIVE_ABCI_QUERY_VERSIONS_V3, // changed: ranked + boolean-HAVING routing gate; the v1 handler also resolves IN_TIME_RANGE from committed block time checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v15.rs b/packages/rs-platform-version/src/version/v15.rs new file mode 100644 index 00000000000..e1c662f0a54 --- /dev/null +++ b/packages/rs-platform-version/src/version/v15.rs @@ -0,0 +1,129 @@ +use crate::version::consensus_versions::ConsensusVersions; +use crate::version::dpp_versions::dpp_asset_lock_versions::v1::DPP_ASSET_LOCK_VERSIONS_V1; +use crate::version::dpp_versions::dpp_contract_versions::v6::CONTRACT_VERSIONS_V6; +use crate::version::dpp_versions::dpp_costs_versions::v1::DPP_COSTS_VERSIONS_V1; +use crate::version::dpp_versions::dpp_document_versions::v4::DOCUMENT_VERSIONS_V4; +use crate::version::dpp_versions::dpp_factory_versions::v1::DPP_FACTORY_VERSIONS_V1; +use crate::version::dpp_versions::dpp_identity_versions::v1::IDENTITY_VERSIONS_V1; +use crate::version::dpp_versions::dpp_method_versions::v3::DPP_METHOD_VERSIONS_V3; +use crate::version::dpp_versions::dpp_state_transition_conversion_versions::v2::STATE_TRANSITION_CONVERSION_VERSIONS_V2; +use crate::version::dpp_versions::dpp_state_transition_method_versions::v1::STATE_TRANSITION_METHOD_VERSIONS_V1; +use crate::version::dpp_versions::dpp_state_transition_serialization_versions::v3::STATE_TRANSITION_SERIALIZATION_VERSIONS_V3; +use crate::version::dpp_versions::dpp_state_transition_versions::v3::STATE_TRANSITION_VERSIONS_V3; +use crate::version::dpp_versions::dpp_token_versions::v2::TOKEN_VERSIONS_V2; +use crate::version::dpp_versions::dpp_validation_versions::v5::DPP_VALIDATION_VERSIONS_V5; +use crate::version::dpp_versions::dpp_voting_versions::v2::VOTING_VERSION_V2; +use crate::version::dpp_versions::DPPVersion; +use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; +use crate::version::drive_abci_versions::drive_abci_method_versions::v11::DRIVE_ABCI_METHOD_VERSIONS_V11; +use crate::version::drive_abci_versions::drive_abci_query_versions::v3::DRIVE_ABCI_QUERY_VERSIONS_V3; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; +use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; +use crate::version::drive_abci_versions::drive_abci_validation_versions::v10::DRIVE_ABCI_VALIDATION_VERSIONS_V10; +use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v3::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V3; +use crate::version::drive_abci_versions::DriveAbciVersion; +use crate::version::drive_versions::v9::DRIVE_VERSION_V9; +use crate::version::fee::v2::FEE_VERSION2; +use crate::version::protocol_version::PlatformVersion; +use crate::version::system_data_contract_versions::v3::SYSTEM_DATA_CONTRACT_VERSIONS_V3; +use crate::version::system_limits::v4::SYSTEM_LIMITS_V4; +use crate::version::ProtocolVersion; + +pub const PROTOCOL_VERSION_15: ProtocolVersion = 15; + +/// v15 enables ABCI state sync: a fresh node can bootstrap from a peer's grovedb +/// snapshot instead of replaying the chain. +/// +/// The consensus changes gate on `DRIVE_ABCI_METHOD_VERSIONS_V11`: +/// +/// * `run_block_proposal` 0 -> 1: every block writes a reduced platform state +/// (`Misc/reduced_saved_state`) into the replicated state just before the root hash is +/// computed, and `validator_set_update` moves above the root-hash computation so the +/// stored reduced state reflects the post-rotation validator set. The full platform +/// state only lives in non-replicated aux storage, so without this a state-synced node +/// would have no way to rebuild its in-memory state. +/// * `consensus_params_update` 1 -> 2: the first block of v15 also emits evidence +/// params sized for state-synced nodes that do not hold full history (issue #2512). +/// * `perform_events_on_first_block_of_protocol_change` writes the initial reduced state +/// at the v15 activation block, so every snapshot taken at or after activation is +/// restorable. Snapshots from before activation lack the key and are not served. +/// +/// Everything else matches v14. The grovedb state sync wire protocol version used for +/// snapshots is `DRIVE_ABCI_STATE_SYNC_VERSIONS_V1.protocol_version` (1), shared by all +/// platform versions. +pub const PLATFORM_V15: PlatformVersion = PlatformVersion { + protocol_version: PROTOCOL_VERSION_15, + drive: DRIVE_VERSION_V9, + drive_abci: DriveAbciVersion { + structs: DRIVE_ABCI_STRUCTURE_VERSIONS_V1, + methods: DRIVE_ABCI_METHOD_VERSIONS_V11, // changed: run_block_proposal v1 (reduced state write + validator rotation above root hash) and consensus_params_update v2 (evidence params on the v15 activation block) + validation_and_processing: DRIVE_ABCI_VALIDATION_VERSIONS_V10, + withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V3, + query: DRIVE_ABCI_QUERY_VERSIONS_V3, + checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, + }, + dpp: DPPVersion { + costs: DPP_COSTS_VERSIONS_V1, + validation: DPP_VALIDATION_VERSIONS_V5, + state_transition_serialization_versions: STATE_TRANSITION_SERIALIZATION_VERSIONS_V3, + state_transition_conversion_versions: STATE_TRANSITION_CONVERSION_VERSIONS_V2, + state_transition_method_versions: STATE_TRANSITION_METHOD_VERSIONS_V1, + state_transitions: STATE_TRANSITION_VERSIONS_V3, + contract_versions: CONTRACT_VERSIONS_V6, + document_versions: DOCUMENT_VERSIONS_V4, + identity_versions: IDENTITY_VERSIONS_V1, + voting_versions: VOTING_VERSION_V2, + token_versions: TOKEN_VERSIONS_V2, + asset_lock_versions: DPP_ASSET_LOCK_VERSIONS_V1, + methods: DPP_METHOD_VERSIONS_V3, + factory_versions: DPP_FACTORY_VERSIONS_V1, + }, + system_data_contracts: SYSTEM_DATA_CONTRACT_VERSIONS_V3, + fee_version: FEE_VERSION2, + system_limits: SYSTEM_LIMITS_V4, + consensus: ConsensusVersions { + tenderdash_consensus_version: 1, + }, +}; + +#[cfg(test)] +mod tests { + use super::*; + use crate::version::v14::PLATFORM_V14; + + /// The state sync consensus changes live in v15's own method table, so a v14 node + /// keeps running run_block_proposal v0 (no reduced-state write, rotation after the + /// root hash) and consensus_params_update v1. Making v14 non-zero here would be + /// consensus-breaking for already-deployed nodes. + #[test] + fn state_sync_consensus_changes_gate_at_v15() { + assert_eq!(PLATFORM_V14.drive_abci.methods.engine.run_block_proposal, 0); + assert_eq!( + PLATFORM_V14 + .drive_abci + .methods + .engine + .consensus_params_update, + 1 + ); + assert_eq!(PLATFORM_V15.drive_abci.methods.engine.run_block_proposal, 1); + assert_eq!( + PLATFORM_V15 + .drive_abci + .methods + .engine + .consensus_params_update, + 2 + ); + } + + /// All platform versions share grovedb state sync wire protocol version 1 until a + /// grovedb wire v2 exists; the supported set lives next to the snapshot types in + /// drive-abci. + #[test] + fn state_sync_wire_protocol_version_is_one() { + assert_eq!(PLATFORM_V15.drive_abci.state_sync.protocol_version, 1); + assert_eq!(PLATFORM_V14.drive_abci.state_sync.protocol_version, 1); + } +} diff --git a/packages/rs-platform-version/src/version/v2.rs b/packages/rs-platform-version/src/version/v2.rs index 93cd7b07232..0bcfcb9fbeb 100644 --- a/packages/rs-platform-version/src/version/v2.rs +++ b/packages/rs-platform-version/src/version/v2.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v1::DRIVE_ABCI_METHOD_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v2::DRIVE_ABCI_VALIDATION_VERSIONS_V2; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v1::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1; @@ -40,6 +41,7 @@ pub const PLATFORM_V2: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v3.rs b/packages/rs-platform-version/src/version/v3.rs index c125b94ff9b..c8bfa8b212d 100644 --- a/packages/rs-platform-version/src/version/v3.rs +++ b/packages/rs-platform-version/src/version/v3.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v2::DRIVE_ABCI_METHOD_VERSIONS_V2; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v2::DRIVE_ABCI_VALIDATION_VERSIONS_V2; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v1::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1; @@ -46,6 +47,7 @@ pub const PLATFORM_V3: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V1, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v4.rs b/packages/rs-platform-version/src/version/v4.rs index dba41251e96..c7268418092 100644 --- a/packages/rs-platform-version/src/version/v4.rs +++ b/packages/rs-platform-version/src/version/v4.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v3::DRIVE_ABCI_METHOD_VERSIONS_V3; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v3::DRIVE_ABCI_VALIDATION_VERSIONS_V3; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V4: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v5.rs b/packages/rs-platform-version/src/version/v5.rs index 3c288cfe63d..e0e9150dfd6 100644 --- a/packages/rs-platform-version/src/version/v5.rs +++ b/packages/rs-platform-version/src/version/v5.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v4::DRIVE_ABCI_METHOD_VERSIONS_V4; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v3::DRIVE_ABCI_VALIDATION_VERSIONS_V3; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V5: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v6.rs b/packages/rs-platform-version/src/version/v6.rs index 7d948da6f00..43def28162d 100644 --- a/packages/rs-platform-version/src/version/v6.rs +++ b/packages/rs-platform-version/src/version/v6.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v4::DRIVE_ABCI_METHOD_VERSIONS_V4; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v4::DRIVE_ABCI_VALIDATION_VERSIONS_V4; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V6: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v7.rs b/packages/rs-platform-version/src/version/v7.rs index 09755d462e1..eabdc58c585 100644 --- a/packages/rs-platform-version/src/version/v7.rs +++ b/packages/rs-platform-version/src/version/v7.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v4::DRIVE_ABCI_METHOD_VERSIONS_V4; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v5::DRIVE_ABCI_VALIDATION_VERSIONS_V5; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V7: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v8.rs b/packages/rs-platform-version/src/version/v8.rs index 2096142ac18..f4c2f9dd1f5 100644 --- a/packages/rs-platform-version/src/version/v8.rs +++ b/packages/rs-platform-version/src/version/v8.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v5::DRIVE_ABCI_METHOD_VERSIONS_V5; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v5::DRIVE_ABCI_VALIDATION_VERSIONS_V5; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -45,6 +46,7 @@ pub const PLATFORM_V8: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, diff --git a/packages/rs-platform-version/src/version/v9.rs b/packages/rs-platform-version/src/version/v9.rs index a27803f6da8..8ee4fc891bf 100644 --- a/packages/rs-platform-version/src/version/v9.rs +++ b/packages/rs-platform-version/src/version/v9.rs @@ -17,6 +17,7 @@ use crate::version::dpp_versions::DPPVersion; use crate::version::drive_abci_versions::drive_abci_checkpoint_parameters::v1::DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1; use crate::version::drive_abci_versions::drive_abci_method_versions::v6::DRIVE_ABCI_METHOD_VERSIONS_V6; use crate::version::drive_abci_versions::drive_abci_query_versions::v0::DRIVE_ABCI_QUERY_VERSIONS_V0; +use crate::version::drive_abci_versions::drive_abci_state_sync_versions::v1::DRIVE_ABCI_STATE_SYNC_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_structure_versions::v1::DRIVE_ABCI_STRUCTURE_VERSIONS_V1; use crate::version::drive_abci_versions::drive_abci_validation_versions::v6::DRIVE_ABCI_VALIDATION_VERSIONS_V6; use crate::version::drive_abci_versions::drive_abci_withdrawal_constants::v2::DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2; @@ -41,6 +42,7 @@ pub const PLATFORM_V9: PlatformVersion = PlatformVersion { withdrawal_constants: DRIVE_ABCI_WITHDRAWAL_CONSTANTS_V2, query: DRIVE_ABCI_QUERY_VERSIONS_V0, checkpoints: DRIVE_ABCI_CHECKPOINT_PARAMETERS_V1, + state_sync: DRIVE_ABCI_STATE_SYNC_VERSIONS_V1, }, dpp: DPPVersion { costs: DPP_COSTS_VERSIONS_V1, From 787c2fddd09d2b8ff0d9900f9e3fdf0073ab7a95 Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 29 Aug 2026 01:01:18 +0200 Subject: [PATCH 04/34] feat(drive-abci): run_block_proposal v1 writes reduced platform state before root hash v1 (gated on drive-abci method versions v11 / protocol v15) is a copy of v0 with validator_set_update moved above the root-hash computation and the reduced platform state written into the replicated state immediately before the root hash, so the stored state carries the post-rotation next validator set and is covered by the block's app hash. Adds the store/fetch_reduced_platform_state execution wrappers and the PlatformState::to_reduced_platform_state conversion (fee versions persisted faithfully by number). A test proves rotation outcomes are unchanged by the reorder: validator_set_update only mutates in-memory block state and reads neither the app hash nor grovedb. Co-Authored-By: Claude Fable 5 --- .../engine/run_block_proposal/mod.rs | 13 +- .../engine/run_block_proposal/v1/mod.rs | 510 ++++++++++++++++++ .../block_end/validator_set_update/mod.rs | 127 +++++ .../fetch_reduced_platform_state/mod.rs | 34 ++ .../fetch_reduced_platform_state/v0/mod.rs | 23 + .../src/execution/storage/mod.rs | 2 + .../store_reduced_platform_state/mod.rs | 32 ++ .../store_reduced_platform_state/v0/mod.rs | 23 + .../src/platform_types/platform_state/mod.rs | 40 ++ 9 files changed, 803 insertions(+), 1 deletion(-) create mode 100644 packages/rs-drive-abci/src/execution/engine/run_block_proposal/v1/mod.rs create mode 100644 packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/mod.rs create mode 100644 packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/v0/mod.rs create mode 100644 packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/mod.rs create mode 100644 packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/v0/mod.rs diff --git a/packages/rs-drive-abci/src/execution/engine/run_block_proposal/mod.rs b/packages/rs-drive-abci/src/execution/engine/run_block_proposal/mod.rs index 68d87275ace..f0063a452f9 100644 --- a/packages/rs-drive-abci/src/execution/engine/run_block_proposal/mod.rs +++ b/packages/rs-drive-abci/src/execution/engine/run_block_proposal/mod.rs @@ -15,6 +15,7 @@ use dpp::version::PlatformVersion; use drive::grovedb::Transaction; mod v0; +mod v1; impl Platform where @@ -154,9 +155,19 @@ Your software version: {}, latest supported protocol version: {}."#, block_platform_version, timer, ), + 1 => self.run_block_proposal_v1( + block_proposal, + known_from_us, + epoch_info, + transaction, + platform_state, + block_platform_state, + block_platform_version, + timer, + ), version => Err(Error::Execution(ExecutionError::UnknownVersionMismatch { method: "run_block_proposal".to_string(), - known_versions: vec![0], + known_versions: vec![0, 1], received: version, })), } diff --git a/packages/rs-drive-abci/src/execution/engine/run_block_proposal/v1/mod.rs b/packages/rs-drive-abci/src/execution/engine/run_block_proposal/v1/mod.rs new file mode 100644 index 00000000000..95a4d03430d --- /dev/null +++ b/packages/rs-drive-abci/src/execution/engine/run_block_proposal/v1/mod.rs @@ -0,0 +1,510 @@ +use dpp::block::epoch::Epoch; + +use dpp::validation::ValidationResult; + +use dpp::version::PlatformVersion; +use drive::grovedb::Transaction; + +use crate::abci::AbciError; +use crate::error::execution::ExecutionError; + +use crate::error::Error; +use crate::execution::types::block_execution_context::v0::{ + BlockExecutionContextV0Getters, BlockExecutionContextV0MutableGetters, +}; +use crate::execution::types::block_execution_context::BlockExecutionContext; +use crate::execution::types::block_fees::v0::BlockFeesV0; +use crate::execution::types::block_state_info::v0::{ + BlockStateInfoV0Getters, BlockStateInfoV0Methods, BlockStateInfoV0Setters, +}; +use crate::execution::types::{block_execution_context, block_state_info}; +use crate::metrics::HistogramTiming; +use crate::platform_types::block_execution_outcome; +use crate::platform_types::block_proposal; +use crate::platform_types::epoch_info::v0::{EpochInfoV0Getters, EpochInfoV0Methods}; +use crate::platform_types::epoch_info::EpochInfo; +use crate::platform_types::platform::Platform; +use crate::platform_types::platform_state::PlatformState; +use crate::platform_types::platform_state::PlatformStateV0Methods; +use crate::platform_types::verify_chain_lock_result::v0::VerifyChainLockResult; +use crate::rpc::core::CoreRPCLike; +use dpp::reduced_platform_state::v0::ReducedBlockInfoV0; + +impl Platform +where + C: CoreRPCLike, +{ + /// Runs a block proposal, either from process proposal or prepare proposal. + /// + /// This function takes a `BlockProposal` and a `Transaction` as input and processes the block + /// proposal. It first validates the block proposal and then processes raw state transitions, + /// withdrawal transactions, and block fees. It also updates the validator set. + /// + /// v1 (protocol v15, state sync): identical to v0 except that + /// `validator_set_update` runs BEFORE the root hash is computed (it only mutates the + /// in-memory block platform state, never grovedb, so the move cannot change the root + /// hash or the rotation outcome), and the reduced platform state — including the + /// post-rotation next validator set — is then written into the replicated grovedb + /// state immediately before the root hash, so it is covered by this block's app hash + /// and a state-synced node can reconstruct the full platform state from it. + /// + /// # Arguments + /// + /// * `block_proposal` - The block proposal to be processed. + /// * `known_from_us` - Do we know that we made this block proposal?. + /// * `transaction` - The transaction associated with the block proposal. + /// + /// # Returns + /// + /// * `Result, Error>` - If the block proposal is + /// successfully processed, it returns a `ValidationResult` containing the `BlockExecutionOutcome`. + /// If the block proposal processing fails, it returns an `Error`. Consensus errors are returned + /// in the `ValidationResult`, while critical system errors are returned in the `Result`. + /// + /// # Errors + /// + /// This function may return an `Error` variant if there is a problem with processing the block + /// proposal, updating the core info, processing raw state transitions, or processing block fees. + /// + #[allow(clippy::too_many_arguments)] + pub(super) fn run_block_proposal_v1( + &self, + block_proposal: block_proposal::v0::BlockProposal, + known_from_us: bool, + epoch_info: EpochInfo, + transaction: &Transaction, + last_committed_platform_state: &PlatformState, + mut block_platform_state: PlatformState, + platform_version: &'static PlatformVersion, + timer: Option<&HistogramTiming>, + ) -> Result, Error> + { + tracing::trace!( + method = "run_block_proposal_v1", + ?block_proposal, + ?epoch_info, + "Running a block proposal for height: {}, round: {}", + block_proposal.height, + block_proposal.round, + ); + + // Run block proposal determines version by itself based on the previous + // state and block time. + // It should provide correct version on prepare proposal to block header + // and validate it on process proposal. + // If version set to 0 (default number value) it means we are on prepare proposal, + // so there is no need for validation. + if !known_from_us + && block_proposal.consensus_versions.app != platform_version.protocol_version as u64 + { + return Ok(ValidationResult::new_with_error( + AbciError::BadRequest(format!( + "received a block proposal with protocol version {}, expected: {}", + block_proposal.consensus_versions.app, platform_version.protocol_version + )) + .into(), + )); + } + + let last_block_time_ms = last_committed_platform_state.last_committed_block_time_ms(); + let last_block_height = last_committed_platform_state.last_committed_known_block_height_or( + self.config.abci.genesis_height.saturating_sub(1), + ); + let last_block_core_height = last_committed_platform_state + .last_committed_known_core_height_or(self.config.abci.genesis_core_height); + + // Init block execution context + let block_state_info = block_state_info::v0::BlockStateInfoV0::from_block_proposal( + &block_proposal, + last_block_time_ms, + ); + + // First let's check that this is the follower to a previous block + if !block_state_info.next_block_to(last_block_height, last_block_core_height)? { + // we are on the wrong height or round + return Ok(ValidationResult::new_with_error(AbciError::WrongBlockReceived(format!( + "received a block proposal for height: {} core height: {}, current height: {} core height: {}", + block_state_info.height, block_state_info.core_chain_locked_height, last_block_height, last_block_core_height + )).into())); + } + + // destructure the block proposal + let block_proposal::v0::BlockProposal { + core_chain_locked_height, + core_chain_lock_update, + proposed_app_version, + proposer_pro_tx_hash, + validator_set_quorum_hash, + raw_state_transitions, + .. + } = block_proposal; + + let block_info = block_state_info.to_block_info( + Epoch::new(epoch_info.current_epoch_index()) + .expect("current epoch index should be in range"), + ); + + if epoch_info.is_epoch_change_but_not_genesis() { + tracing::info!( + epoch_index = epoch_info.current_epoch_index(), + "epoch change occurring from epoch {} to epoch {}", + epoch_info + .previous_epoch_index() + .expect("must be set since we aren't on genesis"), + epoch_info.current_epoch_index(), + ); + } + + // Update block platform state with current and next epoch protocol versions + // if it was proposed + // This is happening only on epoch change + self.upgrade_protocol_version_on_epoch_change( + &block_info, + &epoch_info, + last_committed_platform_state, + &mut block_platform_state, + transaction, + platform_version, + )?; + + // If there is a core chain lock update, we should start by verifying it + if let Some(core_chain_lock_update) = core_chain_lock_update.as_ref() { + if !known_from_us { + let verification_result = self.verify_chain_lock( + block_state_info.round, // the round is to allow us to bypass local verification in case of chain stall + &block_platform_state, + core_chain_lock_update, + true, // if it's not known from us, then we should try submitting it + platform_version, + ); + + let VerifyChainLockResult { + chain_lock_signature_is_deserializable, + found_valid_locally, + found_valid_by_core, + core_is_synced, + } = match verification_result { + Ok(verification_result) => verification_result, + Err(Error::Execution(e)) => { + // This will happen only if an internal version error + return Err(Error::Execution(e)); + } + Err(e) => { + // This will happen only if a core rpc error + return Ok(ValidationResult::new_with_error( + AbciError::InvalidChainLock(e.to_string()).into(), + )); + } + }; + + if !chain_lock_signature_is_deserializable { + return Ok(ValidationResult::new_with_error( + AbciError::InvalidChainLock(format!( + "received a chain lock for height {} that has a signature that can not be deserialized {:?}", + block_info.height, core_chain_lock_update, + )) + .into(), + )); + } + + if let Some(found_valid_locally) = found_valid_locally { + // This means we are able to check if the chain lock is valid + if !found_valid_locally { + // The signature was not valid + return Ok(ValidationResult::new_with_error( + AbciError::InvalidChainLock(format!( + "received a chain lock for height {} that we figured out was invalid based on platform state {:?}", + block_info.height, core_chain_lock_update, + )) + .into(), + )); + } + } + + if let Some(found_valid_by_core) = found_valid_by_core { + // This means we asked core if the chain lock was valid + if !found_valid_by_core { + // Core said it wasn't valid + return Ok(ValidationResult::new_with_error( + AbciError::InvalidChainLock(format!( + "received a chain lock for height {} that is invalid based on a core request {:?}", + block_info.height, core_chain_lock_update, + )) + .into(), + )); + } + } + + if let Some(core_is_synced) = core_is_synced { + // Core is just not synced + if !core_is_synced { + // The submission was not accepted by core + return Ok(ValidationResult::new_with_error( + AbciError::ChainLockedBlockNotKnownByCore(format!( + "received a chain lock for height {} that we could not accept because core is not synced {:?}", + block_info.height, core_chain_lock_update, + )) + .into(), + )); + } + } + } + } + + // Update the masternode list and create masternode identities and also update the active quorums + self.update_core_info( + Some(last_committed_platform_state), + &mut block_platform_state, + core_chain_locked_height, + false, + &block_info, + transaction, + platform_version, + )?; + + // Update the validator proposed app version + // It should be called after protocol version upgrade + self.drive + .update_validator_proposed_app_version( + proposer_pro_tx_hash, + proposed_app_version as u32, + Some(transaction), + &platform_version.drive, + ) + .map_err(|e| { + Error::Execution(ExecutionError::UpdateValidatorProposedAppVersionError(e)) + })?; // This is a system error + + // Rebroadcast expired withdrawals if they exist + // We do that before we mark withdrawals as expired + // to rebroadcast them on the next block but not the same + // one + // TODO: It must be also only on core height change + self.rebroadcast_expired_withdrawal_documents( + &block_info, + last_committed_platform_state, + transaction, + platform_version, + )?; + + // Mark all previously broadcasted and chainlocked withdrawals as complete + // only when we are on a new core height + if block_state_info.core_chain_locked_height() != last_block_core_height { + self.update_broadcasted_withdrawal_statuses( + &block_info, + transaction, + platform_version, + )?; + } + + // Preparing withdrawal transactions for signing and broadcasting + // To process withdrawals we need to dequeue untiled transactions from the withdrawal transactions queue + // Untiled transactions then converted to unsigned transactions, appending current block information + // required for signature verification (core height and quorum hash) + // Then we save unsigned transaction bytes to block execution context + // to be signed (on extend_vote), verified (on verify_vote) and broadcasted (on finalize_block) + // Also, the dequeued untiled transaction added to the broadcasted transaction queue to for further + // resigning in case of failures. + let unsigned_withdrawal_transaction_bytes = self + .dequeue_and_build_unsigned_withdrawal_transactions( + validator_set_quorum_hash, + &block_info, + Some(transaction), + platform_version, + )?; + + // Run all dao platform events, such as vote tallying and distribution of contested documents + // This must be done before state transition processing + // Otherwise we would expect a proof after a successful vote that has since been cleaned up. + self.run_dao_platform_events( + &block_info, + last_committed_platform_state, + &block_platform_state, + Some(transaction), + platform_version, + )?; + + // Process transactions + let state_transitions_result = self.process_raw_state_transitions( + raw_state_transitions, + &block_platform_state, + &block_info, + transaction, + platform_version, + known_from_us, + timer, + )?; + + // Store the address balances to recent block storage + self.store_address_balances_to_recent_block_storage( + &state_transitions_result.address_balances_updated, + &block_info, + transaction, + platform_version, + )?; + + // Clean up expired compacted address balance entries + self.cleanup_recent_block_storage_address_balances( + &block_info, + transaction, + platform_version, + )?; + + // Record shielded pool anchor if the commitment tree changed this block. + // This stores block_height → anchor_bytes so shielded transactions can + // reference a recent anchor for spend authorization. + self.record_shielded_pool_anchor_if_changed( + block_proposal.height, + transaction, + platform_version, + )?; + + // Prune anchors older than the configured retention depth + self.prune_shielded_pool_anchors(block_proposal.height, transaction, platform_version)?; + + // Pool withdrawals into transactions queue + + // Takes queued withdrawals, creates untiled withdrawal transaction payload, saves them to queue + // Corresponding withdrawal documents are changed from queued to pooled + self.pool_withdrawals_into_transactions_queue( + &block_info, + last_committed_platform_state, + Some(transaction), + platform_version, + )?; + + // Cleans up the expired locks for withdrawal amounts + // to update daily withdrawal limit + // This is for example when we make a withdrawal for 30 Dash + // But we can only withdraw 1000 Dash a day + // after the withdrawal we should only be able to withdraw 970 Dash + // But 24 hours later that locked 30 comes back + self.clean_up_expired_locks_of_withdrawal_amounts( + &block_info, + transaction, + platform_version, + )?; + + // Create a new block execution context + + let mut block_execution_context: BlockExecutionContext = + block_execution_context::v0::BlockExecutionContextV0 { + block_state_info: block_state_info.into(), + epoch_info, + unsigned_withdrawal_transactions: unsigned_withdrawal_transaction_bytes, + block_address_balance_changes: std::collections::BTreeMap::new(), + block_platform_state, + proposer_results: None, + } + .into(); + + // while we have the state transitions executed, we now need to process the block fees + let block_fees_v0: BlockFeesV0 = state_transitions_result.aggregated_fees().clone().into(); + + // Process fees + let processed_block_fees = self.process_block_fees_and_validate_sum_trees( + &block_execution_context, + block_fees_v0.into(), + transaction, + platform_version, + )?; + + tracing::debug!(block_fees = ?processed_block_fees, "block fees are processed"); + + // Record the credits this block minted into Platform (asset locks funding state + // transitions, epoch Core rewards) as a credit inflow: the daily withdrawal limit adds + // inflows younger than its day-old base to the daily maximum, so it limits net outflow. + // A system event, so nobody pays fees for the write. + self.record_credit_inflows_for_withdrawals( + state_transitions_result + .credit_mints() + .saturating_add(processed_block_fees.credit_mints), + &block_info, + transaction, + platform_version, + )?; + + // Record the total credits in Platform if this block changed it: the daily withdrawal + // limit is a share of the total credits Platform held a day ago, read from this history. + // This runs after fees and epoch rewards, the last things in a block that can move the + // total, and before the app hash so the entry is part of this block's state. + self.record_total_credits_history_for_withdrawals( + &block_info, + transaction, + platform_version, + )?; + + // Unlike v0, the validator set update happens BEFORE the root hash is computed. + // It only mutates the in-memory block platform state (the rotated + // next_validator_set_quorum_hash) and never touches grovedb, so the rotation + // outcome and the root hash are unaffected by the move; it must come first so + // the reduced platform state written below carries the post-rotation state. + let validator_set_update = self.validator_set_update( + block_proposal.proposer_pro_tx_hash, + last_committed_platform_state, + &mut block_execution_context, + platform_version, + )?; + + // Write the reduced platform state into the replicated grovedb state, immediately + // before the root hash so it is covered by this block's app hash. A state-synced + // node reads it back to reconstruct the full platform state, which otherwise only + // exists in non-replicated aux storage. The app hash, block id hash and signature + // of this block are unknown at this point and are stored as `None`. + let reduced_platform_state = block_execution_context + .block_platform_state() + .to_reduced_platform_state( + ReducedBlockInfoV0 { + basic_info: block_info, + app_hash: None, + quorum_hash: validator_set_quorum_hash.into(), + block_id_hash: None, + proposer_pro_tx_hash: proposer_pro_tx_hash.into(), + signature: None, + round: block_proposal.round, + }, + core_chain_locked_height, + ); + + self.store_reduced_platform_state( + &reduced_platform_state, + Some(transaction), + platform_version, + )?; + + let root_hash = self + .drive + .grove + .root_hash(Some(transaction), &platform_version.drive.grove_version) + .unwrap() + .map_err(|e| Error::Drive(drive::error::Error::from(e)))?; //GroveDb errors are system errors + + block_execution_context + .block_state_info_mut() + .set_app_hash(Some(root_hash)); + + if tracing::enabled!(tracing::Level::TRACE) { + tracing::trace!( + method = "run_block_proposal_v1", + app_hash = hex::encode(root_hash), + block_hash = hex::encode(block_proposal.block_hash.unwrap_or_default()), + platform_state_fingerprint = hex::encode( + block_execution_context + .block_platform_state() + .fingerprint()? + ), + "Block proposal executed successfully", + ); + } + + Ok(ValidationResult::new_with_data( + block_execution_outcome::v0::BlockExecutionOutcome { + app_hash: root_hash, + state_transitions_result, + validator_set_update, + platform_version, + block_execution_context, + }, + )) + } +} diff --git a/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs index 62f1cdfab8c..96cd4e29e79 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs @@ -971,5 +971,132 @@ mod tests { "wrap-around should not trigger when last block was on different quorum" ); } + + /// run_block_proposal v1 (protocol v15) moves `validator_set_update` from AFTER + /// the root-hash computation (its v0 position) to BEFORE it, so the reduced + /// platform state written into the replicated state can carry the post-rotation + /// next validator set. The only observable differences between the two call + /// sites are (a) `block_state_info.app_hash` being set and (b) grovedb having + /// received additional writes in between. Rotation reads neither, and this test + /// proves it: for rotation-triggering and non-triggering scenarios alike, the + /// rotation outcome (returned update and resulting next validator set quorum + /// hash) is identical whether or not the app hash was set and grovedb was + /// written to before the call. + #[test] + fn v2_rotation_outcome_is_independent_of_root_hash_ordering() { + use crate::execution::types::block_execution_context::v0::BlockExecutionContextV0MutableGetters; + use crate::execution::types::block_state_info::v0::BlockStateInfoV0Setters; + use dpp::reduced_platform_state::v0::ReducedBlockInfoV0; + + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let platform_version = PlatformVersion::latest(); + + let mut rng = StdRng::seed_from_u64(57); + let qh1 = quorum_hash_from_seed(1); + let qh2 = quorum_hash_from_seed(2); + let vs1 = make_validator_set(qh1, &[10, 20, 30], &mut rng); + let vs2 = make_validator_set(qh2, &[40, 50, 60], &mut rng); + + let mut validator_sets = IndexMap::new(); + validator_sets.insert(qh1, vs1); + validator_sets.insert(qh2, vs2); + + // Scenarios: (proposer seed, last committed proposer seed, description) + // - proposer 20 after 10: mid-quorum, no rotation + // - proposer 30 after 20: last member, rotation to qh2 + // - proposer 10 after 20: wrap-around, rotation to qh2 + let scenarios: [(u8, u8, &str); 3] = [ + (20, 10, "no rotation"), + (30, 20, "rotation on last member"), + (10, 20, "rotation on wrap-around"), + ]; + + for (proposer_seed, last_proposer_seed, description) in scenarios { + let mut platform_state = platform.state.load().as_ref().clone(); + platform_state.set_current_validator_set_quorum_hash(qh1); + platform_state.set_validator_sets(validator_sets.clone()); + let mut last_proposer = [0u8; 32]; + last_proposer[31] = last_proposer_seed; + platform_state.set_last_committed_block_info(Some(make_extended_block_info( + *qh1.as_byte_array(), + last_proposer, + 5, + ))); + + let mut proposer = [0u8; 32]; + proposer[31] = proposer_seed; + + // v1 ordering: rotation runs before the root hash exists and before any + // reduced-state write. + let mut context_before_root_hash = + make_block_execution_context(platform_state.clone()); + let update_before = platform + .validator_set_update_v2( + proposer, + &platform_state, + &mut context_before_root_hash, + ) + .expect("should succeed before root hash"); + + // v0 ordering: by the time rotation runs, the app hash has been computed + // and set, and grovedb has received the block's writes (simulated here by + // a committed reduced-state write). + let reduced_platform_state = platform_state.to_reduced_platform_state( + ReducedBlockInfoV0 { + basic_info: BlockInfo::default(), + app_hash: None, + quorum_hash: (*qh1.as_byte_array()).into(), + block_id_hash: None, + proposer_pro_tx_hash: proposer.into(), + signature: None, + round: 0, + }, + 1, + ); + platform + .store_reduced_platform_state(&reduced_platform_state, None, platform_version) + .expect("should store reduced platform state"); + let mut context_after_root_hash = + make_block_execution_context(platform_state.clone()); + context_after_root_hash + .block_state_info_mut() + .set_app_hash(Some([9u8; 32])); + let update_after = platform + .validator_set_update_v2( + proposer, + &platform_state, + &mut context_after_root_hash, + ) + .expect("should succeed after root hash"); + + assert_eq!( + update_before, update_after, + "validator set update must not depend on call ordering ({})", + description + ); + assert_eq!( + context_before_root_hash + .block_platform_state() + .next_validator_set_quorum_hash(), + context_after_root_hash + .block_platform_state() + .next_validator_set_quorum_hash(), + "next validator set quorum hash must not depend on call ordering ({})", + description + ); + assert_eq!( + context_before_root_hash + .block_platform_state() + .current_validator_set_quorum_hash(), + context_after_root_hash + .block_platform_state() + .current_validator_set_quorum_hash(), + "current validator set quorum hash must not depend on call ordering ({})", + description + ); + } + } } } diff --git a/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/mod.rs b/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/mod.rs new file mode 100644 index 00000000000..c7d52ae3e10 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/mod.rs @@ -0,0 +1,34 @@ +mod v0; + +use crate::error::execution::ExecutionError; +use crate::error::Error; +use crate::platform_types::platform::Platform; +use dpp::reduced_platform_state::ReducedPlatformState; +use dpp::version::PlatformVersion; +use drive::query::TransactionArg; + +impl Platform { + /// Fetch the reduced platform state from the replicated grovedb state. + /// + /// Returns `Ok(None)` when the reduced state is absent (a snapshot taken before the + /// protocol version that introduced it). + pub fn fetch_reduced_platform_state( + &self, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result, Error> { + match platform_version + .drive_abci + .methods + .platform_state_storage + .fetch_reduced_platform_state + { + 0 => self.fetch_reduced_platform_state_v0(transaction, platform_version), + version => Err(Error::Execution(ExecutionError::UnknownVersionMismatch { + method: "fetch_reduced_platform_state".to_string(), + known_versions: vec![0], + received: version, + })), + } + } +} diff --git a/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/v0/mod.rs b/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/v0/mod.rs new file mode 100644 index 00000000000..29ac4392488 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/v0/mod.rs @@ -0,0 +1,23 @@ +use crate::error::Error; +use crate::platform_types::platform::Platform; +use dpp::reduced_platform_state::ReducedPlatformState; +use dpp::serialization::PlatformDeserializableFromVersionedStructure; +use dpp::version::PlatformVersion; +use drive::query::TransactionArg; + +impl Platform { + pub(super) fn fetch_reduced_platform_state_v0( + &self, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result, Error> { + self.drive + .fetch_reduced_platform_state_bytes(transaction, platform_version) + .map_err(Error::Drive)? + .map(|bytes| { + ReducedPlatformState::versioned_deserialize(&bytes, platform_version) + .map_err(Error::Protocol) + }) + .transpose() + } +} diff --git a/packages/rs-drive-abci/src/execution/storage/mod.rs b/packages/rs-drive-abci/src/execution/storage/mod.rs index 92c2b2417dc..017babf8c28 100644 --- a/packages/rs-drive-abci/src/execution/storage/mod.rs +++ b/packages/rs-drive-abci/src/execution/storage/mod.rs @@ -1,2 +1,4 @@ pub mod fetch_platform_state; +mod fetch_reduced_platform_state; mod store_platform_state; +mod store_reduced_platform_state; diff --git a/packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/mod.rs b/packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/mod.rs new file mode 100644 index 00000000000..5b037ca8d39 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/mod.rs @@ -0,0 +1,32 @@ +mod v0; + +use crate::error::execution::ExecutionError; +use crate::error::Error; +use crate::platform_types::platform::Platform; +use dpp::reduced_platform_state::ReducedPlatformState; +use dpp::version::PlatformVersion; +use drive::query::TransactionArg; + +impl Platform { + /// Store the reduced platform state in the replicated grovedb state + pub fn store_reduced_platform_state( + &self, + state: &ReducedPlatformState, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + match platform_version + .drive_abci + .methods + .platform_state_storage + .store_reduced_platform_state + { + 0 => self.store_reduced_platform_state_v0(state, transaction, platform_version), + version => Err(Error::Execution(ExecutionError::UnknownVersionMismatch { + method: "store_reduced_platform_state".to_string(), + known_versions: vec![0], + received: version, + })), + } + } +} diff --git a/packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/v0/mod.rs b/packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/v0/mod.rs new file mode 100644 index 00000000000..aa4db11a82e --- /dev/null +++ b/packages/rs-drive-abci/src/execution/storage/store_reduced_platform_state/v0/mod.rs @@ -0,0 +1,23 @@ +use crate::error::Error; +use crate::platform_types::platform::Platform; +use dpp::reduced_platform_state::ReducedPlatformState; +use dpp::serialization::PlatformSerializable; +use dpp::version::PlatformVersion; +use drive::query::TransactionArg; + +impl Platform { + pub(super) fn store_reduced_platform_state_v0( + &self, + state: &ReducedPlatformState, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + self.drive + .store_reduced_platform_state_bytes( + &state.serialize_to_bytes()?, + transaction, + platform_version, + ) + .map_err(Error::Drive) + } +} diff --git a/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs b/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs index 81f438fe51a..2562a170b33 100644 --- a/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs @@ -26,6 +26,8 @@ use dpp::block::block_info::BlockInfo; use dpp::dashcore::hashes::Hash; use dpp::dashcore_rpc::json::MasternodeListItem; use dpp::fee::default_costs::CachedEpochIndexFeeVersions; +use dpp::reduced_platform_state::v0::{ReducedBlockInfoV0, ReducedPlatformStateV0}; +use dpp::reduced_platform_state::ReducedPlatformState; use dpp::util::hash::hash_double; use std::collections::BTreeMap; use std::fmt::{Debug, Formatter}; @@ -122,6 +124,44 @@ impl PlatformState { pub fn fingerprint(&self) -> Result<[u8; 32], Error> { Ok(hash_double(self.serialize_to_bytes()?)) } + + /// Builds the reduced platform state that is written into the replicated grovedb + /// state each block so a state-synced node can reconstruct the full platform state. + /// + /// `last_committed_block_info` describes the block currently being processed (it + /// becomes the last committed block once the block finalizes); fields that are not + /// known during proposal processing (app hash, block id hash, signature) are `None`. + /// `quorum_positions` records the order of the validator sets, which is not + /// otherwise recoverable from Core RPC during reconstruction. + pub fn to_reduced_platform_state( + &self, + last_committed_block_info: ReducedBlockInfoV0, + proposed_core_chain_locked_height: u32, + ) -> ReducedPlatformState { + ReducedPlatformState::V0(ReducedPlatformStateV0 { + last_committed_block_info: Some(last_committed_block_info), + current_protocol_version_in_consensus: self.current_protocol_version_in_consensus, + next_epoch_protocol_version: self.next_epoch_protocol_version, + current_validator_set_quorum_hash: self + .current_validator_set_quorum_hash + .to_byte_array() + .into(), + next_validator_set_quorum_hash: self + .next_validator_set_quorum_hash + .map(|quorum_hash| quorum_hash.to_byte_array().into()), + previous_fee_versions: self + .previous_fee_versions + .iter() + .map(|(epoch_index, fee_version)| (*epoch_index, fee_version.fee_version_number)) + .collect(), + quorum_positions: self + .validator_sets + .keys() + .map(|quorum_hash| quorum_hash.to_byte_array().into()) + .collect(), + proposed_core_chain_locked_height, + }) + } /// The default state at init chain pub fn default_with_protocol_versions( current_protocol_version_in_consensus: ProtocolVersion, From 2e04b4f2bf5a024ed3c9dca821f6f22e97e647e6 Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 29 Aug 2026 01:05:02 +0200 Subject: [PATCH 05/34] feat(drive-abci): write initial reduced platform state on transition to v15 transition_to_version_15 stores the reduced platform state built from the last committed platform state under Misc/reduced_saved_state during the v15 activation block, so the key exists in the replicated state from the fork block onward and every snapshot taken at or after activation is restorable. run_block_proposal v1 overwrites it later in the same block with the state of the block being processed. Co-Authored-By: Claude Fable 5 --- .../engine/run_block_proposal/v1/mod.rs | 4 +- .../block_end/validator_set_update/mod.rs | 4 +- .../v0/mod.rs | 95 +++++++++++++++++++ .../src/platform_types/platform_state/mod.rs | 4 +- 4 files changed, 101 insertions(+), 6 deletions(-) diff --git a/packages/rs-drive-abci/src/execution/engine/run_block_proposal/v1/mod.rs b/packages/rs-drive-abci/src/execution/engine/run_block_proposal/v1/mod.rs index 95a4d03430d..cdaf3180ae3 100644 --- a/packages/rs-drive-abci/src/execution/engine/run_block_proposal/v1/mod.rs +++ b/packages/rs-drive-abci/src/execution/engine/run_block_proposal/v1/mod.rs @@ -454,7 +454,7 @@ where let reduced_platform_state = block_execution_context .block_platform_state() .to_reduced_platform_state( - ReducedBlockInfoV0 { + Some(ReducedBlockInfoV0 { basic_info: block_info, app_hash: None, quorum_hash: validator_set_quorum_hash.into(), @@ -462,7 +462,7 @@ where proposer_pro_tx_hash: proposer_pro_tx_hash.into(), signature: None, round: block_proposal.round, - }, + }), core_chain_locked_height, ); diff --git a/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs index 96cd4e29e79..fcb2237a59a 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs @@ -1044,7 +1044,7 @@ mod tests { // and set, and grovedb has received the block's writes (simulated here by // a committed reduced-state write). let reduced_platform_state = platform_state.to_reduced_platform_state( - ReducedBlockInfoV0 { + Some(ReducedBlockInfoV0 { basic_info: BlockInfo::default(), app_hash: None, quorum_hash: (*qh1.as_byte_array()).into(), @@ -1052,7 +1052,7 @@ mod tests { proposer_pro_tx_hash: proposer.into(), signature: None, round: 0, - }, + }), 1, ); platform diff --git a/packages/rs-drive-abci/src/execution/platform_events/protocol_upgrade/perform_events_on_first_block_of_protocol_change/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/protocol_upgrade/perform_events_on_first_block_of_protocol_change/v0/mod.rs index 4f66d59dce8..ab03451de62 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/protocol_upgrade/perform_events_on_first_block_of_protocol_change/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/protocol_upgrade/perform_events_on_first_block_of_protocol_change/v0/mod.rs @@ -4,10 +4,12 @@ use crate::platform_types::platform::Platform; use crate::platform_types::platform_state::PlatformState; use crate::platform_types::platform_state::PlatformStateV0Methods; use dpp::block::block_info::BlockInfo; +use dpp::block::extended_block_info::v0::ExtendedBlockInfoV0Getters; use dpp::dashcore::hashes::Hash; use dpp::data_contracts::SystemDataContract; use dpp::fee::Credits; use dpp::platform_value::Identifier; +use dpp::reduced_platform_state::v0::ReducedBlockInfoV0; use dpp::serialization::PlatformDeserializable; use dpp::system_data_contracts::load_system_data_contract; use dpp::version::PlatformVersion; @@ -119,6 +121,10 @@ impl Platform { self.transition_to_version_14(block_info, transaction, platform_version)?; } + if previous_protocol_version < 15 && platform_version.protocol_version >= 15 { + self.transition_to_version_15(platform_state, transaction, platform_version)?; + } + Ok(()) } @@ -738,6 +744,44 @@ impl Platform { Ok(()) } + + /// When transitioning to version 15 we write the initial reduced platform state (built + /// from the last committed platform state) under `Misc/reduced_saved_state`, so the key + /// exists in the replicated state from the fork block onward. `run_block_proposal` v1 + /// overwrites it later in this same block with the state of the block being processed; + /// this initial write guarantees no v15 block ever commits without the key, which is + /// what makes every snapshot taken at or after activation restorable via state sync. + fn transition_to_version_15( + &self, + platform_state: &PlatformState, + transaction: &Transaction, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + let last_committed_block_info = + platform_state + .last_committed_block_info() + .as_ref() + .map(|extended_block_info| ReducedBlockInfoV0 { + basic_info: *extended_block_info.basic_info(), + app_hash: Some((*extended_block_info.app_hash()).into()), + quorum_hash: (*extended_block_info.quorum_hash()).into(), + block_id_hash: Some((*extended_block_info.block_id_hash()).into()), + proposer_pro_tx_hash: (*extended_block_info.proposer_pro_tx_hash()).into(), + signature: Some(*extended_block_info.signature()), + round: extended_block_info.round(), + }); + + let reduced_platform_state = platform_state.to_reduced_platform_state( + last_committed_block_info, + platform_state.last_committed_core_height(), + ); + + self.store_reduced_platform_state( + &reduced_platform_state, + Some(transaction), + platform_version, + ) + } } #[cfg(test)] @@ -2642,4 +2686,55 @@ mod tests { diffs.join("\n"), ); } + + #[test] + fn test_transition_to_version_15_writes_initial_reduced_platform_state() { + use dpp::reduced_platform_state::ReducedPlatformState; + + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let platform_version = PlatformVersion::latest(); + + let transaction = platform.drive.grove.start_transaction(); + + let platform_state = platform.state.load(); + + // Before the transition, the replicated state must not carry the reduced state key. + let pre_transition = platform + .fetch_reduced_platform_state(Some(&transaction), platform_version) + .expect("fetching an absent reduced platform state should not error"); + assert!( + pre_transition.is_none(), + "reduced platform state must not exist before transition_to_version_15" + ); + + let result = + platform.transition_to_version_15(&platform_state, &transaction, platform_version); + assert!(result.is_ok(), "transition failed: {:?}", result.err()); + + let reduced = platform + .fetch_reduced_platform_state(Some(&transaction), platform_version) + .expect("expected to fetch reduced platform state") + .expect("reduced platform state must exist after transition_to_version_15"); + + let ReducedPlatformState::V0(reduced) = reduced; + assert_eq!( + reduced.current_protocol_version_in_consensus, + platform_state.current_protocol_version_in_consensus() + ); + assert_eq!( + reduced.next_epoch_protocol_version, + platform_state.next_epoch_protocol_version() + ); + assert_eq!( + reduced.quorum_positions.len(), + platform_state.validator_sets().len(), + "quorum positions must mirror the validator set order" + ); + assert_eq!( + reduced.proposed_core_chain_locked_height, + platform_state.last_committed_core_height() + ); + } } diff --git a/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs b/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs index 2562a170b33..b6ea5973ea2 100644 --- a/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs @@ -135,11 +135,11 @@ impl PlatformState { /// otherwise recoverable from Core RPC during reconstruction. pub fn to_reduced_platform_state( &self, - last_committed_block_info: ReducedBlockInfoV0, + last_committed_block_info: Option, proposed_core_chain_locked_height: u32, ) -> ReducedPlatformState { ReducedPlatformState::V0(ReducedPlatformStateV0 { - last_committed_block_info: Some(last_committed_block_info), + last_committed_block_info, current_protocol_version_in_consensus: self.current_protocol_version_in_consensus, next_epoch_protocol_version: self.next_epoch_protocol_version, current_validator_set_quorum_hash: self From 67a080547d26bb7ac62eec15ec918e46477c7232 Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 29 Aug 2026 01:22:20 +0200 Subject: [PATCH 06/34] feat(drive-abci): serve state sync snapshots from the checkpoint registry Adds StateSyncAbciConfig (env contract: SNAPSHOTS_ENABLED, SNAPSHOTS_FREQUENCY_SECONDS, MAX_NUM_SNAPSHOTS, CHECKPOINTS_PATH) which, when enabled, overrides the platform-version-driven checkpoint frequency, retention and directory. list_snapshots and load_snapshot_chunk (on both the tenderdash socket app and the gRPC CheckTx app) serve snapshots directly from drive.checkpoints: only checkpoints containing the reduced platform state are offered (pre-v15 checkpoints are unrestorable), requested wire versions are validated against a single supported-set const, chunk ids are size-capped before decoding (#3773), and served checkpoints are pinned via the existing Arc refcount so pruning cannot delete them mid-transfer. Co-Authored-By: Claude Fable 5 --- packages/rs-drive-abci/.env.local | 7 + packages/rs-drive-abci/.env.mainnet | 7 + packages/rs-drive-abci/.env.testnet | 7 + .../rs-drive-abci/src/abci/app/check_tx.rs | 38 +++- packages/rs-drive-abci/src/abci/app/full.rs | 29 ++- packages/rs-drive-abci/src/abci/app/mod.rs | 8 + packages/rs-drive-abci/src/abci/config.rs | 114 ++++++++++- packages/rs-drive-abci/src/abci/error.rs | 8 + .../src/abci/handler/list_snapshots.rs | 142 ++++++++++++++ .../src/abci/handler/load_snapshot_chunk.rs | 181 ++++++++++++++++++ .../rs-drive-abci/src/abci/handler/mod.rs | 4 + .../create_grovedb_checkpoint/v0/mod.rs | 20 +- .../block_end/should_checkpoint/v0/mod.rs | 20 +- .../block_end/update_checkpoints/v0/mod.rs | 18 +- .../rs-drive-abci/src/platform_types/mod.rs | 2 + .../src/platform_types/snapshot/mod.rs | 104 ++++++++++ packages/rs-drive-abci/src/utils/mod.rs | 1 + .../rs-drive-abci/src/utils/serialization.rs | 23 +++ packages/rs-drive/src/drive/mod.rs | 20 ++ .../rs-drive/src/drive/platform_state/mod.rs | 2 +- 20 files changed, 733 insertions(+), 22 deletions(-) create mode 100644 packages/rs-drive-abci/src/abci/handler/list_snapshots.rs create mode 100644 packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs create mode 100644 packages/rs-drive-abci/src/platform_types/snapshot/mod.rs diff --git a/packages/rs-drive-abci/.env.local b/packages/rs-drive-abci/.env.local index c0e3ac3347a..4eb320b87aa 100644 --- a/packages/rs-drive-abci/.env.local +++ b/packages/rs-drive-abci/.env.local @@ -90,3 +90,10 @@ GROVEDB_VISUALIZER_ENABLED=false GROVEDB_VISUALIZER_ADDRESS=127.0.0.1:8083 NETWORK=regtest + +# ABCI state sync snapshots (serving side; disabled by default) +SNAPSHOTS_ENABLED=false +SNAPSHOTS_FREQUENCY_SECONDS=600 +MAX_NUM_SNAPSHOTS=3 +# CHECKPOINTS_PATH defaults to /checkpoints when unset +#CHECKPOINTS_PATH= diff --git a/packages/rs-drive-abci/.env.mainnet b/packages/rs-drive-abci/.env.mainnet index 65409c1d0a3..214b2b6c001 100644 --- a/packages/rs-drive-abci/.env.mainnet +++ b/packages/rs-drive-abci/.env.mainnet @@ -92,3 +92,10 @@ GROVEDB_VISUALIZER_ADDRESS=127.0.0.1:8083 PROPOSER_TX_PROCESSING_TIME_LIMIT=5000 NETWORK=mainnet + +# ABCI state sync snapshots (serving side; disabled by default) +SNAPSHOTS_ENABLED=false +SNAPSHOTS_FREQUENCY_SECONDS=600 +MAX_NUM_SNAPSHOTS=3 +# CHECKPOINTS_PATH defaults to /checkpoints when unset +#CHECKPOINTS_PATH= diff --git a/packages/rs-drive-abci/.env.testnet b/packages/rs-drive-abci/.env.testnet index 9e85d109c5f..b5a8379edc6 100644 --- a/packages/rs-drive-abci/.env.testnet +++ b/packages/rs-drive-abci/.env.testnet @@ -92,3 +92,10 @@ GROVEDB_VISUALIZER_ADDRESS=127.0.0.1:8083 PROPOSER_TX_PROCESSING_TIME_LIMIT=5000 NETWORK=testnet + +# ABCI state sync snapshots (serving side; disabled by default) +SNAPSHOTS_ENABLED=false +SNAPSHOTS_FREQUENCY_SECONDS=600 +MAX_NUM_SNAPSHOTS=3 +# CHECKPOINTS_PATH defaults to /checkpoints when unset +#CHECKPOINTS_PATH= diff --git a/packages/rs-drive-abci/src/abci/app/check_tx.rs b/packages/rs-drive-abci/src/abci/app/check_tx.rs index 170eb519599..4ed32b2fbed 100644 --- a/packages/rs-drive-abci/src/abci/app/check_tx.rs +++ b/packages/rs-drive-abci/src/abci/app/check_tx.rs @@ -1,7 +1,8 @@ -use crate::abci::app::PlatformApplication; +use crate::abci::app::{PlatformApplication, SnapshotManagerApplication}; use crate::abci::handler; use crate::error::Error; use crate::platform_types::platform::Platform; +use crate::platform_types::snapshot::SnapshotManager; use crate::rpc::core::CoreRPCLike; use crate::utils::spawn_blocking_task_with_name_if_supported; use async_trait::async_trait; @@ -22,6 +23,8 @@ where /// Platform platform: Arc>, core_rpc: Arc, + /// The snapshot manager, pinning checkpoints that are being served to peers + snapshot_manager: SnapshotManager, } impl PlatformApplication for CheckTxAbciApplication @@ -33,13 +36,26 @@ where } } +impl SnapshotManagerApplication for CheckTxAbciApplication +where + C: CoreRPCLike + Send + Sync + 'static, +{ + fn snapshot_manager(&self) -> &SnapshotManager { + &self.snapshot_manager + } +} + impl CheckTxAbciApplication where C: CoreRPCLike + Send + Sync + 'static, { /// Create new ABCI app pub fn new(platform: Arc>, core_rpc: Arc) -> Self { - Self { platform, core_rpc } + Self { + platform, + core_rpc, + snapshot_manager: SnapshotManager::new(), + } } } @@ -92,6 +108,24 @@ where .await .map_err(|error| tonic::Status::internal(format!("check tx panics: {}", error)))? } + + async fn list_snapshots( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + handler::list_snapshots(self, request.into_inner()) + .map(tonic::Response::new) + .map_err(error_into_status) + } + + async fn load_snapshot_chunk( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + handler::load_snapshot_chunk(self, request.into_inner()) + .map(tonic::Response::new) + .map_err(error_into_status) + } } pub fn error_into_status(error: Error) -> tonic::Status { diff --git a/packages/rs-drive-abci/src/abci/app/full.rs b/packages/rs-drive-abci/src/abci/app/full.rs index bd290b87156..539fc07ff29 100644 --- a/packages/rs-drive-abci/src/abci/app/full.rs +++ b/packages/rs-drive-abci/src/abci/app/full.rs @@ -1,10 +1,14 @@ -use crate::abci::app::{BlockExecutionApplication, PlatformApplication, TransactionalApplication}; +use crate::abci::app::{ + BlockExecutionApplication, PlatformApplication, SnapshotManagerApplication, + TransactionalApplication, +}; use crate::abci::handler; use crate::abci::handler::error::error_into_exception; use crate::error::execution::ExecutionError; use crate::error::Error; use crate::execution::types::block_execution_context::BlockExecutionContext; use crate::platform_types::platform::Platform; +use crate::platform_types::snapshot::SnapshotManager; use crate::rpc::core::CoreRPCLike; use dpp::version::PlatformVersion; use drive::grovedb::Transaction; @@ -23,6 +27,8 @@ pub struct FullAbciApplication<'a, C> { pub transaction: RwLock>>, /// The current block execution context pub block_execution_context: RwLock>, + /// The snapshot manager, pinning checkpoints that are being served to peers + pub snapshot_manager: SnapshotManager, } impl<'a, C> FullAbciApplication<'a, C> { @@ -32,6 +38,7 @@ impl<'a, C> FullAbciApplication<'a, C> { platform, transaction: Default::default(), block_execution_context: Default::default(), + snapshot_manager: SnapshotManager::new(), } } } @@ -42,6 +49,12 @@ impl PlatformApplication for FullAbciApplication<'_, C> { } } +impl SnapshotManagerApplication for FullAbciApplication<'_, C> { + fn snapshot_manager(&self) -> &SnapshotManager { + &self.snapshot_manager + } +} + impl BlockExecutionApplication for FullAbciApplication<'_, C> { fn block_execution_context(&self) -> &RwLock> { &self.block_execution_context @@ -241,4 +254,18 @@ where ) -> Result { handler::verify_vote_extension(self, request).map_err(error_into_exception) } + + fn list_snapshots( + &self, + request: proto::RequestListSnapshots, + ) -> Result { + handler::list_snapshots(self, request).map_err(error_into_exception) + } + + fn load_snapshot_chunk( + &self, + request: proto::RequestLoadSnapshotChunk, + ) -> Result { + handler::load_snapshot_chunk(self, request).map_err(error_into_exception) + } } diff --git a/packages/rs-drive-abci/src/abci/app/mod.rs b/packages/rs-drive-abci/src/abci/app/mod.rs index 27d7ef0794e..4410a6d3052 100644 --- a/packages/rs-drive-abci/src/abci/app/mod.rs +++ b/packages/rs-drive-abci/src/abci/app/mod.rs @@ -10,6 +10,7 @@ pub mod execution_result; mod full; use crate::execution::types::block_execution_context::BlockExecutionContext; +use crate::platform_types::snapshot::SnapshotManager; use crate::rpc::core::DefaultCoreRPC; #[cfg(test)] pub(crate) use check_tx::error_into_status; @@ -24,6 +25,13 @@ pub trait PlatformApplication { fn platform(&self) -> &Platform; } +/// ABCI application that serves state sync snapshots +pub trait SnapshotManagerApplication { + /// Returns the snapshot manager, which pins checkpoints that are actively being + /// served so pruning cannot delete them mid-transfer + fn snapshot_manager(&self) -> &SnapshotManager; +} + /// Transactional ABCI application pub trait TransactionalApplication<'a> { /// Creates and keeps a new transaction diff --git a/packages/rs-drive-abci/src/abci/config.rs b/packages/rs-drive-abci/src/abci/config.rs index 7f80ab0e010..da8bda064ab 100644 --- a/packages/rs-drive-abci/src/abci/config.rs +++ b/packages/rs-drive-abci/src/abci/config.rs @@ -1,7 +1,8 @@ //! Configuration of ABCI Application server -use crate::utils::from_opt_str_or_number; +use crate::utils::{from_opt_str_or_number, from_str_or_native}; use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; // We allow changes in the ABCI configuration, but there should be a social process // involved in making this change. @@ -37,6 +38,78 @@ pub struct AbciConfig { /// Maximum time limit (in ms) to process state transitions to prepare proposal #[serde(default, deserialize_with = "from_opt_str_or_number")] pub proposer_tx_processing_time_limit: Option, + + /// State sync snapshot serving configuration + #[serde(flatten)] + pub state_sync: StateSyncAbciConfig, +} + +/// Configuration of ABCI state sync snapshot serving. +/// +/// NOTE: the field names (and thus the environment variable names `SNAPSHOTS_ENABLED`, +/// `SNAPSHOTS_FREQUENCY_SECONDS`, `MAX_NUM_SNAPSHOTS`, `CHECKPOINTS_PATH`) are a contract +/// with dashmate's env generation — do not rename them. +// @append_only +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct StateSyncAbciConfig { + /// Whether snapshots are offered to state-syncing peers. When enabled, the + /// snapshot frequency and retention below override the platform-version-driven + /// checkpoint parameters. + #[serde( + default = "StateSyncAbciConfig::default_snapshots_enabled", + deserialize_with = "from_str_or_native" + )] + pub snapshots_enabled: bool, + + /// How often (in seconds) a snapshot (grovedb checkpoint) is created + #[serde( + default = "StateSyncAbciConfig::default_snapshots_frequency_seconds", + deserialize_with = "from_str_or_native" + )] + pub snapshots_frequency_seconds: u32, + + /// Maximum number of snapshots kept on disk + #[serde( + default = "StateSyncAbciConfig::default_max_num_snapshots", + deserialize_with = "from_str_or_native" + )] + pub max_num_snapshots: usize, + + /// Directory where checkpoints are stored; defaults to `/checkpoints` + #[serde(default)] + pub checkpoints_path: Option, +} + +impl StateSyncAbciConfig { + pub(crate) fn default_snapshots_enabled() -> bool { + false + } + + pub(crate) fn default_snapshots_frequency_seconds() -> u32 { + 600 + } + + pub(crate) fn default_max_num_snapshots() -> usize { + 3 + } + + /// Resolves the checkpoints directory, defaulting to `/checkpoints` + pub fn resolved_checkpoints_path(&self, db_path: &Path) -> PathBuf { + self.checkpoints_path + .clone() + .unwrap_or_else(|| db_path.join("checkpoints")) + } +} + +impl Default for StateSyncAbciConfig { + fn default() -> Self { + Self { + snapshots_enabled: Self::default_snapshots_enabled(), + snapshots_frequency_seconds: Self::default_snapshots_frequency_seconds(), + max_num_snapshots: Self::default_max_num_snapshots(), + checkpoints_path: None, + } + } } impl AbciConfig { @@ -58,6 +131,7 @@ impl Default for AbciConfig { chain_id: "chain_id".to_string(), log: Default::default(), proposer_tx_processing_time_limit: Default::default(), + state_sync: Default::default(), } } } @@ -85,6 +159,42 @@ mod tests { assert_eq!(config.chain_id, "chain_id"); assert!(config.log.is_empty()); assert!(config.proposer_tx_processing_time_limit.is_none()); + assert!(!config.state_sync.snapshots_enabled); + assert_eq!(config.state_sync.snapshots_frequency_seconds, 600); + assert_eq!(config.state_sync.max_num_snapshots, 3); + assert!(config.state_sync.checkpoints_path.is_none()); + } + + #[test] + fn state_sync_config_resolves_default_checkpoints_path_from_db_path() { + let config = StateSyncAbciConfig::default(); + assert_eq!( + config.resolved_checkpoints_path(Path::new("/var/lib/drive/db")), + PathBuf::from("/var/lib/drive/db/checkpoints") + ); + + let config = StateSyncAbciConfig { + checkpoints_path: Some(PathBuf::from("/mnt/checkpoints")), + ..Default::default() + }; + assert_eq!( + config.resolved_checkpoints_path(Path::new("/var/lib/drive/db")), + PathBuf::from("/mnt/checkpoints") + ); + } + + #[test] + fn state_sync_config_deserializes_from_env_style_strings() { + // envy provides every value as a string; the custom deserializers must coerce + let json = r#"{"abci_consensus_bind_address": "tcp://x:1", "snapshots_enabled": "true", "snapshots_frequency_seconds": "120", "max_num_snapshots": "5", "checkpoints_path": "/tmp/checkpoints"}"#; + let config: AbciConfig = serde_json::from_str(json).expect("should deserialize"); + assert!(config.state_sync.snapshots_enabled); + assert_eq!(config.state_sync.snapshots_frequency_seconds, 120); + assert_eq!(config.state_sync.max_num_snapshots, 5); + assert_eq!( + config.state_sync.checkpoints_path, + Some(PathBuf::from("/tmp/checkpoints")) + ); } #[test] @@ -98,6 +208,7 @@ mod tests { chain_id: "test-chain".to_string(), log: Default::default(), proposer_tx_processing_time_limit: None, + state_sync: Default::default(), }; let serialized = serde_json::to_string(&config).expect("should serialize"); @@ -143,6 +254,7 @@ mod tests { chain_id: "clone-test".to_string(), log: Default::default(), proposer_tx_processing_time_limit: Some(1000), + state_sync: Default::default(), }; let cloned = config.clone(); diff --git a/packages/rs-drive-abci/src/abci/error.rs b/packages/rs-drive-abci/src/abci/error.rs index 306e0644956..f8168b2cbac 100644 --- a/packages/rs-drive-abci/src/abci/error.rs +++ b/packages/rs-drive-abci/src/abci/error.rs @@ -54,6 +54,14 @@ pub enum AbciError { #[error("bad commit signature: {0}")] BadCommitSignature(String), + /// Invalid state sync request received from Tenderdash or a peer + #[error("bad request state sync: {0}")] + StateSyncBadRequest(String), + + /// Internal error during state sync + #[error("internal error state sync: {0}")] + StateSyncInternalError(String), + /// The chain lock received was invalid #[error("invalid chain lock: {0}")] InvalidChainLock(String), diff --git a/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs b/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs new file mode 100644 index 00000000000..0b2752685e1 --- /dev/null +++ b/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs @@ -0,0 +1,142 @@ +use crate::abci::app::PlatformApplication; +use crate::abci::AbciError; +use crate::error::Error; +use crate::platform_types::platform_state::PlatformStateV0Methods; +use crate::rpc::core::CoreRPCLike; +use tenderdash_abci::proto::abci as proto; + +/// Lists the state sync snapshots this node can serve. +/// +/// Snapshots are the rocksdb checkpoints Drive already keeps (`drive.checkpoints`). +/// Only checkpoints that contain the reduced platform state are offered: a checkpoint +/// taken before the protocol version that introduced it (v15) cannot be restored, since +/// a state-synced node would have no way to reconstruct its platform state. +pub fn list_snapshots( + app: &A, + _request: proto::RequestListSnapshots, +) -> Result +where + A: PlatformApplication, + C: CoreRPCLike, +{ + tracing::trace!("[state_sync] api list_snapshots called"); + + if !app.platform().config.abci.state_sync.snapshots_enabled { + return Ok(Default::default()); + } + + let platform_state = app.platform().state.load(); + let platform_version = platform_state.current_platform_version()?; + let grove_version = &platform_version.drive.grove_version; + + let checkpoints = app.platform().drive.checkpoints.load(); + + let mut snapshots = Vec::new(); + for (height, checkpoint_info) in checkpoints.iter() { + let checkpoint = &checkpoint_info.checkpoint; + + let restorable = checkpoint + .has_reduced_platform_state(grove_version) + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "list_snapshots unable to inspect checkpoint at height {}: {}", + height, e + )) + })?; + if !restorable { + continue; + } + + let root_hash = checkpoint + .grove_db + .root_hash(None, grove_version) + .unwrap() + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "list_snapshots unable to get root hash of checkpoint at height {}: {}", + height, e + )) + })?; + + snapshots.push(proto::Snapshot { + height: *height, + version: platform_version.drive_abci.state_sync.protocol_version as u32, + hash: root_hash.to_vec(), + metadata: Vec::new(), + }); + } + + Ok(proto::ResponseListSnapshots { snapshots }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::abci::app::FullAbciApplication; + use crate::config::PlatformConfig; + use crate::test::helpers::fast_forward_to_block::fast_forward_to_block; + use crate::test::helpers::setup::TestPlatformBuilder; + use dpp::version::PlatformVersion; + + fn config_with_snapshots_enabled() -> PlatformConfig { + let mut config = PlatformConfig::default_local(); + config.abci.state_sync.snapshots_enabled = true; + config + } + + #[test] + fn list_snapshots_returns_nothing_when_serving_is_disabled() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let app = FullAbciApplication::new(&platform); + + let response = list_snapshots(&app, Default::default()).expect("should list snapshots"); + assert!(response.snapshots.is_empty()); + } + + #[test] + fn list_snapshots_serves_only_checkpoints_with_reduced_platform_state() { + let platform = TestPlatformBuilder::new() + .with_config(config_with_snapshots_enabled()) + .build_with_mock_rpc() + .set_genesis_state(); + let platform_version = PlatformVersion::latest(); + let app = FullAbciApplication::new(&platform); + + // A checkpoint taken before the reduced platform state exists (pre-v15 + // activation) is unrestorable and must not be offered. + fast_forward_to_block(&platform, 1_000_000, 10, 42, 0, false); + platform + .create_grovedb_checkpoint(platform_version) + .expect("should create checkpoint"); + + let response = list_snapshots(&app, Default::default()).expect("should list snapshots"); + assert!( + response.snapshots.is_empty(), + "checkpoints without the reduced platform state must be filtered out" + ); + + // Once the reduced platform state is in the replicated state, new checkpoints + // are restorable and must be offered. + let reduced_platform_state = platform.state.load().to_reduced_platform_state(None, 42); + platform + .store_reduced_platform_state(&reduced_platform_state, None, platform_version) + .expect("should store reduced platform state"); + + fast_forward_to_block(&platform, 2_000_000, 20, 43, 0, false); + platform + .create_grovedb_checkpoint(platform_version) + .expect("should create checkpoint"); + + let response = list_snapshots(&app, Default::default()).expect("should list snapshots"); + assert_eq!(response.snapshots.len(), 1); + let snapshot = &response.snapshots[0]; + assert_eq!(snapshot.height, 20); + assert_eq!( + snapshot.version, + platform_version.drive_abci.state_sync.protocol_version as u32 + ); + assert_eq!(snapshot.hash.len(), 32); + } +} diff --git a/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs b/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs new file mode 100644 index 00000000000..ed6c1652c3a --- /dev/null +++ b/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs @@ -0,0 +1,181 @@ +use crate::abci::app::{PlatformApplication, SnapshotManagerApplication}; +use crate::abci::AbciError; +use crate::error::Error; +use crate::platform_types::platform_state::PlatformStateV0Methods; +use crate::platform_types::snapshot::{ + MAX_STATE_SYNC_CHUNK_ID_SIZE, SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS, +}; +use crate::rpc::core::CoreRPCLike; +use std::sync::Arc; +use tenderdash_abci::proto::abci as proto; + +/// Serves one chunk of a state sync snapshot from the checkpoint registry. +/// +/// The served checkpoint is pinned in the snapshot manager so checkpoint pruning cannot +/// delete it from disk while a peer is still downloading it. +pub fn load_snapshot_chunk( + app: &A, + request: proto::RequestLoadSnapshotChunk, +) -> Result +where + A: PlatformApplication + SnapshotManagerApplication, + C: CoreRPCLike, +{ + tracing::trace!( + height = request.height, + version = request.version, + chunk_id = hex::encode(&request.chunk_id), + "[state_sync] api load_snapshot_chunk", + ); + + if !app.platform().config.abci.state_sync.snapshots_enabled { + return Err(AbciError::StateSyncBadRequest( + "load_snapshot_chunk snapshot serving is disabled".to_string(), + ) + .into()); + } + + // Cap peer-supplied sizes before anything decodes them (issue #3773) + if request.chunk_id.len() > MAX_STATE_SYNC_CHUNK_ID_SIZE { + return Err(AbciError::StateSyncBadRequest(format!( + "load_snapshot_chunk chunk id of {} bytes exceeds the {} byte limit", + request.chunk_id.len(), + MAX_STATE_SYNC_CHUNK_ID_SIZE + )) + .into()); + } + + let wire_version = u16::try_from(request.version) + .ok() + .filter(|version| SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS.contains(version)); + let Some(wire_version) = wire_version else { + return Err(AbciError::StateSyncBadRequest(format!( + "load_snapshot_chunk unsupported state sync protocol version {}, supported: {:?}", + request.version, SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS + )) + .into()); + }; + + let platform_state = app.platform().state.load(); + let platform_version = platform_state.current_platform_version()?; + let grove_version = &platform_version.drive.grove_version; + + // Resolve the checkpoint: from the registry, or — if pruning already dropped it — + // from the pins of transfers already in flight. + let checkpoint = app + .platform() + .drive + .checkpoints + .load() + .get(&request.height) + .map(|checkpoint_info| Arc::clone(&checkpoint_info.checkpoint)) + .or_else(|| app.snapshot_manager().pinned_checkpoint(request.height)) + .ok_or_else(|| { + AbciError::StateSyncBadRequest(format!( + "load_snapshot_chunk no snapshot at height {}", + request.height + )) + })?; + + // Pin (or refresh the pin of) the checkpoint for the duration of the transfer + app.snapshot_manager() + .pin_for_serving(request.height, Arc::clone(&checkpoint)); + + let chunk = checkpoint + .grove_db + .fetch_chunk(&request.chunk_id, None, wire_version, grove_version) + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "load_snapshot_chunk unable to fetch chunk: {}", + e + )) + })?; + + Ok(proto::ResponseLoadSnapshotChunk { chunk }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::abci::app::FullAbciApplication; + use crate::config::PlatformConfig; + use crate::test::helpers::fast_forward_to_block::fast_forward_to_block; + use crate::test::helpers::setup::TestPlatformBuilder; + use dpp::version::PlatformVersion; + + #[test] + fn load_snapshot_chunk_serves_root_chunk_and_rejects_bad_requests() { + let mut config = PlatformConfig::default_local(); + config.abci.state_sync.snapshots_enabled = true; + let platform = TestPlatformBuilder::new() + .with_config(config) + .build_with_mock_rpc() + .set_genesis_state(); + let platform_version = PlatformVersion::latest(); + let app = FullAbciApplication::new(&platform); + + let reduced_platform_state = platform.state.load().to_reduced_platform_state(None, 42); + platform + .store_reduced_platform_state(&reduced_platform_state, None, platform_version) + .expect("should store reduced platform state"); + fast_forward_to_block(&platform, 1_000_000, 10, 42, 0, false); + platform + .create_grovedb_checkpoint(platform_version) + .expect("should create checkpoint"); + + let root_hash = platform + .drive + .grove + .root_hash(None, &platform_version.drive.grove_version) + .unwrap() + .expect("should get root hash"); + + // The root chunk (chunk id == app hash) must be served + let response = load_snapshot_chunk( + &app, + proto::RequestLoadSnapshotChunk { + height: 10, + version: 1, + chunk_id: root_hash.to_vec(), + }, + ) + .expect("should load root chunk"); + assert!(!response.chunk.is_empty()); + + // The served checkpoint must now be pinned against pruning + assert!(app.snapshot_manager.pinned_checkpoint(10).is_some()); + + // Unknown height is rejected + assert!(load_snapshot_chunk( + &app, + proto::RequestLoadSnapshotChunk { + height: 999, + version: 1, + chunk_id: root_hash.to_vec(), + }, + ) + .is_err()); + + // Unsupported wire version is rejected + assert!(load_snapshot_chunk( + &app, + proto::RequestLoadSnapshotChunk { + height: 10, + version: 2, + chunk_id: root_hash.to_vec(), + }, + ) + .is_err()); + + // Oversized chunk id is rejected before any decoding + assert!(load_snapshot_chunk( + &app, + proto::RequestLoadSnapshotChunk { + height: 10, + version: 1, + chunk_id: vec![0u8; MAX_STATE_SYNC_CHUNK_ID_SIZE + 1], + }, + ) + .is_err()); + } +} diff --git a/packages/rs-drive-abci/src/abci/handler/mod.rs b/packages/rs-drive-abci/src/abci/handler/mod.rs index 8acd0737ebe..6b74a74760b 100644 --- a/packages/rs-drive-abci/src/abci/handler/mod.rs +++ b/packages/rs-drive-abci/src/abci/handler/mod.rs @@ -42,6 +42,8 @@ mod extend_vote; mod finalize_block; mod info; mod init_chain; +mod list_snapshots; +mod load_snapshot_chunk; mod prepare_proposal; mod process_proposal; mod verify_vote_extension; @@ -52,6 +54,8 @@ pub use extend_vote::extend_vote; pub use finalize_block::finalize_block; pub use info::info; pub use init_chain::init_chain; +pub use list_snapshots::list_snapshots; +pub use load_snapshot_chunk::load_snapshot_chunk; pub use prepare_proposal::prepare_proposal; pub use process_proposal::process_proposal; pub use verify_vote_extension::verify_vote_extension; diff --git a/packages/rs-drive-abci/src/execution/platform_events/block_end/create_grovedb_checkpoint/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/block_end/create_grovedb_checkpoint/v0/mod.rs index c6b1d43549e..632c68e4e58 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/block_end/create_grovedb_checkpoint/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/block_end/create_grovedb_checkpoint/v0/mod.rs @@ -37,13 +37,19 @@ where let block_height = platform_state.last_committed_block_height(); let block_time = platform_state.last_committed_block_time_ms().unwrap_or(0); - let keep_n = platform_version.drive_abci.checkpoints.num_checkpoints as usize; - - // Build the checkpoint path: db_path/checkpoints/ - let checkpoint_path = self - .config - .db_path - .join("checkpoints") + // When snapshot serving is enabled, the operator-provided state sync + // configuration overrides the platform-version-driven checkpoint retention. + let state_sync_config = &self.config.abci.state_sync; + let keep_n = if state_sync_config.snapshots_enabled { + state_sync_config.max_num_snapshots + } else { + platform_version.drive_abci.checkpoints.num_checkpoints as usize + }; + + // Build the checkpoint path: / + // (defaults to db_path/checkpoints) + let checkpoint_path = state_sync_config + .resolved_checkpoints_path(&self.config.db_path) .join(block_height.to_string()); // Create the parent checkpoints directory if it doesn't exist diff --git a/packages/rs-drive-abci/src/execution/platform_events/block_end/should_checkpoint/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/block_end/should_checkpoint/v0/mod.rs index 79d2034f99c..45d8412b7f6 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/block_end/should_checkpoint/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/block_end/should_checkpoint/v0/mod.rs @@ -39,10 +39,22 @@ where return Ok(None); } - // How often we want a checkpoint - let checkpoint_interval_milliseconds = - platform_version.drive_abci.checkpoints.frequency_seconds as u64 * 1000; - let keep_n = platform_version.drive_abci.checkpoints.num_checkpoints as usize; + // How often we want a checkpoint. When snapshot serving is enabled, the + // operator-provided state sync configuration overrides the + // platform-version-driven checkpoint parameters. + let state_sync_config = &self.config.abci.state_sync; + let (frequency_seconds, keep_n) = if state_sync_config.snapshots_enabled { + ( + state_sync_config.snapshots_frequency_seconds as u64, + state_sync_config.max_num_snapshots, + ) + } else { + ( + platform_version.drive_abci.checkpoints.frequency_seconds as u64, + platform_version.drive_abci.checkpoints.num_checkpoints as usize, + ) + }; + let checkpoint_interval_milliseconds = frequency_seconds * 1000; // If disabled or misconfigured, do nothing. if checkpoint_interval_milliseconds == 0 || keep_n == 0 { diff --git a/packages/rs-drive-abci/src/execution/platform_events/block_end/update_checkpoints/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/block_end/update_checkpoints/v0/mod.rs index f3b297ce7e0..a75d735c551 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/block_end/update_checkpoints/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/block_end/update_checkpoints/v0/mod.rs @@ -33,13 +33,19 @@ where return Ok(false); }; - let keep_n = platform_version.drive_abci.checkpoints.num_checkpoints as usize; + // When snapshot serving is enabled, the operator-provided state sync + // configuration overrides the platform-version-driven checkpoint retention. + let state_sync_config = &self.config.abci.state_sync; + let keep_n = if state_sync_config.snapshots_enabled { + state_sync_config.max_num_snapshots + } else { + platform_version.drive_abci.checkpoints.num_checkpoints as usize + }; - // Build the checkpoint path: db_path/checkpoints/ - let checkpoint_path = self - .config - .db_path - .join("checkpoints") + // Build the checkpoint path: / + // (defaults to db_path/checkpoints) + let checkpoint_path = state_sync_config + .resolved_checkpoints_path(&self.config.db_path) .join(block_height.to_string()); // Create the checkpoints directory if it doesn't exist diff --git a/packages/rs-drive-abci/src/platform_types/mod.rs b/packages/rs-drive-abci/src/platform_types/mod.rs index 0f3b33981de..7f59de23585 100644 --- a/packages/rs-drive-abci/src/platform_types/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/mod.rs @@ -22,6 +22,8 @@ pub mod platform_state; pub mod required_identity_public_key_set; /// Signature verification quorums for Core pub mod signature_verification_quorum_set; +/// ABCI state sync snapshot types +pub mod snapshot; /// The state transition execution result as part of the block execution outcome pub mod state_transitions_processing_result; /// The validator module diff --git a/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs new file mode 100644 index 00000000000..cb0924d5131 --- /dev/null +++ b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs @@ -0,0 +1,104 @@ +//! ABCI state sync snapshot types. +//! +//! Snapshots are served directly from the rocksdb checkpoints Drive already creates +//! (`drive.checkpoints`, populated by `create_grovedb_checkpoint` after each qualifying +//! block is committed); there is no separate snapshot store. + +use drive::drive::Checkpoint; +use std::collections::BTreeMap; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; + +/// The grovedb state sync wire protocol versions this node can serve and consume. +/// +/// This is THE single supported-set constant: when grovedb wire version 2 lands, add it +/// here and add a `DriveAbciStateSyncVersions` const selecting it in rs-platform-version +/// (`drive_abci.state_sync.protocol_version` is the version stamped on snapshots this +/// node offers). +pub const SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS: &[u16] = &[1]; + +/// Maximum accepted size (in bytes) of a single snapshot chunk, enforced before any +/// grovedb decode of peer-supplied data (issue #3773). +pub const MAX_STATE_SYNC_CHUNK_SIZE: usize = 16 * 1024 * 1024; + +/// Maximum accepted size (in bytes) of a chunk id, enforced before any grovedb decode +/// of peer-supplied data (issue #3773). Chunk ids are packed vectors of 32-byte subtree +/// prefixes plus short traversal instructions, so well-formed ids stay far below this. +pub const MAX_STATE_SYNC_CHUNK_ID_SIZE: usize = 64 * 1024; + +/// How long a served checkpoint stays pinned after the last chunk request for it. +/// +/// A state-syncing peer requests chunks continuously; if none arrived for this long the +/// transfer is considered abandoned and the pin is released, allowing a checkpoint that +/// pruning already marked for deletion to be removed from disk. +const SERVING_PIN_INACTIVITY_TTL: Duration = Duration::from_secs(600); + +/// Keeps checkpoints that are actively being served to state-syncing peers alive. +/// +/// Checkpoint pruning marks old checkpoints for deletion and drops them from the +/// registry; the directory is removed when the last `Arc` drops. Holding an +/// `Arc` clone here for every checkpoint a peer is currently downloading extends that +/// refcount, so a checkpoint cannot be deleted mid-transfer. Pins are released after +/// [`SERVING_PIN_INACTIVITY_TTL`] of inactivity. +#[derive(Default)] +pub struct SnapshotManager { + /// Height -> (pinned checkpoint, instant of the most recent chunk request) + serving_pins: RwLock, Instant)>>, +} + +impl SnapshotManager { + /// Creates a new snapshot manager with no active pins + pub fn new() -> Self { + Self::default() + } + + /// Pins a checkpoint that is being served (or refreshes the pin of one that already + /// is), and drops pins whose transfers have been inactive for longer than the TTL. + pub fn pin_for_serving(&self, height: u64, checkpoint: Arc) { + let now = Instant::now(); + let mut pins = self + .serving_pins + .write() + .expect("serving pins lock poisoned"); + pins.retain(|_, (_, last_served)| { + now.saturating_duration_since(*last_served) < SERVING_PIN_INACTIVITY_TTL + }); + pins.insert(height, (checkpoint, now)); + } + + /// Returns a pinned checkpoint for the given height, if the pin is still held. + /// + /// Used to keep serving a snapshot whose checkpoint pruning has already dropped + /// from the registry. + pub fn pinned_checkpoint(&self, height: u64) -> Option> { + self.serving_pins + .read() + .expect("serving pins lock poisoned") + .get(&height) + .map(|(checkpoint, _)| Arc::clone(checkpoint)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn supported_wire_versions_include_the_version_platform_versions_stamp() { + use dpp::version::PlatformVersion; + // Every platform version stamps its state_sync.protocol_version on the + // snapshots it offers; the supported set must accept what we serve. + for platform_version in dpp::version::ALL_VERSIONS + .map(PlatformVersion::get) + .filter_map(Result::ok) + { + assert!( + SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS + .contains(&platform_version.drive_abci.state_sync.protocol_version), + "platform version {} stamps unsupported state sync wire version {}", + platform_version.protocol_version, + platform_version.drive_abci.state_sync.protocol_version + ); + } + } +} diff --git a/packages/rs-drive-abci/src/utils/mod.rs b/packages/rs-drive-abci/src/utils/mod.rs index bfebd6dcd62..05bbdf32e46 100644 --- a/packages/rs-drive-abci/src/utils/mod.rs +++ b/packages/rs-drive-abci/src/utils/mod.rs @@ -4,5 +4,6 @@ mod spawn; pub(crate) use replay::is_historical_block; pub use serialization::from_opt_str_or_number; +pub use serialization::from_str_or_native; pub use serialization::from_str_or_number; pub use spawn::spawn_blocking_task_with_name_if_supported; diff --git a/packages/rs-drive-abci/src/utils/serialization.rs b/packages/rs-drive-abci/src/utils/serialization.rs index 8259ff1dce3..cc5b965d49f 100644 --- a/packages/rs-drive-abci/src/utils/serialization.rs +++ b/packages/rs-drive-abci/src/utils/serialization.rs @@ -13,6 +13,29 @@ where s.parse::().map_err(Error::custom) } +/// Deserialize a value from a string (as provided by envy, where every value is a +/// string) or from its native representation (as in JSON round trips). +pub fn from_str_or_native<'de, D, T>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, + T: serde::Deserialize<'de> + std::str::FromStr, + ::Err: std::fmt::Display, +{ + use serde::de::Error; + + #[derive(Deserialize)] + #[serde(untagged)] + enum NativeOrString { + Native(T), + String(String), + } + + match NativeOrString::::deserialize(deserializer)? { + NativeOrString::Native(value) => Ok(value), + NativeOrString::String(s) => s.parse::().map_err(Error::custom), + } +} + /// Deserialize a value from an optional string or a number pub fn from_opt_str_or_number<'de, D, T>(deserializer: D) -> Result, D::Error> where diff --git a/packages/rs-drive/src/drive/mod.rs b/packages/rs-drive/src/drive/mod.rs index 98eeafa0c24..367ad56a397 100644 --- a/packages/rs-drive/src/drive/mod.rs +++ b/packages/rs-drive/src/drive/mod.rs @@ -106,6 +106,26 @@ impl Checkpoint { } } + /// Returns true if this checkpoint contains the reduced platform state + /// (`Misc/reduced_saved_state`), which a state-syncing node needs to reconstruct the + /// platform state. Checkpoints taken before the protocol version that introduced the + /// reduced state lack the key and cannot be offered as state sync snapshots. + pub fn has_reduced_platform_state( + &self, + grove_version: &grovedb_version::version::GroveVersion, + ) -> Result { + self.grove_db + .get_raw_optional( + (&crate::drive::system::misc_path()).into(), + crate::drive::platform_state::REDUCED_PLATFORM_STATE_KEY, + None, + grove_version, + ) + .unwrap() + .map(|maybe_element| maybe_element.is_some()) + .map_err(Error::from) + } + /// Marks this checkpoint for deletion when it is dropped. pub fn mark_for_deletion(&self) { self.marked_for_deletion diff --git a/packages/rs-drive/src/drive/platform_state/mod.rs b/packages/rs-drive/src/drive/platform_state/mod.rs index 5a0e2fe5d9b..9b100fc239d 100644 --- a/packages/rs-drive/src/drive/platform_state/mod.rs +++ b/packages/rs-drive/src/drive/platform_state/mod.rs @@ -4,7 +4,7 @@ mod store_platform_state_bytes; mod store_reduced_platform_state_bytes; const PLATFORM_STATE_KEY: &[u8; 11] = b"saved_state"; -const REDUCED_PLATFORM_STATE_KEY: &[u8; 19] = b"reduced_saved_state"; +pub(crate) const REDUCED_PLATFORM_STATE_KEY: &[u8; 19] = b"reduced_saved_state"; #[cfg(test)] mod tests { From cfa0b34120a269b6a20d2f5af12ccf1dd7aefc5e Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 29 Aug 2026 01:30:44 +0200 Subject: [PATCH 07/34] feat(drive-abci): consume state sync snapshots via offer and apply chunk handlers Adds the StateSyncApplication trait and a snapshot fetching session (grovedb sync session plus the wire version taken from the offered snapshot) on the Consensus and Full ABCI apps. offer_snapshot validates the offered version against the single supported-set const (REJECT_FORMAT otherwise), wipes grovedb, and answers Accept on both the fresh-session and the replace-with-newer-height paths. apply_snapshot_chunk caps chunk and chunk-id sizes before any decode (#3773), answers RETRY with the failed chunk in refetch_chunks (banning the sender) instead of killing the session when grovedb rejects a chunk, and on completion commits the session, verifies grovedb, reconstructs the platform state (stub until the next commit) and checks the restored root hash against the snapshot app hash. The completion log fires once per transfer. Co-Authored-By: Claude Fable 5 --- .../rs-drive-abci/src/abci/app/consensus.rs | 34 +- packages/rs-drive-abci/src/abci/app/full.rs | 33 +- packages/rs-drive-abci/src/abci/app/mod.rs | 12 +- .../src/abci/handler/apply_snapshot_chunk.rs | 296 ++++++++++++++++++ .../rs-drive-abci/src/abci/handler/mod.rs | 4 + .../src/abci/handler/offer_snapshot.rs | 178 +++++++++++ .../src/execution/platform_events/mod.rs | 2 + .../platform_events/state_sync/mod.rs | 3 + .../reconstruct_platform_state/mod.rs | 29 ++ .../src/platform_types/snapshot/mod.rs | 22 ++ 10 files changed, 607 insertions(+), 6 deletions(-) create mode 100644 packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs create mode 100644 packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs create mode 100644 packages/rs-drive-abci/src/execution/platform_events/state_sync/mod.rs create mode 100644 packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs diff --git a/packages/rs-drive-abci/src/abci/app/consensus.rs b/packages/rs-drive-abci/src/abci/app/consensus.rs index 43b6d518db8..8147a1a4f49 100644 --- a/packages/rs-drive-abci/src/abci/app/consensus.rs +++ b/packages/rs-drive-abci/src/abci/app/consensus.rs @@ -1,10 +1,13 @@ -use crate::abci::app::{BlockExecutionApplication, PlatformApplication, TransactionalApplication}; +use crate::abci::app::{ + BlockExecutionApplication, PlatformApplication, StateSyncApplication, TransactionalApplication, +}; use crate::abci::handler; use crate::abci::handler::error::error_into_exception; use crate::error::execution::ExecutionError; use crate::error::Error; use crate::execution::types::block_execution_context::BlockExecutionContext; use crate::platform_types::platform::Platform; +use crate::platform_types::snapshot::SnapshotFetchingSession; use crate::rpc::core::CoreRPCLike; use dpp::version::PlatformVersion; use drive::grovedb::Transaction; @@ -23,6 +26,8 @@ pub struct ConsensusAbciApplication<'a, C> { transaction: RwLock>>, /// The current block execution context block_execution_context: RwLock>, + /// The state sync transfer currently in progress, if any + snapshot_fetching_session: RwLock>>, } impl<'a, C> ConsensusAbciApplication<'a, C> { @@ -32,6 +37,7 @@ impl<'a, C> ConsensusAbciApplication<'a, C> { platform, transaction: Default::default(), block_execution_context: Default::default(), + snapshot_fetching_session: Default::default(), } } } @@ -42,6 +48,16 @@ impl PlatformApplication for ConsensusAbciApplication<'_, C> { } } +impl<'a, C> StateSyncApplication<'a, C> for ConsensusAbciApplication<'a, C> { + fn snapshot_fetching_session(&self) -> &RwLock>> { + &self.snapshot_fetching_session + } + + fn platform(&self) -> &'a Platform { + self.platform + } +} + impl BlockExecutionApplication for ConsensusAbciApplication<'_, C> { fn block_execution_context(&self) -> &RwLock> { &self.block_execution_context @@ -105,7 +121,7 @@ mod tests { crate::test::helpers::setup::TestPlatformBuilder::new().build_with_mock_rpc(); let app = ConsensusAbciApplication::::new(&platform.platform); - let _platform_ref = app.platform(); + let _platform_ref = PlatformApplication::platform(&app); } #[test] @@ -222,4 +238,18 @@ where ) -> Result { handler::verify_vote_extension(self, request).map_err(error_into_exception) } + + fn offer_snapshot( + &self, + request: proto::RequestOfferSnapshot, + ) -> Result { + handler::offer_snapshot(self, request).map_err(error_into_exception) + } + + fn apply_snapshot_chunk( + &self, + request: proto::RequestApplySnapshotChunk, + ) -> Result { + handler::apply_snapshot_chunk(self, request).map_err(error_into_exception) + } } diff --git a/packages/rs-drive-abci/src/abci/app/full.rs b/packages/rs-drive-abci/src/abci/app/full.rs index 539fc07ff29..e5274f01f30 100644 --- a/packages/rs-drive-abci/src/abci/app/full.rs +++ b/packages/rs-drive-abci/src/abci/app/full.rs @@ -1,6 +1,6 @@ use crate::abci::app::{ BlockExecutionApplication, PlatformApplication, SnapshotManagerApplication, - TransactionalApplication, + StateSyncApplication, TransactionalApplication, }; use crate::abci::handler; use crate::abci::handler::error::error_into_exception; @@ -8,7 +8,7 @@ use crate::error::execution::ExecutionError; use crate::error::Error; use crate::execution::types::block_execution_context::BlockExecutionContext; use crate::platform_types::platform::Platform; -use crate::platform_types::snapshot::SnapshotManager; +use crate::platform_types::snapshot::{SnapshotFetchingSession, SnapshotManager}; use crate::rpc::core::CoreRPCLike; use dpp::version::PlatformVersion; use drive::grovedb::Transaction; @@ -29,6 +29,8 @@ pub struct FullAbciApplication<'a, C> { pub block_execution_context: RwLock>, /// The snapshot manager, pinning checkpoints that are being served to peers pub snapshot_manager: SnapshotManager, + /// The state sync transfer currently in progress, if any + pub snapshot_fetching_session: RwLock>>, } impl<'a, C> FullAbciApplication<'a, C> { @@ -39,6 +41,7 @@ impl<'a, C> FullAbciApplication<'a, C> { transaction: Default::default(), block_execution_context: Default::default(), snapshot_manager: SnapshotManager::new(), + snapshot_fetching_session: Default::default(), } } } @@ -55,6 +58,16 @@ impl SnapshotManagerApplication for FullAbciApplication<'_, C> { } } +impl<'a, C> StateSyncApplication<'a, C> for FullAbciApplication<'a, C> { + fn snapshot_fetching_session(&self) -> &RwLock>> { + &self.snapshot_fetching_session + } + + fn platform(&self) -> &'a Platform { + self.platform + } +} + impl BlockExecutionApplication for FullAbciApplication<'_, C> { fn block_execution_context(&self) -> &RwLock> { &self.block_execution_context @@ -118,7 +131,7 @@ mod tests { crate::test::helpers::setup::TestPlatformBuilder::new().build_with_mock_rpc(); let app = FullAbciApplication::::new(&platform.platform); - let _platform_ref = app.platform(); + let _platform_ref = PlatformApplication::platform(&app); } #[test] @@ -268,4 +281,18 @@ where ) -> Result { handler::load_snapshot_chunk(self, request).map_err(error_into_exception) } + + fn offer_snapshot( + &self, + request: proto::RequestOfferSnapshot, + ) -> Result { + handler::offer_snapshot(self, request).map_err(error_into_exception) + } + + fn apply_snapshot_chunk( + &self, + request: proto::RequestApplySnapshotChunk, + ) -> Result { + handler::apply_snapshot_chunk(self, request).map_err(error_into_exception) + } } diff --git a/packages/rs-drive-abci/src/abci/app/mod.rs b/packages/rs-drive-abci/src/abci/app/mod.rs index 4410a6d3052..fc575f4066f 100644 --- a/packages/rs-drive-abci/src/abci/app/mod.rs +++ b/packages/rs-drive-abci/src/abci/app/mod.rs @@ -10,7 +10,7 @@ pub mod execution_result; mod full; use crate::execution::types::block_execution_context::BlockExecutionContext; -use crate::platform_types::snapshot::SnapshotManager; +use crate::platform_types::snapshot::{SnapshotFetchingSession, SnapshotManager}; use crate::rpc::core::DefaultCoreRPC; #[cfg(test)] pub(crate) use check_tx::error_into_status; @@ -32,6 +32,16 @@ pub trait SnapshotManagerApplication { fn snapshot_manager(&self) -> &SnapshotManager; } +/// ABCI application that can bootstrap its state via state sync +pub trait StateSyncApplication<'p, C = DefaultCoreRPC> { + /// Returns the state sync transfer currently in progress, if any + fn snapshot_fetching_session(&self) -> &RwLock>>; + + /// Returns Platform with the full `'p` lifetime, so a grovedb state sync session + /// borrowing the grove can be stored in the snapshot fetching session + fn platform(&self) -> &'p Platform; +} + /// Transactional ABCI application pub trait TransactionalApplication<'a> { /// Creates and keeps a new transaction diff --git a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs new file mode 100644 index 00000000000..59768dc3115 --- /dev/null +++ b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs @@ -0,0 +1,296 @@ +use crate::abci::app::StateSyncApplication; +use crate::abci::AbciError; +use crate::error::Error; +use crate::platform_types::platform_state::PlatformStateV0Methods; +use crate::platform_types::snapshot::{MAX_STATE_SYNC_CHUNK_ID_SIZE, MAX_STATE_SYNC_CHUNK_SIZE}; +use crate::rpc::core::CoreRPCLike; +use tenderdash_abci::proto::abci as proto; +use tenderdash_abci::proto::abci::response_apply_snapshot_chunk; + +/// Applies one chunk of a state sync snapshot to the grovedb sync session. +/// +/// A chunk grovedb rejects does not kill the whole transfer: Tenderdash is asked to +/// refetch that chunk (from a different peer, if it identified the sender). When the +/// last chunk lands, the session is committed, grovedb is verified against the target +/// app hash, and the platform state is reconstructed from the reduced platform state +/// contained in the restored snapshot. +pub fn apply_snapshot_chunk<'a, 'db: 'a, A, C>( + app: &'a A, + request: proto::RequestApplySnapshotChunk, +) -> Result +where + A: StateSyncApplication<'db, C> + 'db, + C: CoreRPCLike + 'db, +{ + tracing::trace!( + chunk_id = hex::encode(&request.chunk_id), + chunk_len = request.chunk.len(), + "[state_sync] api apply_snapshot_chunk", + ); + + // Cap peer-supplied sizes before anything decodes them (issue #3773) + if request.chunk_id.len() > MAX_STATE_SYNC_CHUNK_ID_SIZE { + return Err(AbciError::StateSyncBadRequest(format!( + "apply_snapshot_chunk chunk id of {} bytes exceeds the {} byte limit", + request.chunk_id.len(), + MAX_STATE_SYNC_CHUNK_ID_SIZE + )) + .into()); + } + if request.chunk.len() > MAX_STATE_SYNC_CHUNK_SIZE { + return Err(AbciError::StateSyncBadRequest(format!( + "apply_snapshot_chunk chunk of {} bytes exceeds the {} byte limit", + request.chunk.len(), + MAX_STATE_SYNC_CHUNK_SIZE + )) + .into()); + } + + let platform_version = app.platform().state.load().current_platform_version()?; + let grove_version = &platform_version.drive.grove_version; + + let mut session_write_guard = app.snapshot_fetching_session().write().map_err(|_| { + AbciError::StateSyncInternalError( + "apply_snapshot_chunk unable to lock session (poisoned)".to_string(), + ) + })?; + + { + let session = session_write_guard + .as_mut() + .ok_or(AbciError::StateSyncBadRequest( + "apply_snapshot_chunk no state sync session in progress".to_string(), + ))?; + + let wire_version = session.wire_version; + let next_chunk_ids = match session.state_sync_info.apply_chunk( + &request.chunk_id, + &request.chunk, + wire_version, + grove_version, + ) { + Ok(next_chunk_ids) => next_chunk_ids, + Err(e) => { + // A chunk grovedb cannot apply (corrupted or tampered data) is + // recoverable: keep the session and ask Tenderdash to refetch the chunk, + // banning the peer that sent it so the refetch goes elsewhere. + tracing::warn!( + chunk_id = hex::encode(&request.chunk_id), + sender = request.sender, + error = ?e, + "[state_sync] apply_snapshot_chunk rejected a chunk, requesting refetch", + ); + let reject_senders = if request.sender.is_empty() { + vec![] + } else { + vec![request.sender] + }; + return Ok(proto::ResponseApplySnapshotChunk { + result: response_apply_snapshot_chunk::Result::Retry.into(), + refetch_chunks: vec![request.chunk_id], + reject_senders, + next_chunks: vec![], + }); + } + }; + + if !session.state_sync_info.is_sync_completed() { + return Ok(proto::ResponseApplySnapshotChunk { + result: response_apply_snapshot_chunk::Result::Accept.into(), + refetch_chunks: vec![], + reject_senders: vec![], + next_chunks: next_chunk_ids, + }); + } + + if !next_chunk_ids.is_empty() { + return Err(AbciError::StateSyncInternalError( + "apply_snapshot_chunk session is completed but next_chunk_ids is not empty" + .to_string(), + ) + .into()); + } + } + + // The transfer is complete: consume the session and commit it + let session = session_write_guard + .take() + .expect("session presence was just checked"); + + app.platform() + .drive + .grove + .commit_session(session.state_sync_info, grove_version) + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "apply_snapshot_chunk unable to commit session: {}", + e + )) + })?; + + tracing::debug!("[state_sync] transfer complete, verifying grovedb"); + + let incorrect_hashes = app + .platform() + .drive + .grove + .verify_grovedb(None, true, false, grove_version) + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "apply_snapshot_chunk unable to verify grovedb: {}", + e + )) + })?; + if !incorrect_hashes.is_empty() { + return Err(AbciError::StateSyncInternalError(format!( + "apply_snapshot_chunk grovedb verification failed with {} incorrect hashes", + incorrect_hashes.len() + )) + .into()); + } + + // Rebuild the in-memory platform state from the reduced platform state contained in + // the restored snapshot. This re-derives masternode lists and quorums from Core and + // must leave the grovedb root hash untouched; the equality check below proves it. + app.platform() + .reconstruct_platform_state(&session.app_hash, platform_version)?; + + let drive_app_hash = app + .platform() + .drive + .grove + .root_hash(None, grove_version) + .unwrap() + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "apply_snapshot_chunk unable to get app hash: {}", + e + )) + })?; + + if drive_app_hash != session.app_hash { + tracing::error!( + state_sync_app_hash = hex::encode(session.app_hash), + drive_app_hash = hex::encode(drive_app_hash), + "[state_sync] restored grovedb root hash does not match the snapshot app hash", + ); + return Err(AbciError::StateSyncInternalError(format!( + "apply_snapshot_chunk grovedb verification failed with incorrect app hash: {}", + hex::encode(drive_app_hash) + )) + .into()); + } + + tracing::info!( + height = session.snapshot.height, + app_hash = hex::encode(session.app_hash), + "state_sync completed", + ); + + Ok(proto::ResponseApplySnapshotChunk { + result: response_apply_snapshot_chunk::Result::CompleteSnapshot.into(), + refetch_chunks: vec![], + reject_senders: vec![], + next_chunks: vec![], + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::abci::app::FullAbciApplication; + use crate::abci::handler::offer_snapshot; + use crate::test::helpers::setup::TestPlatformBuilder; + + #[test] + fn apply_snapshot_chunk_without_session_is_rejected() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let app = FullAbciApplication::new(&platform); + + assert!(apply_snapshot_chunk( + &app, + proto::RequestApplySnapshotChunk { + chunk_id: vec![1u8; 32], + chunk: vec![], + sender: String::new(), + }, + ) + .is_err()); + } + + #[test] + fn apply_snapshot_chunk_caps_sizes_before_decoding() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let app = FullAbciApplication::new(&platform); + + assert!(apply_snapshot_chunk( + &app, + proto::RequestApplySnapshotChunk { + chunk_id: vec![0u8; MAX_STATE_SYNC_CHUNK_ID_SIZE + 1], + chunk: vec![], + sender: String::new(), + }, + ) + .is_err()); + + assert!(apply_snapshot_chunk( + &app, + proto::RequestApplySnapshotChunk { + chunk_id: vec![1u8; 32], + chunk: vec![0u8; MAX_STATE_SYNC_CHUNK_SIZE + 1], + sender: String::new(), + }, + ) + .is_err()); + } + + #[test] + fn apply_snapshot_chunk_asks_for_refetch_of_a_bad_chunk() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let app = FullAbciApplication::new(&platform); + + let target_app_hash = vec![7u8; 32]; + offer_snapshot( + &app, + proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height: 100, + version: 1, + hash: target_app_hash.clone(), + metadata: vec![], + }), + app_hash: target_app_hash.clone(), + }, + ) + .expect("should accept offer"); + + // Garbage bytes for the root chunk: grovedb rejects them, and the session must + // survive with a Retry + refetch of exactly that chunk, banning the sender. + let response = apply_snapshot_chunk( + &app, + proto::RequestApplySnapshotChunk { + chunk_id: target_app_hash.clone(), + chunk: vec![0xde, 0xad, 0xbe, 0xef], + sender: "peer-1".to_string(), + }, + ) + .expect("bad chunk should not error the session"); + + assert_eq!( + response.result, + i32::from(response_apply_snapshot_chunk::Result::Retry) + ); + assert_eq!(response.refetch_chunks, vec![target_app_hash]); + assert_eq!(response.reject_senders, vec!["peer-1".to_string()]); + assert!( + app.snapshot_fetching_session.read().unwrap().is_some(), + "the session must survive a bad chunk" + ); + } +} diff --git a/packages/rs-drive-abci/src/abci/handler/mod.rs b/packages/rs-drive-abci/src/abci/handler/mod.rs index 6b74a74760b..443758734a3 100644 --- a/packages/rs-drive-abci/src/abci/handler/mod.rs +++ b/packages/rs-drive-abci/src/abci/handler/mod.rs @@ -35,6 +35,7 @@ //! can only make changes that are backwards compatible. Otherwise new calls must be made instead. //! +mod apply_snapshot_chunk; mod check_tx; mod echo; pub mod error; @@ -44,10 +45,12 @@ mod info; mod init_chain; mod list_snapshots; mod load_snapshot_chunk; +mod offer_snapshot; mod prepare_proposal; mod process_proposal; mod verify_vote_extension; +pub use apply_snapshot_chunk::apply_snapshot_chunk; pub use check_tx::check_tx; pub use echo::echo; pub use extend_vote::extend_vote; @@ -56,6 +59,7 @@ pub use info::info; pub use init_chain::init_chain; pub use list_snapshots::list_snapshots; pub use load_snapshot_chunk::load_snapshot_chunk; +pub use offer_snapshot::offer_snapshot; pub use prepare_proposal::prepare_proposal; pub use process_proposal::process_proposal; pub use verify_vote_extension::verify_vote_extension; diff --git a/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs b/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs new file mode 100644 index 00000000000..c7838e972ae --- /dev/null +++ b/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs @@ -0,0 +1,178 @@ +use crate::abci::app::StateSyncApplication; +use crate::abci::AbciError; +use crate::error::Error; +use crate::platform_types::platform_state::PlatformStateV0Methods; +use crate::platform_types::snapshot::{ + SnapshotFetchingSession, STATE_SYNC_SUBTREES_BATCH_SIZE, SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS, +}; +use crate::rpc::core::CoreRPCLike; +use tenderdash_abci::proto::abci as proto; +use tenderdash_abci::proto::abci::response_offer_snapshot; + +/// Handles a snapshot offered by Tenderdash during state sync. +/// +/// Accepting an offer wipes the local grovedb and opens a grovedb state sync session +/// targeting the light-client-verified app hash. A later offer for a higher height +/// replaces a session already in progress (also answered with Accept); an offer for a +/// lower or equal height than the session in progress is rejected. +pub fn offer_snapshot<'a, 'db: 'a, A, C: 'db>( + app: &'a A, + request: proto::RequestOfferSnapshot, +) -> Result +where + A: StateSyncApplication<'db, C> + 'db, + C: CoreRPCLike, +{ + let request_app_hash: [u8; 32] = request.app_hash.try_into().map_err(|_| { + AbciError::StateSyncBadRequest("offer_snapshot invalid app_hash length".to_string()) + })?; + let offered_snapshot = request.snapshot.ok_or(AbciError::StateSyncBadRequest( + "offer_snapshot empty snapshot in request".to_string(), + ))?; + + tracing::debug!( + height = offered_snapshot.height, + version = offered_snapshot.version, + "[state_sync] api offer_snapshot", + ); + + // The grovedb wire version of the whole transfer is the OFFERED snapshot's version, + // validated against the single supported set. Unsupported versions ask Tenderdash to + // reject every snapshot of this format and try others. + let wire_version = u16::try_from(offered_snapshot.version) + .ok() + .filter(|version| SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS.contains(version)); + let Some(wire_version) = wire_version else { + tracing::warn!( + height = offered_snapshot.height, + version = offered_snapshot.version, + supported = ?SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS, + "[state_sync] offer_snapshot rejecting unsupported snapshot version", + ); + return Ok(proto::ResponseOfferSnapshot { + result: response_offer_snapshot::Result::RejectFormat.into(), + }); + }; + + let platform_version = app.platform().state.load().current_platform_version()?; + + let mut session_write_guard = app.snapshot_fetching_session().write().map_err(|_| { + AbciError::StateSyncInternalError( + "offer_snapshot unable to lock session (poisoned)".to_string(), + ) + })?; + + if let Some(session) = session_write_guard.as_ref() { + if offered_snapshot.height <= session.snapshot.height { + return Err(AbciError::StateSyncBadRequest(format!( + "offer_snapshot already syncing snapshot at height {}, offered height {} is not newer", + session.snapshot.height, offered_snapshot.height + )) + .into()); + } + tracing::warn!( + current_height = session.snapshot.height, + offered_height = offered_snapshot.height, + "[state_sync] offer_snapshot replacing session in progress with newer snapshot", + ); + } + + // Both the fresh-session and the replace-session paths wipe grovedb, start a new + // grovedb sync session, and answer Accept. + app.platform().drive.grove.wipe().map_err(|e| { + AbciError::StateSyncInternalError(format!("offer_snapshot unable to wipe grovedb: {}", e)) + })?; + + let state_sync_info = app + .platform() + .drive + .grove + .start_snapshot_syncing( + request_app_hash, + STATE_SYNC_SUBTREES_BATCH_SIZE, + wire_version, + &platform_version.drive.grove_version, + ) + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "offer_snapshot unable to start snapshot syncing session: {}", + e + )) + })?; + + *session_write_guard = Some(SnapshotFetchingSession { + snapshot: offered_snapshot, + app_hash: request_app_hash, + wire_version, + state_sync_info, + }); + + Ok(proto::ResponseOfferSnapshot { + result: response_offer_snapshot::Result::Accept.into(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::abci::app::FullAbciApplication; + use crate::test::helpers::setup::TestPlatformBuilder; + + fn offer_at(height: u64, version: u32) -> proto::RequestOfferSnapshot { + proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height, + version, + hash: vec![7u8; 32], + metadata: vec![], + }), + app_hash: vec![7u8; 32], + } + } + + #[test] + fn offer_snapshot_rejects_unsupported_version_with_reject_format() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let app = FullAbciApplication::new(&platform); + + let response = offer_snapshot(&app, offer_at(100, 999)).expect("should not error"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::RejectFormat) + ); + assert!(app.snapshot_fetching_session.read().unwrap().is_none()); + } + + #[test] + fn offer_snapshot_accepts_fresh_and_replacing_offers() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let app = FullAbciApplication::new(&platform); + + // Fresh session is accepted + let response = offer_snapshot(&app, offer_at(100, 1)).expect("should accept fresh offer"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::Accept) + ); + + // A lower-or-equal height while syncing is rejected + assert!(offer_snapshot(&app, offer_at(100, 1)).is_err()); + assert!(offer_snapshot(&app, offer_at(50, 1)).is_err()); + + // A newer snapshot replaces the session and MUST also answer Accept + // (the old prototype returned the default UNKNOWN result here) + let response = offer_snapshot(&app, offer_at(200, 1)).expect("should accept newer offer"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::Accept) + ); + let session_guard = app.snapshot_fetching_session.read().unwrap(); + let session = session_guard.as_ref().expect("session must exist"); + assert_eq!(session.snapshot.height, 200); + assert_eq!(session.wire_version, 1); + } +} diff --git a/packages/rs-drive-abci/src/execution/platform_events/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/mod.rs index 1ac0715b9d2..32461d6a15e 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/mod.rs @@ -20,6 +20,8 @@ pub(in crate::execution) mod fee_pool_outwards_distribution; pub(in crate::execution) mod initialization; /// Protocol upgrade events pub(in crate::execution) mod protocol_upgrade; +/// State sync platform state reconstruction +pub(in crate::execution) mod state_sync; /// State transition processing pub(in crate::execution) mod state_transition_processing; mod tokens; diff --git a/packages/rs-drive-abci/src/execution/platform_events/state_sync/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/state_sync/mod.rs new file mode 100644 index 00000000000..f48dae65527 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/platform_events/state_sync/mod.rs @@ -0,0 +1,3 @@ +//! State sync events: reconstruction of the platform state after a snapshot restore. + +mod reconstruct_platform_state; diff --git a/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs new file mode 100644 index 00000000000..f2c3562c1c8 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs @@ -0,0 +1,29 @@ +use crate::abci::AbciError; +use crate::error::Error; +use crate::platform_types::platform::Platform; +use crate::rpc::core::CoreRPCLike; +use dpp::version::PlatformVersion; + +impl Platform +where + C: CoreRPCLike, +{ + /// Reconstructs the full in-memory platform state after a state sync snapshot + /// restore, from the reduced platform state contained in the restored grovedb + /// state, and persists it to aux storage so it survives restarts. + /// + /// Must not change the grovedb root hash: the caller compares the root hash against + /// the snapshot app hash after this returns. + pub fn reconstruct_platform_state( + &self, + _app_hash: &[u8; 32], + _platform_version: &PlatformVersion, + ) -> Result<(), Error> { + // TODO(state-sync): implemented in the follow-up commit that ports the platform + // state reconstruction (reduced state fetch + update_core_info re-derivation). + Err(AbciError::StateSyncInternalError( + "platform state reconstruction is not implemented yet".to_string(), + ) + .into()) + } +} diff --git a/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs index cb0924d5131..58f83e2aad8 100644 --- a/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs @@ -5,9 +5,12 @@ //! block is committed); there is no separate snapshot store. use drive::drive::Checkpoint; +use drive::grovedb::replication::MultiStateSyncSession; use std::collections::BTreeMap; +use std::pin::Pin; use std::sync::{Arc, RwLock}; use std::time::{Duration, Instant}; +use tenderdash_abci::proto::abci; /// The grovedb state sync wire protocol versions this node can serve and consume. /// @@ -26,6 +29,25 @@ pub const MAX_STATE_SYNC_CHUNK_SIZE: usize = 16 * 1024 * 1024; /// prefixes plus short traversal instructions, so well-formed ids stay far below this. pub const MAX_STATE_SYNC_CHUNK_ID_SIZE: usize = 64 * 1024; +/// Maximum number of subtrees processed in a single batch of a grovedb state sync +/// session on the consuming side. +pub const STATE_SYNC_SUBTREES_BATCH_SIZE: usize = 64; + +/// A state sync transfer in progress on the consuming side. +pub struct SnapshotFetchingSession<'db> { + /// The snapshot being restored + pub snapshot: abci::Snapshot, + /// The light-client-verified app hash for the snapshot height, from Tenderdash + pub app_hash: [u8; 32], + /// The grovedb state sync wire protocol version this transfer speaks — taken from + /// the offered snapshot's `version`, validated against + /// [`SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS`], and used for every chunk of the + /// transfer. + pub wire_version: u16, + /// The grovedb state sync session + pub state_sync_info: Pin>>, +} + /// How long a served checkpoint stays pinned after the last chunk request for it. /// /// A state-syncing peer requests chunks continuously; if none arrived for this long the From 31d5d58b2bf673f2986fc5d0c0716b72fb62fb0e Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 29 Aug 2026 01:36:46 +0200 Subject: [PATCH 08/34] feat(drive-abci): reconstruct platform state from the reduced state after snapshot restore reconstruct_platform_state reads the reduced platform state out of the restored grovedb, restores scalar fields and fee versions faithfully by version number, re-derives masternode lists, identities and quorums from Core via update_core_info with start_from_scratch=true (idempotent re-derivation, proven by the caller's root-hash equality check), restores the recorded validator set order, and advances the state to the snapshot block via update_state_cache so the info handler reports the snapshot height and app hash across restarts. update_core_info now passes is_init_chain through to update_quorum_info (its only effect is skipping the same-core-height short-circuit, required for init chain and reconstruction; the normal block path is unchanged), and update_masternode_list's early return is likewise guarded. Co-Authored-By: Claude Fable 5 --- .../update_core_info/v0/mod.rs | 9 +- .../update_masternode_list/v0/mod.rs | 24 +- .../reconstruct_platform_state/mod.rs | 263 +++++++++++++++++- 3 files changed, 273 insertions(+), 23 deletions(-) diff --git a/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/v0/mod.rs index 91564d2db1c..e4075cb173e 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/v0/mod.rs @@ -59,11 +59,18 @@ where platform_version, )?; + // `is_init_chain` doubles as `start_from_scratch`: on init chain and on state + // sync reconstruction the quorums must be built even if the (freshly + // constructed) block state happens to already report the requested core height. + // The flag's only effect inside update_quorum_info is to skip that + // same-core-height short-circuit; on the normal block path (`is_init_chain = + // false`) behavior is unchanged. The previous hardcoded `false` only worked + // because those flows start from a state whose derived core height is 0. self.update_quorum_info( platform_state, block_platform_state, core_block_height, - false, + is_init_chain, platform_version, ) } diff --git a/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_masternode_list/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_masternode_list/v0/mod.rs index 3bc37499fa7..e7842b5fbb9 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_masternode_list/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_masternode_list/v0/mod.rs @@ -41,16 +41,20 @@ where transaction: &Transaction, platform_version: &PlatformVersion, ) -> Result<(), Error> { - if let Some(last_committed_block_info) = - block_platform_state.last_committed_block_info().as_ref() - { - if core_block_height == last_committed_block_info.basic_info().core_height { - tracing::debug!( - method = "update_masternode_list_v0", - "no update mnl at height {}", - core_block_height, - ); - return Ok(()); // no need to do anything + // On init chain and on state sync reconstruction the masternode list must be + // built from scratch even if the block state already reports this core height. + if !is_init_chain { + if let Some(last_committed_block_info) = + block_platform_state.last_committed_block_info().as_ref() + { + if core_block_height == last_committed_block_info.basic_info().core_height { + tracing::debug!( + method = "update_masternode_list_v0", + "no update mnl at height {}", + core_block_height, + ); + return Ok(()); // no need to do anything + } } } tracing::debug!( diff --git a/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs index f2c3562c1c8..b8f177daecb 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs @@ -1,29 +1,268 @@ use crate::abci::AbciError; use crate::error::Error; use crate::platform_types::platform::Platform; +use crate::platform_types::platform_state::{PlatformState, PlatformStateV0Methods}; +use crate::platform_types::signature_verification_quorum_set::SignatureVerificationQuorumSet; +use crate::platform_types::validator_set::ValidatorSet; use crate::rpc::core::CoreRPCLike; +use dpp::block::extended_block_info::v0::{ExtendedBlockInfoV0, ExtendedBlockInfoV0Getters}; +use dpp::block::extended_block_info::ExtendedBlockInfo; +use dpp::dashcore::hashes::Hash; +use dpp::dashcore::QuorumHash; +use dpp::fee::default_costs::CachedEpochIndexFeeVersions; +use dpp::platform_value::Bytes32; +use dpp::reduced_platform_state::ReducedPlatformState; +use dpp::version::fee::FeeVersion; use dpp::version::PlatformVersion; +use indexmap::IndexMap; +use std::collections::BTreeMap; impl Platform where C: CoreRPCLike, { /// Reconstructs the full in-memory platform state after a state sync snapshot - /// restore, from the reduced platform state contained in the restored grovedb - /// state, and persists it to aux storage so it survives restarts. + /// restore, and persists it to aux storage so it survives restarts. /// - /// Must not change the grovedb root hash: the caller compares the root hash against - /// the snapshot app hash after this returns. + /// ## Expected state + /// + /// The restored grovedb contains the reduced platform state that + /// `run_block_proposal` v1 wrote while processing the snapshot block, i.e. the + /// state after the whole block including `validator_set_update`, immediately + /// before the root hash was computed. Reconstruction: + /// + /// 1. restores the scalar fields (protocol versions, quorum hashes, fee versions) + /// directly from the reduced state; + /// 2. re-derives the masternode lists, masternode identities and quorums from Core + /// via `update_core_info` with `start_from_scratch = true` — the identity writes + /// are re-derivations of data already present in the restored state, so the + /// grovedb root hash MUST NOT change (the caller's root-hash equality check is + /// the proof of that idempotence); + /// 3. restores the validator set order recorded by the source (`quorum_positions`), + /// which cannot be recovered from Core RPC; + /// 4. advances the state to the snapshot block via `update_state_cache`, which + /// performs the same next-into-current validator set rotation the source node + /// performed when it finalized that block, persists the state to aux storage and + /// publishes it, so the `info` handler reports the snapshot height and app hash + /// after both this restore and any later restart. pub fn reconstruct_platform_state( &self, - _app_hash: &[u8; 32], - _platform_version: &PlatformVersion, + app_hash: &[u8; 32], + platform_version: &PlatformVersion, ) -> Result<(), Error> { - // TODO(state-sync): implemented in the follow-up commit that ports the platform - // state reconstruction (reduced state fetch + update_core_info re-derivation). - Err(AbciError::StateSyncInternalError( - "platform state reconstruction is not implemented yet".to_string(), - ) - .into()) + let reduced_platform_state = self + .fetch_reduced_platform_state(None, platform_version)? + .ok_or_else(|| { + AbciError::StateSyncInternalError( + "reconstruct_platform_state restored snapshot does not contain a reduced \ + platform state (was it taken before the v15 activation height?)" + .to_string(), + ) + })?; + let ReducedPlatformState::V0(saved) = reduced_platform_state; + + // Everything below runs with the platform version the snapshot's chain was + // actually on, which may lag the version this binary considers latest. + let state_platform_version = + PlatformVersion::get(saved.current_protocol_version_in_consensus)?; + + // Restore the fee versions of previous epochs faithfully, by version number + let previous_fee_versions: CachedEpochIndexFeeVersions = saved + .previous_fee_versions + .iter() + .map(|(epoch_index, fee_version_number)| { + Ok((*epoch_index, FeeVersion::get(*fee_version_number)?)) + }) + .collect::>()?; + + let mut platform_state = PlatformState { + genesis_block_info: None, + last_committed_block_info: None, + current_protocol_version_in_consensus: saved.current_protocol_version_in_consensus, + next_epoch_protocol_version: saved.next_epoch_protocol_version, + current_validator_set_quorum_hash: QuorumHash::from_byte_array( + saved.current_validator_set_quorum_hash.to_buffer(), + ), + next_validator_set_quorum_hash: saved + .next_validator_set_quorum_hash + .map(|quorum_hash| QuorumHash::from_byte_array(quorum_hash.to_buffer())), + validator_sets: Default::default(), + chain_lock_validating_quorums: SignatureVerificationQuorumSet::new( + &self.config.chain_lock, + state_platform_version, + )?, + instant_lock_validating_quorums: SignatureVerificationQuorumSet::new( + &self.config.instant_lock, + state_platform_version, + )?, + full_masternode_list: Default::default(), + hpmn_masternode_list: Default::default(), + previous_fee_versions, + }; + + let saved_block_info = + saved + .last_committed_block_info + .ok_or(AbciError::StateSyncInternalError( + "reconstruct_platform_state reduced platform state has no last committed \ + block info" + .to_string(), + ))?; + + // The reduced state is written before the block's root hash exists, so its app + // hash is normally None and the snapshot app hash fills it in. If it does carry + // one, it must agree with the snapshot. + if let Some(saved_app_hash) = saved_block_info.app_hash { + if saved_app_hash.to_buffer() != *app_hash { + return Err(AbciError::StateSyncInternalError(format!( + "reconstruct_platform_state reduced platform state app hash {} does not \ + match snapshot app hash {}", + hex::encode(saved_app_hash.to_buffer()), + hex::encode(app_hash), + )) + .into()); + } + } + + let current_block_info: ExtendedBlockInfo = ExtendedBlockInfoV0 { + basic_info: saved_block_info.basic_info, + app_hash: *app_hash, + quorum_hash: saved_block_info.quorum_hash.to_buffer(), + // Not known during proposal processing, and not needed for consensus after + // a restore; restored as zeroes. + block_id_hash: saved_block_info + .block_id_hash + .map(|hash| hash.to_buffer()) + .unwrap_or_default(), + proposer_pro_tx_hash: saved_block_info.proposer_pro_tx_hash.to_buffer(), + // Same: unknown at store time, restored as zeroes when absent. + signature: saved_block_info.signature.unwrap_or([0u8; 96]), + round: saved_block_info.round, + } + .into(); + + // Re-derive masternode lists, masternode identities and quorums from Core, from + // scratch, at the core height the snapshot block ran with. The identity writes + // must be byte-identical to what is already in the restored state. + let transaction = self.drive.grove.start_transaction(); + self.update_core_info( + None, + &mut platform_state, + saved.proposed_core_chain_locked_height, + true, + current_block_info.basic_info(), + &transaction, + state_platform_version, + )?; + + // Core RPC returns quorums in an order that need not match the incremental + // order the source node maintained; restore the recorded order. + sort_validator_sets_by_saved_positions( + platform_state.validator_sets_mut(), + &saved.quorum_positions, + ); + + let block_height = platform_state.last_committed_block_height(); + + // Advance the state to the snapshot block: rotates next-into-current exactly as + // the source did on finalization, persists to aux storage and publishes the + // state for the info handler. + self.update_state_cache( + current_block_info, + platform_state, + &transaction, + state_platform_version, + )?; + + self.drive + .grove + .commit_transaction(transaction) + .unwrap() + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "reconstruct_platform_state unable to commit transaction: {}", + e + )) + })?; + + tracing::debug!( + block_height, + app_hash = hex::encode(app_hash), + "[state_sync] platform state reconstructed", + ); + + Ok(()) + } +} + +/// Sorts the validator sets into the order recorded in the reduced platform state. +/// +/// Validator sets not present in the recorded order (which should not happen when the +/// reduced state and Core agree on the quorum list) sort last, preserving their +/// relative order. +fn sort_validator_sets_by_saved_positions( + validator_sets: &mut IndexMap, + quorum_positions: &[Bytes32], +) { + let lookup_table: BTreeMap<&[u8], usize> = quorum_positions + .iter() + .enumerate() + .map(|(position, quorum_hash)| (quorum_hash.as_slice(), position)) + .collect(); + + validator_sets.sort_by(|a_hash, _, b_hash, _| { + let a_position = lookup_table + .get(a_hash.as_byte_array().as_slice()) + .unwrap_or(&usize::MAX); + let b_position = lookup_table + .get(b_hash.as_byte_array().as_slice()) + .unwrap_or(&usize::MAX); + + a_position.cmp(b_position) + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn quorum_hash(seed: u8) -> QuorumHash { + let mut bytes = [0u8; 32]; + bytes[31] = seed; + QuorumHash::from_byte_array(bytes) + } + + #[test] + fn should_sort_validator_sets_into_saved_positions() { + use dpp::bls_signatures::{Bls12381G2Impl, SecretKey}; + use dpp::core_types::validator_set::v0::ValidatorSetV0; + use rand::rngs::StdRng; + use rand::SeedableRng; + + let mut rng = StdRng::seed_from_u64(7); + let mut validator_sets: IndexMap = IndexMap::new(); + for seed in [1u8, 2, 3] { + validator_sets.insert( + quorum_hash(seed), + ValidatorSet::V0(ValidatorSetV0 { + quorum_hash: quorum_hash(seed), + quorum_index: None, + core_height: 100, + members: Default::default(), + threshold_public_key: SecretKey::::random(&mut rng) + .public_key(), + }), + ); + } + + let saved_positions: Vec = [3u8, 1, 2] + .into_iter() + .map(|seed| quorum_hash(seed).to_byte_array().into()) + .collect(); + + sort_validator_sets_by_saved_positions(&mut validator_sets, &saved_positions); + + let order: Vec = validator_sets.keys().copied().collect(); + assert_eq!(order, vec![quorum_hash(3), quorum_hash(1), quorum_hash(2)]); } } From 110c0c759028638b90c54b881569a5bced58ebdd Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 29 Aug 2026 01:40:41 +0200 Subject: [PATCH 09/34] feat(drive-abci): consensus_params_update v2 emits evidence params on v15 activation The first block of protocol v15 additionally emits EvidenceParams (max age 15000 blocks / 20 days, max bytes 1 MiB) per issue #2512, in named constants. A review-flag comment notes that 15000 blocks (~1 day at 6s blocks) vs 20 days look inconsistent, since evidence expires at the earlier bound, and must be confirmed before release. Co-Authored-By: Claude Fable 5 --- .../engine/consensus_params_update/mod.rs | 79 ++++++++++++++++++- .../engine/consensus_params_update/v2/mod.rs | 60 ++++++++++++++ 2 files changed, 137 insertions(+), 2 deletions(-) create mode 100644 packages/rs-drive-abci/src/execution/engine/consensus_params_update/v2/mod.rs diff --git a/packages/rs-drive-abci/src/execution/engine/consensus_params_update/mod.rs b/packages/rs-drive-abci/src/execution/engine/consensus_params_update/mod.rs index e86a9ddb3eb..e7f9df5b222 100644 --- a/packages/rs-drive-abci/src/execution/engine/consensus_params_update/mod.rs +++ b/packages/rs-drive-abci/src/execution/engine/consensus_params_update/mod.rs @@ -8,6 +8,7 @@ use tenderdash_abci::proto::types::ConsensusParams; mod v0; mod v1; +mod v2; pub(crate) fn consensus_params_update( network: Network, @@ -33,9 +34,15 @@ pub(crate) fn consensus_params_update( new_platform_version, epoch_info, )), + 2 => Ok(v2::consensus_params_update_v2( + network, + original_platform_version, + new_platform_version, + epoch_info, + )), version => Err(Error::Execution(ExecutionError::UnknownVersionMismatch { method: "consensus_params_update".to_string(), - known_versions: vec![0, 1], + known_versions: vec![0, 1, 2], received: version, })), } @@ -143,7 +150,7 @@ mod tests { received, })) => { assert_eq!(method, "consensus_params_update"); - assert_eq!(known_versions, vec![0, 1]); + assert_eq!(known_versions, vec![0, 1, 2]); assert_eq!(received, 99); } other => panic!("expected UnknownVersionMismatch error, got: {:?}", other), @@ -587,4 +594,72 @@ mod tests { assert!(result.is_none()); } } + + mod v2_evidence_params { + use super::*; + + /// Crossing to v15 (whose method table selects consensus_params_update v2) must + /// emit both the new app version and the evidence params from issue #2512. + #[test] + fn crossing_to_v15_emits_evidence_params() { + let platform_v14 = PlatformVersion::get(14).expect("v14 exists"); + let platform_v15 = PlatformVersion::get(15).expect("v15 exists"); + let epoch_info = epoch_change_to(10); + + let params = + consensus_params_update(Network::Devnet, platform_v14, platform_v15, &epoch_info) + .expect("should not error") + .expect("crossing to v15 must emit consensus params"); + + let version = params.version.expect("version params must be set"); + assert_eq!(version.app_version, 15); + + let evidence = params.evidence.expect("evidence params must be set"); + assert_eq!(evidence.max_age_num_blocks, 15_000); + assert_eq!( + evidence + .max_age_duration + .expect("max age duration must be set") + .seconds, + 20 * 24 * 60 * 60 + ); + assert_eq!(evidence.max_bytes, 1_048_576); + } + + /// Once the network is on v15, a block without a version change emits nothing: + /// the evidence params are a one-shot emission on the activation block. + #[test] + fn steady_state_v15_emits_nothing() { + let platform_v15 = PlatformVersion::get(15).expect("v15 exists"); + let epoch_info = mid_epoch(11); + + let result = + consensus_params_update(Network::Devnet, platform_v15, platform_v15, &epoch_info) + .expect("should not error"); + assert!(result.is_none()); + } + + /// A version change that does not cross the v15 boundary must not attach + /// evidence params even when dispatched through v2. + #[test] + fn non_crossing_version_change_has_no_evidence_params() { + let platform_v13 = PlatformVersion::get(13).expect("v13 exists"); + let platform_v14 = PlatformVersion::get(14).expect("v14 exists"); + let epoch_info = epoch_change_to(9); + + let params = v2::consensus_params_update_v2( + Network::Devnet, + platform_v13, + platform_v14, + &epoch_info, + ) + .expect("version change must emit consensus params"); + + assert!(params.version.is_some()); + assert!( + params.evidence.is_none(), + "evidence params are only for the v15 crossing" + ); + } + } } diff --git a/packages/rs-drive-abci/src/execution/engine/consensus_params_update/v2/mod.rs b/packages/rs-drive-abci/src/execution/engine/consensus_params_update/v2/mod.rs new file mode 100644 index 00000000000..8f1c4663a04 --- /dev/null +++ b/packages/rs-drive-abci/src/execution/engine/consensus_params_update/v2/mod.rs @@ -0,0 +1,60 @@ +use crate::execution::engine::consensus_params_update::v1::consensus_params_update_v1; +use crate::platform_types::epoch_info::EpochInfo; +use dpp::dashcore::Network; +use dpp::version::v15::PROTOCOL_VERSION_15; +use dpp::version::PlatformVersion; +use tenderdash_abci::proto::google::protobuf::Duration; +use tenderdash_abci::proto::types::{ConsensusParams, EvidenceParams}; + +/// Maximum evidence age in blocks, applied when the network crosses to protocol +/// version 15 (state sync). Value proposed in issue #2512 for nodes that bootstrap +/// from snapshots and do not hold full history. +/// +/// REVIEW BEFORE RELEASE: at ~6s blocks, 15_000 blocks is roughly one day, while +/// [`V15_EVIDENCE_MAX_AGE_DURATION_SECONDS`] below is 20 days. Evidence expires when +/// EITHER bound is exceeded, so the effective window is the smaller (~1 day) — the two +/// values from #2512 look inconsistent and need to be confirmed before this ships. +const V15_EVIDENCE_MAX_AGE_NUM_BLOCKS: i64 = 15_000; + +/// Maximum evidence age in time: 20 days, per issue #2512. See the review note on +/// [`V15_EVIDENCE_MAX_AGE_NUM_BLOCKS`]. +const V15_EVIDENCE_MAX_AGE_DURATION_SECONDS: i64 = 20 * 24 * 60 * 60; + +/// Maximum total evidence per block in bytes. Tenderdash's default (1 MiB); #2512 does +/// not change it, but the whole evidence section must be populated when it is emitted. +const V15_EVIDENCE_MAX_BYTES: i64 = 1_048_576; + +/// Same as v1, but the first block of protocol version 15 additionally emits evidence +/// params sized for a network whose nodes may have bootstrapped via state sync +/// (issue #2512). +#[inline(always)] +pub(super) fn consensus_params_update_v2( + network: Network, + original_platform_version: &PlatformVersion, + new_platform_version: &PlatformVersion, + epoch_info: &EpochInfo, +) -> Option { + let mut consensus_params = consensus_params_update_v1( + network, + original_platform_version, + new_platform_version, + epoch_info, + )?; + + // Crossing to v15 implies a protocol version change, so v1 always emits params on + // the activation block and we only need to attach the evidence section. + let is_crossing_to_v15 = original_platform_version.protocol_version < PROTOCOL_VERSION_15 + && new_platform_version.protocol_version >= PROTOCOL_VERSION_15; + if is_crossing_to_v15 { + consensus_params.evidence = Some(EvidenceParams { + max_age_num_blocks: V15_EVIDENCE_MAX_AGE_NUM_BLOCKS, + max_age_duration: Some(Duration { + seconds: V15_EVIDENCE_MAX_AGE_DURATION_SECONDS, + nanos: 0, + }), + max_bytes: V15_EVIDENCE_MAX_BYTES, + }); + } + + Some(consensus_params) +} From 67bfa6f8a860caa43c9b578010940690f327e974 Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 29 Aug 2026 02:04:52 +0200 Subject: [PATCH 10/34] test(drive-abci): two-instance state sync integration tests A source chain runs past several checkpoints via the strategy harness with snapshot serving enabled, and a fresh target restores its newest snapshot through the real offer/load/apply chunk loop (modeled on grovedb's run_sync driver) with mocked Core RPC. Findings baked into the tests: grovedb wire v1 at the pinned rev cannot faithfully restore sum trees (root hash reproduces but recomputation diverges - latent corruption that the strict post-restore verify_grovedb correctly refuses), pinned by a minimal tripwire reproducer plus an active test asserting the refusal; the full happy-path test is ignored until the grovedb pin gains the fixed wire version. The reconstruction path itself is fully validated by an active test running it against the source's own grove: it is byte-idempotent (root hash unchanged by the masternode identity re-derivation) and reproduces the complete platform state including validator set order, masternode lists and fee versions, satisfying the info handler. A tampered chunk yields RETRY with a refetch and sender ban; since grovedb drops a chunk id from its pending set before processing, a refetch it can no longer honor yields RETRY_SNAPSHOT, and offer_snapshot now accepts same-height re-offers so Tenderdash snapshot restarts work. Pre-v15 snapshots are not offered and cannot be restored. Co-Authored-By: Claude Fable 5 --- .../src/abci/handler/apply_snapshot_chunk.rs | 41 +- .../src/abci/handler/offer_snapshot.rs | 20 +- .../tests/strategy_tests/test_cases/mod.rs | 1 + .../test_cases/state_sync_tests.rs | 730 ++++++++++++++++++ .../tests/sum_tree_sync_probe.rs | 125 +++ 5 files changed, 905 insertions(+), 12 deletions(-) create mode 100644 packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs create mode 100644 packages/rs-drive-abci/tests/sum_tree_sync_probe.rs diff --git a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs index 59768dc3115..e7b55c26d24 100644 --- a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs +++ b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs @@ -71,6 +71,32 @@ where ) { Ok(next_chunk_ids) => next_chunk_ids, Err(e) => { + let reject_senders = if request.sender.is_empty() { + vec![] + } else { + vec![request.sender.clone()] + }; + + // grovedb removes a chunk id from its pending set before processing it, + // so a chunk it has already seen (e.g. the refetch of one it rejected) + // cannot be re-applied within this session: ask Tenderdash to restart + // the snapshot instead (a same-height re-offer, which we accept). + if matches!(&e, drive::grovedb::Error::InternalError(message) if message.contains("not expected")) + { + tracing::warn!( + chunk_id = hex::encode(&request.chunk_id), + sender = request.sender, + error = ?e, + "[state_sync] apply_snapshot_chunk cannot re-apply a chunk in this session, requesting snapshot restart", + ); + return Ok(proto::ResponseApplySnapshotChunk { + result: response_apply_snapshot_chunk::Result::RetrySnapshot.into(), + refetch_chunks: vec![], + reject_senders, + next_chunks: vec![], + }); + } + // A chunk grovedb cannot apply (corrupted or tampered data) is // recoverable: keep the session and ask Tenderdash to refetch the chunk, // banning the peer that sent it so the refetch goes elsewhere. @@ -80,11 +106,6 @@ where error = ?e, "[state_sync] apply_snapshot_chunk rejected a chunk, requesting refetch", ); - let reject_senders = if request.sender.is_empty() { - vec![] - } else { - vec![request.sender] - }; return Ok(proto::ResponseApplySnapshotChunk { result: response_apply_snapshot_chunk::Result::Retry.into(), refetch_chunks: vec![request.chunk_id], @@ -142,9 +163,15 @@ where )) })?; if !incorrect_hashes.is_empty() { + let paths: Vec = incorrect_hashes + .keys() + .take(5) + .map(|path| path.iter().map(hex::encode).collect::>().join("/")) + .collect(); return Err(AbciError::StateSyncInternalError(format!( - "apply_snapshot_chunk grovedb verification failed with {} incorrect hashes", - incorrect_hashes.len() + "apply_snapshot_chunk grovedb verification failed with {} incorrect hashes, first paths: [{}]", + incorrect_hashes.len(), + paths.join(", ") )) .into()); } diff --git a/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs b/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs index c7838e972ae..4d228040f7d 100644 --- a/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs +++ b/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs @@ -63,9 +63,12 @@ where })?; if let Some(session) = session_write_guard.as_ref() { - if offered_snapshot.height <= session.snapshot.height { + // An offer at the same height is a legitimate snapshot restart (Tenderdash's + // RETRY_SNAPSHOT flow) and replaces the session; only strictly older offers are + // rejected. + if offered_snapshot.height < session.snapshot.height { return Err(AbciError::StateSyncBadRequest(format!( - "offer_snapshot already syncing snapshot at height {}, offered height {} is not newer", + "offer_snapshot already syncing snapshot at height {}, offered height {} is older", session.snapshot.height, offered_snapshot.height )) .into()); @@ -73,7 +76,7 @@ where tracing::warn!( current_height = session.snapshot.height, offered_height = offered_snapshot.height, - "[state_sync] offer_snapshot replacing session in progress with newer snapshot", + "[state_sync] offer_snapshot replacing session in progress", ); } @@ -159,10 +162,17 @@ mod tests { i32::from(response_offer_snapshot::Result::Accept) ); - // A lower-or-equal height while syncing is rejected - assert!(offer_snapshot(&app, offer_at(100, 1)).is_err()); + // A strictly lower height while syncing is rejected assert!(offer_snapshot(&app, offer_at(50, 1)).is_err()); + // A same-height re-offer is a snapshot restart (Tenderdash RETRY_SNAPSHOT): + // the session is replaced and the offer accepted + let response = offer_snapshot(&app, offer_at(100, 1)).expect("should accept restart"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::Accept) + ); + // A newer snapshot replaces the session and MUST also answer Accept // (the old prototype returned the default UNKNOWN result here) let response = offer_snapshot(&app, offer_at(200, 1)).expect("should accept newer offer"); diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/mod.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/mod.rs index 1e963cb1cbd..ab6960ced35 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/test_cases/mod.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/mod.rs @@ -11,6 +11,7 @@ mod process_proposal_collision_tests; mod required_since_update_tests; // TODO: re-enable once OperationType shielded variants are implemented // mod shielded_tests; +mod state_sync_tests; mod token_tests; mod top_up_tests; mod update_identities_tests; diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs new file mode 100644 index 00000000000..f65dac93d43 --- /dev/null +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs @@ -0,0 +1,730 @@ +//! Two-instance ABCI state sync integration tests: a source chain serves snapshots from +//! its checkpoint registry and a fresh target restores one chunk by chunk, then +//! reconstructs its platform state. +//! +//! KNOWN LIMITATION at the pinned grovedb revision (6c882c3): state sync wire protocol +//! version 1 does not faithfully restore SumTree subtrees — the copied node hashes +//! reproduce the source root hash, but re-opening a restored sum tree recomputes a +//! different root (latent corruption), which the strict `verify_grovedb` call in +//! `apply_snapshot_chunk` correctly refuses. See `tests/sum_tree_sync_probe.rs` for the +//! minimal upstream reproducer. The full happy-path test below is therefore `#[ignore]`d +//! until the grovedb pin gains the fixed wire version, and an active test pins today's +//! refusal behavior instead. + +#[cfg(test)] +mod tests { + use crate::execution::run_chain_for_strategy; + use crate::strategy::{ChainExecutionOutcome, NetworkStrategy}; + use dpp::dashcore::hashes::Hash; + use dpp::dashcore::{BlockHash, QuorumHash}; + use dpp::dashcore_rpc::dashcore_rpc_json::{ + ExtendedQuorumDetails, MasternodeListDiff, MasternodeListItem, QuorumInfoResult, + }; + use dpp::dashcore_rpc::json::{ExtendedQuorumListResult, QuorumType}; + use dpp::version::PlatformVersion; + use drive_abci::abci::app::FullAbciApplication; + use drive_abci::config::{ + ChainLockConfig, ExecutionConfig, InstantLockConfig, PlatformConfig, PlatformTestConfig, + ValidatorSetConfig, + }; + use drive_abci::mimic::test_quorum::TestQuorumInfo; + use drive_abci::platform_types::platform::Platform; + use drive_abci::platform_types::platform_state::PlatformStateV0Methods; + use drive_abci::rpc::core::MockCoreRPCLike; + use drive_abci::test::helpers::setup::TestPlatformBuilder; + use std::collections::{BTreeMap, HashMap, VecDeque}; + use strategy_tests::frequency::Frequency; + use strategy_tests::{IdentityInsertInfo, StartAddresses, StartIdentities, Strategy}; + use tenderdash_abci::proto::abci as proto; + use tenderdash_abci::proto::abci::{response_apply_snapshot_chunk, response_offer_snapshot}; + use tenderdash_abci::Application; + + /// A quiet chain with a trickle of identity inserts, no masternode churn and no + /// quorum rotation, so the target's from-scratch Core re-derivation sees exactly + /// the same masternodes and quorums the source chain ran with. + fn state_sync_network_strategy() -> NetworkStrategy { + NetworkStrategy { + strategy: Strategy { + start_contracts: vec![], + operations: vec![], + start_identities: StartIdentities::default(), + start_addresses: StartAddresses::default(), + identity_inserts: IdentityInsertInfo { + frequency: Frequency { + times_per_block_range: 1..2, + chance_per_block: None, + }, + ..Default::default() + }, + identity_contract_nonce_gaps: None, + signer: None, + }, + total_hpmns: 100, + extra_normal_mns: 0, + validator_quorum_count: 24, + chain_lock_quorum_count: 24, + upgrading_info: None, + proposer_strategy: Default::default(), + rotate_quorums: false, + failure_testing: None, + query_testing: None, + verify_state_transition_results: false, + ..Default::default() + } + } + + /// Snapshot serving on with a 1s frequency (every 3s block crosses the boundary, + /// so every block after the first creates a checkpoint), keeping 3 checkpoints. + fn state_sync_platform_config() -> PlatformConfig { + let mut testing_configs = PlatformTestConfig::default_minimal_verifications(); + testing_configs.disable_checkpoints = false; + testing_configs.store_platform_state = true; + + let mut config = PlatformConfig { + validator_set: ValidatorSetConfig::default_100_67(), + chain_lock: ChainLockConfig::default_100_67(), + instant_lock: InstantLockConfig::default_100_67(), + execution: ExecutionConfig { + verify_sum_trees: true, + ..ExecutionConfig::default() + }, + block_spacing_ms: 3000, + testing_configs, + ..Default::default() + }; + config.abci.state_sync.snapshots_enabled = true; + config.abci.state_sync.snapshots_frequency_seconds = 1; + config.abci.state_sync.max_num_snapshots = 3; + config + } + + /// Installs on a fresh target the Core RPC answers its platform state + /// reconstruction will ask for: the full masternode list (the target requests it + /// from scratch, base height None) and the same quorums the source ran with. + fn install_reconstruction_core_mocks( + platform: &mut Platform, + masternodes: Vec, + validator_quorums: &BTreeMap, + ) { + platform + .core_rpc + .expect_get_protx_diff_with_masternodes() + .returning(move |base_block, block| { + assert!( + base_block.is_none(), + "state reconstruction must request the full masternode list from scratch" + ); + Ok(MasternodeListDiff { + base_height: 0, + block_height: block, + added_mns: masternodes.clone(), + removed_mns: vec![], + updated_mns: vec![], + }) + }); + + let quorum_details: Vec<(QuorumHash, ExtendedQuorumDetails)> = validator_quorums + .keys() + .map(|quorum_hash| { + ( + *quorum_hash, + ExtendedQuorumDetails { + creation_height: 0, + quorum_index: None, + mined_block_hash: BlockHash::all_zeros(), + num_valid_members: 0, + health_ratio: 0.0, + }, + ) + }) + .collect(); + platform + .core_rpc + .expect_get_quorum_listextended() + .returning(move |_| { + Ok(ExtendedQuorumListResult { + quorums_by_type: HashMap::from([( + QuorumType::Llmq100_67, + quorum_details.clone().into_iter().collect(), + )]), + }) + }); + + let quorum_infos: HashMap = validator_quorums + .iter() + .map(|(quorum_hash, test_quorum_info)| (*quorum_hash, test_quorum_info.into())) + .collect(); + platform.core_rpc.expect_get_quorum_info().returning( + move |_, quorum_hash: &QuorumHash, _| { + Ok(quorum_infos + .get::(quorum_hash) + .unwrap_or_else(|| { + panic!("expected to get quorum {}", hex::encode(quorum_hash)) + }) + .clone()) + }, + ); + } + + /// Drives the chunk transfer loop between a serving app and a restoring app, + /// modeled on grovedb's run_sync driver: start from the root chunk (id == app + /// hash) and keep requesting whatever the target asks for next. + /// + /// When `tamper_with_first_chunk` is set, the first served chunk is corrupted to + /// prove the target answers RETRY with a refetch of exactly that chunk (banning + /// the sender) instead of killing the session. At the current grovedb revision the + /// refetched chunk cannot be re-applied within the session (grovedb removes a + /// chunk id from its pending set before processing), so the target then answers + /// RETRY_SNAPSHOT; the driver handles that the way Tenderdash would, by + /// re-offering the same snapshot and restarting the transfer. + fn sync_snapshot( + source_app: &FullAbciApplication, + target_app: &FullAbciApplication, + snapshot: &proto::Snapshot, + tamper_with_first_chunk: bool, + ) -> Result<(), proto::ResponseException> { + let mut tamper_next = tamper_with_first_chunk; + let mut restarts = 0usize; + + 'snapshot_attempt: loop { + let offer_response = target_app.offer_snapshot(proto::RequestOfferSnapshot { + snapshot: Some(snapshot.clone()), + app_hash: snapshot.hash.clone(), + })?; + assert_eq!( + offer_response.result, + i32::from(response_offer_snapshot::Result::Accept), + "target must accept the offered snapshot" + ); + + let mut chunk_queue: VecDeque> = VecDeque::from([snapshot.hash.clone()]); + + while let Some(chunk_id) = chunk_queue.pop_front() { + let chunk = source_app + .load_snapshot_chunk(proto::RequestLoadSnapshotChunk { + height: snapshot.height, + version: snapshot.version, + chunk_id: chunk_id.clone(), + })? + .chunk; + + if tamper_next { + tamper_next = false; + let mut tampered = chunk.clone(); + let last = tampered.len() - 1; + tampered[last] ^= 0xff; + + let response = + target_app.apply_snapshot_chunk(proto::RequestApplySnapshotChunk { + chunk_id: chunk_id.clone(), + chunk: tampered, + sender: "malicious-peer".to_string(), + })?; + assert_eq!( + response.result, + i32::from(response_apply_snapshot_chunk::Result::Retry), + "a tampered chunk must be answered with a retry, not kill the session" + ); + assert_eq!( + response.refetch_chunks, + vec![chunk_id.clone()], + "the tampered chunk must be refetched" + ); + assert_eq!(response.reject_senders, vec!["malicious-peer".to_string()]); + assert!( + target_app + .snapshot_fetching_session + .read() + .unwrap() + .is_some(), + "the session must survive a tampered chunk" + ); + } + + let response = + target_app.apply_snapshot_chunk(proto::RequestApplySnapshotChunk { + chunk_id, + chunk, + sender: "honest-peer".to_string(), + })?; + + match response.result { + result + if result == i32::from(response_apply_snapshot_chunk::Result::Accept) => + { + chunk_queue.extend(response.next_chunks); + } + result + if result + == i32::from( + response_apply_snapshot_chunk::Result::CompleteSnapshot, + ) => + { + assert!( + chunk_queue.is_empty(), + "transfer completed with chunks still queued" + ); + return Ok(()); + } + result + if result + == i32::from(response_apply_snapshot_chunk::Result::RetrySnapshot) => + { + restarts += 1; + assert!(restarts <= 2, "too many snapshot restarts"); + continue 'snapshot_attempt; + } + other => panic!("unexpected apply_snapshot_chunk result {}", other), + } + } + + panic!("chunk transfer ran out of chunks without completing"); + } + } + + struct SourceChain<'a> { + source_app: FullAbciApplication<'a, MockCoreRPCLike>, + proposers: Vec, + validator_quorums: BTreeMap, + snapshot: proto::Snapshot, + } + + /// Runs the source chain past several checkpoints and picks its newest offered + /// snapshot. + async fn run_source_chain<'a>( + source_platform: &'a mut drive_abci::test::helpers::setup::TempPlatform, + config: &PlatformConfig, + ) -> SourceChain<'a> { + let ChainExecutionOutcome { + abci_app: source_app, + proposers, + validator_quorums, + .. + } = run_chain_for_strategy( + source_platform, + 15, + state_sync_network_strategy(), + config.clone(), + 15, + &mut None, + &mut None, + ) + .await; + + let snapshots = source_app + .list_snapshots(Default::default()) + .expect("source should list snapshots") + .snapshots; + assert!( + !snapshots.is_empty(), + "the source chain must have produced at least one restorable snapshot" + ); + let snapshot = snapshots + .iter() + .max_by_key(|snapshot| snapshot.height) + .expect("at least one snapshot") + .clone(); + + SourceChain { + source_app, + proposers: proposers + .iter() + .map(|proposer| proposer.masternode.clone()) + .collect(), + validator_quorums, + snapshot, + } + } + + /// End to end: run a source chain past several checkpoints, serve its newest + /// snapshot, restore it chunk by chunk on a fresh target (with one tampered chunk + /// along the way to prove refetch/restart recovery), reconstruct the target + /// platform state, and verify the target matches the source checkpoint exactly. + #[tokio::test] + #[ignore = "grovedb state sync wire v1 (rev 6c882c3) cannot faithfully restore sum trees; \ + unignore when the grovedb pin gains the fixed wire version — see \ + tests/sum_tree_sync_probe.rs and state_sync_transfer_detects_sum_tree_restore_defect"] + async fn run_state_sync_between_two_platforms() { + let config = state_sync_platform_config(); + let mut source_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let source = run_source_chain(&mut source_platform, &config).await; + let snapshot = &source.snapshot; + + // The platform state the source had at exactly the snapshot height + let source_platform_state = source + .source_app + .platform + .checkpoint_platform_states + .load() + .get(&snapshot.height) + .expect("source must cache the platform state of its checkpoint") + .clone(); + + // A fresh target node, knowing nothing but Core RPC + let mut target_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + install_reconstruction_core_mocks( + &mut target_platform.platform, + source.proposers.clone(), + &source.validator_quorums, + ); + let target_app = FullAbciApplication::new(&target_platform); + + sync_snapshot(&source.source_app, &target_app, snapshot, true) + .expect("state sync must complete"); + + let platform_version = PlatformVersion::latest(); + let grove_version = &platform_version.drive.grove_version; + + // Grove roots agree between source checkpoint and target + let target_root_hash = target_platform + .drive + .grove + .root_hash(None, grove_version) + .unwrap() + .expect("target root hash"); + assert_eq!(target_root_hash.to_vec(), snapshot.hash); + + // The restored grovedb is internally consistent + let verification_issues = target_platform + .drive + .grove + .verify_grovedb(None, true, false, grove_version) + .expect("expected to verify grovedb"); + assert!( + verification_issues.is_empty(), + "restored grovedb must verify cleanly: {:?}", + verification_issues + ); + + // The reconstructed platform state matches the source's state at the snapshot + // height, except for the fields that are not replicated (block signature and + // block id hash restore as zeroes) + let target_state = target_platform.state.load(); + assert_eq!( + target_state.current_protocol_version_in_consensus(), + source_platform_state.current_protocol_version_in_consensus() + ); + assert_eq!( + target_state.next_epoch_protocol_version(), + source_platform_state.next_epoch_protocol_version() + ); + assert_eq!( + target_state.last_committed_block_height(), + snapshot.height, + "target must be at the snapshot height" + ); + assert_eq!( + target_state.last_committed_block_app_hash(), + source_platform_state.last_committed_block_app_hash() + ); + assert_eq!( + target_state.current_validator_set_quorum_hash(), + source_platform_state.current_validator_set_quorum_hash() + ); + assert_eq!( + target_state.next_validator_set_quorum_hash(), + source_platform_state.next_validator_set_quorum_hash() + ); + assert_eq!( + target_state.validator_sets().keys().collect::>(), + source_platform_state + .validator_sets() + .keys() + .collect::>(), + "validator set order must be restored from the recorded quorum positions" + ); + assert_eq!( + target_state.validator_sets(), + source_platform_state.validator_sets(), + "validator sets must match" + ); + assert_eq!( + target_state.full_masternode_list(), + source_platform_state.full_masternode_list() + ); + assert_eq!( + target_state.hpmn_masternode_list(), + source_platform_state.hpmn_masternode_list() + ); + assert_eq!( + target_state.previous_fee_versions(), + source_platform_state.previous_fee_versions(), + "fee versions of previous epochs must be restored faithfully" + ); + + // The target's info handler must pass its own app-hash consistency check and + // report the snapshot height and hash to Tenderdash's post-sync verifyApp + let info = target_app + .info(proto::RequestInfo { + version: tenderdash_abci::proto::meta::TENDERDASH_VERSION.to_string(), + block_version: 0, + p2p_version: 0, + abci_version: tenderdash_abci::proto::meta::ABCI_VERSION.to_string(), + }) + .expect("target info handler must succeed"); + assert_eq!(info.last_block_height as u64, snapshot.height); + assert_eq!(info.last_block_app_hash, snapshot.hash); + } + + /// Pins today's behavior at the pinned grovedb revision: the transfer itself + /// completes (including recovery from a tampered chunk via RETRY and a snapshot + /// restart), but the strict post-restore verification detects that wire v1 did not + /// faithfully restore the sum trees and refuses the snapshot instead of accepting + /// latent corruption. When this test starts failing because the sync SUCCEEDS, + /// grovedb has been fixed: un-ignore `run_state_sync_between_two_platforms` and + /// drop this pin. + #[tokio::test] + async fn state_sync_transfer_detects_sum_tree_restore_defect() { + let config = state_sync_platform_config(); + let mut source_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let source = run_source_chain(&mut source_platform, &config).await; + + let mut target_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + install_reconstruction_core_mocks( + &mut target_platform.platform, + source.proposers.clone(), + &source.validator_quorums, + ); + let target_app = FullAbciApplication::new(&target_platform); + + let error = sync_snapshot(&source.source_app, &target_app, &source.snapshot, true) + .expect_err( + "at grovedb rev 6c882c3 the restored sum trees must fail verification — if \ + this now succeeds, grovedb is fixed: un-ignore \ + run_state_sync_between_two_platforms and remove this pin", + ); + assert!( + error.error.contains("incorrect hashes"), + "the refusal must come from the post-restore grovedb verification, got: {}", + error.error + ); + + // The target refused the snapshot: it never advanced past genesis + assert_eq!( + target_platform.state.load().last_committed_block_height(), + 0 + ); + } + + /// Exercises the platform state reconstruction end to end without going through + /// the (currently defective, see above) grovedb chunk restore: the source chain's + /// own grovedb IS a faithfully "restored" snapshot of itself, so reconstructing + /// on it must (a) not change the grovedb root hash — the proof that re-deriving + /// masternode identities from Core is byte-idempotent — and (b) reproduce the + /// source's in-memory platform state from the reduced platform state alone. + #[tokio::test] + async fn platform_state_reconstruction_is_idempotent_and_matches_source_state() { + let config = state_sync_platform_config(); + let mut source_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let source = run_source_chain(&mut source_platform, &config).await; + let platform = source.source_app.platform; + + let platform_version = PlatformVersion::latest(); + let grove_version = &platform_version.drive.grove_version; + + let original_state = platform.state.load().clone(); + let tip_app_hash = platform + .drive + .grove + .root_hash(None, grove_version) + .unwrap() + .expect("source root hash"); + assert_eq!( + original_state.last_committed_block_app_hash(), + Some(tip_app_hash), + "sanity: chain tip state matches grove root" + ); + + // The run_chain mocks already answer the from-scratch masternode/quorum + // requests reconstruction makes, exactly as they did for the chain itself. + platform + .reconstruct_platform_state(&tip_app_hash, platform_version) + .expect("platform state reconstruction must succeed"); + + // (a) idempotence: re-deriving masternode identities wrote nothing new + let root_hash_after = platform + .drive + .grove + .root_hash(None, grove_version) + .unwrap() + .expect("source root hash after reconstruction"); + assert_eq!( + root_hash_after, tip_app_hash, + "reconstruction must not change the grovedb root hash" + ); + + // (b) the reconstructed state matches the original, except the fields the + // reduced state cannot carry (block signature / block id hash) + let reconstructed_state = platform.state.load(); + assert_eq!( + reconstructed_state.current_protocol_version_in_consensus(), + original_state.current_protocol_version_in_consensus() + ); + assert_eq!( + reconstructed_state.next_epoch_protocol_version(), + original_state.next_epoch_protocol_version() + ); + assert_eq!( + reconstructed_state.last_committed_block_height(), + original_state.last_committed_block_height() + ); + assert_eq!( + reconstructed_state.last_committed_block_app_hash(), + original_state.last_committed_block_app_hash() + ); + assert_eq!( + reconstructed_state.last_committed_core_height(), + original_state.last_committed_core_height() + ); + assert_eq!( + reconstructed_state.current_validator_set_quorum_hash(), + original_state.current_validator_set_quorum_hash() + ); + assert_eq!( + reconstructed_state.next_validator_set_quorum_hash(), + original_state.next_validator_set_quorum_hash() + ); + assert_eq!( + reconstructed_state + .validator_sets() + .keys() + .collect::>(), + original_state.validator_sets().keys().collect::>(), + "validator set order must be restored from the recorded quorum positions" + ); + assert_eq!( + reconstructed_state.validator_sets(), + original_state.validator_sets() + ); + assert_eq!( + reconstructed_state.full_masternode_list(), + original_state.full_masternode_list() + ); + assert_eq!( + reconstructed_state.hpmn_masternode_list(), + original_state.hpmn_masternode_list() + ); + assert_eq!( + reconstructed_state.previous_fee_versions(), + original_state.previous_fee_versions() + ); + + // The info handler accepts the reconstructed state (it panics on an app-hash + // mismatch between the in-memory state and the grove root) + let info = source + .source_app + .info(proto::RequestInfo { + version: tenderdash_abci::proto::meta::TENDERDASH_VERSION.to_string(), + block_version: 0, + p2p_version: 0, + abci_version: tenderdash_abci::proto::meta::ABCI_VERSION.to_string(), + }) + .expect("info handler must accept the reconstructed state"); + assert_eq!( + info.last_block_height as u64, + original_state.last_committed_block_height() + ); + assert_eq!(info.last_block_app_hash, tip_app_hash.to_vec()); + } + + /// A snapshot from a chain that never wrote the reduced platform state (pre-v15) + /// is not offered by the source, and a target driven at it anyway refuses to + /// restore it. + #[tokio::test] + async fn pre_v15_snapshot_is_not_served_and_cannot_be_restored() { + let config = state_sync_platform_config(); + + let mut source_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .with_initial_protocol_version(14) + .build_with_mock_rpc(); + + let ChainExecutionOutcome { + abci_app: source_app, + proposers, + validator_quorums, + .. + } = run_chain_for_strategy( + &mut source_platform, + 15, + state_sync_network_strategy(), + config.clone(), + 15, + &mut None, + &mut None, + ) + .await; + + // The v14 chain created checkpoints, but none carries the reduced platform + // state, so none may be offered. + assert!( + !source_app.platform.drive.checkpoints.load().is_empty(), + "the source must have created checkpoints" + ); + let snapshots = source_app + .list_snapshots(Default::default()) + .expect("source should list snapshots") + .snapshots; + assert!( + snapshots.is_empty(), + "pre-v15 checkpoints are unrestorable and must not be offered" + ); + + // Even if a peer maliciously offers such a snapshot, the target must refuse to + // restore it. (At the current grovedb revision the refusal comes from the + // post-restore verification; once grovedb faithfully restores sum trees it + // comes from the missing reduced platform state at the reconstruction step. + // Either way the snapshot must not be accepted.) + let (height, checkpoint) = { + let checkpoints = source_app.platform.drive.checkpoints.load(); + let (height, info) = checkpoints + .last_key_value() + .expect("at least one checkpoint"); + (*height, std::sync::Arc::clone(&info.checkpoint)) + }; + let platform_version = PlatformVersion::latest(); + let checkpoint_root = checkpoint + .grove_db + .root_hash(None, &platform_version.drive.grove_version) + .unwrap() + .expect("checkpoint root hash"); + let forged_snapshot = proto::Snapshot { + height, + version: 1, + hash: checkpoint_root.to_vec(), + metadata: vec![], + }; + + let mut target_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + install_reconstruction_core_mocks( + &mut target_platform.platform, + proposers + .iter() + .map(|proposer| proposer.masternode.clone()) + .collect(), + &validator_quorums, + ); + let target_app = FullAbciApplication::new(&target_platform); + + sync_snapshot(&source_app, &target_app, &forged_snapshot, false) + .expect_err("a snapshot without the reduced platform state must be refused"); + + // The target holds no usable platform state: it never advanced past genesis + assert_eq!( + target_platform.state.load().last_committed_block_height(), + 0 + ); + } +} diff --git a/packages/rs-drive-abci/tests/sum_tree_sync_probe.rs b/packages/rs-drive-abci/tests/sum_tree_sync_probe.rs new file mode 100644 index 00000000000..e8a45af0940 --- /dev/null +++ b/packages/rs-drive-abci/tests/sum_tree_sync_probe.rs @@ -0,0 +1,125 @@ +//! Minimal reproducer / tripwire for a grovedb state sync limitation at the pinned +//! revision (6c882c3): wire protocol version 1 does not faithfully restore SumTree +//! subtrees. The chunk transfer copies the source's node hashes, so the restored +//! database reproduces the source ROOT hash — but re-opening the restored sum tree +//! and recomputing its root yields a different hash, i.e. the corruption is latent +//! and `verify_grovedb` detects it. +//! +//! This is why `apply_snapshot_chunk` runs the strict `verify_grovedb` check after +//! committing a state sync session, and why the full two-instance state sync +//! integration test (`run_state_sync_between_two_platforms`) is `#[ignore]`d. +//! +//! WHEN THIS TEST STARTS FAILING because no verification issues are reported, the +//! grovedb pin has been fixed: delete this tripwire and un-ignore the full +//! integration test. + +use drive::grovedb::{Element, GroveDb}; +use drive::grovedb_path::SubtreePath; +use platform_version::version::PlatformVersion; +use std::collections::VecDeque; + +#[test] +fn sum_tree_state_sync_restore_is_latently_corrupt_at_pinned_grovedb() { + let grove_version = &PlatformVersion::latest().drive.grove_version; + let source_dir = tempfile::tempdir().unwrap(); + let source = GroveDb::open(source_dir.path()).unwrap(); + + let root: SubtreePath<[u8; 0]> = SubtreePath::empty(); + + source + .insert( + root.clone(), + b"s", + Element::empty_sum_tree(), + None, + None, + grove_version, + ) + .unwrap() + .unwrap(); + let sum_path: &[&[u8]] = &[b"s"]; + for (key, value) in [(b"a", 5i64), (b"b", 7i64)] { + source + .insert( + sum_path, + key, + Element::new_sum_item(value), + None, + None, + grove_version, + ) + .unwrap() + .unwrap(); + } + // A normal tree with an item, for contrast: it restores cleanly. + source + .insert( + root.clone(), + b"n", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .unwrap(); + let normal_path: &[&[u8]] = &[b"n"]; + source + .insert( + normal_path, + b"k", + Element::new_item(b"v".to_vec()), + None, + None, + grove_version, + ) + .unwrap() + .unwrap(); + + let app_hash = source.root_hash(None, grove_version).unwrap().unwrap(); + + let target_dir = tempfile::tempdir().unwrap(); + let target = GroveDb::open(target_dir.path()).unwrap(); + let mut session = target + .start_snapshot_syncing(app_hash, 64, 1, grove_version) + .unwrap(); + + let mut queue: VecDeque> = VecDeque::from([app_hash.to_vec()]); + while let Some(chunk_id) = queue.pop_front() { + let chunk = source + .fetch_chunk(&chunk_id, None, 1, grove_version) + .unwrap(); + let next = session + .apply_chunk(&chunk_id, &chunk, 1, grove_version) + .unwrap(); + queue.extend(next); + if session.is_sync_completed() { + break; + } + } + assert!(session.is_sync_completed()); + target.commit_session(session, grove_version).unwrap(); + + // The copied node hashes reproduce the source root hash exactly... + let target_root = target.root_hash(None, grove_version).unwrap().unwrap(); + assert_eq!( + target_root, app_hash, + "restored root hash must match the source" + ); + + // ...but recomputing the restored sum tree exposes the latent corruption. + let issues = target + .verify_grovedb(None, true, false, grove_version) + .unwrap(); + let paths: Vec = issues + .keys() + .map(|path| path.iter().map(hex::encode).collect::>().join("/")) + .collect(); + assert_eq!( + paths, + vec!["73".to_string()], // hex of b"s", the sum tree + "expected exactly the sum tree to fail verification — if no issues are \ + reported, grovedb has been fixed: delete this tripwire and un-ignore \ + run_state_sync_between_two_platforms" + ); +} From cff70ebb161ec13d40b7e8cc8196efc47f68dc9c Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 29 Aug 2026 02:51:32 +0200 Subject: [PATCH 11/34] fix(drive-abci): commit state sync re-derivation before publishing reconstructed state Review follow-up: reconstruct_platform_state now commits the update_core_info re-derivation before update_state_cache publishes the in-memory state, so a commit failure propagates without the info handler ever reporting a snapshot height grovedb never persisted. Aux writes (not part of the root hash) commit in their own transaction afterwards. Also documents that the RetrySnapshot string-match fallback is safe if grovedb's error wording changes. Co-Authored-By: Claude Fable 5 --- .../src/abci/handler/apply_snapshot_chunk.rs | 4 +++ .../reconstruct_platform_state/mod.rs | 27 +++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs index e7b55c26d24..20c3572e5e2 100644 --- a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs +++ b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs @@ -81,6 +81,10 @@ where // so a chunk it has already seen (e.g. the refetch of one it rejected) // cannot be re-applied within this session: ask Tenderdash to restart // the snapshot instead (a same-height re-offer, which we accept). + // The string match is brittle by necessity (grovedb only exposes + // InternalError(String) here); if the wording ever changes, the fallback + // below is still safe — Tenderdash retries the chunk until it gives up + // and restarts the snapshot itself. if matches!(&e, drive::grovedb::Error::InternalError(message) if message.contains("not expected")) { tracing::warn!( diff --git a/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs index b8f177daecb..311a96276c3 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs @@ -164,23 +164,40 @@ where let block_height = platform_state.last_committed_block_height(); + // Commit the re-derivation BEFORE the in-memory state is published: if this + // commit fails, nothing has been published and the error propagates with the + // node's observable state unchanged. (Publishing first, as normal block + // finalization does, would leave the info handler reporting a snapshot height + // that grovedb never persisted.) + self.drive + .grove + .commit_transaction(transaction) + .unwrap() + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "reconstruct_platform_state unable to commit transaction: {}", + e + )) + })?; + // Advance the state to the snapshot block: rotates next-into-current exactly as // the source did on finalization, persists to aux storage and publishes the - // state for the info handler. + // state for the info handler. Aux writes are not part of the root hash, so + // committing them separately cannot change the app hash the caller verifies. + let aux_transaction = self.drive.grove.start_transaction(); self.update_state_cache( current_block_info, platform_state, - &transaction, + &aux_transaction, state_platform_version, )?; - self.drive .grove - .commit_transaction(transaction) + .commit_transaction(aux_transaction) .unwrap() .map_err(|e| { AbciError::StateSyncInternalError(format!( - "reconstruct_platform_state unable to commit transaction: {}", + "reconstruct_platform_state unable to commit aux transaction: {}", e )) })?; From 4968a6813e15e303efcd842df4b7261f568d39b8 Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 29 Aug 2026 15:56:08 +0200 Subject: [PATCH 12/34] =?UTF-8?q?fix(drive-abci):=20FEATURE=20FIX=20?= =?UTF-8?q?=E2=80=94=20clear=20Drive=20caches=20when=20offer=5Fsnapshot=20?= =?UTF-8?q?wipes=20grovedb?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit *** THIS COMMIT CHANGES FEATURE CODE, NOT TESTS — please review it on its own. *** offer_snapshot calls drive.grove.wipe() and then restores a snapshot, but Drive's lazily-loaded in-memory caches were left pointing at the state that was just destroyed. The protocol version counter is the damaging one: ProtocolVersionsCache keeps a 'loaded' flag, so load_if_needed never re-reads the restored version counters, and the first block after the restore writes vote counts derived from the WIPED chain. The result is an immediate app hash fork against every other node. Reproduced by state_synced_and_replayed_nodes_stay_converged: with a node whose caches had been touched before the snapshot offer, the synced node and the replayed node disagreed on the app hash at the very first block after the sync, with the divergence isolated to the Versions tree (RootTree::Versions and Versions/0). The test passes with this fix. Reset the counter wholesale rather than calling clear_global_cache, so the loaded flag is cleared too and the cache reloads from the restored state. Also clear the data contract cache and the cached genesis time, for the same reason. system_data_contracts is deliberately left alone: those are compiled-in, version-keyed contracts that never come from grovedb. Reachability: Tenderdash normally offers a snapshot only at startup, before any block has been processed, so on today's code paths the caches are usually still empty and the fork is not reachable in production. This is a latent landmine rather than a live incident — but offer_snapshot performs a destructive wipe and must not leave derived state behind. Co-Authored-By: Claude Fable 5 --- .../src/abci/handler/offer_snapshot.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs b/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs index 4d228040f7d..225c0dee428 100644 --- a/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs +++ b/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs @@ -86,6 +86,21 @@ where AbciError::StateSyncInternalError(format!("offer_snapshot unable to wipe grovedb: {}", e)) })?; + // The wipe destroyed the state every lazily-loaded Drive cache was built from. Left + // in place, those caches would be silently merged into the RESTORED state and fork the + // node: `ProtocolVersionsCache` in particular keeps a `loaded` flag, so + // `load_if_needed` would never re-read the restored version counters and the next block + // would write vote counts derived from the wiped chain instead. Resetting the counter + // wholesale (rather than `clear_global_cache`) is deliberate — it also clears that + // flag, so the cache reloads from the restored state on first use. + // + // `system_data_contracts` is deliberately NOT cleared: those are compiled-in, + // version-keyed contracts that never come from grovedb. + let drive = &app.platform().drive; + *drive.cache.protocol_versions_counter.write() = Default::default(); + drive.cache.data_contracts.clear(); + *drive.cache.genesis_time_ms.write() = None; + let state_sync_info = app .platform() .drive From 78bd27bf005c56b6bbe033f3ee79089e821dfa05 Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 29 Aug 2026 16:36:36 +0200 Subject: [PATCH 13/34] fix(drive-abci): never wedge a node on an interrupted or unusable state sync restore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit *** THIS COMMIT CHANGES FEATURE CODE, NOT TESTS — please review it on its own. *** A restore destroys the database before it rebuilds it, and the rebuild is not atomic with the platform state that has to describe it. Two paths left a node holding a database its platform state knew nothing about, and the info handler panics on exactly that mismatch, so drive-abci crash-looped on the first ABCI call and restarting only reloaded the state causing it: a crash between commit_session and reconstruct_platform_state, and a snapshot that turns out to be unusable — which any peer can cause by offering a pre-v15 one. Restore sentinel. offer_snapshot writes a marker file BEFORE it wipes, so there is no window where the database is destroyed and nothing says so. Platform::open_with_client treats a surviving marker as an unfinished restore: wipe, drop the caches derived from what was wiped, come up empty, clear the marker. The marker is a plain file in db_path, NOT aux storage, because GroveDb::wipe() clears the aux column family too — a sentinel there would be destroyed by the very wipe it exists to survive. It is outside everything grovedb touches and can never affect the app hash. Rejection path. Every failure after commit_session now goes through reject_restored_snapshot: wipe back to a clean slate and answer REJECT_SNAPSHOT rather than returning an error, so Tenderdash discards this snapshot, tries the next, and falls back to block sync when it runs out. An ABCI exception there would abort state sync altogether. Detecting an unusable snapshot BEFORE the commit would be better, but grovedb keeps MultiStateSyncSession::transaction private, so the Misc tree cannot be probed before it lands; that is a follow-up for grovedb #840. Clear points. The marker is cleared when the node is provably self-consistent: after a completed restore, after startup recovery has wiped, and at the end of init_chain — the last of these is what stops an abandoned restore from making the next restart wipe a perfectly good block-synced chain. It is deliberately kept on the rejection path, because an empty database plus a stale in-memory platform state is not yet consistent. The wipe-and-clear-caches helper is now shared by the offer path and the recovery path so the two cannot drift. Co-Authored-By: Claude Fable 5 --- .../src/abci/handler/apply_snapshot_chunk.rs | 154 ++++++++++++++---- .../src/abci/handler/init_chain.rs | 14 ++ .../src/abci/handler/offer_snapshot.rs | 41 ++--- .../src/platform_types/platform/mod.rs | 35 ++++ .../src/platform_types/snapshot/mod.rs | 93 ++++++++++- 5 files changed, 285 insertions(+), 52 deletions(-) diff --git a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs index 20c3572e5e2..f10e87413d2 100644 --- a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs +++ b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs @@ -2,7 +2,10 @@ use crate::abci::app::StateSyncApplication; use crate::abci::AbciError; use crate::error::Error; use crate::platform_types::platform_state::PlatformStateV0Methods; -use crate::platform_types::snapshot::{MAX_STATE_SYNC_CHUNK_ID_SIZE, MAX_STATE_SYNC_CHUNK_SIZE}; +use crate::platform_types::snapshot::{ + clear_restore_sentinel, wipe_drive_for_restore, MAX_STATE_SYNC_CHUNK_ID_SIZE, + MAX_STATE_SYNC_CHUNK_SIZE, +}; use crate::rpc::core::CoreRPCLike; use tenderdash_abci::proto::abci as proto; use tenderdash_abci::proto::abci::response_apply_snapshot_chunk; @@ -155,49 +158,77 @@ where tracing::debug!("[state_sync] transfer complete, verifying grovedb"); - let incorrect_hashes = app - .platform() - .drive - .grove - .verify_grovedb(None, true, false, grove_version) - .map_err(|e| { - AbciError::StateSyncInternalError(format!( - "apply_snapshot_chunk unable to verify grovedb: {}", - e - )) - })?; + // From here on the session is COMMITTED: grovedb durably holds the restored state + // while the platform state still describes the node from before the sync. Every + // failure below must therefore go through `reject_restored_snapshot`, which puts the + // node back to an empty, self-consistent slate and asks Tenderdash for another + // snapshot. Returning an error instead would leave the node holding a database its + // platform state knows nothing about, and the `info` handler panics on exactly that + // mismatch — a crash loop that no restart can clear. + let incorrect_hashes = + match app + .platform() + .drive + .grove + .verify_grovedb(None, true, false, grove_version) + { + Ok(incorrect_hashes) => incorrect_hashes, + Err(e) => { + return reject_restored_snapshot( + app, + &format!("unable to verify the restored grovedb: {}", e), + ); + } + }; if !incorrect_hashes.is_empty() { let paths: Vec = incorrect_hashes .keys() .take(5) .map(|path| path.iter().map(hex::encode).collect::>().join("/")) .collect(); - return Err(AbciError::StateSyncInternalError(format!( - "apply_snapshot_chunk grovedb verification failed with {} incorrect hashes, first paths: [{}]", - incorrect_hashes.len(), - paths.join(", ") - )) - .into()); + return reject_restored_snapshot( + app, + &format!( + "grovedb verification failed with {} incorrect hashes, first paths: [{}]", + incorrect_hashes.len(), + paths.join(", ") + ), + ); } // Rebuild the in-memory platform state from the reduced platform state contained in // the restored snapshot. This re-derives masternode lists and quorums from Core and // must leave the grovedb root hash untouched; the equality check below proves it. - app.platform() - .reconstruct_platform_state(&session.app_hash, platform_version)?; + // + // This is also where a snapshot taken before the reduced platform state existed + // (pre-v15) is refused. Refusing earlier would be better, but grovedb does not expose + // the session's transaction, so the Misc tree cannot be probed before the commit — + // see the note on `reject_restored_snapshot`. + if let Err(e) = app + .platform() + .reconstruct_platform_state(&session.app_hash, platform_version) + { + return reject_restored_snapshot( + app, + &format!("unable to reconstruct the platform state: {}", e), + ); + } - let drive_app_hash = app + let drive_app_hash = match app .platform() .drive .grove .root_hash(None, grove_version) .unwrap() - .map_err(|e| { - AbciError::StateSyncInternalError(format!( - "apply_snapshot_chunk unable to get app hash: {}", - e - )) - })?; + { + Ok(drive_app_hash) => drive_app_hash, + Err(e) => { + return reject_restored_snapshot( + app, + &format!("unable to get the restored app hash: {}", e), + ); + } + }; if drive_app_hash != session.app_hash { tracing::error!( @@ -205,13 +236,25 @@ where drive_app_hash = hex::encode(drive_app_hash), "[state_sync] restored grovedb root hash does not match the snapshot app hash", ); - return Err(AbciError::StateSyncInternalError(format!( - "apply_snapshot_chunk grovedb verification failed with incorrect app hash: {}", - hex::encode(drive_app_hash) - )) - .into()); + return reject_restored_snapshot( + app, + &format!( + "grovedb verification failed with incorrect app hash: {}", + hex::encode(drive_app_hash) + ), + ); } + // The restore is complete and the node is self-consistent again, so the marker that + // tells a restarting process to wipe can go. This is deliberately the LAST step, after + // `reconstruct_platform_state` has committed the platform state to aux storage. + clear_restore_sentinel(&app.platform().config.db_path).map_err(|e| { + AbciError::StateSyncInternalError(format!( + "apply_snapshot_chunk unable to clear the restore sentinel: {}", + e + )) + })?; + tracing::info!( height = session.snapshot.height, app_hash = hex::encode(session.app_hash), @@ -226,6 +269,53 @@ where }) } +/// Puts the node back to an empty, self-consistent slate after a restore that was already +/// committed to grovedb turned out to be unusable, and asks Tenderdash to try a different +/// snapshot. +/// +/// Ideally an unusable snapshot would be detected BEFORE `commit_session`, by probing the +/// Misc tree through the session's still-open transaction. grovedb keeps that transaction +/// private (`MultiStateSyncSession::transaction`, no accessor), so there is no way to read +/// the restored state before it lands. Until grovedb exposes it, this is the containment: +/// undo the commit by wiping, and let Tenderdash pick another snapshot. +/// +/// The restore sentinel is deliberately LEFT IN PLACE. The database is empty, but the +/// in-memory platform state may still describe the chain the offer wiped, so the node is +/// not yet provably consistent. Everything that can happen next resolves it: another +/// `offer_snapshot` re-wipes and re-marks, a successful restore clears it, an `init_chain` +/// clears it, and a restart before any of those wipes and comes up empty. +/// +/// `REJECT_SNAPSHOT` rather than an error is what keeps Tenderdash walking its ladder: it +/// discards this snapshot, tries the next, and falls back to block sync when it runs out. +/// An ABCI exception here would abort state sync altogether. +fn reject_restored_snapshot<'a, 'db: 'a, A, C>( + app: &'a A, + reason: &str, +) -> Result +where + A: StateSyncApplication<'db, C> + 'db, + C: CoreRPCLike + 'db, +{ + tracing::error!( + reason, + "[state_sync] restored snapshot is unusable, wiping and asking for another one", + ); + + wipe_drive_for_restore(&app.platform().drive).map_err(|e| { + AbciError::StateSyncInternalError(format!( + "apply_snapshot_chunk unable to wipe after rejecting a snapshot ({}): {}", + reason, e + )) + })?; + + Ok(proto::ResponseApplySnapshotChunk { + result: response_apply_snapshot_chunk::Result::RejectSnapshot.into(), + refetch_chunks: vec![], + reject_senders: vec![], + next_chunks: vec![], + }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/packages/rs-drive-abci/src/abci/handler/init_chain.rs b/packages/rs-drive-abci/src/abci/handler/init_chain.rs index 0573b05e6b3..a7bf050698e 100644 --- a/packages/rs-drive-abci/src/abci/handler/init_chain.rs +++ b/packages/rs-drive-abci/src/abci/handler/init_chain.rs @@ -1,5 +1,7 @@ use crate::abci::app::{BlockExecutionApplication, PlatformApplication, TransactionalApplication}; +use crate::abci::AbciError; use crate::error::Error; +use crate::platform_types::snapshot::clear_restore_sentinel; use crate::rpc::core::CoreRPCLike; use tenderdash_abci::proto::abci as proto; @@ -32,6 +34,18 @@ where let app_hash = hex::encode(&response.app_hash); + // Genesis has just been created, so the node is self-consistent again. If a state sync + // restore had been abandoned (every offered snapshot rejected, Tenderdash falling back + // to block sync), its marker is still on disk and would make the NEXT restart wipe this + // perfectly good chain. Clear it here — this is the block-sync arm of the same recovery + // that `Platform::open_with_client` performs for an interrupted restore. + clear_restore_sentinel(&app.platform().config.db_path).map_err(|e| { + AbciError::StateSyncInternalError(format!( + "init_chain unable to clear the restore sentinel: {}", + e + )) + })?; + tracing::info!( app_hash, chain_id, diff --git a/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs b/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs index 225c0dee428..5231371d6f0 100644 --- a/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs +++ b/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs @@ -3,7 +3,8 @@ use crate::abci::AbciError; use crate::error::Error; use crate::platform_types::platform_state::PlatformStateV0Methods; use crate::platform_types::snapshot::{ - SnapshotFetchingSession, STATE_SYNC_SUBTREES_BATCH_SIZE, SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS, + wipe_drive_for_restore, write_restore_sentinel, SnapshotFetchingSession, + STATE_SYNC_SUBTREES_BATCH_SIZE, SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS, }; use crate::rpc::core::CoreRPCLike; use tenderdash_abci::proto::abci as proto; @@ -80,26 +81,28 @@ where ); } - // Both the fresh-session and the replace-session paths wipe grovedb, start a new - // grovedb sync session, and answer Accept. - app.platform().drive.grove.wipe().map_err(|e| { - AbciError::StateSyncInternalError(format!("offer_snapshot unable to wipe grovedb: {}", e)) + // Mark the database as under restore BEFORE destroying it. From here until the + // restore completes, the node may be in a state that cannot serve consensus, and the + // only thing that can tell a restarted process so is this marker: without it, startup + // finds a database that disagrees with its platform state and cannot distinguish an + // interrupted restore from corruption. See `Platform::open_with_client`. + write_restore_sentinel( + &app.platform().config.db_path, + &request_app_hash, + offered_snapshot.height, + ) + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "offer_snapshot unable to record the restore sentinel: {}", + e + )) })?; - // The wipe destroyed the state every lazily-loaded Drive cache was built from. Left - // in place, those caches would be silently merged into the RESTORED state and fork the - // node: `ProtocolVersionsCache` in particular keeps a `loaded` flag, so - // `load_if_needed` would never re-read the restored version counters and the next block - // would write vote counts derived from the wiped chain instead. Resetting the counter - // wholesale (rather than `clear_global_cache`) is deliberate — it also clears that - // flag, so the cache reloads from the restored state on first use. - // - // `system_data_contracts` is deliberately NOT cleared: those are compiled-in, - // version-keyed contracts that never come from grovedb. - let drive = &app.platform().drive; - *drive.cache.protocol_versions_counter.write() = Default::default(); - drive.cache.data_contracts.clear(); - *drive.cache.genesis_time_ms.write() = None; + // Both the fresh-session and the replace-session paths wipe grovedb (dropping the + // caches derived from it), start a new grovedb sync session, and answer Accept. + wipe_drive_for_restore(&app.platform().drive).map_err(|e| { + AbciError::StateSyncInternalError(format!("offer_snapshot unable to wipe grovedb: {}", e)) + })?; let state_sync_info = app .platform() diff --git a/packages/rs-drive-abci/src/platform_types/platform/mod.rs b/packages/rs-drive-abci/src/platform_types/platform/mod.rs index 972e530a1ce..a59abe9d7e8 100644 --- a/packages/rs-drive-abci/src/platform_types/platform/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/platform/mod.rs @@ -10,6 +10,9 @@ use std::fmt::{Debug, Formatter}; use crate::platform_types::check_tx_proof_verifier::CheckTxProofVerifier; use crate::platform_types::platform_state::{PlatformState, PlatformStateV0Methods}; +use crate::platform_types::snapshot::{ + clear_restore_sentinel, restore_sentinel_exists, wipe_drive_for_restore, +}; use arc_swap::ArcSwap; use dpp::prelude::BlockHeight; use dpp::serialization::PlatformDeserializableFromVersionedStructure; @@ -147,6 +150,38 @@ impl Platform { let (drive, current_platform_version) = Drive::open(&config.db_path, Some(config.drive.clone())).map_err(Error::Drive)?; + // A state sync restore that never finished leaves grovedb holding state the + // platform state knows nothing about. That is not recoverable by restarting — + // the `info` handler panics on the mismatch, so the node would crash-loop — and it + // cannot be told apart from corruption without a marker. `offer_snapshot` writes + // one before it wipes; if it is still here, the restore did not finish. + // + // Recovery is to become an empty node: wipe, drop the caches derived from what was + // wiped, and come up as if freshly installed, so Tenderdash can offer another + // snapshot or fall back to block sync. Clearing the marker afterwards is safe + // precisely because an empty database with no saved state is self-consistent. + let current_platform_version = if restore_sentinel_exists(&config.db_path) { + tracing::warn!( + db_path = ?config.db_path, + "[state_sync] an unfinished state sync restore was found on startup; wiping \ + and coming up empty so the node can sync again", + ); + + wipe_drive_for_restore(&drive).map_err(Error::Drive)?; + clear_restore_sentinel(&config.db_path).map_err(|e| { + Error::Drive(drive::error::Error::IOErrorWithInfoString( + e.into(), + "trying to clear the state sync restore sentinel".to_owned(), + )) + })?; + + // The wipe removed the stored protocol version along with everything else, so + // the saved-state branch below must not be taken. + None + } else { + current_platform_version + }; + if let Some(platform_version) = current_platform_version { let Some(execution_state) = Platform::::fetch_platform_state(&drive, None, platform_version)? diff --git a/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs index 58f83e2aad8..2979d0736ba 100644 --- a/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs @@ -4,14 +4,105 @@ //! (`drive.checkpoints`, populated by `create_grovedb_checkpoint` after each qualifying //! block is committed); there is no separate snapshot store. -use drive::drive::Checkpoint; +use drive::drive::{Checkpoint, Drive}; use drive::grovedb::replication::MultiStateSyncSession; use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::{Arc, RwLock}; use std::time::{Duration, Instant}; use tenderdash_abci::proto::abci; +/// Name of the marker file that records "a state sync restore is in progress". +/// +/// ## Why a plain file, and not aux storage +/// +/// The obvious home would be grovedb's aux column family, which is not part of the +/// provable tree. It cannot be used here: `GroveDb::wipe()` clears the aux column family +/// along with `default`, `roots` and `meta` +/// (`grovedb/storage/src/rocksdb_storage/storage.rs`, `wipe()` iterates all four). Since +/// wiping is exactly what both the offer path and the recovery path do, a sentinel living +/// in aux would be destroyed by the very operations it exists to survive, and its +/// lifetime would depend on subtle ordering between the write and the wipe. +/// +/// A file next to the database has none of those problems: it is outside everything +/// grovedb touches, it survives any wipe, it costs one `stat` at startup, and an operator +/// can see it. It is deliberately NOT in the provable tree either — it is node-local +/// recovery bookkeeping and must never affect the app hash. +pub const RESTORE_IN_PROGRESS_FILE_NAME: &str = "state_sync_restore_in_progress"; + +/// Path of the restore sentinel for a given database directory. +pub fn restore_sentinel_path(db_path: &Path) -> PathBuf { + db_path.join(RESTORE_IN_PROGRESS_FILE_NAME) +} + +/// Records that a state sync restore has started and the database is therefore allowed to +/// be inconsistent until it finishes. +/// +/// Written BEFORE the wipe, so the window in which the database has been destroyed but +/// nothing marks it as such is empty. The contents are for operators only; the code cares +/// solely about the file's presence. +pub fn write_restore_sentinel( + db_path: &Path, + app_hash: &[u8; 32], + height: u64, +) -> std::io::Result<()> { + std::fs::create_dir_all(db_path)?; + std::fs::write( + restore_sentinel_path(db_path), + format!( + "state sync restore in progress\nheight: {}\napp_hash: {}\n", + height, + hex::encode(app_hash) + ), + ) +} + +/// Clears the restore sentinel. Only ever called once the node is in a self-consistent +/// state: after a restore has fully completed, after startup recovery has wiped, or after +/// a genesis initialization. +pub fn clear_restore_sentinel(db_path: &Path) -> std::io::Result<()> { + match std::fs::remove_file(restore_sentinel_path(db_path)) { + Ok(()) => Ok(()), + // Absent is the normal case on every path that clears defensively. + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error), + } +} + +/// Whether a restore was in progress when this node last stopped. +pub fn restore_sentinel_exists(db_path: &Path) -> bool { + restore_sentinel_path(db_path).exists() +} + +/// Drops every Drive cache that was derived from grovedb. +/// +/// A wipe destroys the state these caches were built from. Left in place they would be +/// silently merged into whatever replaces it: `ProtocolVersionsCache` in particular keeps +/// a `loaded` flag, so `load_if_needed` would never re-read the new version counters and +/// the next block would write vote counts derived from the wiped chain — an immediate app +/// hash fork. Resetting the counter wholesale (rather than `clear_global_cache`) is +/// deliberate: it clears that flag too, so the cache reloads on first use. +/// +/// `system_data_contracts` is deliberately NOT cleared — those are compiled-in, +/// version-keyed contracts that never come from grovedb. +pub fn reset_drive_caches_after_wipe(drive: &Drive) { + *drive.cache.protocol_versions_counter.write() = Default::default(); + drive.cache.data_contracts.clear(); + *drive.cache.genesis_time_ms.write() = None; +} + +/// Wipes grovedb and drops the caches derived from it, leaving the node an empty but +/// entirely self-consistent slate. +/// +/// This is the single place both the offer path and the crash-recovery path go through, +/// so the two can never drift apart. +pub fn wipe_drive_for_restore(drive: &Drive) -> Result<(), drive::error::Error> { + drive.grove.wipe()?; + reset_drive_caches_after_wipe(drive); + Ok(()) +} + /// The grovedb state sync wire protocol versions this node can serve and consume. /// /// This is THE single supported-set constant: when grovedb wire version 2 lands, add it From 9d6653472f03eec262ae5fd6fd02f3fdef291d9d Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 29 Aug 2026 16:37:16 +0200 Subject: [PATCH 14/34] test(drive-abci): cover the state sync restore sentinel and the never-wedge guarantee MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regression tests for the preceding fix. Deliberately free of any assertion that a restore SUCCEEDS, so they are green at BOTH grovedb pins: Dash Platform state always contains sum trees, so a full successful restore needs dashpay/grovedb#840, but a restore that FAILS exercises the same recovery path either way — at the unpatched revision the sum-tree defect supplies the failure for free. Covers: offer_snapshot records the sentinel before it wipes; a rejected offer records none, so a peer cannot make a healthy node wipe itself on the next restart just by offering a format it cannot speak; a restart mid-restore wipes, comes up empty and passes the info handshake instead of crash-looping; a NORMAL restart keeps its state, which is the regression that matters most if startup recovery ever fires unconditionally; init_chain clears a sentinel left by an abandoned restore, so the block-sync fallback's chain survives the next restart; and end to end, an unusable snapshot offered by a peer leaves the node empty, recoverable and able to sync. The shared chunk-loop driver now reports REJECT_SNAPSHOT as a SnapshotSyncOutcome::Rejected rather than treating it as an unexpected result code, and the two existing tests that relied on the old error-returning refusal assert the rejection plus the new wipe-back-to-clean behaviour. Co-Authored-By: Claude Fable 5 --- .../tests/strategy_tests/test_cases/mod.rs | 3 +- .../test_cases/state_sync_sentinel_tests.rs | 521 ++++++++++++++++++ .../test_cases/state_sync_tests.rs | 107 +++- 3 files changed, 608 insertions(+), 23 deletions(-) create mode 100644 packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/mod.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/mod.rs index ab6960ced35..3a6b59a8e27 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/test_cases/mod.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/mod.rs @@ -11,7 +11,8 @@ mod process_proposal_collision_tests; mod required_since_update_tests; // TODO: re-enable once OperationType shielded variants are implemented // mod shielded_tests; -mod state_sync_tests; +mod state_sync_sentinel_tests; +pub(crate) mod state_sync_tests; mod token_tests; mod top_up_tests; mod update_identities_tests; diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs new file mode 100644 index 00000000000..41b83964dfe --- /dev/null +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs @@ -0,0 +1,521 @@ +//! State sync QA: the restore sentinel and the never-wedge guarantee. +//! +//! A state sync restore destroys the node's database before it rebuilds it, and the +//! rebuild is not atomic with the platform state that has to describe it. Two things can +//! therefore leave a node holding a database its platform state knows nothing about: +//! +//! * the process dies between `commit_session` and `reconstruct_platform_state`; +//! * the restored snapshot turns out to be unusable (a pre-v15 snapshot with no reduced +//! platform state, or one that fails verification) — which any peer can cause. +//! +//! Both used to wedge the node permanently, because the `info` handler panics on an +//! app-hash mismatch and restarting reloads exactly the state that causes the panic. The +//! fix is a sentinel file written next to the database before the wipe, cleared only when +//! the node is self-consistent again, plus a rejection path that wipes back to a clean +//! slate instead of returning an error. +//! +//! # These tests do not need the patched grovedb +//! +//! Everything here holds at BOTH grovedb pins. Nothing asserts that a restore SUCCEEDS — +//! the tests that do live in `state_sync_equivalence_tests` and need dashpay/grovedb#840, +//! because Dash Platform state always contains sum trees. What is asserted here is that a +//! restore which does not succeed leaves a recoverable node, and at the unpinned revision +//! the sum-tree defect simply supplies the failure for free: the transfer commits, the +//! post-restore verification fails, and the same rejection path runs. + +#[cfg(test)] +mod tests { + use crate::execution::run_chain_for_strategy; + use crate::strategy::{ChainExecutionOutcome, NetworkStrategy}; + use crate::test_cases::state_sync_tests::tests::{ + install_reconstruction_core_mocks, sync_snapshot, SnapshotSyncOutcome, + }; + use dpp::version::PlatformVersion; + use drive_abci::abci::app::FullAbciApplication; + use drive_abci::config::{ + ChainLockConfig, ExecutionConfig, InstantLockConfig, PlatformConfig, PlatformTestConfig, + ValidatorSetConfig, + }; + use drive_abci::platform_types::platform::Platform; + use drive_abci::platform_types::platform_state::PlatformStateV0Methods; + use drive_abci::platform_types::snapshot::{ + restore_sentinel_exists, write_restore_sentinel, RESTORE_IN_PROGRESS_FILE_NAME, + }; + use drive_abci::rpc::core::MockCoreRPCLike; + use drive_abci::test::helpers::setup::{TempPlatform, TestPlatformBuilder}; + use strategy_tests::frequency::Frequency; + use strategy_tests::{IdentityInsertInfo, StartAddresses, StartIdentities, Strategy}; + use tenderdash_abci::proto::abci as proto; + use tenderdash_abci::proto::abci::response_offer_snapshot; + use tenderdash_abci::Application; + + const SOURCE_CHAIN_BLOCKS: u64 = 6; + const SOURCE_CHAIN_SEED: u64 = 15; + + fn sentinel_platform_config() -> PlatformConfig { + let mut testing_configs = PlatformTestConfig::default_minimal_verifications(); + testing_configs.disable_checkpoints = false; + testing_configs.store_platform_state = true; + + let mut config = PlatformConfig { + validator_set: ValidatorSetConfig::default_100_67(), + chain_lock: ChainLockConfig::default_100_67(), + instant_lock: InstantLockConfig::default_100_67(), + execution: ExecutionConfig { + verify_sum_trees: true, + ..ExecutionConfig::default() + }, + block_spacing_ms: 3000, + testing_configs, + ..Default::default() + }; + config.abci.state_sync.snapshots_enabled = true; + config.abci.state_sync.snapshots_frequency_seconds = 1; + config.abci.state_sync.max_num_snapshots = 3; + config + } + + fn sentinel_strategy() -> NetworkStrategy { + NetworkStrategy { + strategy: Strategy { + start_contracts: vec![], + operations: vec![], + start_identities: StartIdentities::default(), + start_addresses: StartAddresses::default(), + identity_inserts: IdentityInsertInfo { + frequency: Frequency { + times_per_block_range: 1..3, + chance_per_block: None, + }, + ..Default::default() + }, + identity_contract_nonce_gaps: None, + signer: None, + }, + total_hpmns: 100, + extra_normal_mns: 0, + validator_quorum_count: 24, + chain_lock_quorum_count: 24, + upgrading_info: None, + proposer_strategy: Default::default(), + rotate_quorums: false, + failure_testing: None, + query_testing: None, + verify_state_transition_results: false, + ..Default::default() + } + } + + fn root_hash(platform: &Platform) -> [u8; 32] { + platform + .drive + .grove + .root_hash(None, &PlatformVersion::latest().drive.grove_version) + .unwrap() + .expect("root hash") + } + + fn info_request() -> proto::RequestInfo { + proto::RequestInfo { + version: tenderdash_abci::proto::meta::TENDERDASH_VERSION.to_string(), + block_version: 0, + p2p_version: 0, + abci_version: tenderdash_abci::proto::meta::ABCI_VERSION.to_string(), + } + } + + /// Models process death: drops the `Platform` (releasing grovedb's lock and every + /// in-memory session, cache and platform state) and re-opens the SAME directory, which + /// is what a restarted drive-abci does. Only what was durably written survives. + fn restart( + target: TempPlatform, + config: &PlatformConfig, + ) -> TempPlatform { + let TempPlatform { + platform, tempdir, .. + } = target; + drop(platform); + TempPlatform::open_with_tempdir(tempdir, config.clone()) + } + + /// Calling `info` must not panic. The handler panics on an app-hash mismatch between + /// the platform state and grovedb, which is the exact shape of the wedge, so "did it + /// panic" is the property under test rather than the returned value. + fn info_does_not_panic(platform: &TempPlatform) -> bool { + let app = FullAbciApplication::new(platform); + let previous_hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let result = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| app.info(info_request()))); + std::panic::set_hook(previous_hook); + result.is_ok() + } + + /// `offer_snapshot` must record the sentinel BEFORE it wipes, so there is no window in + /// which the database has been destroyed and nothing says so. + #[tokio::test] + async fn offer_snapshot_records_the_restore_sentinel_before_wiping() { + let config = sentinel_platform_config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let outcome = run_chain_for_strategy( + &mut platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + let app = outcome.abci_app; + let db_path = app.platform.config.db_path.clone(); + + assert!( + !restore_sentinel_exists(&db_path), + "a node that never state-synced must not carry the sentinel" + ); + + let response = app + .offer_snapshot(proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height: 1000, + version: 1, + hash: vec![7u8; 32], + metadata: vec![], + }), + app_hash: vec![7u8; 32], + }) + .expect("the offer must be accepted"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::Accept) + ); + + assert!( + restore_sentinel_exists(&db_path), + "accepting an offer wipes the database, so it must first record that a restore \ + is in progress" + ); + // The sentinel is a plain file NEXT TO the database, not aux storage: `wipe()` + // clears the aux column family too, so a sentinel stored there would be destroyed + // by the very wipe it exists to survive. + assert!( + db_path.join(RESTORE_IN_PROGRESS_FILE_NAME).is_file(), + "the sentinel must live outside everything grovedb wipes" + ); + } + + /// A rejected offer must not record a sentinel — nothing was wiped, so nothing needs + /// recovering. Without this, any peer could make a healthy node wipe itself on the next + /// restart just by offering a snapshot in a format it cannot speak. + #[tokio::test] + async fn a_rejected_offer_records_no_sentinel() { + let config = sentinel_platform_config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let outcome = run_chain_for_strategy( + &mut platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + let app = outcome.abci_app; + let db_path = app.platform.config.db_path.clone(); + + let response = app + .offer_snapshot(proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height: 1000, + version: u32::MAX, + hash: vec![7u8; 32], + metadata: vec![], + }), + app_hash: vec![7u8; 32], + }) + .expect("an unsupported version must be answered, not error"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::RejectFormat) + ); + assert!( + !restore_sentinel_exists(&db_path), + "a rejected offer wipes nothing and must leave no sentinel behind" + ); + } + + /// The startup recovery itself: a node whose sentinel is still present comes up EMPTY + /// rather than crash-looping. + /// + /// The database here is deliberately a healthy, fully populated chain — the strongest + /// form of "grovedb holds state the platform state will not describe". Recovery must + /// throw it away, because there is no way to tell how far an interrupted restore got. + #[tokio::test] + async fn a_node_restarting_mid_restore_wipes_and_comes_up_empty() { + let config = sentinel_platform_config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let outcome = run_chain_for_strategy( + &mut platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + let db_path = outcome.abci_app.platform.config.db_path.clone(); + assert_eq!( + outcome + .abci_app + .platform + .state + .load() + .last_committed_block_height(), + SOURCE_CHAIN_BLOCKS + ); + drop(outcome); + + // A restore was in progress when the process died. + write_restore_sentinel(&db_path, &[9u8; 32], 1000).expect("write sentinel"); + + let restarted = restart(platform, &config); + + assert_eq!( + restarted.state.load().last_committed_block_height(), + 0, + "an unfinished restore must not leave the node claiming a height it cannot back up" + ); + assert_eq!( + root_hash(&restarted.platform), + [0u8; 32], + "the database must have been wiped to an empty, self-consistent state" + ); + assert!( + !restore_sentinel_exists(&db_path), + "once the node is empty it is self-consistent again, so the sentinel is cleared" + ); + assert!( + info_does_not_panic(&restarted), + "THE WHOLE POINT: the info handshake must succeed, so the node can be offered \ + another snapshot or fall back to block sync instead of crash-looping" + ); + } + + /// The complement, and the regression that matters most: a node WITHOUT a sentinel + /// must never be wiped. If startup recovery ever fires unconditionally it would + /// silently destroy every node's chain on restart. + #[tokio::test] + async fn a_normal_restart_keeps_its_state() { + let config = sentinel_platform_config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let outcome = run_chain_for_strategy( + &mut platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + let db_path = outcome.abci_app.platform.config.db_path.clone(); + let healthy_root_hash = root_hash(outcome.abci_app.platform); + drop(outcome); + + assert!(!restore_sentinel_exists(&db_path)); + let restarted = restart(platform, &config); + + assert_eq!( + restarted.state.load().last_committed_block_height(), + SOURCE_CHAIN_BLOCKS, + "a normal restart must come back at the tip" + ); + assert_eq!( + root_hash(&restarted.platform), + healthy_root_hash, + "a normal restart must not touch the database" + ); + assert!(info_does_not_panic(&restarted)); + } + + /// The block-sync arm of the recovery. If every offered snapshot is rejected, + /// Tenderdash gives up on state sync and block-syncs from genesis. `init_chain` is + /// where the node becomes self-consistent again, so it must clear a sentinel left over + /// from the abandoned restore — otherwise the NEXT restart would wipe a perfectly good + /// chain. + #[tokio::test] + async fn init_chain_clears_a_sentinel_left_by_an_abandoned_restore() { + let config = sentinel_platform_config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let db_path = platform.platform.config.db_path.clone(); + + // An abandoned restore: the marker is present and the database is empty. + write_restore_sentinel(&db_path, &[9u8; 32], 1000).expect("write sentinel"); + + // Block sync from genesis, which begins with init_chain. + let outcome = run_chain_for_strategy( + &mut platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + + assert_eq!( + outcome + .abci_app + .platform + .state + .load() + .last_committed_block_height(), + SOURCE_CHAIN_BLOCKS, + "the node must block-sync normally after an abandoned restore" + ); + assert!( + !restore_sentinel_exists(&db_path), + "init_chain makes the node self-consistent, so it must clear the sentinel — \ + otherwise the next restart would wipe this chain" + ); + drop(outcome); + + // And prove it: a restart keeps the block-synced chain. + let restarted = restart(platform, &config); + assert_eq!( + restarted.state.load().last_committed_block_height(), + SOURCE_CHAIN_BLOCKS, + "the chain built after an abandoned restore must survive a restart" + ); + } + + /// End to end for the remotely-triggerable case: a peer offers a snapshot this node + /// cannot use, and the node must end up able to sync rather than wedged. + /// + /// The snapshot here is a pre-v15 one (a v14 chain's checkpoint, which carries no + /// reduced platform state). Nothing stops a peer from advertising it: `proto::Snapshot` + /// carries a height, a wire version and a hash, and no protocol version at all. + /// + /// This test is pin-agnostic on purpose. With grovedb #840 the refusal comes from the + /// missing reduced platform state; at the unpatched revision the sum-tree defect makes + /// the post-restore verification fail first. Either way the snapshot is refused AFTER + /// the session was committed, which is precisely the path that has to leave the node + /// recoverable. + #[tokio::test] + async fn an_unusable_snapshot_leaves_the_node_able_to_sync() { + let config = sentinel_platform_config(); + + let mut source_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .with_initial_protocol_version(14) + .build_with_mock_rpc(); + let ChainExecutionOutcome { + abci_app: source_app, + proposers, + validator_quorums, + .. + } = run_chain_for_strategy( + &mut source_platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + + let (height, checkpoint) = { + let checkpoints = source_app.platform.drive.checkpoints.load(); + let (height, info) = checkpoints + .last_key_value() + .expect("at least one checkpoint"); + (*height, std::sync::Arc::clone(&info.checkpoint)) + }; + let platform_version = PlatformVersion::latest(); + let checkpoint_root = checkpoint + .grove_db + .root_hash(None, &platform_version.drive.grove_version) + .unwrap() + .expect("checkpoint root hash"); + let forged_snapshot = proto::Snapshot { + height, + version: platform_version.drive_abci.state_sync.protocol_version as u32, + hash: checkpoint_root.to_vec(), + metadata: vec![], + }; + + let mut target_platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + install_reconstruction_core_mocks( + &mut target_platform.platform, + proposers + .iter() + .map(|proposer| proposer.masternode.clone()) + .collect(), + &validator_quorums, + ); + let db_path = target_platform.platform.config.db_path.clone(); + + { + let target_app = FullAbciApplication::new(&target_platform); + let outcome = sync_snapshot(&source_app, &target_app, &forged_snapshot, false) + .expect("an unusable snapshot must be answered, not errored"); + assert_eq!( + outcome, + SnapshotSyncOutcome::Rejected, + "an unusable snapshot must be answered with REJECT_SNAPSHOT so Tenderdash \ + tries the next one instead of aborting state sync" + ); + + // The node wiped itself back to a clean slate rather than keeping state it + // cannot use... + assert_ne!( + root_hash(&target_platform.platform).to_vec(), + forged_snapshot.hash, + "the refused snapshot must not be left on disk" + ); + assert_eq!( + root_hash(&target_platform.platform), + [0u8; 32], + "the refusal must leave an empty database" + ); + // ...and the sentinel stays, because the in-memory platform state may still + // describe the chain the offer wiped. Whatever happens next resolves it. + assert!( + restore_sentinel_exists(&db_path), + "the node is empty but not yet provably consistent, so the marker stays \ + until a restore succeeds, an init_chain runs, or a restart wipes" + ); + } + + // A restart is the worst case, and it recovers. + let restarted = restart(target_platform, &config); + assert_eq!(restarted.state.load().last_committed_block_height(), 0); + assert_eq!(root_hash(&restarted.platform), [0u8; 32]); + assert!( + !restore_sentinel_exists(&db_path), + "startup recovery leaves the node self-consistent and clears the marker" + ); + assert!( + info_does_not_panic(&restarted), + "THE FIX: a peer offering an unusable snapshot must not be able to wedge this \ + node. Before the fix, info panicked here and drive-abci crash-looped." + ); + } +} diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs index f65dac93d43..fd982a4d14e 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs @@ -12,7 +12,7 @@ //! refusal behavior instead. #[cfg(test)] -mod tests { +pub(crate) mod tests { use crate::execution::run_chain_for_strategy; use crate::strategy::{ChainExecutionOutcome, NetworkStrategy}; use dpp::dashcore::hashes::Hash; @@ -101,7 +101,7 @@ mod tests { /// Installs on a fresh target the Core RPC answers its platform state /// reconstruction will ask for: the full masternode list (the target requests it /// from scratch, base height None) and the same quorums the source ran with. - fn install_reconstruction_core_mocks( + pub(crate) fn install_reconstruction_core_mocks( platform: &mut Platform, masternodes: Vec, validator_quorums: &BTreeMap, @@ -177,12 +177,25 @@ mod tests { /// chunk id from its pending set before processing), so the target then answers /// RETRY_SNAPSHOT; the driver handles that the way Tenderdash would, by /// re-offering the same snapshot and restarting the transfer. - fn sync_snapshot( + /// How a snapshot transfer ended. + /// + /// `Rejected` is not an error: the target restored the snapshot, found it unusable, + /// wiped itself back to a clean slate and asked Tenderdash for a different one. The + /// driver reports it so tests can tell a clean refusal from a transport failure. + #[derive(Debug, PartialEq, Eq)] + pub(crate) enum SnapshotSyncOutcome { + /// The target restored and accepted the snapshot. + Completed, + /// The target answered REJECT_SNAPSHOT; Tenderdash would move on to the next one. + Rejected, + } + + pub(crate) fn sync_snapshot( source_app: &FullAbciApplication, target_app: &FullAbciApplication, snapshot: &proto::Snapshot, tamper_with_first_chunk: bool, - ) -> Result<(), proto::ResponseException> { + ) -> Result { let mut tamper_next = tamper_with_first_chunk; let mut restarts = 0usize; @@ -264,7 +277,24 @@ mod tests { chunk_queue.is_empty(), "transfer completed with chunks still queued" ); - return Ok(()); + return Ok(SnapshotSyncOutcome::Completed); + } + result + if result + == i32::from(response_apply_snapshot_chunk::Result::RejectSnapshot) => + { + // The target restored the snapshot, found it unusable and wiped + // itself back to a clean slate. Tenderdash would try the next + // snapshot; there is nothing more for this driver to do. + assert!( + target_app + .snapshot_fetching_session + .read() + .unwrap() + .is_none(), + "a rejected snapshot must not leave a session open" + ); + return Ok(SnapshotSyncOutcome::Rejected); } result if result @@ -341,8 +371,8 @@ mod tests { /// along the way to prove refetch/restart recovery), reconstruct the target /// platform state, and verify the target matches the source checkpoint exactly. #[tokio::test] - #[ignore = "grovedb state sync wire v1 (rev 6c882c3) cannot faithfully restore sum trees; \ - unignore when the grovedb pin gains the fixed wire version — see \ + #[ignore = "the pinned grovedb (6c882c3) cannot faithfully restore sum trees; un-ignore \ + when the pin includes the sum-tree restore fix (dashpay/grovedb#840) — see \ tests/sum_tree_sync_probe.rs and state_sync_transfer_detects_sum_tree_restore_defect"] async fn run_state_sync_between_two_platforms() { let config = state_sync_platform_config(); @@ -373,8 +403,12 @@ mod tests { ); let target_app = FullAbciApplication::new(&target_platform); - sync_snapshot(&source.source_app, &target_app, snapshot, true) - .expect("state sync must complete"); + assert_eq!( + sync_snapshot(&source.source_app, &target_app, snapshot, true) + .expect("state sync must not error"), + SnapshotSyncOutcome::Completed, + "state sync must complete" + ); let platform_version = PlatformVersion::latest(); let grove_version = &platform_version.drive.grove_version; @@ -495,23 +529,34 @@ mod tests { ); let target_app = FullAbciApplication::new(&target_platform); - let error = sync_snapshot(&source.source_app, &target_app, &source.snapshot, true) - .expect_err( - "at grovedb rev 6c882c3 the restored sum trees must fail verification — if \ - this now succeeds, grovedb is fixed: un-ignore \ - run_state_sync_between_two_platforms and remove this pin", - ); - assert!( - error.error.contains("incorrect hashes"), - "the refusal must come from the post-restore grovedb verification, got: {}", - error.error + let outcome = sync_snapshot(&source.source_app, &target_app, &source.snapshot, true) + .expect("a refused snapshot is answered, not errored"); + assert_eq!( + outcome, + SnapshotSyncOutcome::Rejected, + "at grovedb rev 6c882c3 the restored sum trees must fail verification — if this \ + now completes, grovedb is fixed: un-ignore run_state_sync_between_two_platforms \ + and remove this pin" ); - // The target refused the snapshot: it never advanced past genesis + // The target refused the snapshot: it never advanced past genesis, and — since the + // refusal happens after the session was already committed — it wiped itself back to + // a clean slate rather than keeping the unusable state. assert_eq!( target_platform.state.load().last_committed_block_height(), 0 ); + assert_ne!( + target_platform + .drive + .grove + .root_hash(None, &PlatformVersion::latest().drive.grove_version) + .unwrap() + .expect("target root hash") + .to_vec(), + source.snapshot.hash, + "a refused snapshot must not be left on disk" + ); } /// Exercises the platform state reconstruction end to end without going through @@ -718,13 +763,31 @@ mod tests { ); let target_app = FullAbciApplication::new(&target_platform); - sync_snapshot(&source_app, &target_app, &forged_snapshot, false) - .expect_err("a snapshot without the reduced platform state must be refused"); + let outcome = sync_snapshot(&source_app, &target_app, &forged_snapshot, false) + .expect("a refused snapshot is answered, not errored"); + assert_eq!( + outcome, + SnapshotSyncOutcome::Rejected, + "a snapshot without the reduced platform state must be refused" + ); // The target holds no usable platform state: it never advanced past genesis assert_eq!( target_platform.state.load().last_committed_block_height(), 0 ); + // ...and it did not keep the state it could not use: the refusal wipes back to a + // clean slate so Tenderdash can offer another snapshot or fall back to block sync. + assert_ne!( + target_platform + .drive + .grove + .root_hash(None, &platform_version.drive.grove_version) + .unwrap() + .expect("target root hash") + .to_vec(), + forged_snapshot.hash, + "a refused snapshot must not be left on disk" + ); } } From e3e9f1d8be8c0058191e113f217d7af86544b7cc Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 29 Aug 2026 16:39:43 +0200 Subject: [PATCH 15/34] docs(platform-version): flag the FEE_VERSION2 fee_version_number collision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No behaviour change — this only makes an existing landmine visible. FEE_VERSION2, which protocol versions 9 and later actually run with, declares fee_version_number 1, the same number FEE_VERSION1 declares, and is absent from FEE_VERSIONS. FeeVersion::get resolves numbers through that list, so FeeVersion::get(1) can only ever return FEE_VERSION1 — never FEE_VERSION2 — even though the two differ in data_contract_registration. That makes every number-only round trip of a fee version silently lossy, and there are two: PlatformStateForSavingV1 stores previous_fee_versions as (epoch index -> number), so a node that RESTARTS rehydrates previous epochs' fees as FEE_VERSION1; ReducedPlatformStateV0 does the same, so a node that STATE-SYNCS gets the substitution without even restarting. It is latent rather than a live fork only because previous_fee_versions is consulted solely to price storage refunds and the two constants have identical storage fees. It becomes a consensus fork the moment a future FeeVersion changes a storage or processing fee without taking a distinct number. Documents the rule — every FeeVersion constant must have a unique fee_version_number and be listed in FEE_VERSIONS at the index its number implies — and adds fee_version_numbers_are_unique_and_resolvable to enforce it. The test is #[ignore]d because it fails today; running it with --ignored reproduces the defect. Un-ignore it as part of giving FEE_VERSION2 its own number, which is protocol-visible and needs a migration rather than an in-place edit. Co-Authored-By: Claude Fable 5 --- .../src/version/fee/mod.rs | 68 +++++++++++++++++++ .../rs-platform-version/src/version/fee/v2.rs | 38 +++++++++++ 2 files changed, 106 insertions(+) diff --git a/packages/rs-platform-version/src/version/fee/mod.rs b/packages/rs-platform-version/src/version/fee/mod.rs index 8b603e2ddba..1df42b6beb0 100644 --- a/packages/rs-platform-version/src/version/fee/mod.rs +++ b/packages/rs-platform-version/src/version/fee/mod.rs @@ -29,6 +29,16 @@ pub mod vote_resolution_fund_fees; pub type FeeVersionNumber = u32; +/// The fee schedules [`FeeVersion::get`] can resolve, indexed by `fee_version_number - 1`. +/// +/// # This list is INCOMPLETE, and that is a known defect +/// +/// `FEE_VERSION2` — what protocol versions 9 and later actually run with — is missing, and +/// declares `fee_version_number: 1`, colliding with `FEE_VERSION1`. Since the fee version +/// NUMBER is the only thing persisted (`PlatformStateForSavingV1` and +/// `ReducedPlatformStateV0` both store `epoch index -> number`), every node that restarts +/// or state-syncs rehydrates previous epochs' fees as `FEE_VERSION1`. See the doc comment +/// on [`v2::FEE_VERSION2`] for why that is currently latent and what fixing it requires. pub const FEE_VERSIONS: &[FeeVersion] = &[FEE_VERSION1]; #[derive(Clone, Debug, Encode, Decode, Default, PartialEq, Eq)] @@ -116,3 +126,61 @@ impl From for FeeVersion { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::version::fee::v2::FEE_VERSION2; + + /// Every `FeeVersion` constant must carry a distinct `fee_version_number`, and + /// `FEE_VERSIONS` must contain all of them, because the number is the ONLY thing + /// persisted: `PlatformStateForSavingV1` and `ReducedPlatformStateV0` both store + /// `(epoch index -> fee version number)` and rehydrate through `FeeVersion::get`. A + /// number that does not resolve back to the constant it came from silently substitutes + /// a different fee schedule on any node that restarts or state-syncs. + /// + /// This test FAILS today, which is why it is ignored: `FEE_VERSION2` declares + /// `fee_version_number: 1`, the same as `FEE_VERSION1`, and is absent from + /// `FEE_VERSIONS`, so `FeeVersion::get(1)` returns `FEE_VERSION1` even for the epochs + /// that ran on `FEE_VERSION2`. See the doc comment on `FEE_VERSION2`. + /// + /// Un-ignore it as part of giving `FEE_VERSION2` its own number and adding it to + /// `FEE_VERSIONS`. That is protocol-visible and needs a migration, which is why the + /// defect is pinned here rather than fixed in place. + #[test] + #[ignore = "known defect: FEE_VERSION2 reuses fee_version_number 1 and is absent from \ + FEE_VERSIONS; fixing it is protocol-visible - see the FEE_VERSION2 docs"] + fn fee_version_numbers_are_unique_and_resolvable() { + let all_fee_versions = [&FEE_VERSION1, &FEE_VERSION2]; + + for fee_version in all_fee_versions { + let resolved = FeeVersion::get(fee_version.fee_version_number).unwrap_or_else(|_| { + panic!( + "fee version number {} does not resolve through FEE_VERSIONS", + fee_version.fee_version_number + ) + }); + assert_eq!( + resolved, fee_version, + "FeeVersion::get({}) returned a DIFFERENT fee schedule than the constant \ + declaring that number. Every number-only round trip - a node restarting, a \ + node state-syncing - would substitute this wrong schedule.", + fee_version.fee_version_number + ); + } + + let mut numbers: Vec = all_fee_versions + .iter() + .map(|fee_version| fee_version.fee_version_number) + .collect(); + numbers.sort_unstable(); + let mut deduped = numbers.clone(); + deduped.dedup(); + assert_eq!( + deduped.len(), + numbers.len(), + "two FeeVersion constants share a fee_version_number: {:?}", + numbers + ); + } +} diff --git a/packages/rs-platform-version/src/version/fee/v2.rs b/packages/rs-platform-version/src/version/fee/v2.rs index fe82ac5534f..24f09b06a49 100644 --- a/packages/rs-platform-version/src/version/fee/v2.rs +++ b/packages/rs-platform-version/src/version/fee/v2.rs @@ -9,7 +9,45 @@ use crate::version::fee::vote_resolution_fund_fees::v1::VOTE_RESOLUTION_FUND_FEE use crate::version::fee::FeeVersion; /// Introduced in protocol version 9 (2.0) +/// +/// # WARNING: `fee_version_number` collides with [`FEE_VERSION1`], and this one is not +/// reachable by number +/// +/// [`FeeVersion::get`] resolves a number through [`FEE_VERSIONS`], which contains only +/// `FEE_VERSION1`. This constant declares the SAME `fee_version_number: 1`, so +/// `FeeVersion::get(1)` can only ever return `FEE_VERSION1` — never this one, even though +/// this is what protocol versions 9 and later actually run with, and the two differ in +/// `data_contract_registration`. +/// +/// That makes every number-only round trip of a fee version silently lossy. Two exist: +/// +/// * `PlatformStateForSavingV1` stores `previous_fee_versions` as +/// `(epoch index -> fee version number)`, so a node that RESTARTS rehydrates previous +/// epochs' fees as `FEE_VERSION1`; +/// * `ReducedPlatformStateV0` does the same, so a node that STATE-SYNCS gets the same +/// substitution without even restarting. +/// +/// It is latent rather than a live consensus fork only because `previous_fee_versions` is +/// consulted solely to price storage refunds (`rs-drive/src/fees/op.rs`), and +/// `FEE_VERSION1` and `FEE_VERSION2` have IDENTICAL `storage` fees. It becomes a fork the +/// moment a future `FeeVersion` changes a storage or processing fee without also taking a +/// distinct number. +/// +/// ## The rule +/// +/// **Every `FeeVersion` constant must have a unique `fee_version_number`, and must be +/// listed in [`FEE_VERSIONS`] at the index its number implies.** Fixing this constant to +/// `fee_version_number: 2` and adding it to `FEE_VERSIONS` is protocol-visible (it changes +/// what a restarted or state-synced node computes for old epochs), so it needs a +/// versioned migration rather than an in-place edit — which is why this is documented here +/// instead of changed. `fee_version_numbers_are_unique` in `super` is the enforcement, and +/// is `#[ignore]`d until then. +/// +/// [`FEE_VERSION1`]: crate::version::fee::v1::FEE_VERSION1 +/// [`FEE_VERSIONS`]: crate::version::fee::FEE_VERSIONS +/// [`FeeVersion::get`]: crate::version::fee::FeeVersion::get pub const FEE_VERSION2: FeeVersion = FeeVersion { + // BUG: must be 2. See the doc comment above — changing it is protocol-visible. fee_version_number: 1, uses_version_fee_multiplier_permille: Some(1000), //No action storage: FEE_STORAGE_VERSION1, From 89c5e077284a9d03622431bafc61e1d1776198c8 Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 29 Aug 2026 17:02:36 +0200 Subject: [PATCH 16/34] fix(drive-abci): clear the checkpoint registry on wipe and stop failing on sentinel cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit *** THIS COMMIT CHANGES FEATURE CODE, NOT TESTS — please review it on its own. *** Three findings from an independent review of the preceding fix. 1. The wipe did not clear drive.checkpoints. That registry is populated by Drive::open and is what list_snapshots serves to peers — it is not a value cache that merely goes stale. Left in place across a wipe, a node that discarded a chain kept advertising snapshots of it, so a peer could state-sync from state this node no longer had. Now cleared, with the entries marked for deletion first so their directories are removed rather than leaking on disk. Regression test: a_wiped_node_stops_serving_snapshots_of_the_discarded_chain. 2. Clearing the sentinel at the two points where the node is ALREADY self-consistent — the end of a completed restore, and the end of init_chain — propagated I/O errors, so a failed remove_file turned a fully successful restore or a working genesis into a hard ABCI error. Now best-effort with a loud error log: the cost of not removing it is one unnecessary wipe-and-resync on a later restart, which is bounded and safe, unlike failing the operation. 3. commit_session's own failure still returned an ABCI exception rather than going through the recovery path. grovedb only makes the session durable once its internal root-hash check passes, so nothing is committed on that error — but the database is still WIPED from the offer, so the node must not be left as it is, and an exception stalls Tenderdash's snapshot ladder where REJECT_SNAPSHOT keeps it moving. Co-Authored-By: Claude Fable 5 --- .../src/abci/handler/apply_snapshot_chunk.rs | 30 +++++----- .../src/abci/handler/init_chain.rs | 13 ++-- .../src/platform_types/snapshot/mod.rs | 32 ++++++++++ .../test_cases/state_sync_sentinel_tests.rs | 59 +++++++++++++++++++ 4 files changed, 111 insertions(+), 23 deletions(-) diff --git a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs index f10e87413d2..8888b1307dc 100644 --- a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs +++ b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs @@ -3,7 +3,7 @@ use crate::abci::AbciError; use crate::error::Error; use crate::platform_types::platform_state::PlatformStateV0Methods; use crate::platform_types::snapshot::{ - clear_restore_sentinel, wipe_drive_for_restore, MAX_STATE_SYNC_CHUNK_ID_SIZE, + clear_restore_sentinel_best_effort, wipe_drive_for_restore, MAX_STATE_SYNC_CHUNK_ID_SIZE, MAX_STATE_SYNC_CHUNK_SIZE, }; use crate::rpc::core::CoreRPCLike; @@ -145,16 +145,19 @@ where .take() .expect("session presence was just checked"); - app.platform() + // grovedb only makes the session durable once its own root-hash check passes, so a + // failure here leaves nothing committed — but the database is still WIPED from the + // offer, so the node cannot be left as it is. Route it through the same recovery path + // as every later failure, which also keeps Tenderdash's snapshot ladder moving instead + // of aborting state sync with an exception. + if let Err(e) = app + .platform() .drive .grove .commit_session(session.state_sync_info, grove_version) - .map_err(|e| { - AbciError::StateSyncInternalError(format!( - "apply_snapshot_chunk unable to commit session: {}", - e - )) - })?; + { + return reject_restored_snapshot(app, &format!("unable to commit the session: {}", e)); + } tracing::debug!("[state_sync] transfer complete, verifying grovedb"); @@ -247,13 +250,10 @@ where // The restore is complete and the node is self-consistent again, so the marker that // tells a restarting process to wipe can go. This is deliberately the LAST step, after - // `reconstruct_platform_state` has committed the platform state to aux storage. - clear_restore_sentinel(&app.platform().config.db_path).map_err(|e| { - AbciError::StateSyncInternalError(format!( - "apply_snapshot_chunk unable to clear the restore sentinel: {}", - e - )) - })?; + // `reconstruct_platform_state` has committed the platform state to aux storage, and + // deliberately best-effort: a successful restore must not be turned into an ABCI error + // by a `remove_file` hiccup. + clear_restore_sentinel_best_effort(&app.platform().config.db_path); tracing::info!( height = session.snapshot.height, diff --git a/packages/rs-drive-abci/src/abci/handler/init_chain.rs b/packages/rs-drive-abci/src/abci/handler/init_chain.rs index a7bf050698e..5923aefd11f 100644 --- a/packages/rs-drive-abci/src/abci/handler/init_chain.rs +++ b/packages/rs-drive-abci/src/abci/handler/init_chain.rs @@ -1,7 +1,6 @@ use crate::abci::app::{BlockExecutionApplication, PlatformApplication, TransactionalApplication}; -use crate::abci::AbciError; use crate::error::Error; -use crate::platform_types::snapshot::clear_restore_sentinel; +use crate::platform_types::snapshot::clear_restore_sentinel_best_effort; use crate::rpc::core::CoreRPCLike; use tenderdash_abci::proto::abci as proto; @@ -39,12 +38,10 @@ where // to block sync), its marker is still on disk and would make the NEXT restart wipe this // perfectly good chain. Clear it here — this is the block-sync arm of the same recovery // that `Platform::open_with_client` performs for an interrupted restore. - clear_restore_sentinel(&app.platform().config.db_path).map_err(|e| { - AbciError::StateSyncInternalError(format!( - "init_chain unable to clear the restore sentinel: {}", - e - )) - })?; + // Best-effort: failing to remove a marker file must not turn a working genesis into a + // failed init_chain. The worst case is one unnecessary wipe-and-resync on a later + // restart, which is loud but safe. + clear_restore_sentinel_best_effort(&app.platform().config.db_path); tracing::info!( app_hash, diff --git a/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs index 2979d0736ba..4186b0ec6e4 100644 --- a/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs @@ -58,6 +58,26 @@ pub fn write_restore_sentinel( ) } +/// Clears the restore sentinel at a point where the node is already self-consistent — +/// after a completed restore, or after a genesis initialization — WITHOUT being able to +/// fail the operation that got it there. +/// +/// Propagating an I/O error from here would turn a fully successful restore (or a working +/// genesis) into a hard ABCI error over nothing but a `remove_file` hiccup. The cost of +/// failing to remove it is bounded and safe in the other direction: the next startup sees +/// a sentinel, wipes, and re-syncs. Loud, but never a wedge. +pub fn clear_restore_sentinel_best_effort(db_path: &Path) { + if let Err(error) = clear_restore_sentinel(db_path) { + tracing::error!( + ?error, + path = ?restore_sentinel_path(db_path), + "[state_sync] could not clear the state sync restore sentinel; the node is \ + consistent, but the next restart will wipe and re-sync unnecessarily. Remove \ + the file by hand to avoid that.", + ); + } +} + /// Clears the restore sentinel. Only ever called once the node is in a self-consistent /// state: after a restore has fully completed, after startup recovery has wiped, or after /// a genesis initialization. @@ -86,10 +106,22 @@ pub fn restore_sentinel_exists(db_path: &Path) -> bool { /// /// `system_data_contracts` is deliberately NOT cleared — those are compiled-in, /// version-keyed contracts that never come from grovedb. +/// +/// The checkpoint registry goes too, and it is not merely a cache: `Drive::open` populates +/// `drive.checkpoints` before any wipe can run, and `list_snapshots` serves whatever is in +/// it to peers. Left alone, a node that wiped and re-synced would keep offering snapshots +/// of the chain it just discarded. The entries are marked for deletion first so their +/// directories are removed when the last `Arc` drops, rather than leaking on disk. pub fn reset_drive_caches_after_wipe(drive: &Drive) { *drive.cache.protocol_versions_counter.write() = Default::default(); drive.cache.data_contracts.clear(); *drive.cache.genesis_time_ms.write() = None; + + let checkpoints = drive.checkpoints.load(); + for checkpoint_info in checkpoints.values() { + checkpoint_info.checkpoint.mark_for_deletion(); + } + drive.checkpoints.store(Arc::new(BTreeMap::new())); } /// Wipes grovedb and drops the caches derived from it, leaving the node an empty but diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs index 41b83964dfe..b2753a574e2 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs @@ -207,6 +207,65 @@ mod tests { ); } + /// A wipe must also drop the CHECKPOINT REGISTRY, not just the value caches. + /// + /// `drive.checkpoints` is populated by `Drive::open` and is what `list_snapshots` + /// serves to peers. It is not a cache that merely goes stale: left in place across a + /// wipe, a node that discarded a chain would keep advertising snapshots of it, and a + /// peer state-syncing from those would restore a chain this node no longer has and + /// cannot vouch for. + #[tokio::test] + async fn a_wiped_node_stops_serving_snapshots_of_the_discarded_chain() { + let config = sentinel_platform_config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + let outcome = run_chain_for_strategy( + &mut platform.platform, + SOURCE_CHAIN_BLOCKS, + sentinel_strategy(), + config.clone(), + SOURCE_CHAIN_SEED, + &mut None, + &mut None, + ) + .await; + let app = outcome.abci_app; + + assert!( + !app.list_snapshots(Default::default()) + .expect("should list snapshots") + .snapshots + .is_empty(), + "sanity: the chain must have produced servable snapshots to begin with" + ); + + // Accepting an offer wipes the database out from under those checkpoints. + app.offer_snapshot(proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height: 1000, + version: 1, + hash: vec![7u8; 32], + metadata: vec![], + }), + app_hash: vec![7u8; 32], + }) + .expect("the offer must be accepted"); + + assert!( + app.list_snapshots(Default::default()) + .expect("should list snapshots") + .snapshots + .is_empty(), + "after a wipe the node must stop advertising snapshots of the chain it just \ + discarded — otherwise a peer would state-sync from state this node no longer has" + ); + assert!( + app.platform.drive.checkpoints.load().is_empty(), + "the checkpoint registry itself must be cleared, not just filtered at serve time" + ); + } + /// A rejected offer must not record a sentinel — nothing was wiped, so nothing needs /// recovering. Without this, any peer could make a healthy node wipe itself on the next /// restart just by offering a snapshot in a format it cannot speak. From 6c2bf401747a373363efbf76ea517a6c2e7ce2aa Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 29 Aug 2026 17:34:33 +0200 Subject: [PATCH 17/34] docs(drive-abci): drop the future wire-v2 framing from state sync docs Maintainer ruling: the original state sync never shipped, so grovedb updates its replication protocol in place and stays at version 1 - there is no v2. The supported-set constant and the offered-snapshot validation remain so any future incompatible protocol change fails fast on both sides; comments now say exactly that instead of describing a version bump that will not happen. No behavior changes. Co-Authored-By: Claude Fable 5 --- .../src/platform_types/snapshot/mod.rs | 12 +++++++----- .../test_cases/state_sync_tests.rs | 18 +++++++++--------- .../rs-drive-abci/tests/sum_tree_sync_probe.rs | 4 ++-- .../drive_abci_state_sync_versions/mod.rs | 10 +++++----- .../rs-platform-version/src/version/v15.rs | 11 ++++++----- 5 files changed, 29 insertions(+), 26 deletions(-) diff --git a/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs index 4186b0ec6e4..6290c334630 100644 --- a/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs @@ -135,12 +135,14 @@ pub fn wipe_drive_for_restore(drive: &Drive) -> Result<(), drive::error::Error> Ok(()) } -/// The grovedb state sync wire protocol versions this node can serve and consume. +/// The grovedb state sync protocol versions this node can serve and consume. /// -/// This is THE single supported-set constant: when grovedb wire version 2 lands, add it -/// here and add a `DriveAbciStateSyncVersions` const selecting it in rs-platform-version -/// (`drive_abci.state_sync.protocol_version` is the version stamped on snapshots this -/// node offers). +/// Exactly one protocol version exists: state sync never shipped, so grovedb updates +/// its replication protocol in place and stays at version 1. This single supported-set +/// constant and the offered-snapshot validation against it exist so that any future +/// incompatible protocol change fails fast on both the serving and consuming side +/// instead of producing a corrupt restore. (`drive_abci.state_sync.protocol_version` +/// is the version stamped on snapshots this node offers.) pub const SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS: &[u16] = &[1]; /// Maximum accepted size (in bytes) of a single snapshot chunk, enforced before any diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs index fd982a4d14e..567cc391aa6 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs @@ -2,14 +2,14 @@ //! its checkpoint registry and a fresh target restores one chunk by chunk, then //! reconstructs its platform state. //! -//! KNOWN LIMITATION at the pinned grovedb revision (6c882c3): state sync wire protocol -//! version 1 does not faithfully restore SumTree subtrees — the copied node hashes -//! reproduce the source root hash, but re-opening a restored sum tree recomputes a -//! different root (latent corruption), which the strict `verify_grovedb` call in -//! `apply_snapshot_chunk` correctly refuses. See `tests/sum_tree_sync_probe.rs` for the -//! minimal upstream reproducer. The full happy-path test below is therefore `#[ignore]`d -//! until the grovedb pin gains the fixed wire version, and an active test pins today's -//! refusal behavior instead. +//! KNOWN LIMITATION at the pinned grovedb revision (6c882c3): state sync does not +//! faithfully restore SumTree subtrees — the copied node hashes reproduce the source +//! root hash, but re-opening a restored sum tree recomputes a different root (latent +//! corruption), which the strict `verify_grovedb` call in `apply_snapshot_chunk` +//! correctly refuses. See `tests/sum_tree_sync_probe.rs` for the minimal upstream +//! reproducer. The full happy-path test below is therefore `#[ignore]`d until the +//! grovedb pin includes the sum-tree restore fix (dashpay/grovedb#840), and an active +//! test pins today's refusal behavior instead. #[cfg(test)] pub(crate) mod tests { @@ -506,7 +506,7 @@ pub(crate) mod tests { /// Pins today's behavior at the pinned grovedb revision: the transfer itself /// completes (including recovery from a tampered chunk via RETRY and a snapshot - /// restart), but the strict post-restore verification detects that wire v1 did not + /// restart), but the strict post-restore verification detects that grovedb did not /// faithfully restore the sum trees and refuses the snapshot instead of accepting /// latent corruption. When this test starts failing because the sync SUCCEEDS, /// grovedb has been fixed: un-ignore `run_state_sync_between_two_platforms` and diff --git a/packages/rs-drive-abci/tests/sum_tree_sync_probe.rs b/packages/rs-drive-abci/tests/sum_tree_sync_probe.rs index e8a45af0940..3a747a69d7c 100644 --- a/packages/rs-drive-abci/tests/sum_tree_sync_probe.rs +++ b/packages/rs-drive-abci/tests/sum_tree_sync_probe.rs @@ -1,6 +1,6 @@ //! Minimal reproducer / tripwire for a grovedb state sync limitation at the pinned -//! revision (6c882c3): wire protocol version 1 does not faithfully restore SumTree -//! subtrees. The chunk transfer copies the source's node hashes, so the restored +//! revision (6c882c3): the replication protocol at this revision does not faithfully +//! restore SumTree subtrees. The chunk transfer copies the source's node hashes, so the restored //! database reproduces the source ROOT hash — but re-opening the restored sum tree //! and recomputing its root yields a different hash, i.e. the corruption is latent //! and `verify_grovedb` detects it. diff --git a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/mod.rs b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/mod.rs index bffcad033a1..f0158bc6ad9 100644 --- a/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/mod.rs +++ b/packages/rs-platform-version/src/version/drive_abci_versions/drive_abci_state_sync_versions/mod.rs @@ -5,10 +5,10 @@ use versioned_feature_core::FeatureVersion; /// Versions for ABCI state sync (snapshot serving and consumption). #[derive(Clone, Debug, Default)] pub struct DriveAbciStateSyncVersions { - /// The grovedb state sync wire protocol version used for snapshots this node - /// creates and serves. Snapshots offered by peers are validated against the - /// supported set in `drive-abci`'s snapshot module; bumping to a new grovedb - /// wire version means adding a new `DriveAbciStateSyncVersions` const here and - /// extending that supported set. + /// The grovedb state sync protocol version used for snapshots this node creates + /// and serves. Exactly one version exists (grovedb updates its replication + /// protocol in place and stays at version 1); snapshots offered by peers are + /// validated against the supported set in `drive-abci`'s snapshot module so any + /// future incompatible protocol change fails fast on both sides. pub protocol_version: FeatureVersion, } diff --git a/packages/rs-platform-version/src/version/v15.rs b/packages/rs-platform-version/src/version/v15.rs index e1c662f0a54..8b84f8d8757 100644 --- a/packages/rs-platform-version/src/version/v15.rs +++ b/packages/rs-platform-version/src/version/v15.rs @@ -48,9 +48,10 @@ pub const PROTOCOL_VERSION_15: ProtocolVersion = 15; /// at the v15 activation block, so every snapshot taken at or after activation is /// restorable. Snapshots from before activation lack the key and are not served. /// -/// Everything else matches v14. The grovedb state sync wire protocol version used for +/// Everything else matches v14. The grovedb state sync protocol version used for /// snapshots is `DRIVE_ABCI_STATE_SYNC_VERSIONS_V1.protocol_version` (1), shared by all -/// platform versions. +/// platform versions; grovedb updates its replication protocol in place, so exactly one +/// version exists. pub const PLATFORM_V15: PlatformVersion = PlatformVersion { protocol_version: PROTOCOL_VERSION_15, drive: DRIVE_VERSION_V9, @@ -118,9 +119,9 @@ mod tests { ); } - /// All platform versions share grovedb state sync wire protocol version 1 until a - /// grovedb wire v2 exists; the supported set lives next to the snapshot types in - /// drive-abci. + /// All platform versions share grovedb state sync protocol version 1 — the only + /// version that exists, since grovedb updates its replication protocol in place. + /// The supported set lives next to the snapshot types in drive-abci. #[test] fn state_sync_wire_protocol_version_is_one() { assert_eq!(PLATFORM_V15.drive_abci.state_sync.protocol_version, 1); From 3ed472463585268f7a2e2d41852e4e94889dade4 Mon Sep 17 00:00:00 2001 From: pasta Date: Sun, 30 Aug 2026 00:44:41 +0200 Subject: [PATCH 18/34] fix(drive): do not clobber genesis evidence params at InitChain InitChain passes PlatformVersion::first() as the original version so Tenderdash learns the real app version, but that fiction also makes a chain STARTING on protocol v15+ look like it just crossed to v15 - and consensus_params_update_v2 then emits the 15000-block evidence window meant for chains upgrading with pre-state-sync genesis documents (#2512), silently overriding the evidence params of the genesis document being initialized. At genesis the operator's genesis document is authoritative; strip the evidence section from the InitChain update so it stays in force. Mid-chain crossings to v15 keep the override. Co-Authored-By: Claude Fable 5 --- .../initialization/init_chain/v0/mod.rs | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/packages/rs-drive-abci/src/execution/engine/initialization/init_chain/v0/mod.rs b/packages/rs-drive-abci/src/execution/engine/initialization/init_chain/v0/mod.rs index e34fee97d36..fd74c21372d 100644 --- a/packages/rs-drive-abci/src/execution/engine/initialization/init_chain/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/engine/initialization/init_chain/v0/mod.rs @@ -144,13 +144,28 @@ where self.config.execution.epoch_time_length_s, )?); + let mut consensus_params = consensus_params_update( + self.config.network, + first_platform_version, + platform_version, + &epoch_info, + )?; + + // `first_platform_version` above is a fiction for Tenderdash's benefit — it + // makes the update carry the real app version, since Tenderdash starts genesis + // assuming the first one. But it also makes a chain that STARTS on protocol + // v15+ look like it just crossed to v15, and the update then carries the + // evidence window override meant for chains upgrading with pre-state-sync + // genesis documents (#2512) — silently clobbering the evidence params of the + // genesis document being initialized right now. At genesis the operator's + // genesis document is authoritative, so the evidence section must not be + // emitted here; a `None` section leaves the genesis values in force. + if let Some(params) = consensus_params.as_mut() { + params.evidence = None; + } + Ok(ResponseInitChain { - consensus_params: consensus_params_update( - self.config.network, - first_platform_version, - platform_version, - &epoch_info, - )?, + consensus_params, app_hash: app_hash.to_vec(), validator_set_update: Some(validator_set), next_core_chain_lock_update: None, From ff325145c497260f00a1ff9bc799e66b573457de Mon Sep 17 00:00:00 2001 From: pasta Date: Mon, 31 Aug 2026 00:14:46 +0200 Subject: [PATCH 19/34] fix(drive-abci): reload checkpoints from the configured CHECKPOINTS_PATH Checkpoint creation honors the operator-configured checkpoints directory, but startup was hard-coded to /checkpoints in both Drive::open and the platform_state.bin load. A node running with a custom CHECKPOINTS_PATH therefore came back from a restart with an empty registry: it stopped advertising the snapshots it had retained, and the directories it had written could never be pruned. Thread the resolved path through Drive::open_with_checkpoints_path and Platform::open_with_client so creation and reload always agree. Co-Authored-By: Claude Fable 5 --- .../src/platform_types/platform/mod.rs | 22 +++++++++++---- .../src/open/load_current_checkpoints.rs | 17 +++++++---- packages/rs-drive/src/open/mod.rs | 28 +++++++++++++++++-- 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/packages/rs-drive-abci/src/platform_types/platform/mod.rs b/packages/rs-drive-abci/src/platform_types/platform/mod.rs index a59abe9d7e8..f2387f70734 100644 --- a/packages/rs-drive-abci/src/platform_types/platform/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/platform/mod.rs @@ -147,8 +147,22 @@ impl Platform { } }; - let (drive, current_platform_version) = - Drive::open(&config.db_path, Some(config.drive.clone())).map_err(Error::Drive)?; + // Checkpoints are created under the operator-configured `CHECKPOINTS_PATH` + // (defaulting to `/checkpoints`); startup MUST read them back from the + // same place, or a node with a custom path comes up with an empty registry: + // it would stop advertising the snapshots it retained and could never prune the + // directories it wrote. + let checkpoints_path = config + .abci + .state_sync + .resolved_checkpoints_path(&config.db_path); + + let (drive, current_platform_version) = Drive::open_with_checkpoints_path( + &config.db_path, + Some(config.drive.clone()), + Some(&checkpoints_path), + ) + .map_err(Error::Drive)?; // A state sync restore that never finished leaves grovedb holding state the // platform state knows nothing about. That is not recoverable by restarting — @@ -195,9 +209,7 @@ impl Platform { let mut checkpoint_platform_states = BTreeMap::new(); let checkpoints = drive.checkpoints.load(); for (&block_height, _checkpoint_info) in checkpoints.iter() { - let checkpoint_state_path = config - .db_path - .join("checkpoints") + let checkpoint_state_path = checkpoints_path .join(block_height.to_string()) .join("platform_state.bin"); diff --git a/packages/rs-drive/src/open/load_current_checkpoints.rs b/packages/rs-drive/src/open/load_current_checkpoints.rs index 99792a5f059..c94a0b794fc 100644 --- a/packages/rs-drive/src/open/load_current_checkpoints.rs +++ b/packages/rs-drive/src/open/load_current_checkpoints.rs @@ -11,17 +11,24 @@ use crate::error::Error; /// Loads existing checkpoints from the checkpoints directory. /// -/// This function scans the `/checkpoints/` directory for existing checkpoint +/// This function scans the given checkpoints directory for existing checkpoint /// subdirectories (named by block height), opens each one as a GroveDb, and returns /// an ArcSwap containing the loaded checkpoints. /// +/// The directory is passed in rather than derived, because checkpoints may be configured +/// to live outside the database directory (`CHECKPOINTS_PATH`). Deriving it here would +/// leave a node restarted with a custom path holding an empty registry: it would stop +/// advertising its retained snapshots and could never prune the directories it wrote. +/// /// # Arguments -/// * `db_path` - The path to the database directory (parent of the checkpoints directory) +/// * `checkpoints_dir` - The directory checkpoints are written to /// /// # Returns /// * An `ArcSwap` containing a `BTreeMap` of checkpoints keyed by block height -pub fn load_current_checkpoints>(db_path: P) -> Result { - let checkpoints_dir = db_path.as_ref().join("checkpoints"); +pub fn load_current_checkpoints>( + checkpoints_dir: P, +) -> Result { + let checkpoints_dir = checkpoints_dir.as_ref(); let mut checkpoints = BTreeMap::new(); @@ -31,7 +38,7 @@ pub fn load_current_checkpoints>(db_path: P) -> Result entries, Err(_) => return Ok(ArcSwap::from_pointee(checkpoints)), }; diff --git a/packages/rs-drive/src/open/mod.rs b/packages/rs-drive/src/open/mod.rs index e51fb1aab19..b213933ae67 100644 --- a/packages/rs-drive/src/open/mod.rs +++ b/packages/rs-drive/src/open/mod.rs @@ -30,6 +30,27 @@ impl Drive { pub fn open>( path: P, config: Option, + ) -> Result<(Self, Option<&'static PlatformVersion>), Error> { + Self::open_with_checkpoints_path(path, config, None::<&Path>) + } + + /// Opens GroveDB database, loading the checkpoint registry from an explicit directory. + /// + /// Checkpoints may be configured to live outside the database directory + /// (`CHECKPOINTS_PATH`). Whoever knows that configuration must pass the same directory + /// checkpoint creation writes to, otherwise the registry comes up empty after a + /// restart and the checkpoints on disk are neither advertised nor prunable. + /// `checkpoints_path = None` keeps the historical `/checkpoints` default. + /// + /// # Arguments + /// + /// * `path` - The path to the GroveDB. + /// * `config` - An `Option` which contains `DriveConfig`. If not specified, default configuration is used. + /// * `checkpoints_path` - The directory checkpoints are written to, if not the default. + pub fn open_with_checkpoints_path, Q: AsRef>( + path: P, + config: Option, + checkpoints_path: Option, ) -> Result<(Self, Option<&'static PlatformVersion>), Error> { let config = config.unwrap_or_default(); let db_path = path.as_ref(); @@ -52,8 +73,11 @@ impl Drive { }) .transpose()?; - // Load existing checkpoints from the checkpoints directory - let checkpoints = load_current_checkpoints(db_path)?; + // Load existing checkpoints from the configured checkpoints directory + let checkpoints = match checkpoints_path.as_ref() { + Some(checkpoints_path) => load_current_checkpoints(checkpoints_path.as_ref())?, + None => load_current_checkpoints(db_path.join("checkpoints"))?, + }; let drive = Drive { grove, From 5e6f30af54d641ac86d0ccd9bc78dd5f310b1920 Mon Sep 17 00:00:00 2001 From: pasta Date: Mon, 31 Aug 2026 00:15:03 +0200 Subject: [PATCH 20/34] fix(drive-abci)!: restore snapshots under the version they were produced at The consuming node picked grove_version from its own in-memory platform state. A node that state syncs has no saved state, so it sits at the initial protocol version (Drive v1 / GROVE_V1) while snapshots are only restorable from v15 (Drive v9 / GROVE_V4). grovedb replication, tree opening, root hashing and restore are all version gated, so generation and verification ran under different rules. list_snapshots now stamps the checkpoint's own protocol version into the snapshot metadata and serves each checkpoint under that version; offer_snapshot decodes it, refuses anything that is not a known version >= v15, and pins it on the session so every grovedb call of the transfer uses the same table. The value is peer-supplied but untrusted-safe: a lie fails verification against the light-client-verified app hash and lands on REJECT_SNAPSHOT. Also in the snapshot lifecycle: any accepted-format offer now replaces the session in progress (refusing a lower height let a peer advertise a high snapshot, withhold its chunks and block Tenderdash's fallback to an honest older one); oversized chunks and chunk ids answer RETRY/RETRY_SNAPSHOT with the sender rejected instead of throwing an ABCI exception that would abort state sync on a wiped database; and serving pins gained an absolute lifetime, a count cap, expiry on read, a per-block sweep, and are only taken after a chunk was actually served. Co-Authored-By: Claude Fable 5 --- packages/rs-drive-abci/src/abci/app/full.rs | 7 + .../src/abci/handler/apply_snapshot_chunk.rs | 158 +++++++--- .../src/abci/handler/list_snapshots.rs | 43 ++- .../src/abci/handler/load_snapshot_chunk.rs | 42 ++- .../src/abci/handler/offer_snapshot.rs | 138 +++++++-- .../src/platform_types/snapshot/mod.rs | 288 +++++++++++++++++- packages/rs-drive/src/drive/mod.rs | 12 + 7 files changed, 596 insertions(+), 92 deletions(-) diff --git a/packages/rs-drive-abci/src/abci/app/full.rs b/packages/rs-drive-abci/src/abci/app/full.rs index e5274f01f30..c49ec51443b 100644 --- a/packages/rs-drive-abci/src/abci/app/full.rs +++ b/packages/rs-drive-abci/src/abci/app/full.rs @@ -244,6 +244,13 @@ where &self, request: proto::RequestFinalizeBlock, ) -> Result { + // Autonomous expiry of serving pins: an abandoned state sync transfer stops + // making chunk requests, so nothing on the serving path would ever release its + // pin and the pruned checkpoint directory would stay on disk forever. A block is + // the one thing that reliably keeps happening. This is node-local bookkeeping and + // touches no consensus state. + self.snapshot_manager.release_expired_pins(); + handler::finalize_block(self, request).map_err(error_into_exception) } diff --git a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs index 8888b1307dc..025d2659618 100644 --- a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs +++ b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs @@ -1,7 +1,6 @@ use crate::abci::app::StateSyncApplication; use crate::abci::AbciError; use crate::error::Error; -use crate::platform_types::platform_state::PlatformStateV0Methods; use crate::platform_types::snapshot::{ clear_restore_sentinel_best_effort, wipe_drive_for_restore, MAX_STATE_SYNC_CHUNK_ID_SIZE, MAX_STATE_SYNC_CHUNK_SIZE, @@ -31,39 +30,76 @@ where "[state_sync] api apply_snapshot_chunk", ); - // Cap peer-supplied sizes before anything decodes them (issue #3773) - if request.chunk_id.len() > MAX_STATE_SYNC_CHUNK_ID_SIZE { - return Err(AbciError::StateSyncBadRequest(format!( - "apply_snapshot_chunk chunk id of {} bytes exceeds the {} byte limit", - request.chunk_id.len(), - MAX_STATE_SYNC_CHUNK_ID_SIZE - )) - .into()); - } - if request.chunk.len() > MAX_STATE_SYNC_CHUNK_SIZE { - return Err(AbciError::StateSyncBadRequest(format!( - "apply_snapshot_chunk chunk of {} bytes exceeds the {} byte limit", - request.chunk.len(), - MAX_STATE_SYNC_CHUNK_SIZE - )) - .into()); - } - - let platform_version = app.platform().state.load().current_platform_version()?; - let grove_version = &platform_version.drive.grove_version; - let mut session_write_guard = app.snapshot_fetching_session().write().map_err(|_| { AbciError::StateSyncInternalError( "apply_snapshot_chunk unable to lock session (poisoned)".to_string(), ) })?; + // The version the SNAPSHOT was produced at, pinned when the offer was accepted. Using + // `self.state`'s version here would be wrong: a state-syncing node has no saved state, + // so it is still on the initial protocol version and would decode, verify and hash the + // restored trees under a different (older) grovedb table than the one that generated + // the chunks. + let platform_version = session_write_guard + .as_ref() + .ok_or(AbciError::StateSyncBadRequest( + "apply_snapshot_chunk no state sync session in progress".to_string(), + ))? + .platform_version; + let grove_version = &platform_version.drive.grove_version; + { let session = session_write_guard .as_mut() - .ok_or(AbciError::StateSyncBadRequest( - "apply_snapshot_chunk no state sync session in progress".to_string(), - ))?; + .expect("session presence was just checked"); + + let reject_senders = if request.sender.is_empty() { + vec![] + } else { + vec![request.sender.clone()] + }; + + // Cap peer-supplied sizes before anything decodes them (issue #3773). + // + // These are TRANSFER faults, not reasons to abort state sync: an application + // error here would reach Tenderdash as an ABCI exception, killing the whole + // restore and leaving the node on the wiped database the offer created (with the + // restore sentinel still set). Both caps are therefore answered with the same + // recoverable ladder the other malformed-chunk paths use. + if request.chunk.len() > MAX_STATE_SYNC_CHUNK_SIZE { + // Oversized chunk DATA: the chunk id itself is still fine, so ban the sender + // and have Tenderdash refetch exactly this chunk from someone else. + tracing::warn!( + chunk_id = hex::encode(&request.chunk_id), + sender = request.sender, + chunk_len = request.chunk.len(), + limit = MAX_STATE_SYNC_CHUNK_SIZE, + "[state_sync] apply_snapshot_chunk oversized chunk, rejecting the sender and requesting refetch", + ); + return Ok(proto::ResponseApplySnapshotChunk { + result: response_apply_snapshot_chunk::Result::Retry.into(), + refetch_chunks: vec![request.chunk_id], + reject_senders, + next_chunks: vec![], + }); + } + if request.chunk_id.len() > MAX_STATE_SYNC_CHUNK_ID_SIZE { + // An oversized chunk id is intrinsically invalid — refetching it would ask for + // the same impossible id again — so restart the snapshot instead. + tracing::warn!( + chunk_id_len = request.chunk_id.len(), + sender = request.sender, + limit = MAX_STATE_SYNC_CHUNK_ID_SIZE, + "[state_sync] apply_snapshot_chunk oversized chunk id, requesting snapshot restart", + ); + return Ok(proto::ResponseApplySnapshotChunk { + result: response_apply_snapshot_chunk::Result::RetrySnapshot.into(), + refetch_chunks: vec![], + reject_senders, + next_chunks: vec![], + }); + } let wire_version = session.wire_version; let next_chunk_ids = match session.state_sync_info.apply_chunk( @@ -74,12 +110,6 @@ where ) { Ok(next_chunk_ids) => next_chunk_ids, Err(e) => { - let reject_senders = if request.sender.is_empty() { - vec![] - } else { - vec![request.sender.clone()] - }; - // grovedb removes a chunk id from its pending set before processing it, // so a chunk it has already seen (e.g. the refetch of one it rejected) // cannot be re-applied within this session: ask Tenderdash to restart @@ -321,7 +351,9 @@ mod tests { use super::*; use crate::abci::app::FullAbciApplication; use crate::abci::handler::offer_snapshot; + use crate::platform_types::snapshot::encode_snapshot_metadata; use crate::test::helpers::setup::TestPlatformBuilder; + use dpp::version::v15::PROTOCOL_VERSION_15; #[test] fn apply_snapshot_chunk_without_session_is_rejected() { @@ -341,32 +373,72 @@ mod tests { .is_err()); } + fn offer_a_snapshot(app: &FullAbciApplication) -> Vec { + let target_app_hash = vec![7u8; 32]; + offer_snapshot( + app, + proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height: 100, + version: 1, + hash: target_app_hash.clone(), + metadata: encode_snapshot_metadata(PROTOCOL_VERSION_15), + }), + app_hash: target_app_hash.clone(), + }, + ) + .expect("should accept offer"); + target_app_hash + } + + /// An oversized chunk or chunk id is a recoverable transfer fault, not a reason to + /// abort state sync with an ABCI exception: the session must survive and Tenderdash + /// must be given a way forward. #[test] - fn apply_snapshot_chunk_caps_sizes_before_decoding() { + fn apply_snapshot_chunk_caps_sizes_without_aborting_the_transfer() { let platform = TestPlatformBuilder::new() .build_with_mock_rpc() .set_genesis_state(); let app = FullAbciApplication::new(&platform); + let target_app_hash = offer_a_snapshot(&app); - assert!(apply_snapshot_chunk( + // Oversized chunk data: ban the sender and refetch exactly this chunk + let response = apply_snapshot_chunk( &app, proto::RequestApplySnapshotChunk { - chunk_id: vec![0u8; MAX_STATE_SYNC_CHUNK_ID_SIZE + 1], - chunk: vec![], - sender: String::new(), + chunk_id: target_app_hash.clone(), + chunk: vec![0u8; MAX_STATE_SYNC_CHUNK_SIZE + 1], + sender: "fat-peer".to_string(), }, ) - .is_err()); + .expect("an oversized chunk must not abort the ABCI request"); + assert_eq!( + response.result, + i32::from(response_apply_snapshot_chunk::Result::Retry) + ); + assert_eq!(response.refetch_chunks, vec![target_app_hash]); + assert_eq!(response.reject_senders, vec!["fat-peer".to_string()]); + assert!( + app.snapshot_fetching_session.read().unwrap().is_some(), + "the session must survive an oversized chunk" + ); - assert!(apply_snapshot_chunk( + // Oversized chunk id: intrinsically invalid, so restart the snapshot + let response = apply_snapshot_chunk( &app, proto::RequestApplySnapshotChunk { - chunk_id: vec![1u8; 32], - chunk: vec![0u8; MAX_STATE_SYNC_CHUNK_SIZE + 1], - sender: String::new(), + chunk_id: vec![0u8; MAX_STATE_SYNC_CHUNK_ID_SIZE + 1], + chunk: vec![], + sender: "fat-peer".to_string(), }, ) - .is_err()); + .expect("an oversized chunk id must not abort the ABCI request"); + assert_eq!( + response.result, + i32::from(response_apply_snapshot_chunk::Result::RetrySnapshot) + ); + assert!(response.refetch_chunks.is_empty()); + assert_eq!(response.reject_senders, vec!["fat-peer".to_string()]); } #[test] @@ -384,7 +456,7 @@ mod tests { height: 100, version: 1, hash: target_app_hash.clone(), - metadata: vec![], + metadata: encode_snapshot_metadata(PROTOCOL_VERSION_15), }), app_hash: target_app_hash.clone(), }, diff --git a/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs b/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs index 0b2752685e1..89648e6110b 100644 --- a/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs +++ b/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs @@ -1,8 +1,9 @@ use crate::abci::app::PlatformApplication; use crate::abci::AbciError; use crate::error::Error; -use crate::platform_types::platform_state::PlatformStateV0Methods; +use crate::platform_types::snapshot::encode_snapshot_metadata; use crate::rpc::core::CoreRPCLike; +use dpp::version::PlatformVersion; use tenderdash_abci::proto::abci as proto; /// Lists the state sync snapshots this node can serve. @@ -25,16 +26,33 @@ where return Ok(Default::default()); } - let platform_state = app.platform().state.load(); - let platform_version = platform_state.current_platform_version()?; - let grove_version = &platform_version.drive.grove_version; - let checkpoints = app.platform().drive.checkpoints.load(); let mut snapshots = Vec::new(); for (height, checkpoint_info) in checkpoints.iter() { let checkpoint = &checkpoint_info.checkpoint; + // Read the checkpoint under the version IT was written at, not this node's + // current one: a node that has since upgraded still serves older checkpoints, and + // grovedb's tree opening and root-hash rules are version gated. The same version + // is stamped into the snapshot metadata so the consuming node — which has no way + // to derive it — restores under exactly these rules. + let Some(snapshot_protocol_version) = + checkpoint.current_protocol_version().map_err(|e| { + AbciError::StateSyncInternalError(format!( + "list_snapshots unable to read the protocol version of the checkpoint at \ + height {}: {}", + height, e + )) + })? + else { + continue; + }; + let Ok(snapshot_platform_version) = PlatformVersion::get(snapshot_protocol_version) else { + continue; + }; + let grove_version = &snapshot_platform_version.drive.grove_version; + let restorable = checkpoint .has_reduced_platform_state(grove_version) .map_err(|e| { @@ -60,9 +78,12 @@ where snapshots.push(proto::Snapshot { height: *height, - version: platform_version.drive_abci.state_sync.protocol_version as u32, + version: snapshot_platform_version + .drive_abci + .state_sync + .protocol_version as u32, hash: root_hash.to_vec(), - metadata: Vec::new(), + metadata: encode_snapshot_metadata(snapshot_protocol_version), }); } @@ -124,6 +145,14 @@ mod tests { .store_reduced_platform_state(&reduced_platform_state, None, platform_version) .expect("should store reduced platform state"); + // A real node always has its protocol version in aux (Drive::open reads it to + // decide whether there is saved state at all); snapshots are stamped with the + // checkpoint's own version, so the test platform has to have one too. + platform + .drive + .store_current_protocol_version(platform_version.protocol_version, None) + .expect("should store protocol version"); + fast_forward_to_block(&platform, 2_000_000, 20, 43, 0, false); platform .create_grovedb_checkpoint(platform_version) diff --git a/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs b/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs index ed6c1652c3a..892813ccb4f 100644 --- a/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs +++ b/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs @@ -1,11 +1,11 @@ use crate::abci::app::{PlatformApplication, SnapshotManagerApplication}; use crate::abci::AbciError; use crate::error::Error; -use crate::platform_types::platform_state::PlatformStateV0Methods; use crate::platform_types::snapshot::{ MAX_STATE_SYNC_CHUNK_ID_SIZE, SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS, }; use crate::rpc::core::CoreRPCLike; +use dpp::version::PlatformVersion; use std::sync::Arc; use tenderdash_abci::proto::abci as proto; @@ -56,10 +56,6 @@ where .into()); }; - let platform_state = app.platform().state.load(); - let platform_version = platform_state.current_platform_version()?; - let grove_version = &platform_version.drive.grove_version; - // Resolve the checkpoint: from the registry, or — if pruning already dropped it — // from the pins of transfers already in flight. let checkpoint = app @@ -77,9 +73,26 @@ where )) })?; - // Pin (or refresh the pin of) the checkpoint for the duration of the transfer - app.snapshot_manager() - .pin_for_serving(request.height, Arc::clone(&checkpoint)); + // Chunks must be generated under the version the checkpoint was WRITTEN at — the same + // one `list_snapshots` stamped into the snapshot metadata and the consuming node + // restores under. This node's own current version may have moved on since. + let snapshot_protocol_version = checkpoint + .current_protocol_version() + .map_err(|e| { + AbciError::StateSyncInternalError(format!( + "load_snapshot_chunk unable to read the protocol version of the checkpoint at \ + height {}: {}", + request.height, e + )) + })? + .ok_or_else(|| { + AbciError::StateSyncInternalError(format!( + "load_snapshot_chunk checkpoint at height {} has no protocol version", + request.height + )) + })?; + let snapshot_platform_version = PlatformVersion::get(snapshot_protocol_version)?; + let grove_version = &snapshot_platform_version.drive.grove_version; let chunk = checkpoint .grove_db @@ -91,6 +104,12 @@ where )) })?; + // Pin (or refresh the pin of) the checkpoint only once a chunk was actually served. + // Pinning before the fetch would let a peer keep a checkpoint — and its directory — + // alive with a stream of requests that never succeed. + app.snapshot_manager() + .pin_for_serving(request.height, checkpoint); + Ok(proto::ResponseLoadSnapshotChunk { chunk }) } @@ -118,6 +137,13 @@ mod tests { platform .store_reduced_platform_state(&reduced_platform_state, None, platform_version) .expect("should store reduced platform state"); + // Snapshots are served under the checkpoint's OWN protocol version, which a real + // node always has in aux. + platform + .drive + .store_current_protocol_version(platform_version.protocol_version, None) + .expect("should store protocol version"); + fast_forward_to_block(&platform, 1_000_000, 10, 42, 0, false); platform .create_grovedb_checkpoint(platform_version) diff --git a/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs b/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs index 5231371d6f0..d83c6a7aebd 100644 --- a/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs +++ b/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs @@ -1,21 +1,22 @@ use crate::abci::app::StateSyncApplication; use crate::abci::AbciError; use crate::error::Error; -use crate::platform_types::platform_state::PlatformStateV0Methods; use crate::platform_types::snapshot::{ - wipe_drive_for_restore, write_restore_sentinel, SnapshotFetchingSession, - STATE_SYNC_SUBTREES_BATCH_SIZE, SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS, + decode_snapshot_metadata, wipe_drive_for_restore, write_restore_sentinel, + SnapshotFetchingSession, STATE_SYNC_SUBTREES_BATCH_SIZE, + SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS, }; use crate::rpc::core::CoreRPCLike; +use dpp::version::v15::PROTOCOL_VERSION_15; +use dpp::version::PlatformVersion; use tenderdash_abci::proto::abci as proto; use tenderdash_abci::proto::abci::response_offer_snapshot; /// Handles a snapshot offered by Tenderdash during state sync. /// /// Accepting an offer wipes the local grovedb and opens a grovedb state sync session -/// targeting the light-client-verified app hash. A later offer for a higher height -/// replaces a session already in progress (also answered with Accept); an offer for a -/// lower or equal height than the session in progress is rejected. +/// targeting the light-client-verified app hash. Any accepted-format offer replaces a +/// session already in progress (also answered with Accept), whatever height it carries. pub fn offer_snapshot<'a, 'db: 'a, A, C: 'db>( app: &'a A, request: proto::RequestOfferSnapshot, @@ -55,7 +56,29 @@ where }); }; - let platform_version = app.platform().state.load().current_platform_version()?; + // The Platform version of the SNAPSHOT, which is what every grovedb call of this + // transfer must run under. It cannot come from `self.state`: a node that state syncs + // has no saved state, so its in-memory platform state is still at the initial protocol + // version and would hand grovedb the wrong (much older) version table than the one the + // serving node generated the chunks with. + let snapshot_platform_version = + decode_snapshot_metadata(&offered_snapshot.metadata).and_then(|protocol_version| { + // Only versions that write the reduced platform state can be restored at all; + // anything else is refused here rather than after a full transfer. + (protocol_version >= PROTOCOL_VERSION_15) + .then(|| PlatformVersion::get(protocol_version).ok()) + .flatten() + }); + let Some(snapshot_platform_version) = snapshot_platform_version else { + tracing::warn!( + height = offered_snapshot.height, + metadata = hex::encode(&offered_snapshot.metadata), + "[state_sync] offer_snapshot rejecting a snapshot without a usable platform version in its metadata", + ); + return Ok(proto::ResponseOfferSnapshot { + result: response_offer_snapshot::Result::Reject.into(), + }); + }; let mut session_write_guard = app.snapshot_fetching_session().write().map_err(|_| { AbciError::StateSyncInternalError( @@ -64,16 +87,16 @@ where })?; if let Some(session) = session_write_guard.as_ref() { - // An offer at the same height is a legitimate snapshot restart (Tenderdash's - // RETRY_SNAPSHOT flow) and replaces the session; only strictly older offers are - // rejected. - if offered_snapshot.height < session.snapshot.height { - return Err(AbciError::StateSyncBadRequest(format!( - "offer_snapshot already syncing snapshot at height {}, offered height {} is older", - session.snapshot.height, offered_snapshot.height - )) - .into()); - } + // Every offer Tenderdash makes is Tenderdash resetting the transfer, so it always + // replaces the session in progress — including one for a LOWER height. + // + // The height in a snapshot descriptor is peer-supplied and untrusted (only the + // `app_hash` is light-client verified), so refusing to go backwards would hand a + // peer a wedge: advertise a high snapshot, withhold its chunks, and Tenderdash's + // fallback to an honest peer's older checkpoint would then be answered with an + // ABCI exception that aborts state sync altogether. Replacing is safe because the + // restore is only ever accepted against the verified app hash of whatever offer + // won. tracing::warn!( current_height = session.snapshot.height, offered_height = offered_snapshot.height, @@ -112,7 +135,7 @@ where request_app_hash, STATE_SYNC_SUBTREES_BATCH_SIZE, wire_version, - &platform_version.drive.grove_version, + &snapshot_platform_version.drive.grove_version, ) .map_err(|e| { AbciError::StateSyncInternalError(format!( @@ -125,6 +148,7 @@ where snapshot: offered_snapshot, app_hash: request_app_hash, wire_version, + platform_version: snapshot_platform_version, state_sync_info, }); @@ -139,18 +163,65 @@ mod tests { use crate::abci::app::FullAbciApplication; use crate::test::helpers::setup::TestPlatformBuilder; + use crate::platform_types::snapshot::encode_snapshot_metadata; + fn offer_at(height: u64, version: u32) -> proto::RequestOfferSnapshot { + offer_at_with_metadata( + height, + version, + encode_snapshot_metadata(PROTOCOL_VERSION_15), + ) + } + + fn offer_at_with_metadata( + height: u64, + version: u32, + metadata: Vec, + ) -> proto::RequestOfferSnapshot { proto::RequestOfferSnapshot { snapshot: Some(proto::Snapshot { height, version, hash: vec![7u8; 32], - metadata: vec![], + metadata, }), app_hash: vec![7u8; 32], } } + /// The snapshot's Platform version drives every grovedb call of the transfer, so an + /// offer that does not carry a usable one must be refused BEFORE the database is + /// wiped — and refused as a per-snapshot Reject, so Tenderdash keeps walking its + /// ladder instead of aborting state sync. + #[test] + fn offer_snapshot_rejects_offers_without_a_usable_platform_version() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + let app = FullAbciApplication::new(&platform); + + for metadata in [ + vec![], // absent + vec![0u8; 3], // wrong length + encode_snapshot_metadata(1), // pre-v15, cannot be restored + encode_snapshot_metadata(u32::MAX), // unknown version + encode_snapshot_metadata(PROTOCOL_VERSION_15 - 1), // last version before v15 + ] { + let response = offer_snapshot(&app, offer_at_with_metadata(100, 1, metadata.clone())) + .expect("should not error"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::Reject), + "metadata {:?} must be rejected", + metadata + ); + assert!( + app.snapshot_fetching_session.read().unwrap().is_none(), + "a rejected offer must not open a session", + ); + } + } + #[test] fn offer_snapshot_rejects_unsupported_version_with_reject_format() { let platform = TestPlatformBuilder::new() @@ -180,8 +251,29 @@ mod tests { i32::from(response_offer_snapshot::Result::Accept) ); - // A strictly lower height while syncing is rejected - assert!(offer_snapshot(&app, offer_at(50, 1)).is_err()); + // A LOWER height while syncing is Tenderdash falling back to another available + // snapshot after the higher one turned out to be unservable. It must replace the + // session and be accepted, otherwise a peer that advertises a high snapshot and + // then withholds its chunks could block the fallback. + let response = + offer_snapshot(&app, offer_at(50, 1)).expect("should accept an older fallback offer"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::Accept) + ); + assert_eq!( + app.snapshot_fetching_session + .read() + .unwrap() + .as_ref() + .expect("session must exist") + .snapshot + .height, + 50, + ); + + // Bring the session back up to 100 for the restart check below + offer_snapshot(&app, offer_at(100, 1)).expect("should accept offer"); // A same-height re-offer is a snapshot restart (Tenderdash RETRY_SNAPSHOT): // the session is replaced and the offer accepted @@ -202,5 +294,9 @@ mod tests { let session = session_guard.as_ref().expect("session must exist"); assert_eq!(session.snapshot.height, 200); assert_eq!(session.wire_version, 1); + assert_eq!( + session.platform_version.protocol_version, PROTOCOL_VERSION_15, + "the session must run under the SNAPSHOT's platform version", + ); } } diff --git a/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs index 6290c334630..48cb3105209 100644 --- a/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs @@ -4,6 +4,8 @@ //! (`drive.checkpoints`, populated by `create_grovedb_checkpoint` after each qualifying //! block is committed); there is no separate snapshot store. +use dpp::util::deserializer::ProtocolVersion; +use dpp::version::PlatformVersion; use drive::drive::{Checkpoint, Drive}; use drive::grovedb::replication::MultiStateSyncSession; use std::collections::BTreeMap; @@ -158,6 +160,31 @@ pub const MAX_STATE_SYNC_CHUNK_ID_SIZE: usize = 64 * 1024; /// session on the consuming side. pub const STATE_SYNC_SUBTREES_BATCH_SIZE: usize = 64; +/// Encodes the Platform protocol version a snapshot was produced at into the ABCI +/// snapshot `metadata` field. +/// +/// The consuming node cannot derive this: a node that state syncs has no saved state, so +/// its in-memory platform state is still at [`dpp::version::INITIAL_PROTOCOL_VERSION`] +/// and its Drive version table — including `grove_version` — is the wrong one for the +/// snapshot. grovedb's replication, tree opening and root-hash rules are version gated, +/// so serving and consuming MUST use the same table or the restore is decoded under +/// different rules than it was generated with. +/// +/// The value is peer-supplied and therefore untrusted, which is safe: the restore is only +/// ever accepted against the light-client-verified app hash, so a lie produces a failed +/// verification and a `REJECT_SNAPSHOT`, never a silently wrong database. +pub fn encode_snapshot_metadata(protocol_version: ProtocolVersion) -> Vec { + protocol_version.to_be_bytes().to_vec() +} + +/// Decodes the Platform protocol version out of an ABCI snapshot's `metadata` field. +/// Returns `None` for anything that is not exactly the encoding above. +pub fn decode_snapshot_metadata(metadata: &[u8]) -> Option { + <[u8; 4]>::try_from(metadata) + .ok() + .map(ProtocolVersion::from_be_bytes) +} + /// A state sync transfer in progress on the consuming side. pub struct SnapshotFetchingSession<'db> { /// The snapshot being restored @@ -169,6 +196,11 @@ pub struct SnapshotFetchingSession<'db> { /// [`SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS`], and used for every chunk of the /// transfer. pub wire_version: u16, + /// The Platform version the snapshot was PRODUCED at, decoded from the offered + /// snapshot's metadata (see [`encode_snapshot_metadata`]). Every grovedb call of this + /// transfer — session start, chunk application, commit, verification and the final + /// root hash — uses this version's table, not the fresh node's own. + pub platform_version: &'static PlatformVersion, /// The grovedb state sync session pub state_sync_info: Pin>>, } @@ -180,17 +212,58 @@ pub struct SnapshotFetchingSession<'db> { /// pruning already marked for deletion to be removed from disk. const SERVING_PIN_INACTIVITY_TTL: Duration = Duration::from_secs(600); +/// Absolute lifetime of a serving pin, regardless of activity. +/// +/// The inactivity TTL alone is refreshable, so a peer that keeps touching a height keeps +/// its checkpoint alive forever. A real restore is far quicker than this — a full +/// mainnet-state restore measures in seconds — so an hour is generous for any honest +/// transfer while putting a ceiling on how long abuse can hold a directory back. +const SERVING_PIN_MAX_LIFETIME: Duration = Duration::from_secs(3600); + +/// Hard cap on how many checkpoints may be pinned for serving at once. +/// +/// Without it, a peer could keep one transfer alive per height it ever touched: pruning +/// keeps advancing, the peer keeps refreshing, and the number of checkpoint directories +/// held back from deletion grows without bound regardless of `MAX_NUM_SNAPSHOTS`. Eight +/// concurrent transfers is far more than the handful of snapshots a node advertises, so +/// the cap only ever bites on abuse; when it does, the least recently served pin goes +/// first. +const MAX_SERVING_PINS: usize = 8; + +/// A checkpoint held back from deletion for a transfer in flight. +struct ServingPin { + checkpoint: Arc, + /// When the pin was first taken — bounds its absolute lifetime + pinned_at: Instant, + /// When a chunk was last successfully served from it — bounds its idle lifetime + last_served: Instant, +} + +impl ServingPin { + fn is_live(&self, now: Instant) -> bool { + now.saturating_duration_since(self.last_served) < SERVING_PIN_INACTIVITY_TTL + && now.saturating_duration_since(self.pinned_at) < SERVING_PIN_MAX_LIFETIME + } +} + /// Keeps checkpoints that are actively being served to state-syncing peers alive. /// /// Checkpoint pruning marks old checkpoints for deletion and drops them from the /// registry; the directory is removed when the last `Arc` drops. Holding an /// `Arc` clone here for every checkpoint a peer is currently downloading extends that -/// refcount, so a checkpoint cannot be deleted mid-transfer. Pins are released after -/// [`SERVING_PIN_INACTIVITY_TTL`] of inactivity. +/// refcount, so a checkpoint cannot be deleted mid-transfer. +/// +/// A pin must not be something a remote peer can hold open indefinitely, so it is bounded +/// three ways: [`SERVING_PIN_INACTIVITY_TTL`] since the last chunk actually served, +/// [`SERVING_PIN_MAX_LIFETIME`] since it was taken (not refreshable), and +/// [`MAX_SERVING_PINS`] in total. Expiry also must not depend on peers making further +/// requests, or an abandoned transfer would hold its directory forever: +/// [`SnapshotManager::release_expired_pins`] runs once per block and every read of a pin +/// re-checks both deadlines. #[derive(Default)] pub struct SnapshotManager { - /// Height -> (pinned checkpoint, instant of the most recent chunk request) - serving_pins: RwLock, Instant)>>, + /// Height -> the pin held for a transfer of that snapshot + serving_pins: RwLock>, } impl SnapshotManager { @@ -200,35 +273,224 @@ impl SnapshotManager { } /// Pins a checkpoint that is being served (or refreshes the pin of one that already - /// is), and drops pins whose transfers have been inactive for longer than the TTL. + /// is), dropping expired pins and enforcing [`MAX_SERVING_PINS`]. + /// + /// Call this only AFTER a chunk was successfully served: a request that could not be + /// answered must not be able to keep a checkpoint alive. pub fn pin_for_serving(&self, height: u64, checkpoint: Arc) { let now = Instant::now(); let mut pins = self .serving_pins .write() .expect("serving pins lock poisoned"); - pins.retain(|_, (_, last_served)| { - now.saturating_duration_since(*last_served) < SERVING_PIN_INACTIVITY_TTL - }); - pins.insert(height, (checkpoint, now)); + retain_live_pins(&mut pins, now); + + if let Some(pin) = pins.get_mut(&height) { + // Refresh the idle deadline only — `pinned_at` is deliberately untouched so + // the absolute lifetime cannot be extended by activity. + pin.last_served = now; + return; + } + + // Evict the least recently served pin to make room for a genuinely new one + while pins.len() >= MAX_SERVING_PINS { + let Some(coldest) = pins + .iter() + .min_by_key(|(_, pin)| pin.last_served) + .map(|(pinned_height, _)| *pinned_height) + else { + break; + }; + tracing::warn!( + evicted_height = coldest, + new_height = height, + "[state_sync] serving pin limit reached, releasing the least recently served pin", + ); + pins.remove(&coldest); + } + + pins.insert( + height, + ServingPin { + checkpoint, + pinned_at: now, + last_served: now, + }, + ); } - /// Returns a pinned checkpoint for the given height, if the pin is still held. + /// Returns a pinned checkpoint for the given height, if the pin is still held AND + /// still live. /// - /// Used to keep serving a snapshot whose checkpoint pruning has already dropped - /// from the registry. + /// Used to keep serving a snapshot whose checkpoint pruning has already dropped from + /// the registry. An expired pin is dropped rather than returned: handing one out + /// would let a peer resurrect (and then indefinitely refresh) a checkpoint whose + /// transfer was abandoned long ago. pub fn pinned_checkpoint(&self, height: u64) -> Option> { + let now = Instant::now(); + let mut pins = self + .serving_pins + .write() + .expect("serving pins lock poisoned"); + retain_live_pins(&mut pins, now); + pins.get(&height).map(|pin| Arc::clone(&pin.checkpoint)) + } + + /// Releases every pin that has passed either of its deadlines. + /// + /// Called once per block so an abandoned transfer cannot hold a pruned checkpoint + /// directory on disk forever while waiting for a chunk request that never comes, and + /// so an over-long one is cut off even while it keeps requesting. + pub fn release_expired_pins(&self) { + let now = Instant::now(); + let mut pins = self + .serving_pins + .write() + .expect("serving pins lock poisoned"); + retain_live_pins(&mut pins, now); + } + + /// Number of checkpoints currently pinned for serving. + #[cfg(test)] + pub fn pinned_count(&self) -> usize { + self.serving_pins + .read() + .expect("serving pins lock poisoned") + .len() + } + + /// Test-only: the two deadlines of a pin, as `(pinned_at, last_served)`. + #[cfg(test)] + fn pin_instants(&self, height: u64) -> Option<(Instant, Instant)> { self.serving_pins .read() .expect("serving pins lock poisoned") .get(&height) - .map(|(checkpoint, _)| Arc::clone(checkpoint)) + .map(|pin| (pin.pinned_at, pin.last_served)) + } + + /// Test-only: backdates a pin's deadlines so expiry can be exercised without waiting. + #[cfg(test)] + fn backdate_pin(&self, height: u64, pinned_at: Option, last_served: Option) { + let mut pins = self + .serving_pins + .write() + .expect("serving pins lock poisoned"); + if let Some(pin) = pins.get_mut(&height) { + if let Some(pinned_at) = pinned_at { + pin.pinned_at = pinned_at; + } + if let Some(last_served) = last_served { + pin.last_served = last_served; + } + } } } +fn retain_live_pins(pins: &mut BTreeMap, now: Instant) { + pins.retain(|_, pin| pin.is_live(now)); +} + #[cfg(test)] mod tests { use super::*; + use drive::grovedb::GroveDb; + + /// Each checkpoint needs its own directory: rocksdb takes an exclusive lock on the + /// one it opens. + fn checkpoint_in(dir: &tempfile::TempDir, height: u64) -> Arc { + let path = dir.path().join(height.to_string()); + let grove_db = GroveDb::open(&path).expect("should open grovedb"); + Arc::new(Checkpoint::new(grove_db, path)) + } + + /// An abandoned transfer stops making chunk requests, so the pin must expire on its + /// own — both when swept from the block path and when a later request tries to read + /// it (otherwise a peer could resurrect and then indefinitely refresh a checkpoint + /// pruning already dropped). + #[test] + fn expired_pins_are_released_without_any_further_chunk_request() { + let dir = tempfile::tempdir().expect("should create temp dir"); + let manager = SnapshotManager::new(); + manager.pin_for_serving(10, checkpoint_in(&dir, 10)); + assert!(manager.pinned_checkpoint(10).is_some()); + + let Some(long_ago) = Instant::now().checked_sub(SERVING_PIN_INACTIVITY_TTL * 2) else { + // Monotonic clock too young to backdate; nothing to assert on this platform. + return; + }; + manager.backdate_pin(10, None, Some(long_ago)); + + assert!( + manager.pinned_checkpoint(10).is_none(), + "an expired pin must not be handed out", + ); + + manager.pin_for_serving(11, checkpoint_in(&dir, 11)); + manager.backdate_pin(11, None, Some(long_ago)); + manager.release_expired_pins(); + assert_eq!( + manager.pinned_count(), + 0, + "the block-driven sweep must release expired pins with no peer activity", + ); + } + + /// The inactivity TTL is refreshable, so on its own it lets a peer hold a checkpoint + /// forever by touching it periodically. The absolute lifetime is not refreshable and + /// must cut such a pin off even while requests keep arriving. + #[test] + fn constant_touching_cannot_extend_a_pin_past_its_absolute_lifetime() { + let dir = tempfile::tempdir().expect("should create temp dir"); + let manager = SnapshotManager::new(); + let checkpoint = checkpoint_in(&dir, 10); + manager.pin_for_serving(10, Arc::clone(&checkpoint)); + let (pinned_at, _) = manager.pin_instants(10).expect("pin must exist"); + + // Continued activity refreshes the idle deadline but must NOT reset `pinned_at`, + // or the absolute deadline could be pushed out forever. + manager.pin_for_serving(10, Arc::clone(&checkpoint)); + let (pinned_at_after_touch, last_served) = + manager.pin_instants(10).expect("pin must exist"); + assert_eq!( + pinned_at, pinned_at_after_touch, + "serving another chunk must not extend the absolute lifetime", + ); + assert!(last_served >= pinned_at); + + // Once that deadline passes the pin goes, even though it was just touched. The + // peer can only get it back while the checkpoint is still in the registry — a + // pruned one is gone for good, which is the retention this bounds. + let Some(long_ago) = Instant::now().checked_sub(SERVING_PIN_MAX_LIFETIME * 2) else { + return; + }; + manager.backdate_pin(10, Some(long_ago), None); + assert!( + manager.pinned_checkpoint(10).is_none(), + "an over-long pin must expire despite continued activity", + ); + } + + /// A peer that keeps touching new heights as pruning advances must not be able to + /// hold an unbounded number of checkpoint directories back from deletion. + #[test] + fn serving_pins_are_capped() { + let dir = tempfile::tempdir().expect("should create temp dir"); + let manager = SnapshotManager::new(); + + for height in 0..(MAX_SERVING_PINS as u64 + 4) { + manager.pin_for_serving(height, checkpoint_in(&dir, height)); + } + + assert_eq!(manager.pinned_count(), MAX_SERVING_PINS); + assert!( + manager.pinned_checkpoint(0).is_none(), + "the least recently served pin must be evicted first", + ); + assert!(manager + .pinned_checkpoint(MAX_SERVING_PINS as u64 + 3) + .is_some()); + } #[test] fn supported_wire_versions_include_the_version_platform_versions_stamp() { diff --git a/packages/rs-drive/src/drive/mod.rs b/packages/rs-drive/src/drive/mod.rs index 367ad56a397..060c2292d79 100644 --- a/packages/rs-drive/src/drive/mod.rs +++ b/packages/rs-drive/src/drive/mod.rs @@ -11,6 +11,8 @@ use crate::config::DriveConfig; use arc_swap::ArcSwap; #[cfg(feature = "server")] use dpp::prelude::{BlockHeight, TimestampMillis}; +#[cfg(feature = "server")] +use dpp::util::deserializer::ProtocolVersion; #[cfg(any(feature = "server", feature = "verify"))] use grovedb::GroveDb; use std::fmt; @@ -126,6 +128,16 @@ impl Checkpoint { .map_err(Error::from) } + /// The Platform protocol version the chain was running at when this checkpoint was + /// taken, as recorded in the checkpoint's own aux storage. + /// + /// State sync must serve and consume a snapshot under the version table the snapshot + /// was PRODUCED with — the consuming node is typically a fresh node whose in-memory + /// version is still the initial one — so this is the authoritative source for it. + pub fn current_protocol_version(&self) -> Result, Error> { + Drive::fetch_current_protocol_version_with_grovedb(&self.grove_db, None) + } + /// Marks this checkpoint for deletion when it is dropped. pub fn mark_for_deletion(&self) { self.marked_for_deletion From cbd4288fd2ad4d7025a0cfcb2404eb364e7795d8 Mon Sep 17 00:00:00 2001 From: pasta Date: Mon, 31 Aug 2026 00:15:20 +0200 Subject: [PATCH 21/34] fix(drive-abci)!: carry the quorum-set history and pin validator sets in state sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reconstruction built both signature-verification quorum sets empty and called update_core_info with platform_state = None, so a restored node had no previous quorums at all. For chain locks that degrades safely (verify_chain_lock_locally returns Ok(None) when there is no history and defers to Core), but instant lock verification has no Core fallback by design: a node restored within SIGN_OFFSET core blocks of a quorum change would judge an InstantAssetLockProof against a different quorum than a node that replayed the chain, and reject a state transition the network accepted. That history cannot be re-derived from Core — get_quorum_listextended answers which quorums exist at a height, not when this node observed the set change — so ReducedPlatformStateV0 now carries the superseded quorums and their three core heights for both sets, and reconstruction reinstates them verbatim (previous_change_height included, which set_previous_past_quorums would have derived wrongly). The current sets are still re-derived from Core, where the answer is exact. Reconstruction also treated saved.quorum_positions as a sorting hint only, dropping saved hashes it did not see and appending unexpected Core ones. Validator sets live in the platform state, not grovedb, so the app-hash check cannot catch that: require an exact hash-set match and refuse the snapshot otherwise. Co-Authored-By: Claude Fable 5 --- .../rs-dpp/src/reduced_platform_state/mod.rs | 25 ++- .../src/reduced_platform_state/v0/mod.rs | 41 ++++ .../reconstruct_platform_state/mod.rs | 197 +++++++++++++++++- .../src/platform_types/platform_state/mod.rs | 42 +++- .../signature_verification_quorum_set/mod.rs | 27 ++- .../v0/quorum_set.rs | 57 +++++ 6 files changed, 382 insertions(+), 7 deletions(-) diff --git a/packages/rs-dpp/src/reduced_platform_state/mod.rs b/packages/rs-dpp/src/reduced_platform_state/mod.rs index 05ab4151e77..62b3c2444d1 100644 --- a/packages/rs-dpp/src/reduced_platform_state/mod.rs +++ b/packages/rs-dpp/src/reduced_platform_state/mod.rs @@ -59,7 +59,10 @@ impl PlatformDeserializableFromVersionedStructure for ReducedPlatformState { #[cfg(test)] mod tests { - use super::v0::{ReducedBlockInfoV0, ReducedPlatformStateV0}; + use super::v0::{ + ReducedBlockInfoV0, ReducedPlatformStateV0, ReducedPreviousQuorumsV0, + ReducedVerificationQuorumV0, + }; use super::*; use crate::block::block_info::BlockInfo; @@ -82,6 +85,26 @@ mod tests { previous_fee_versions: [(0u16, 1u32)].into_iter().collect(), quorum_positions: vec![[4u8; 32].into(), [5u8; 32].into()], proposed_core_chain_locked_height: 1000, + previous_chain_lock_quorums: Some(ReducedPreviousQuorumsV0 { + quorums: vec![ReducedVerificationQuorumV0 { + quorum_hash: [6u8; 32].into(), + public_key: [7u8; 48], + index: None, + }], + last_active_core_height: 990, + updated_at_core_height: 995, + previous_change_height: Some(900), + }), + previous_instant_lock_quorums: Some(ReducedPreviousQuorumsV0 { + quorums: vec![ReducedVerificationQuorumV0 { + quorum_hash: [8u8; 32].into(), + public_key: [9u8; 48], + index: Some(3), + }], + last_active_core_height: 991, + updated_at_core_height: 996, + previous_change_height: None, + }), }); let bytes = state.serialize_to_bytes().expect("should serialize"); diff --git a/packages/rs-dpp/src/reduced_platform_state/v0/mod.rs b/packages/rs-dpp/src/reduced_platform_state/v0/mod.rs index edda9bd2cdb..60814fe3160 100644 --- a/packages/rs-dpp/src/reduced_platform_state/v0/mod.rs +++ b/packages/rs-dpp/src/reduced_platform_state/v0/mod.rs @@ -28,6 +28,40 @@ pub struct ReducedBlockInfoV0 { pub round: u32, } +/// One quorum of a signature-verification quorum set, as persisted in the reduced +/// platform state. +#[derive(Clone, Debug, PartialEq, Encode, Decode)] +pub struct ReducedVerificationQuorumV0 { + /// The quorum hash + pub quorum_hash: Bytes32, + /// The quorum's threshold BLS public key, compressed (48 bytes) + pub public_key: [u8; 48], + /// The DIP24 rotation index, for rotating quorum types + pub index: Option, +} + +/// The superseded quorums of a signature-verification quorum set, together with the core +/// heights that define the window they are still authoritative for. +/// +/// This history CANNOT be recovered from Core: `get_quorum_listextended` answers "which +/// quorums exist at height h", not "when did this node observe the set change". It is +/// nonetheless consensus-relevant — `select_quorums` picks the previous set for locks +/// signed within `SIGN_OFFSET` core blocks of a change — so it has to travel with the +/// snapshot. Without it a restored node would judge an instant lock against a different +/// quorum than a node that replayed the chain, and reject a state transition the network +/// accepted. +#[derive(Clone, Debug, PartialEq, Encode, Decode)] +pub struct ReducedPreviousQuorumsV0 { + /// The superseded quorums + pub quorums: Vec, + /// The core height at which these quorums were last active + pub last_active_core_height: u32, + /// The core height at which the quorums were changed + pub updated_at_core_height: u32, + /// The core height at which the set before these became active + pub previous_change_height: Option, +} + /// Reduced Platform State V0. /// /// This minimal version of the Platform state is written into GroveDB (under the Misc @@ -58,4 +92,11 @@ pub struct ReducedPlatformStateV0 { /// note this can differ from the one in RequestPrepareProposal, as it can be /// modified by the proposer. pub proposed_core_chain_locked_height: u32, + /// The superseded chain lock validating quorums, if any. The CURRENT set is + /// re-derived from Core during reconstruction (it is exactly the quorum list at + /// `proposed_core_chain_locked_height`); only the history has to be carried. + pub previous_chain_lock_quorums: Option, + /// The superseded instant lock validating quorums, if any. See + /// [`ReducedPreviousQuorumsV0`] for why this cannot be left to reconstruction. + pub previous_instant_lock_quorums: Option, } diff --git a/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs index 311a96276c3..cf05dc7a596 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs @@ -2,20 +2,25 @@ use crate::abci::AbciError; use crate::error::Error; use crate::platform_types::platform::Platform; use crate::platform_types::platform_state::{PlatformState, PlatformStateV0Methods}; -use crate::platform_types::signature_verification_quorum_set::SignatureVerificationQuorumSet; +use crate::platform_types::signature_verification_quorum_set::{ + Quorums, SignatureVerificationQuorumSet, SignatureVerificationQuorumSetV0Methods, + VerificationQuorum, +}; use crate::platform_types::validator_set::ValidatorSet; use crate::rpc::core::CoreRPCLike; use dpp::block::extended_block_info::v0::{ExtendedBlockInfoV0, ExtendedBlockInfoV0Getters}; use dpp::block::extended_block_info::ExtendedBlockInfo; +use dpp::bls_signatures::PublicKey as BlsPublicKey; use dpp::dashcore::hashes::Hash; use dpp::dashcore::QuorumHash; use dpp::fee::default_costs::CachedEpochIndexFeeVersions; use dpp::platform_value::Bytes32; +use dpp::reduced_platform_state::v0::ReducedPreviousQuorumsV0; use dpp::reduced_platform_state::ReducedPlatformState; use dpp::version::fee::FeeVersion; use dpp::version::PlatformVersion; use indexmap::IndexMap; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; impl Platform where @@ -155,6 +160,39 @@ where state_platform_version, )?; + // The validator sets live in the platform state, NOT in grovedb, so the caller's + // app-hash equality check cannot see a disagreement between what Core just handed + // us and what the snapshot source actually ran with. `quorum_positions` is the + // consensus-covered list of validator set hashes from the source, so require an + // exact match before publishing anything: a restored node running with validator + // sets the chain never agreed on is worse than no restore at all, and the caller + // turns this error into a REJECT_SNAPSHOT. + let derived_validator_sets: BTreeSet<[u8; 32]> = platform_state + .validator_sets() + .keys() + .map(|quorum_hash| quorum_hash.to_byte_array()) + .collect(); + let saved_validator_sets: BTreeSet<[u8; 32]> = saved + .quorum_positions + .iter() + .map(|quorum_hash| quorum_hash.to_buffer()) + .collect(); + if derived_validator_sets != saved_validator_sets { + return Err(AbciError::StateSyncInternalError(format!( + "reconstruct_platform_state validator sets re-derived from Core do not match the \ + snapshot: {} derived, {} saved, {} only in Core, {} only in the snapshot", + derived_validator_sets.len(), + saved_validator_sets.len(), + derived_validator_sets + .difference(&saved_validator_sets) + .count(), + saved_validator_sets + .difference(&derived_validator_sets) + .count(), + )) + .into()); + } + // Core RPC returns quorums in an order that need not match the incremental // order the source node maintained; restore the recorded order. sort_validator_sets_by_saved_positions( @@ -162,6 +200,22 @@ where &saved.quorum_positions, ); + // Reinstate the signature-verification quorum HISTORY. `update_core_info` above + // rebuilt the current sets from Core — which is exact, the quorums of a type at a + // core height are whatever Core reports — but it was given `platform_state = None` + // and so could not produce any previous set. That history is consensus-relevant: + // `select_quorums` uses the previous set for locks signed within `SIGN_OFFSET` + // core blocks of a change, and for instant locks there is no Core fallback, so a + // restored node missing it would reject an asset lock proof the network accepted. + restore_previous_quorums( + platform_state.chain_lock_validating_quorums_mut(), + saved.previous_chain_lock_quorums.as_ref(), + )?; + restore_previous_quorums( + platform_state.instant_lock_validating_quorums_mut(), + saved.previous_instant_lock_quorums.as_ref(), + )?; + let block_height = platform_state.last_committed_block_height(); // Commit the re-derivation BEFORE the in-memory state is published: if this @@ -212,6 +266,52 @@ where } } +/// Reinstates the superseded quorums of a signature-verification quorum set exactly as the +/// snapshot source held them. +/// +/// `None` is a legitimate answer (the source had seen no quorum change yet) and leaves the +/// set without a history, which is what the source had. +fn restore_previous_quorums( + quorum_set: &mut SignatureVerificationQuorumSet, + saved: Option<&ReducedPreviousQuorumsV0>, +) -> Result<(), Error> { + let Some(saved) = saved else { + return Ok(()); + }; + + let quorums = saved + .quorums + .iter() + .map(|quorum| { + let public_key = BlsPublicKey::try_from(quorum.public_key.as_slice()).map_err(|e| { + AbciError::StateSyncInternalError(format!( + "reconstruct_platform_state previous quorum {} has an undeserializable public \ + key: {}", + hex::encode(quorum.quorum_hash.to_buffer()), + e + )) + })?; + + Ok(( + QuorumHash::from_byte_array(quorum.quorum_hash.to_buffer()), + VerificationQuorum { + public_key, + index: quorum.index, + }, + )) + }) + .collect::, Error>>()?; + + quorum_set.restore_previous_past_quorums( + quorums, + saved.last_active_core_height, + saved.updated_at_core_height, + saved.previous_change_height, + ); + + Ok(()) +} + /// Sorts the validator sets into the order recorded in the reduced platform state. /// /// Validator sets not present in the recorded order (which should not happen when the @@ -249,6 +349,99 @@ mod tests { QuorumHash::from_byte_array(bytes) } + /// The quorum-set history that travels with a snapshot must come back byte for byte, + /// including `previous_change_height` — `set_previous_past_quorums` DERIVES that field + /// from whatever the set already holds, which on a freshly reconstructed set is + /// nothing, so restoring through it would silently lose it and change which quorums + /// `select_quorums` considers verifiable. + #[test] + fn should_restore_the_previous_quorum_history_verbatim() { + use crate::config::ChainLockConfig; + use crate::platform_types::platform_state::to_reduced_previous_quorums; + use dpp::bls_signatures::{Bls12381G2Impl, SecretKey}; + use rand::rngs::StdRng; + use rand::SeedableRng; + + let mut rng = StdRng::seed_from_u64(11); + let quorums: Quorums = [(1u8, None), (2u8, Some(3u32))] + .into_iter() + .map(|(seed, index)| { + ( + quorum_hash(seed), + VerificationQuorum { + public_key: SecretKey::::random(&mut rng).public_key(), + index, + }, + ) + }) + .collect(); + + let mut source = SignatureVerificationQuorumSet::new( + &ChainLockConfig::default_100_67(), + PlatformVersion::latest(), + ) + .expect("should build quorum set"); + // Two changes, so `previous_change_height` is populated and can be lost + source.set_previous_past_quorums(quorums.clone(), 900, 950); + source.set_previous_past_quorums(quorums.clone(), 990, 995); + + let saved = to_reduced_previous_quorums(&source).expect("should capture the history"); + + let mut restored = SignatureVerificationQuorumSet::new( + &ChainLockConfig::default_100_67(), + PlatformVersion::latest(), + ) + .expect("should build quorum set"); + restore_previous_quorums(&mut restored, Some(&saved)).expect("should restore"); + + let source_previous = source.previous_past_quorums().expect("source has history"); + let restored_previous = restored + .previous_past_quorums() + .expect("restored must have history"); + + assert_eq!( + restored_previous.last_active_core_height, + source_previous.last_active_core_height + ); + assert_eq!( + restored_previous.updated_at_core_height, + source_previous.updated_at_core_height + ); + assert_eq!( + restored_previous.previous_change_height, + source_previous.previous_change_height + ); + assert_eq!(restored_previous.previous_change_height, Some(950)); + + assert_eq!( + restored_previous.quorums.len(), + source_previous.quorums.len() + ); + for (quorum_hash, source_quorum) in source_previous.quorums.iter() { + let restored_quorum = restored_previous + .quorums + .get(quorum_hash) + .expect("every quorum must be restored"); + assert_eq!(restored_quorum.public_key, source_quorum.public_key); + assert_eq!(restored_quorum.index, source_quorum.index); + } + } + + /// A set with no history restores to no history, not to an empty one — an empty + /// previous set would make `select_quorums` consider locks verifiable against nothing. + #[test] + fn should_leave_a_set_without_history_alone() { + use crate::config::ChainLockConfig; + + let mut restored = SignatureVerificationQuorumSet::new( + &ChainLockConfig::default_100_67(), + PlatformVersion::latest(), + ) + .expect("should build quorum set"); + restore_previous_quorums(&mut restored, None).expect("should restore"); + assert!(!restored.has_previous_past_quorums()); + } + #[test] fn should_sort_validator_sets_into_saved_positions() { use dpp::bls_signatures::{Bls12381G2Impl, SecretKey}; diff --git a/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs b/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs index b6ea5973ea2..ca4edfd610f 100644 --- a/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/platform_state/mod.rs @@ -21,12 +21,17 @@ use crate::error::execution::ExecutionError; pub use crate::platform_types::platform_state::accessors::PlatformStateV0Methods; use crate::platform_types::platform_state::platform_state_for_saving::v1::PlatformStateForSavingV1; use crate::platform_types::platform_state::platform_state_for_saving::PlatformStateForSaving; -use crate::platform_types::signature_verification_quorum_set::SignatureVerificationQuorumSet; +use crate::platform_types::signature_verification_quorum_set::{ + SignatureVerificationQuorumSet, SignatureVerificationQuorumSetV0Methods, +}; use dpp::block::block_info::BlockInfo; use dpp::dashcore::hashes::Hash; use dpp::dashcore_rpc::json::MasternodeListItem; use dpp::fee::default_costs::CachedEpochIndexFeeVersions; -use dpp::reduced_platform_state::v0::{ReducedBlockInfoV0, ReducedPlatformStateV0}; +use dpp::reduced_platform_state::v0::{ + ReducedBlockInfoV0, ReducedPlatformStateV0, ReducedPreviousQuorumsV0, + ReducedVerificationQuorumV0, +}; use dpp::reduced_platform_state::ReducedPlatformState; use dpp::util::hash::hash_double; use std::collections::BTreeMap; @@ -160,6 +165,12 @@ impl PlatformState { .map(|quorum_hash| quorum_hash.to_byte_array().into()) .collect(), proposed_core_chain_locked_height, + previous_chain_lock_quorums: to_reduced_previous_quorums( + &self.chain_lock_validating_quorums, + ), + previous_instant_lock_quorums: to_reduced_previous_quorums( + &self.instant_lock_validating_quorums, + ), }) } /// The default state at init chain @@ -195,6 +206,33 @@ impl PlatformState { } } +/// Captures the superseded quorums of a signature-verification quorum set for the reduced +/// platform state. +/// +/// The CURRENT quorums are deliberately not captured: reconstruction re-derives them from +/// Core, where the set at a given core height is exactly what Core reports. The history is +/// the part Core cannot answer, so it is the part that has to be carried. +pub(crate) fn to_reduced_previous_quorums( + quorum_set: &SignatureVerificationQuorumSet, +) -> Option { + let previous = quorum_set.previous_past_quorums()?; + + Some(ReducedPreviousQuorumsV0 { + quorums: previous + .quorums + .iter() + .map(|(quorum_hash, quorum)| ReducedVerificationQuorumV0 { + quorum_hash: quorum_hash.to_byte_array().into(), + public_key: quorum.public_key.0.to_compressed(), + index: quorum.index, + }) + .collect(), + last_active_core_height: previous.last_active_core_height, + updated_at_core_height: previous.updated_at_core_height, + previous_change_height: previous.previous_change_height, + }) +} + impl PlatformSerializable for PlatformState { type Error = Error; diff --git a/packages/rs-drive-abci/src/platform_types/signature_verification_quorum_set/mod.rs b/packages/rs-drive-abci/src/platform_types/signature_verification_quorum_set/mod.rs index 1d4a9e71967..df36b83b9f6 100644 --- a/packages/rs-drive-abci/src/platform_types/signature_verification_quorum_set/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/signature_verification_quorum_set/mod.rs @@ -7,8 +7,8 @@ use crate::platform_types::signature_verification_quorum_set::v0::for_saving_v0: use crate::platform_types::signature_verification_quorum_set::v0::for_saving_v1::SignatureVerificationQuorumSetForSavingV1; use crate::platform_types::signature_verification_quorum_set::v0::for_saving_v2::SignatureVerificationQuorumSetForSavingV2; pub use crate::platform_types::signature_verification_quorum_set::v0::quorum_set::{ - QuorumConfig, QuorumsWithConfig, SelectedQuorumSetIterator, SignatureVerificationQuorumSetV0, - SignatureVerificationQuorumSetV0Methods, SIGN_OFFSET, + PreviousPastQuorums, QuorumConfig, QuorumsWithConfig, SelectedQuorumSetIterator, + SignatureVerificationQuorumSetV0, SignatureVerificationQuorumSetV0Methods, SIGN_OFFSET, }; pub use crate::platform_types::signature_verification_quorum_set::v0::quorums::{ Quorum, Quorums, SigningQuorum, ThresholdBlsPublicKey, VerificationQuorum, @@ -76,6 +76,29 @@ impl SignatureVerificationQuorumSetV0Methods for SignatureVerificationQuorumSet } } + fn previous_past_quorums(&self) -> Option> { + match self { + Self::V0(v0) => v0.previous_past_quorums(), + } + } + + fn restore_previous_past_quorums( + &mut self, + previous_quorums: Quorums, + last_active_core_height: u32, + updated_at_core_height: u32, + previous_change_height: Option, + ) { + match self { + Self::V0(v0) => v0.restore_previous_past_quorums( + previous_quorums, + last_active_core_height, + updated_at_core_height, + previous_change_height, + ), + } + } + fn replace_quorums( &mut self, quorums: Quorums, diff --git a/packages/rs-drive-abci/src/platform_types/signature_verification_quorum_set/v0/quorum_set.rs b/packages/rs-drive-abci/src/platform_types/signature_verification_quorum_set/v0/quorum_set.rs index 881347bc6ae..854f47087a0 100644 --- a/packages/rs-drive-abci/src/platform_types/signature_verification_quorum_set/v0/quorum_set.rs +++ b/packages/rs-drive-abci/src/platform_types/signature_verification_quorum_set/v0/quorum_set.rs @@ -23,6 +23,18 @@ pub(super) struct PreviousPastQuorumsV0 { pub(super) previous_change_height: Option, } +/// A borrowed view of the superseded quorums of a set, for callers outside this module. +pub struct PreviousPastQuorums<'q> { + /// The superseded quorums + pub quorums: &'q Quorums, + /// The core height at which these quorums were last active + pub last_active_core_height: u32, + /// The core height at which the quorums were changed + pub updated_at_core_height: u32, + /// The core height at which the set before these became active + pub previous_change_height: Option, +} + /// Quorums with keys for signature verification #[derive(Debug, Clone)] pub struct SignatureVerificationQuorumSetV0 { @@ -55,6 +67,27 @@ pub trait SignatureVerificationQuorumSetV0Methods { /// Has previous quorums? fn has_previous_past_quorums(&self) -> bool; + /// The superseded quorums and the core heights that bound their validity, if any. + /// + /// This history exists only in the platform state — it cannot be re-derived from Core + /// — so it has to be readable to travel with a state sync snapshot. + fn previous_past_quorums(&self) -> Option>; + + /// Restores the superseded quorums verbatim, including the change height of the set + /// before them. + /// + /// Unlike [`SignatureVerificationQuorumSetV0Methods::set_previous_past_quorums`], this + /// does NOT derive `previous_change_height` from whatever this set currently holds: it + /// is for reinstating a history that was captured elsewhere (state sync reconstruction), + /// where deriving would silently produce a different one. + fn restore_previous_past_quorums( + &mut self, + previous_quorums: Quorums, + last_active_core_height: u32, + updated_at_core_height: u32, + previous_change_height: Option, + ); + /// Set last quorums keys and update previous quorums fn replace_quorums( &mut self, @@ -172,6 +205,30 @@ impl SignatureVerificationQuorumSetV0Methods for SignatureVerificationQuorumSetV self.previous.is_some() } + fn previous_past_quorums(&self) -> Option> { + self.previous.as_ref().map(|previous| PreviousPastQuorums { + quorums: &previous.quorums, + last_active_core_height: previous.last_active_core_height, + updated_at_core_height: previous.updated_at_core_height, + previous_change_height: previous.previous_change_height, + }) + } + + fn restore_previous_past_quorums( + &mut self, + previous_quorums: Quorums, + last_active_core_height: u32, + updated_at_core_height: u32, + previous_change_height: Option, + ) { + self.previous = Some(PreviousPastQuorumsV0 { + quorums: previous_quorums, + last_active_core_height, + updated_at_core_height, + previous_change_height, + }); + } + fn replace_quorums( &mut self, quorums: Quorums, From 9671bf61d3970db0bef9400d7cd15737233abf41 Mon Sep 17 00:00:00 2001 From: pasta Date: Mon, 31 Aug 2026 00:15:30 +0200 Subject: [PATCH 22/34] test(drive-abci): cover custom checkpoint paths and snapshot version metadata Adds a restart test proving checkpoints written under a configured CHECKPOINTS_PATH are reloaded and still advertised, and asserts that a snapshot honestly declaring a pre-v15 protocol version is refused at the offer, before anything is wiped. Co-Authored-By: Claude Fable 5 --- .../test_cases/state_sync_sentinel_tests.rs | 10 +- .../test_cases/state_sync_tests.rs | 109 ++++++++++++++++-- 2 files changed, 108 insertions(+), 11 deletions(-) diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs index b2753a574e2..0682e311204 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs @@ -30,6 +30,7 @@ mod tests { use crate::test_cases::state_sync_tests::tests::{ install_reconstruction_core_mocks, sync_snapshot, SnapshotSyncOutcome, }; + use dpp::version::v15::PROTOCOL_VERSION_15; use dpp::version::PlatformVersion; use drive_abci::abci::app::FullAbciApplication; use drive_abci::config::{ @@ -39,7 +40,8 @@ mod tests { use drive_abci::platform_types::platform::Platform; use drive_abci::platform_types::platform_state::PlatformStateV0Methods; use drive_abci::platform_types::snapshot::{ - restore_sentinel_exists, write_restore_sentinel, RESTORE_IN_PROGRESS_FILE_NAME, + encode_snapshot_metadata, restore_sentinel_exists, write_restore_sentinel, + RESTORE_IN_PROGRESS_FILE_NAME, }; use drive_abci::rpc::core::MockCoreRPCLike; use drive_abci::test::helpers::setup::{TempPlatform, TestPlatformBuilder}; @@ -183,7 +185,7 @@ mod tests { height: 1000, version: 1, hash: vec![7u8; 32], - metadata: vec![], + metadata: encode_snapshot_metadata(PROTOCOL_VERSION_15), }), app_hash: vec![7u8; 32], }) @@ -246,7 +248,7 @@ mod tests { height: 1000, version: 1, hash: vec![7u8; 32], - metadata: vec![], + metadata: encode_snapshot_metadata(PROTOCOL_VERSION_15), }), app_hash: vec![7u8; 32], }) @@ -515,7 +517,7 @@ mod tests { height, version: platform_version.drive_abci.state_sync.protocol_version as u32, hash: checkpoint_root.to_vec(), - metadata: vec![], + metadata: encode_snapshot_metadata(platform_version.protocol_version), }; let mut target_platform = TestPlatformBuilder::new() diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs index 567cc391aa6..889d5c9e96d 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs @@ -21,6 +21,7 @@ pub(crate) mod tests { ExtendedQuorumDetails, MasternodeListDiff, MasternodeListItem, QuorumInfoResult, }; use dpp::dashcore_rpc::json::{ExtendedQuorumListResult, QuorumType}; + use dpp::version::v15::PROTOCOL_VERSION_15; use dpp::version::PlatformVersion; use drive_abci::abci::app::FullAbciApplication; use drive_abci::config::{ @@ -30,8 +31,9 @@ pub(crate) mod tests { use drive_abci::mimic::test_quorum::TestQuorumInfo; use drive_abci::platform_types::platform::Platform; use drive_abci::platform_types::platform_state::PlatformStateV0Methods; + use drive_abci::platform_types::snapshot::encode_snapshot_metadata; use drive_abci::rpc::core::MockCoreRPCLike; - use drive_abci::test::helpers::setup::TestPlatformBuilder; + use drive_abci::test::helpers::setup::{TempPlatform, TestPlatformBuilder}; use std::collections::{BTreeMap, HashMap, VecDeque}; use strategy_tests::frequency::Frequency; use strategy_tests::{IdentityInsertInfo, StartAddresses, StartIdentities, Strategy}; @@ -682,6 +684,78 @@ pub(crate) mod tests { assert_eq!(info.last_block_app_hash, tip_app_hash.to_vec()); } + /// Checkpoints are created under the operator-configured `CHECKPOINTS_PATH`, so + /// startup has to read them back from the SAME place. When the reload path was + /// hard-coded to `/checkpoints`, a node configured with a custom path came + /// back from a restart with an empty registry: it stopped advertising the snapshots it + /// had retained, and could never prune the directories it had written. + #[tokio::test] + async fn checkpoints_in_a_custom_path_are_reloaded_after_a_restart() { + let checkpoints_dir = tempfile::tempdir().expect("should create a checkpoints dir"); + let mut config = state_sync_platform_config(); + config.abci.state_sync.checkpoints_path = Some(checkpoints_dir.path().to_path_buf()); + + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + + let (heights_before, db_path) = { + let outcome = run_chain_for_strategy( + &mut platform.platform, + 15, + state_sync_network_strategy(), + config.clone(), + 15, + &mut None, + &mut None, + ) + .await; + + let source = outcome.abci_app.platform; + let heights: Vec = source.drive.checkpoints.load().keys().copied().collect(); + (heights, source.config.db_path.clone()) + }; + + assert!( + !heights_before.is_empty(), + "the chain must have created checkpoints" + ); + assert!( + checkpoints_dir + .path() + .join(heights_before[0].to_string()) + .is_dir(), + "checkpoints must be written under the configured path" + ); + assert!( + !db_path.join("checkpoints").exists(), + "nothing must be written to the default path when one is configured" + ); + + let TempPlatform { + platform: original, + tempdir, + .. + } = platform; + drop(original); + let restarted = TempPlatform::open_with_tempdir(tempdir, config.clone()); + + let heights_after: Vec = restarted.drive.checkpoints.load().keys().copied().collect(); + assert_eq!( + heights_after, heights_before, + "a restart must reload the checkpoints from the configured path" + ); + + let app = FullAbciApplication::new(&restarted); + assert!( + !app.list_snapshots(Default::default()) + .expect("should list snapshots") + .snapshots + .is_empty(), + "a restarted node must keep advertising the snapshots it retained" + ); + } + /// A snapshot from a chain that never wrote the reduced platform state (pre-v15) /// is not offered by the source, and a target driven at it anyway refuses to /// restore it. @@ -725,11 +799,23 @@ pub(crate) mod tests { "pre-v15 checkpoints are unrestorable and must not be offered" ); - // Even if a peer maliciously offers such a snapshot, the target must refuse to - // restore it. (At the current grovedb revision the refusal comes from the - // post-restore verification; once grovedb faithfully restores sum trees it - // comes from the missing reduced platform state at the reconstruction step. - // Either way the snapshot must not be accepted.) + // Offered honestly — with its real, pre-v15 protocol version — such a snapshot is + // refused before anything is wiped. + let honest_pre_v15_offer = proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height: 1, + version: 1, + hash: vec![7u8; 32], + metadata: encode_snapshot_metadata(14), + }), + app_hash: vec![7u8; 32], + }; + + // Even if a peer maliciously offers such a snapshot — lying in the metadata that + // it is restorable — the target must refuse to restore it. (At the current grovedb + // revision the refusal comes from the post-restore verification; once grovedb + // faithfully restores sum trees it comes from the missing reduced platform state + // at the reconstruction step. Either way the snapshot must not be accepted.) let (height, checkpoint) = { let checkpoints = source_app.platform.drive.checkpoints.load(); let (height, info) = checkpoints @@ -747,7 +833,7 @@ pub(crate) mod tests { height, version: 1, hash: checkpoint_root.to_vec(), - metadata: vec![], + metadata: encode_snapshot_metadata(PROTOCOL_VERSION_15), }; let mut target_platform = TestPlatformBuilder::new() @@ -763,6 +849,15 @@ pub(crate) mod tests { ); let target_app = FullAbciApplication::new(&target_platform); + let honest_offer_response = target_app + .offer_snapshot(honest_pre_v15_offer) + .expect("an honestly-labelled pre-v15 offer is answered, not errored"); + assert_eq!( + honest_offer_response.result, + i32::from(response_offer_snapshot::Result::Reject), + "a snapshot that declares a pre-v15 protocol version must be refused up front" + ); + let outcome = sync_snapshot(&source_app, &target_app, &forged_snapshot, false) .expect("a refused snapshot is answered, not errored"); assert_eq!( From febc8f7802a02904861a64b9cfcf5d778ee6c6e7 Mon Sep 17 00:00:00 2001 From: pasta Date: Mon, 31 Aug 2026 00:38:30 +0200 Subject: [PATCH 23/34] refactor(drive-abci): tie the serving-pin cap to retention and tidy snapshot serving MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups from reviewing the state sync fixes: the pin count cap now tracks MAX_NUM_SNAPSHOTS plus slack instead of a fixed 8, and the absolute pin lifetime moves to six hours — the count cap is the real bound on how many checkpoint directories can be held back, so the lifetime only needs to be a backstop, and an hour risked cutting off an honest slow peer whose checkpoint was pruned mid-transfer. Also: a new Checkpoint::platform_version collapses the version resolution duplicated across list_snapshots and load_snapshot_chunk; Drive::open_with_checkpoints_path takes the directory directly instead of an Option; list_snapshots logs when it declines to advertise a checkpoint instead of skipping it silently. Co-Authored-By: Claude Fable 5 --- .../src/abci/handler/apply_snapshot_chunk.rs | 15 +--- .../src/abci/handler/list_snapshots.rs | 29 ++++---- .../src/abci/handler/load_snapshot_chunk.rs | 17 ++--- .../src/abci/handler/offer_snapshot.rs | 14 ++-- .../src/platform_types/platform/mod.rs | 2 +- .../src/platform_types/snapshot/mod.rs | 72 ++++++++++++------- packages/rs-drive/src/drive/mod.rs | 13 ++++ packages/rs-drive/src/open/mod.rs | 13 ++-- 8 files changed, 97 insertions(+), 78 deletions(-) diff --git a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs index 025d2659618..0395ffcb260 100644 --- a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs +++ b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs @@ -448,20 +448,7 @@ mod tests { .set_genesis_state(); let app = FullAbciApplication::new(&platform); - let target_app_hash = vec![7u8; 32]; - offer_snapshot( - &app, - proto::RequestOfferSnapshot { - snapshot: Some(proto::Snapshot { - height: 100, - version: 1, - hash: target_app_hash.clone(), - metadata: encode_snapshot_metadata(PROTOCOL_VERSION_15), - }), - app_hash: target_app_hash.clone(), - }, - ) - .expect("should accept offer"); + let target_app_hash = offer_a_snapshot(&app); // Garbage bytes for the root chunk: grovedb rejects them, and the session must // survive with a Retry + refetch of exactly that chunk, banning the sender. diff --git a/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs b/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs index 89648e6110b..d3799ea4c57 100644 --- a/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs +++ b/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs @@ -3,7 +3,6 @@ use crate::abci::AbciError; use crate::error::Error; use crate::platform_types::snapshot::encode_snapshot_metadata; use crate::rpc::core::CoreRPCLike; -use dpp::version::PlatformVersion; use tenderdash_abci::proto::abci as proto; /// Lists the state sync snapshots this node can serve. @@ -37,18 +36,22 @@ where // grovedb's tree opening and root-hash rules are version gated. The same version // is stamped into the snapshot metadata so the consuming node — which has no way // to derive it — restores under exactly these rules. - let Some(snapshot_protocol_version) = - checkpoint.current_protocol_version().map_err(|e| { - AbciError::StateSyncInternalError(format!( - "list_snapshots unable to read the protocol version of the checkpoint at \ - height {}: {}", - height, e - )) - })? + let Some(snapshot_platform_version) = checkpoint.platform_version().map_err(|e| { + AbciError::StateSyncInternalError(format!( + "list_snapshots unable to read the protocol version of the checkpoint at \ + height {}: {}", + height, e + )) + })? else { - continue; - }; - let Ok(snapshot_platform_version) = PlatformVersion::get(snapshot_protocol_version) else { + // Unreachable on a healthy node — `store_platform_state` writes the protocol + // version in the block transaction that commits before the checkpoint is + // taken — so say so rather than silently dropping the snapshot from the list. + tracing::warn!( + height, + "[state_sync] not offering the checkpoint at this height: it records no \ + protocol version, or one this binary does not know", + ); continue; }; let grove_version = &snapshot_platform_version.drive.grove_version; @@ -83,7 +86,7 @@ where .state_sync .protocol_version as u32, hash: root_hash.to_vec(), - metadata: encode_snapshot_metadata(snapshot_protocol_version), + metadata: encode_snapshot_metadata(snapshot_platform_version.protocol_version), }); } diff --git a/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs b/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs index 892813ccb4f..a634ec4ab86 100644 --- a/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs +++ b/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs @@ -2,10 +2,9 @@ use crate::abci::app::{PlatformApplication, SnapshotManagerApplication}; use crate::abci::AbciError; use crate::error::Error; use crate::platform_types::snapshot::{ - MAX_STATE_SYNC_CHUNK_ID_SIZE, SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS, + max_serving_pins, MAX_STATE_SYNC_CHUNK_ID_SIZE, SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS, }; use crate::rpc::core::CoreRPCLike; -use dpp::version::PlatformVersion; use std::sync::Arc; use tenderdash_abci::proto::abci as proto; @@ -76,8 +75,8 @@ where // Chunks must be generated under the version the checkpoint was WRITTEN at — the same // one `list_snapshots` stamped into the snapshot metadata and the consuming node // restores under. This node's own current version may have moved on since. - let snapshot_protocol_version = checkpoint - .current_protocol_version() + let snapshot_platform_version = checkpoint + .platform_version() .map_err(|e| { AbciError::StateSyncInternalError(format!( "load_snapshot_chunk unable to read the protocol version of the checkpoint at \ @@ -87,11 +86,10 @@ where })? .ok_or_else(|| { AbciError::StateSyncInternalError(format!( - "load_snapshot_chunk checkpoint at height {} has no protocol version", + "load_snapshot_chunk checkpoint at height {} has no usable protocol version", request.height )) })?; - let snapshot_platform_version = PlatformVersion::get(snapshot_protocol_version)?; let grove_version = &snapshot_platform_version.drive.grove_version; let chunk = checkpoint @@ -107,8 +105,11 @@ where // Pin (or refresh the pin of) the checkpoint only once a chunk was actually served. // Pinning before the fetch would let a peer keep a checkpoint — and its directory — // alive with a stream of requests that never succeed. - app.snapshot_manager() - .pin_for_serving(request.height, checkpoint); + app.snapshot_manager().pin_for_serving( + request.height, + checkpoint, + max_serving_pins(app.platform().config.abci.state_sync.max_num_snapshots), + ); Ok(proto::ResponseLoadSnapshotChunk { chunk }) } diff --git a/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs b/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs index d83c6a7aebd..7b9c2b8dc83 100644 --- a/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs +++ b/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs @@ -61,14 +61,12 @@ where // has no saved state, so its in-memory platform state is still at the initial protocol // version and would hand grovedb the wrong (much older) version table than the one the // serving node generated the chunks with. - let snapshot_platform_version = - decode_snapshot_metadata(&offered_snapshot.metadata).and_then(|protocol_version| { - // Only versions that write the reduced platform state can be restored at all; - // anything else is refused here rather than after a full transfer. - (protocol_version >= PROTOCOL_VERSION_15) - .then(|| PlatformVersion::get(protocol_version).ok()) - .flatten() - }); + // + // Only versions that write the reduced platform state can be restored at all, so + // anything below v15 is refused here rather than after a full transfer. + let snapshot_platform_version = decode_snapshot_metadata(&offered_snapshot.metadata) + .filter(|protocol_version| *protocol_version >= PROTOCOL_VERSION_15) + .and_then(|protocol_version| PlatformVersion::get(protocol_version).ok()); let Some(snapshot_platform_version) = snapshot_platform_version else { tracing::warn!( height = offered_snapshot.height, diff --git a/packages/rs-drive-abci/src/platform_types/platform/mod.rs b/packages/rs-drive-abci/src/platform_types/platform/mod.rs index f2387f70734..514fbabce09 100644 --- a/packages/rs-drive-abci/src/platform_types/platform/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/platform/mod.rs @@ -160,7 +160,7 @@ impl Platform { let (drive, current_platform_version) = Drive::open_with_checkpoints_path( &config.db_path, Some(config.drive.clone()), - Some(&checkpoints_path), + &checkpoints_path, ) .map_err(Error::Drive)?; diff --git a/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs index 48cb3105209..9dd61ba12fa 100644 --- a/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs @@ -215,20 +215,36 @@ const SERVING_PIN_INACTIVITY_TTL: Duration = Duration::from_secs(600); /// Absolute lifetime of a serving pin, regardless of activity. /// /// The inactivity TTL alone is refreshable, so a peer that keeps touching a height keeps -/// its checkpoint alive forever. A real restore is far quicker than this — a full -/// mainnet-state restore measures in seconds — so an hour is generous for any honest -/// transfer while putting a ceiling on how long abuse can hold a directory back. -const SERVING_PIN_MAX_LIFETIME: Duration = Duration::from_secs(3600); - -/// Hard cap on how many checkpoints may be pinned for serving at once. +/// its checkpoint alive forever; this deadline is deliberately NOT refreshable. +/// +/// It is the backstop, not the primary bound — [`max_serving_pins`] is what actually +/// limits how many directories can be held back at once — so it is set well above any +/// plausible honest transfer rather than tight. A full mainnet-state restore measures in +/// seconds; six hours leaves enormous room for a slow or rate-limited peer whose +/// checkpoint gets pruned mid-transfer, while still bounding how long a pinned directory +/// can outlive its checkpoint. +const SERVING_PIN_MAX_LIFETIME: Duration = Duration::from_secs(6 * 3600); + +/// How many pins are allowed on top of the number of snapshots the node retains. +/// +/// The interesting pins are the ones for checkpoints pruning has ALREADY dropped from the +/// registry — those are the directories a pin holds back from deletion. There can only +/// ever be a handful of them legitimately (a transfer that started before the checkpoint +/// aged out), so the retained count plus this slack is generous. +const SERVING_PIN_SLACK: usize = 4; + +/// Cap on how many checkpoints may be pinned for serving at once, given how many snapshots +/// the node is configured to retain. /// -/// Without it, a peer could keep one transfer alive per height it ever touched: pruning +/// Without a cap, a peer could keep one transfer alive per height it ever touched: pruning /// keeps advancing, the peer keeps refreshing, and the number of checkpoint directories -/// held back from deletion grows without bound regardless of `MAX_NUM_SNAPSHOTS`. Eight -/// concurrent transfers is far more than the handful of snapshots a node advertises, so -/// the cap only ever bites on abuse; when it does, the least recently served pin goes -/// first. -const MAX_SERVING_PINS: usize = 8; +/// held back from deletion grows without bound regardless of `MAX_NUM_SNAPSHOTS`. The cap +/// only ever bites on abuse; when it does, the least recently served pin goes first, and a +/// peer whose pin is evicted can still resolve the checkpoint from the registry if it is +/// still there. +pub fn max_serving_pins(max_num_snapshots: usize) -> usize { + max_num_snapshots.saturating_add(SERVING_PIN_SLACK) +} /// A checkpoint held back from deletion for a transfer in flight. struct ServingPin { @@ -256,7 +272,7 @@ impl ServingPin { /// A pin must not be something a remote peer can hold open indefinitely, so it is bounded /// three ways: [`SERVING_PIN_INACTIVITY_TTL`] since the last chunk actually served, /// [`SERVING_PIN_MAX_LIFETIME`] since it was taken (not refreshable), and -/// [`MAX_SERVING_PINS`] in total. Expiry also must not depend on peers making further +/// [`max_serving_pins`] in total. Expiry also must not depend on peers making further /// requests, or an abandoned transfer would hold its directory forever: /// [`SnapshotManager::release_expired_pins`] runs once per block and every read of a pin /// re-checks both deadlines. @@ -273,11 +289,12 @@ impl SnapshotManager { } /// Pins a checkpoint that is being served (or refreshes the pin of one that already - /// is), dropping expired pins and enforcing [`MAX_SERVING_PINS`]. + /// is), dropping expired pins and holding the total to `max_pins` (see + /// [`max_serving_pins`]). /// /// Call this only AFTER a chunk was successfully served: a request that could not be /// answered must not be able to keep a checkpoint alive. - pub fn pin_for_serving(&self, height: u64, checkpoint: Arc) { + pub fn pin_for_serving(&self, height: u64, checkpoint: Arc, max_pins: usize) { let now = Instant::now(); let mut pins = self .serving_pins @@ -293,7 +310,7 @@ impl SnapshotManager { } // Evict the least recently served pin to make room for a genuinely new one - while pins.len() >= MAX_SERVING_PINS { + while pins.len() >= max_pins.max(1) { let Some(coldest) = pins .iter() .min_by_key(|(_, pin)| pin.last_served) @@ -412,7 +429,7 @@ mod tests { fn expired_pins_are_released_without_any_further_chunk_request() { let dir = tempfile::tempdir().expect("should create temp dir"); let manager = SnapshotManager::new(); - manager.pin_for_serving(10, checkpoint_in(&dir, 10)); + manager.pin_for_serving(10, checkpoint_in(&dir, 10), max_serving_pins(3)); assert!(manager.pinned_checkpoint(10).is_some()); let Some(long_ago) = Instant::now().checked_sub(SERVING_PIN_INACTIVITY_TTL * 2) else { @@ -426,7 +443,7 @@ mod tests { "an expired pin must not be handed out", ); - manager.pin_for_serving(11, checkpoint_in(&dir, 11)); + manager.pin_for_serving(11, checkpoint_in(&dir, 11), max_serving_pins(3)); manager.backdate_pin(11, None, Some(long_ago)); manager.release_expired_pins(); assert_eq!( @@ -444,12 +461,12 @@ mod tests { let dir = tempfile::tempdir().expect("should create temp dir"); let manager = SnapshotManager::new(); let checkpoint = checkpoint_in(&dir, 10); - manager.pin_for_serving(10, Arc::clone(&checkpoint)); + manager.pin_for_serving(10, Arc::clone(&checkpoint), max_serving_pins(3)); let (pinned_at, _) = manager.pin_instants(10).expect("pin must exist"); // Continued activity refreshes the idle deadline but must NOT reset `pinned_at`, // or the absolute deadline could be pushed out forever. - manager.pin_for_serving(10, Arc::clone(&checkpoint)); + manager.pin_for_serving(10, Arc::clone(&checkpoint), max_serving_pins(3)); let (pinned_at_after_touch, last_served) = manager.pin_instants(10).expect("pin must exist"); assert_eq!( @@ -478,18 +495,21 @@ mod tests { let dir = tempfile::tempdir().expect("should create temp dir"); let manager = SnapshotManager::new(); - for height in 0..(MAX_SERVING_PINS as u64 + 4) { - manager.pin_for_serving(height, checkpoint_in(&dir, height)); + // The cap tracks how many snapshots the node retains, plus slack for transfers + // that started before their checkpoint aged out. + let max_pins = max_serving_pins(3); + assert_eq!(max_pins, 3 + SERVING_PIN_SLACK); + + for height in 0..(max_pins as u64 + 4) { + manager.pin_for_serving(height, checkpoint_in(&dir, height), max_pins); } - assert_eq!(manager.pinned_count(), MAX_SERVING_PINS); + assert_eq!(manager.pinned_count(), max_pins); assert!( manager.pinned_checkpoint(0).is_none(), "the least recently served pin must be evicted first", ); - assert!(manager - .pinned_checkpoint(MAX_SERVING_PINS as u64 + 3) - .is_some()); + assert!(manager.pinned_checkpoint(max_pins as u64 + 3).is_some()); } #[test] diff --git a/packages/rs-drive/src/drive/mod.rs b/packages/rs-drive/src/drive/mod.rs index 060c2292d79..f2e20d43327 100644 --- a/packages/rs-drive/src/drive/mod.rs +++ b/packages/rs-drive/src/drive/mod.rs @@ -13,6 +13,8 @@ use arc_swap::ArcSwap; use dpp::prelude::{BlockHeight, TimestampMillis}; #[cfg(feature = "server")] use dpp::util::deserializer::ProtocolVersion; +#[cfg(feature = "server")] +use dpp::version::PlatformVersion; #[cfg(any(feature = "server", feature = "verify"))] use grovedb::GroveDb; use std::fmt; @@ -138,6 +140,17 @@ impl Checkpoint { Drive::fetch_current_protocol_version_with_grovedb(&self.grove_db, None) } + /// The Platform version this checkpoint must be read under, or `None` when the + /// checkpoint records no protocol version or one this binary does not know. + /// + /// Both are reasons not to serve the checkpoint as a state sync snapshot rather than + /// errors: an unknown version is simply a newer node's checkpoint. + pub fn platform_version(&self) -> Result, Error> { + Ok(self + .current_protocol_version()? + .and_then(|protocol_version| PlatformVersion::get(protocol_version).ok())) + } + /// Marks this checkpoint for deletion when it is dropped. pub fn mark_for_deletion(&self) { self.marked_for_deletion diff --git a/packages/rs-drive/src/open/mod.rs b/packages/rs-drive/src/open/mod.rs index b213933ae67..ede4e2f00cd 100644 --- a/packages/rs-drive/src/open/mod.rs +++ b/packages/rs-drive/src/open/mod.rs @@ -31,7 +31,8 @@ impl Drive { path: P, config: Option, ) -> Result<(Self, Option<&'static PlatformVersion>), Error> { - Self::open_with_checkpoints_path(path, config, None::<&Path>) + let checkpoints_path = path.as_ref().join("checkpoints"); + Self::open_with_checkpoints_path(path, config, checkpoints_path) } /// Opens GroveDB database, loading the checkpoint registry from an explicit directory. @@ -40,17 +41,16 @@ impl Drive { /// (`CHECKPOINTS_PATH`). Whoever knows that configuration must pass the same directory /// checkpoint creation writes to, otherwise the registry comes up empty after a /// restart and the checkpoints on disk are neither advertised nor prunable. - /// `checkpoints_path = None` keeps the historical `/checkpoints` default. /// /// # Arguments /// /// * `path` - The path to the GroveDB. /// * `config` - An `Option` which contains `DriveConfig`. If not specified, default configuration is used. - /// * `checkpoints_path` - The directory checkpoints are written to, if not the default. + /// * `checkpoints_path` - The directory checkpoints are written to. pub fn open_with_checkpoints_path, Q: AsRef>( path: P, config: Option, - checkpoints_path: Option, + checkpoints_path: Q, ) -> Result<(Self, Option<&'static PlatformVersion>), Error> { let config = config.unwrap_or_default(); let db_path = path.as_ref(); @@ -74,10 +74,7 @@ impl Drive { .transpose()?; // Load existing checkpoints from the configured checkpoints directory - let checkpoints = match checkpoints_path.as_ref() { - Some(checkpoints_path) => load_current_checkpoints(checkpoints_path.as_ref())?, - None => load_current_checkpoints(db_path.join("checkpoints"))?, - }; + let checkpoints = load_current_checkpoints(checkpoints_path)?; let drive = Drive { grove, From feee5b9b093aeaf0572ab758494a44b8b00e7791 Mon Sep 17 00:00:00 2001 From: pasta Date: Mon, 31 Aug 2026 12:12:51 +0200 Subject: [PATCH 24/34] fix(drive-abci): open the query height gate once a state sync restore succeeds The query service refuses to execute while the published state's height differs from committed_block_height_guard. A fresh node's guard is 0 and only finalize_block ever stored to it, so a completed restore published the reconstructed state at the snapshot height while the guard stayed at 0, leaving every query unserviceable until the first post-restore block finalized. Store the height into the guard in apply_snapshot_chunk, strictly after grovedb reconstruction, aux persistence and the final app-hash check succeed; a rejected restore leaves the gate closed (covered by a new assertion in the sum-tree-defect test). Also move the query service's wait counter out of the inner loop: declared inside it, it was reset on every pass, so the intended 1-second budget never expired and a query hitting a state/guard mismatch would spin forever instead of restarting and eventually returning NotServiceable. Co-Authored-By: Claude Fable 5 --- .../src/abci/handler/apply_snapshot_chunk.rs | 16 ++++++++++++++++ packages/rs-drive-abci/src/query/service.rs | 7 ++++++- .../test_cases/state_sync_tests.rs | 14 ++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs index 0395ffcb260..8456736fe60 100644 --- a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs +++ b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs @@ -1,11 +1,13 @@ use crate::abci::app::StateSyncApplication; use crate::abci::AbciError; use crate::error::Error; +use crate::platform_types::platform_state::PlatformStateV0Methods; use crate::platform_types::snapshot::{ clear_restore_sentinel_best_effort, wipe_drive_for_restore, MAX_STATE_SYNC_CHUNK_ID_SIZE, MAX_STATE_SYNC_CHUNK_SIZE, }; use crate::rpc::core::CoreRPCLike; +use std::sync::atomic::Ordering; use tenderdash_abci::proto::abci as proto; use tenderdash_abci::proto::abci::response_apply_snapshot_chunk; @@ -278,6 +280,20 @@ where ); } + // The query service only serves while `committed_block_height_guard` matches the + // published state's height. A fresh node's guard is still 0 (nothing was ever + // finalized through it), while `reconstruct_platform_state` just published the + // state at the snapshot height — left alone, that mismatch keeps every query + // unserviceable until the first post-restore block finalizes. Open the gate only + // HERE, after grovedb reconstruction, aux persistence and the final app-hash check + // have all succeeded: on any earlier failure the guard stays 0, exactly as it must + // for a node whose restore was rejected. (A restart re-derives the guard from the + // persisted state, so this store also matches what the next boot would compute.) + app.platform().committed_block_height_guard.store( + app.platform().state.load().last_committed_block_height(), + Ordering::Relaxed, + ); + // The restore is complete and the node is self-consistent again, so the marker that // tells a restarting process to wipe can go. This is deliberately the LAST step, after // `reconstruct_platform_state` has committed the platform state to aux storage, and diff --git a/packages/rs-drive-abci/src/query/service.rs b/packages/rs-drive-abci/src/query/service.rs index 3c6ce47b5a3..37711b899b7 100644 --- a/packages/rs-drive-abci/src/query/service.rs +++ b/packages/rs-drive-abci/src/query/service.rs @@ -126,11 +126,16 @@ impl QueryService { // that query is executed only after/before both states are updated. let mut needs_restart = false; + // The wait budget must survive iterations of the loop below: declared + // inside it, the counter was reset on every pass and the 1 second + // timeout could never fire, so a query arriving while the two states + // disagreed would spin here forever instead of restarting. + let mut counter = 0; + loop { let committed_block_height_guard = platform .committed_block_height_guard .load(Ordering::Relaxed); - let mut counter = 0; if platform_state.last_committed_block_height() == committed_block_height_guard { break; diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs index 889d5c9e96d..0bbeadf04af 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs @@ -453,6 +453,13 @@ pub(crate) mod tests { snapshot.height, "target must be at the snapshot height" ); + assert_eq!( + target_platform + .committed_block_height_guard + .load(std::sync::atomic::Ordering::Relaxed), + snapshot.height, + "a completed restore must open the query height gate at the snapshot height" + ); assert_eq!( target_state.last_committed_block_app_hash(), source_platform_state.last_committed_block_app_hash() @@ -548,6 +555,13 @@ pub(crate) mod tests { target_platform.state.load().last_committed_block_height(), 0 ); + assert_eq!( + target_platform + .committed_block_height_guard + .load(std::sync::atomic::Ordering::Relaxed), + 0, + "a rejected restore must not open the query height gate" + ); assert_ne!( target_platform .drive From 86e5dd1afd0e99998f113509087ce42769902d9c Mon Sep 17 00:00:00 2001 From: pasta Date: Mon, 31 Aug 2026 12:21:54 +0200 Subject: [PATCH 25/34] fix(drive-abci): refuse proofs from a state that has no block proof metadata yet A state restored via state sync stores an all-zero block id hash and quorum signature for the snapshot block. That is structural, not an omission: the reduced platform state is written into grovedb immediately before the root hash is computed, and the block's commit signature signs that root hash, so the signature can never be part of the state it signs. response_proof_v0 copied those zeroes into every current-state proof, and rs-drive-proof-verifier (correctly) rejects an all-zero signature, so between a completed restore (or a restart from the persisted restored state) and the first finalized block the node served proofs no client could ever authenticate. Refuse to build such a proof instead: response_proof_v0 now returns a dedicated error when the state has a committed block but an all-zero signature, and the query service maps it to gRPC UNAVAILABLE so clients retry (or re-query without a proof) rather than report verification failures. The first block finalized after the restore stores real metadata, persists it, and reopens proof serving; queries without proofs are unaffected. Height 0 stays exempt (a chain with no committed block has no signature for anyone), and so do test chains that run with block signing disabled (feature-gated testing-config, not part of production builds) - they finalize every block unsigned and their proofs were never verifiable. Co-Authored-By: Claude Fable 5 --- packages/rs-drive-abci/src/abci/error.rs | 6 + .../src/query/response_metadata/v0/mod.rs | 124 ++++++++++++++++++ packages/rs-drive-abci/src/query/service.rs | 10 +- 3 files changed, 139 insertions(+), 1 deletion(-) diff --git a/packages/rs-drive-abci/src/abci/error.rs b/packages/rs-drive-abci/src/abci/error.rs index f8168b2cbac..ae1678218c3 100644 --- a/packages/rs-drive-abci/src/abci/error.rs +++ b/packages/rs-drive-abci/src/abci/error.rs @@ -98,6 +98,12 @@ pub enum AbciError { /// Generic with code should only be used in tests #[error("invalid state transition error: {0}")] InvalidStateTransition(#[from] ConsensusError), + + /// The current state was restored via state sync and carries no block id hash or + /// quorum signature yet, so a proof built from it could never authenticate; the + /// metadata arrives with the first block finalized after the restore. + #[error("state sync proof metadata not yet available: {0}")] + StateSyncProofMetadataUnavailable(String), } #[cfg(test)] diff --git a/packages/rs-drive-abci/src/query/response_metadata/v0/mod.rs b/packages/rs-drive-abci/src/query/response_metadata/v0/mod.rs index 19621528234..461e651a32a 100644 --- a/packages/rs-drive-abci/src/query/response_metadata/v0/mod.rs +++ b/packages/rs-drive-abci/src/query/response_metadata/v0/mod.rs @@ -1,3 +1,4 @@ +use crate::abci::AbciError; use crate::error::Error; use crate::platform_types::platform::Platform; use crate::platform_types::platform_state::PlatformState; @@ -7,6 +8,44 @@ use dapi_grpc::platform::v0::{Proof, ResponseMetadata}; use drive::error::drive::DriveError; use drive::util::grove_operations::GroveDBToUse; +impl Platform { + /// Refuses to build a proof from a state that has no block proof metadata. + /// + /// A state restored via state sync has an all-zero block id hash and quorum signature + /// at the snapshot height: the reduced platform state is written into grovedb BEFORE + /// the block's root hash exists, and the block's commit signature signs that root + /// hash, so the signature can never be part of the state it signs. + /// `rs-drive-proof-verifier` (correctly) rejects an all-zero signature, meaning a + /// proof built from such a state could never authenticate — refuse it with a + /// retryable error instead. The first block finalized after the restore stores real + /// metadata and reopens proof serving. + /// + /// Height 0 is exempt: a chain that has not committed a block yet has no signature + /// either, which predates state sync and is left as is. + fn ensure_block_proof_metadata_is_available(&self, state: &PlatformState) -> Result<(), Error> { + // A test chain running with block signing disabled finalizes every block with an + // all-zero signature; its proofs were never verifiable, and gating them would + // break the strategy test harness's proof plumbing checks. + #[cfg(feature = "testing-config")] + if !self.config.testing_configs.block_signing { + return Ok(()); + } + + if state.last_committed_block_height() > 0 + && state.last_committed_block_signature() == [0u8; 96] + { + return Err(AbciError::StateSyncProofMetadataUnavailable(format!( + "the state at height {} was restored via state sync and its block signature \ + only becomes known when the next block is finalized; retry shortly, or \ + repeat the query without requesting a proof", + state.last_committed_block_height() + )) + .into()); + } + Ok(()) + } +} + impl Platform { /// Returns response metadata for the given GroveDB that was used. /// @@ -46,6 +85,7 @@ impl Platform { ) -> Result<(CheckpointUsed, Proof), Error> { match grovedb_to_use { GroveDBToUse::Current => { + self.ensure_block_proof_metadata_is_available(platform_state)?; let proof = Proof { grovedb_proof: proof, quorum_hash: platform_state.last_committed_quorum_hash().to_vec(), @@ -74,6 +114,7 @@ impl Platform { })? .clone(); + self.ensure_block_proof_metadata_is_available(&checkpoint_state)?; let proof = Proof { grovedb_proof: proof, quorum_hash: checkpoint_state.last_committed_quorum_hash().to_vec(), @@ -95,6 +136,7 @@ impl Platform { })? .clone(); + self.ensure_block_proof_metadata_is_available(&checkpoint_state)?; let proof = Proof { grovedb_proof: proof, quorum_hash: checkpoint_state.last_committed_quorum_hash().to_vec(), @@ -108,3 +150,85 @@ impl Platform { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::test::helpers::setup::TestPlatformBuilder; + use dpp::block::block_info::BlockInfo; + use dpp::block::extended_block_info::v0::ExtendedBlockInfoV0; + use dpp::block::extended_block_info::ExtendedBlockInfo; + + fn block_info_with_signature(height: u64, signature: [u8; 96]) -> ExtendedBlockInfo { + ExtendedBlockInfo::V0(ExtendedBlockInfoV0 { + basic_info: BlockInfo { + time_ms: 1_000_000, + height, + core_height: 42, + epoch: Default::default(), + }, + app_hash: [1u8; 32], + quorum_hash: [2u8; 32], + block_id_hash: [3u8; 32], + proposer_pro_tx_hash: [4u8; 32], + signature, + round: 0, + }) + } + + /// A state restored via state sync stores an all-zero block signature until the next + /// block finalizes; a proof built from it can never authenticate (the verifier + /// rejects an all-zero signature), so it must be refused rather than served. + #[test] + fn should_refuse_a_proof_from_a_state_without_block_proof_metadata() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + + let mut state = platform.state.load().as_ref().clone(); + state.set_last_committed_block_info(Some(block_info_with_signature(10, [0u8; 96]))); + + let result = platform.response_proof_v0(&state, vec![], GroveDBToUse::Current); + let error = result.expect_err("a zero-signature state must not produce a proof"); + assert!( + matches!( + error, + Error::Abci(AbciError::StateSyncProofMetadataUnavailable(_)) + ), + "expected StateSyncProofMetadataUnavailable, got: {error}" + ); + } + + /// A normally finalized block always carries a real signature; proofs must be served. + #[test] + fn should_serve_a_proof_from_a_state_with_block_proof_metadata() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + + let mut state = platform.state.load().as_ref().clone(); + state.set_last_committed_block_info(Some(block_info_with_signature(10, [5u8; 96]))); + + let (_, proof) = platform + .response_proof_v0(&state, vec![], GroveDBToUse::Current) + .expect("a signed state must produce a proof"); + assert_eq!(proof.signature, vec![5u8; 96]); + assert_eq!(proof.block_id_hash, vec![3u8; 32]); + } + + /// A chain that has not committed a block yet has no signature for anyone — that + /// predates state sync and stays as it was. + #[test] + fn should_leave_the_pre_genesis_state_exempt() { + let platform = TestPlatformBuilder::new() + .build_with_mock_rpc() + .set_genesis_state(); + + let state = platform.state.load(); + assert_eq!(state.last_committed_block_height(), 0, "sanity: no blocks"); + + platform + .response_proof_v0(&state, vec![], GroveDBToUse::Current) + .expect("the pre-genesis state must remain servable"); + } +} diff --git a/packages/rs-drive-abci/src/query/service.rs b/packages/rs-drive-abci/src/query/service.rs index 37711b899b7..ac59915dcdf 100644 --- a/packages/rs-drive-abci/src/query/service.rs +++ b/packages/rs-drive-abci/src/query/service.rs @@ -1006,7 +1006,15 @@ fn query_error_into_status(error: QueryError) -> Status { } fn error_into_status(error: Error) -> Status { - Status::internal(format!("query: {}", error)) + match error { + // Not a server fault: the state was restored via state sync and the block proof + // metadata arrives with the first block finalized after the restore. UNAVAILABLE + // tells clients to retry (or drop the proof request) rather than report a bug. + Error::Abci(crate::abci::AbciError::StateSyncProofMetadataUnavailable(message)) => { + Status::unavailable(message) + } + error => Status::internal(format!("query: {}", error)), + } } fn validate_path_elements_request(request: &GetPathElementsRequest) -> Result<(), Status> { From c6f65bd6c5d8a071ceb620e444efd5055d874571 Mon Sep 17 00:00:00 2001 From: pasta Date: Mon, 31 Aug 2026 12:22:07 +0200 Subject: [PATCH 26/34] fix(drive-abci): expire serving pins autonomously and serve snapshots off the async workers Production snapshot serving runs in CheckTxAbciApplication (the gRPC app server.rs registers), whose SnapshotManager was private and whose process never finalizes blocks - the once-per-block release_expired_pins in FullAbciApplication::finalize_block only covers the all-in-one test application. An abandoned transfer therefore kept its checkpoint Arc alive, holding an already-pruned full-state directory on disk until another serving request or shutdown. The serving SnapshotManager is now shared (Arc) with a small sweep task server.rs spawns next to the gRPC server, which releases expired pins once a minute regardless of peer activity or blocks. list_snapshots and load_snapshot_chunk were also running their synchronous rocksdb/Merk work (checkpoint metadata reads, chunk generation and encoding) directly on Tokio async workers of a tonic handler. Requests are peer-controlled, so concurrent snapshot consumers could occupy the runtime and delay unrelated gRPC traffic. Both handlers now run on the blocking pool, following the adjacent check_tx pattern; they take the platform and snapshot manager directly (owned Arcs clone into the blocking closure), which also retires the now-unused SnapshotManagerApplication trait. Co-Authored-By: Claude Fable 5 --- .../rs-drive-abci/src/abci/app/check_tx.rs | 74 +++++++++++++------ packages/rs-drive-abci/src/abci/app/full.rs | 14 +--- packages/rs-drive-abci/src/abci/app/mod.rs | 9 +-- .../src/abci/handler/list_snapshots.rs | 32 ++++---- .../src/abci/handler/load_snapshot_chunk.rs | 48 ++++++------ .../src/platform_types/snapshot/mod.rs | 14 +++- packages/rs-drive-abci/src/server.rs | 27 ++++++- 7 files changed, 134 insertions(+), 84 deletions(-) diff --git a/packages/rs-drive-abci/src/abci/app/check_tx.rs b/packages/rs-drive-abci/src/abci/app/check_tx.rs index 4ed32b2fbed..d4193d97464 100644 --- a/packages/rs-drive-abci/src/abci/app/check_tx.rs +++ b/packages/rs-drive-abci/src/abci/app/check_tx.rs @@ -1,4 +1,4 @@ -use crate::abci::app::{PlatformApplication, SnapshotManagerApplication}; +use crate::abci::app::PlatformApplication; use crate::abci::handler; use crate::error::Error; use crate::platform_types::platform::Platform; @@ -23,8 +23,12 @@ where /// Platform platform: Arc>, core_rpc: Arc, - /// The snapshot manager, pinning checkpoints that are being served to peers - snapshot_manager: SnapshotManager, + /// The snapshot manager, pinning checkpoints that are being served to peers. + /// + /// Shared (`Arc`) rather than owned: this application never sees blocks, so it + /// cannot expire the pins of abandoned transfers itself — `server::start` keeps a + /// clone and sweeps expired pins on a timer. + snapshot_manager: Arc, } impl PlatformApplication for CheckTxAbciApplication @@ -36,25 +40,20 @@ where } } -impl SnapshotManagerApplication for CheckTxAbciApplication -where - C: CoreRPCLike + Send + Sync + 'static, -{ - fn snapshot_manager(&self) -> &SnapshotManager { - &self.snapshot_manager - } -} - impl CheckTxAbciApplication where C: CoreRPCLike + Send + Sync + 'static, { /// Create new ABCI app - pub fn new(platform: Arc>, core_rpc: Arc) -> Self { + pub fn new( + platform: Arc>, + core_rpc: Arc, + snapshot_manager: Arc, + ) -> Self { Self { platform, core_rpc, - snapshot_manager: SnapshotManager::new(), + snapshot_manager, } } } @@ -113,18 +112,41 @@ where &self, request: tonic::Request, ) -> Result, tonic::Status> { - handler::list_snapshots(self, request.into_inner()) - .map(tonic::Response::new) - .map_err(error_into_status) + // Checkpoint metadata reads are synchronous rocksdb work; requests are + // peer-controlled, so keep them off the async workers (same pattern as check_tx) + let platform = Arc::clone(&self.platform); + let proto_request = request.into_inner(); + + spawn_blocking_task_with_name_if_supported("list_snapshots", move || { + handler::list_snapshots(platform.as_ref(), proto_request) + .map(tonic::Response::new) + .map_err(error_into_status) + })? + .await + .map_err(|error| tonic::Status::internal(format!("list snapshots panics: {}", error)))? } async fn load_snapshot_chunk( &self, request: tonic::Request, ) -> Result, tonic::Status> { - handler::load_snapshot_chunk(self, request.into_inner()) - .map(tonic::Response::new) - .map_err(error_into_status) + // Chunk generation traverses the checkpoint's grovedb and encodes a replication + // chunk — synchronous, potentially large, and peer-controlled. Run it on the + // blocking pool so concurrent snapshot consumers cannot occupy the async workers + // and delay unrelated gRPC traffic (same pattern as check_tx). + let platform = Arc::clone(&self.platform); + let snapshot_manager = Arc::clone(&self.snapshot_manager); + let proto_request = request.into_inner(); + + spawn_blocking_task_with_name_if_supported("load_snapshot_chunk", move || { + handler::load_snapshot_chunk(platform.as_ref(), &snapshot_manager, proto_request) + .map(tonic::Response::new) + .map_err(error_into_status) + })? + .await + .map_err(|error| { + tonic::Status::internal(format!("load snapshot chunk panics: {}", error)) + })? } } @@ -180,7 +202,11 @@ mod tests { let core_rpc = MockCoreRPCLike::new(); - let app = CheckTxAbciApplication::new(Arc::new(platform.platform), Arc::new(core_rpc)); + let app = CheckTxAbciApplication::new( + Arc::new(platform.platform), + Arc::new(core_rpc), + Arc::new(SnapshotManager::new()), + ); let debug_str = format!("{:?}", app); assert_eq!(debug_str, ""); @@ -193,7 +219,11 @@ mod tests { let core_rpc = MockCoreRPCLike::new(); - let app = CheckTxAbciApplication::new(Arc::new(platform.platform), Arc::new(core_rpc)); + let app = CheckTxAbciApplication::new( + Arc::new(platform.platform), + Arc::new(core_rpc), + Arc::new(SnapshotManager::new()), + ); // Just verify we can call platform() without panicking let _platform_ref = app.platform(); diff --git a/packages/rs-drive-abci/src/abci/app/full.rs b/packages/rs-drive-abci/src/abci/app/full.rs index c49ec51443b..01d225a9751 100644 --- a/packages/rs-drive-abci/src/abci/app/full.rs +++ b/packages/rs-drive-abci/src/abci/app/full.rs @@ -1,6 +1,5 @@ use crate::abci::app::{ - BlockExecutionApplication, PlatformApplication, SnapshotManagerApplication, - StateSyncApplication, TransactionalApplication, + BlockExecutionApplication, PlatformApplication, StateSyncApplication, TransactionalApplication, }; use crate::abci::handler; use crate::abci::handler::error::error_into_exception; @@ -52,12 +51,6 @@ impl PlatformApplication for FullAbciApplication<'_, C> { } } -impl SnapshotManagerApplication for FullAbciApplication<'_, C> { - fn snapshot_manager(&self) -> &SnapshotManager { - &self.snapshot_manager - } -} - impl<'a, C> StateSyncApplication<'a, C> for FullAbciApplication<'a, C> { fn snapshot_fetching_session(&self) -> &RwLock>> { &self.snapshot_fetching_session @@ -279,14 +272,15 @@ where &self, request: proto::RequestListSnapshots, ) -> Result { - handler::list_snapshots(self, request).map_err(error_into_exception) + handler::list_snapshots(self.platform, request).map_err(error_into_exception) } fn load_snapshot_chunk( &self, request: proto::RequestLoadSnapshotChunk, ) -> Result { - handler::load_snapshot_chunk(self, request).map_err(error_into_exception) + handler::load_snapshot_chunk(self.platform, &self.snapshot_manager, request) + .map_err(error_into_exception) } fn offer_snapshot( diff --git a/packages/rs-drive-abci/src/abci/app/mod.rs b/packages/rs-drive-abci/src/abci/app/mod.rs index fc575f4066f..209608df0d8 100644 --- a/packages/rs-drive-abci/src/abci/app/mod.rs +++ b/packages/rs-drive-abci/src/abci/app/mod.rs @@ -10,7 +10,7 @@ pub mod execution_result; mod full; use crate::execution::types::block_execution_context::BlockExecutionContext; -use crate::platform_types::snapshot::{SnapshotFetchingSession, SnapshotManager}; +use crate::platform_types::snapshot::SnapshotFetchingSession; use crate::rpc::core::DefaultCoreRPC; #[cfg(test)] pub(crate) use check_tx::error_into_status; @@ -25,13 +25,6 @@ pub trait PlatformApplication { fn platform(&self) -> &Platform; } -/// ABCI application that serves state sync snapshots -pub trait SnapshotManagerApplication { - /// Returns the snapshot manager, which pins checkpoints that are actively being - /// served so pruning cannot delete them mid-transfer - fn snapshot_manager(&self) -> &SnapshotManager; -} - /// ABCI application that can bootstrap its state via state sync pub trait StateSyncApplication<'p, C = DefaultCoreRPC> { /// Returns the state sync transfer currently in progress, if any diff --git a/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs b/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs index d3799ea4c57..2eab450dafe 100644 --- a/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs +++ b/packages/rs-drive-abci/src/abci/handler/list_snapshots.rs @@ -1,8 +1,7 @@ -use crate::abci::app::PlatformApplication; use crate::abci::AbciError; use crate::error::Error; +use crate::platform_types::platform::Platform; use crate::platform_types::snapshot::encode_snapshot_metadata; -use crate::rpc::core::CoreRPCLike; use tenderdash_abci::proto::abci as proto; /// Lists the state sync snapshots this node can serve. @@ -11,21 +10,20 @@ use tenderdash_abci::proto::abci as proto; /// Only checkpoints that contain the reduced platform state are offered: a checkpoint /// taken before the protocol version that introduced it (v15) cannot be restored, since /// a state-synced node would have no way to reconstruct its platform state. -pub fn list_snapshots( - app: &A, +/// +/// Takes the platform directly rather than an application trait so the gRPC serving +/// application can run it on the blocking pool from an owned `Arc`. +pub fn list_snapshots( + platform: &Platform, _request: proto::RequestListSnapshots, -) -> Result -where - A: PlatformApplication, - C: CoreRPCLike, -{ +) -> Result { tracing::trace!("[state_sync] api list_snapshots called"); - if !app.platform().config.abci.state_sync.snapshots_enabled { + if !platform.config.abci.state_sync.snapshots_enabled { return Ok(Default::default()); } - let checkpoints = app.platform().drive.checkpoints.load(); + let checkpoints = platform.drive.checkpoints.load(); let mut snapshots = Vec::new(); for (height, checkpoint_info) in checkpoints.iter() { @@ -96,7 +94,6 @@ where #[cfg(test)] mod tests { use super::*; - use crate::abci::app::FullAbciApplication; use crate::config::PlatformConfig; use crate::test::helpers::fast_forward_to_block::fast_forward_to_block; use crate::test::helpers::setup::TestPlatformBuilder; @@ -113,9 +110,9 @@ mod tests { let platform = TestPlatformBuilder::new() .build_with_mock_rpc() .set_genesis_state(); - let app = FullAbciApplication::new(&platform); - let response = list_snapshots(&app, Default::default()).expect("should list snapshots"); + let response = + list_snapshots(&platform, Default::default()).expect("should list snapshots"); assert!(response.snapshots.is_empty()); } @@ -126,7 +123,6 @@ mod tests { .build_with_mock_rpc() .set_genesis_state(); let platform_version = PlatformVersion::latest(); - let app = FullAbciApplication::new(&platform); // A checkpoint taken before the reduced platform state exists (pre-v15 // activation) is unrestorable and must not be offered. @@ -135,7 +131,8 @@ mod tests { .create_grovedb_checkpoint(platform_version) .expect("should create checkpoint"); - let response = list_snapshots(&app, Default::default()).expect("should list snapshots"); + let response = + list_snapshots(&platform, Default::default()).expect("should list snapshots"); assert!( response.snapshots.is_empty(), "checkpoints without the reduced platform state must be filtered out" @@ -161,7 +158,8 @@ mod tests { .create_grovedb_checkpoint(platform_version) .expect("should create checkpoint"); - let response = list_snapshots(&app, Default::default()).expect("should list snapshots"); + let response = + list_snapshots(&platform, Default::default()).expect("should list snapshots"); assert_eq!(response.snapshots.len(), 1); let snapshot = &response.snapshots[0]; assert_eq!(snapshot.height, 20); diff --git a/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs b/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs index a634ec4ab86..688e70ff39f 100644 --- a/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs +++ b/packages/rs-drive-abci/src/abci/handler/load_snapshot_chunk.rs @@ -1,10 +1,10 @@ -use crate::abci::app::{PlatformApplication, SnapshotManagerApplication}; use crate::abci::AbciError; use crate::error::Error; +use crate::platform_types::platform::Platform; use crate::platform_types::snapshot::{ - max_serving_pins, MAX_STATE_SYNC_CHUNK_ID_SIZE, SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS, + max_serving_pins, SnapshotManager, MAX_STATE_SYNC_CHUNK_ID_SIZE, + SUPPORTED_STATE_SYNC_PROTOCOL_VERSIONS, }; -use crate::rpc::core::CoreRPCLike; use std::sync::Arc; use tenderdash_abci::proto::abci as proto; @@ -12,14 +12,14 @@ use tenderdash_abci::proto::abci as proto; /// /// The served checkpoint is pinned in the snapshot manager so checkpoint pruning cannot /// delete it from disk while a peer is still downloading it. -pub fn load_snapshot_chunk( - app: &A, +/// +/// Takes the platform and snapshot manager directly rather than an application trait so +/// the gRPC serving application can run it on the blocking pool from owned `Arc`s. +pub fn load_snapshot_chunk( + platform: &Platform, + snapshot_manager: &SnapshotManager, request: proto::RequestLoadSnapshotChunk, -) -> Result -where - A: PlatformApplication + SnapshotManagerApplication, - C: CoreRPCLike, -{ +) -> Result { tracing::trace!( height = request.height, version = request.version, @@ -27,7 +27,7 @@ where "[state_sync] api load_snapshot_chunk", ); - if !app.platform().config.abci.state_sync.snapshots_enabled { + if !platform.config.abci.state_sync.snapshots_enabled { return Err(AbciError::StateSyncBadRequest( "load_snapshot_chunk snapshot serving is disabled".to_string(), ) @@ -57,14 +57,13 @@ where // Resolve the checkpoint: from the registry, or — if pruning already dropped it — // from the pins of transfers already in flight. - let checkpoint = app - .platform() + let checkpoint = platform .drive .checkpoints .load() .get(&request.height) .map(|checkpoint_info| Arc::clone(&checkpoint_info.checkpoint)) - .or_else(|| app.snapshot_manager().pinned_checkpoint(request.height)) + .or_else(|| snapshot_manager.pinned_checkpoint(request.height)) .ok_or_else(|| { AbciError::StateSyncBadRequest(format!( "load_snapshot_chunk no snapshot at height {}", @@ -105,10 +104,10 @@ where // Pin (or refresh the pin of) the checkpoint only once a chunk was actually served. // Pinning before the fetch would let a peer keep a checkpoint — and its directory — // alive with a stream of requests that never succeed. - app.snapshot_manager().pin_for_serving( + snapshot_manager.pin_for_serving( request.height, checkpoint, - max_serving_pins(app.platform().config.abci.state_sync.max_num_snapshots), + max_serving_pins(platform.config.abci.state_sync.max_num_snapshots), ); Ok(proto::ResponseLoadSnapshotChunk { chunk }) @@ -117,7 +116,6 @@ where #[cfg(test)] mod tests { use super::*; - use crate::abci::app::FullAbciApplication; use crate::config::PlatformConfig; use crate::test::helpers::fast_forward_to_block::fast_forward_to_block; use crate::test::helpers::setup::TestPlatformBuilder; @@ -132,7 +130,7 @@ mod tests { .build_with_mock_rpc() .set_genesis_state(); let platform_version = PlatformVersion::latest(); - let app = FullAbciApplication::new(&platform); + let snapshot_manager = SnapshotManager::new(); let reduced_platform_state = platform.state.load().to_reduced_platform_state(None, 42); platform @@ -159,7 +157,8 @@ mod tests { // The root chunk (chunk id == app hash) must be served let response = load_snapshot_chunk( - &app, + &platform, + &snapshot_manager, proto::RequestLoadSnapshotChunk { height: 10, version: 1, @@ -170,11 +169,12 @@ mod tests { assert!(!response.chunk.is_empty()); // The served checkpoint must now be pinned against pruning - assert!(app.snapshot_manager.pinned_checkpoint(10).is_some()); + assert!(snapshot_manager.pinned_checkpoint(10).is_some()); // Unknown height is rejected assert!(load_snapshot_chunk( - &app, + &platform, + &snapshot_manager, proto::RequestLoadSnapshotChunk { height: 999, version: 1, @@ -185,7 +185,8 @@ mod tests { // Unsupported wire version is rejected assert!(load_snapshot_chunk( - &app, + &platform, + &snapshot_manager, proto::RequestLoadSnapshotChunk { height: 10, version: 2, @@ -196,7 +197,8 @@ mod tests { // Oversized chunk id is rejected before any decoding assert!(load_snapshot_chunk( - &app, + &platform, + &snapshot_manager, proto::RequestLoadSnapshotChunk { height: 10, version: 1, diff --git a/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs index 9dd61ba12fa..43b2d7b8862 100644 --- a/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs +++ b/packages/rs-drive-abci/src/platform_types/snapshot/mod.rs @@ -225,6 +225,14 @@ const SERVING_PIN_INACTIVITY_TTL: Duration = Duration::from_secs(600); /// can outlive its checkpoint. const SERVING_PIN_MAX_LIFETIME: Duration = Duration::from_secs(6 * 3600); +/// How often the autonomous sweep task releases expired serving pins. +/// +/// Expiry must not depend on peers making further requests (an abandoned transfer makes +/// none) nor on this process seeing blocks (the gRPC serving application does not). +/// The interval only bounds how long an expired pin lingers past its deadline, so it is +/// uncritical; once a minute is nothing next to the TTLs it enforces. +pub const SERVING_PIN_SWEEP_INTERVAL: Duration = Duration::from_secs(60); + /// How many pins are allowed on top of the number of snapshots the node retains. /// /// The interesting pins are the ones for checkpoints pruning has ALREADY dropped from the @@ -274,8 +282,10 @@ impl ServingPin { /// [`SERVING_PIN_MAX_LIFETIME`] since it was taken (not refreshable), and /// [`max_serving_pins`] in total. Expiry also must not depend on peers making further /// requests, or an abandoned transfer would hold its directory forever: -/// [`SnapshotManager::release_expired_pins`] runs once per block and every read of a pin -/// re-checks both deadlines. +/// [`SnapshotManager::release_expired_pins`] runs autonomously (every +/// [`SERVING_PIN_SWEEP_INTERVAL`] from the sweep task `server::start` spawns next to the +/// gRPC serving application, and once per block in the all-in-one test application) and +/// every read of a pin re-checks both deadlines. #[derive(Default)] pub struct SnapshotManager { /// Height -> the pin held for a transfer of that snapshot diff --git a/packages/rs-drive-abci/src/server.rs b/packages/rs-drive-abci/src/server.rs index 3baf33f5c2a..bf8d1b8e3e1 100644 --- a/packages/rs-drive-abci/src/server.rs +++ b/packages/rs-drive-abci/src/server.rs @@ -5,6 +5,7 @@ use crate::abci::app::CheckTxAbciApplication; use crate::abci::app::ConsensusAbciApplication; use crate::config::PlatformConfig; use crate::platform_types::platform::Platform; +use crate::platform_types::snapshot::{SnapshotManager, SERVING_PIN_SWEEP_INTERVAL}; use crate::query::QueryService; use crate::rpc::core::DefaultCoreRPC; use std::sync::Arc; @@ -31,8 +32,30 @@ pub fn start( ) .expect("failed to open check tx core rpc"); - let check_tx_service = - CheckTxAbciApplication::new(Arc::clone(&platform), Arc::new(check_tx_core_rpc)); + // The snapshot manager pins the checkpoints being served to state-syncing peers. + // It lives with the gRPC application below (which answers ListSnapshots and + // LoadSnapshotChunk), but that application never sees blocks, so nothing on its + // request path would ever release the pin of an ABANDONED transfer — the peer + // simply stops asking. A shared handle and a timer task make expiry autonomous. + let snapshot_manager = Arc::new(SnapshotManager::new()); + + let serving_pin_sweep_cancel = cancel.clone(); + let serving_pin_sweep_manager = Arc::clone(&snapshot_manager); + runtime.spawn(async move { + let mut interval = tokio::time::interval(SERVING_PIN_SWEEP_INTERVAL); + loop { + tokio::select! { + _ = serving_pin_sweep_cancel.cancelled() => break, + _ = interval.tick() => serving_pin_sweep_manager.release_expired_pins(), + } + } + }); + + let check_tx_service = CheckTxAbciApplication::new( + Arc::clone(&platform), + Arc::new(check_tx_core_rpc), + snapshot_manager, + ); let grpc_server = dapi_grpc::tonic::transport::Server::builder() .add_service( From 5c1dd87939519828ea58e632b278b4bb072f7fbc Mon Sep 17 00:00:00 2001 From: pasta Date: Mon, 7 Sep 2026 23:06:29 -0500 Subject: [PATCH 27/34] fix(drive-abci): answer a bad chunk with RETRY_SNAPSHOT and drop the sum-tree tripwire grovedb invalidates its restore session on a failed chunk, so the target asks Tenderdash to restart the snapshot instead of refetching one chunk. The sum-tree probe is removed; the full two-instance round trip stays ignored until the grovedb pin carries dashpay/grovedb#840, which arrives with the GroveDB 6.0.0 bump in #4635. --- .../src/abci/handler/apply_snapshot_chunk.rs | 85 ++++++------ .../test_cases/state_sync_tests.rs | 108 +++------------ .../tests/sum_tree_sync_probe.rs | 125 ------------------ 3 files changed, 59 insertions(+), 259 deletions(-) delete mode 100644 packages/rs-drive-abci/tests/sum_tree_sync_probe.rs diff --git a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs index 8456736fe60..07ddcff8a48 100644 --- a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs +++ b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs @@ -13,11 +13,12 @@ use tenderdash_abci::proto::abci::response_apply_snapshot_chunk; /// Applies one chunk of a state sync snapshot to the grovedb sync session. /// -/// A chunk grovedb rejects does not kill the whole transfer: Tenderdash is asked to -/// refetch that chunk (from a different peer, if it identified the sender). When the -/// last chunk lands, the session is committed, grovedb is verified against the target -/// app hash, and the platform state is reconstructed from the reduced platform state -/// contained in the restored snapshot. +/// A chunk grovedb rejects does not kill the whole transfer: the sender is banned and +/// Tenderdash is asked to restart the snapshot (grovedb invalidates the session on a +/// failed chunk, so the restore starts over from a fresh session on the re-offer). When +/// the last chunk lands, the session is committed, grovedb is verified against the +/// target app hash, and the platform state is reconstructed from the reduced platform +/// state contained in the restored snapshot. pub fn apply_snapshot_chunk<'a, 'db: 'a, A, C>( app: &'a A, request: proto::RequestApplySnapshotChunk, @@ -112,42 +113,23 @@ where ) { Ok(next_chunk_ids) => next_chunk_ids, Err(e) => { - // grovedb removes a chunk id from its pending set before processing it, - // so a chunk it has already seen (e.g. the refetch of one it rejected) - // cannot be re-applied within this session: ask Tenderdash to restart - // the snapshot instead (a same-height re-offer, which we accept). - // The string match is brittle by necessity (grovedb only exposes - // InternalError(String) here); if the wording ever changes, the fallback - // below is still safe — Tenderdash retries the chunk until it gives up - // and restarts the snapshot itself. - if matches!(&e, drive::grovedb::Error::InternalError(message) if message.contains("not expected")) - { - tracing::warn!( - chunk_id = hex::encode(&request.chunk_id), - sender = request.sender, - error = ?e, - "[state_sync] apply_snapshot_chunk cannot re-apply a chunk in this session, requesting snapshot restart", - ); - return Ok(proto::ResponseApplySnapshotChunk { - result: response_apply_snapshot_chunk::Result::RetrySnapshot.into(), - refetch_chunks: vec![], - reject_senders, - next_chunks: vec![], - }); - } - - // A chunk grovedb cannot apply (corrupted or tampered data) is - // recoverable: keep the session and ask Tenderdash to refetch the chunk, - // banning the peer that sent it so the refetch goes elsewhere. + // A chunk grovedb cannot apply (corrupted or tampered data) permanently + // invalidates the grovedb session: every later `apply_chunk` and the + // final commit refuse, and grovedb does not expose whether a given error + // poisoned the session or was caught before any write. The transfer is + // still recoverable — ban the sender and ask Tenderdash to restart the + // snapshot (a re-offer, which `offer_snapshot` answers by wiping and + // opening a fresh session) rather than refetch a chunk this session can + // no longer accept. tracing::warn!( chunk_id = hex::encode(&request.chunk_id), sender = request.sender, error = ?e, - "[state_sync] apply_snapshot_chunk rejected a chunk, requesting refetch", + "[state_sync] apply_snapshot_chunk rejected a chunk, requesting snapshot restart", ); return Ok(proto::ResponseApplySnapshotChunk { - result: response_apply_snapshot_chunk::Result::Retry.into(), - refetch_chunks: vec![request.chunk_id], + result: response_apply_snapshot_chunk::Result::RetrySnapshot.into(), + refetch_chunks: vec![], reject_senders, next_chunks: vec![], }); @@ -370,6 +352,7 @@ mod tests { use crate::platform_types::snapshot::encode_snapshot_metadata; use crate::test::helpers::setup::TestPlatformBuilder; use dpp::version::v15::PROTOCOL_VERSION_15; + use tenderdash_abci::proto::abci::response_offer_snapshot; #[test] fn apply_snapshot_chunk_without_session_is_rejected() { @@ -458,7 +441,7 @@ mod tests { } #[test] - fn apply_snapshot_chunk_asks_for_refetch_of_a_bad_chunk() { + fn apply_snapshot_chunk_asks_for_a_snapshot_restart_on_a_bad_chunk() { let platform = TestPlatformBuilder::new() .build_with_mock_rpc() .set_genesis_state(); @@ -466,8 +449,9 @@ mod tests { let target_app_hash = offer_a_snapshot(&app); - // Garbage bytes for the root chunk: grovedb rejects them, and the session must - // survive with a Retry + refetch of exactly that chunk, banning the sender. + // Garbage bytes for the root chunk: grovedb rejects them and invalidates its + // session, so the answer is a snapshot restart that bans the sender — not an + // ABCI exception, and not a refetch this session could no longer apply. let response = apply_snapshot_chunk( &app, proto::RequestApplySnapshotChunk { @@ -480,13 +464,32 @@ mod tests { assert_eq!( response.result, - i32::from(response_apply_snapshot_chunk::Result::Retry) + i32::from(response_apply_snapshot_chunk::Result::RetrySnapshot) ); - assert_eq!(response.refetch_chunks, vec![target_app_hash]); + assert!(response.refetch_chunks.is_empty()); assert_eq!(response.reject_senders, vec!["peer-1".to_string()]); assert!( app.snapshot_fetching_session.read().unwrap().is_some(), - "the session must survive a bad chunk" + "the session stays in place until the re-offer replaces it" + ); + + // The re-offer Tenderdash answers with must be accepted and start over + let response = offer_snapshot( + &app, + proto::RequestOfferSnapshot { + snapshot: Some(proto::Snapshot { + height: 100, + version: 1, + hash: target_app_hash.clone(), + metadata: encode_snapshot_metadata(PROTOCOL_VERSION_15), + }), + app_hash: target_app_hash, + }, + ) + .expect("re-offer after a bad chunk must not error"); + assert_eq!( + response.result, + i32::from(response_offer_snapshot::Result::Accept) ); } } diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs index 0bbeadf04af..aad81a2c5e6 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs @@ -1,15 +1,6 @@ //! Two-instance ABCI state sync integration tests: a source chain serves snapshots from //! its checkpoint registry and a fresh target restores one chunk by chunk, then //! reconstructs its platform state. -//! -//! KNOWN LIMITATION at the pinned grovedb revision (6c882c3): state sync does not -//! faithfully restore SumTree subtrees — the copied node hashes reproduce the source -//! root hash, but re-opening a restored sum tree recomputes a different root (latent -//! corruption), which the strict `verify_grovedb` call in `apply_snapshot_chunk` -//! correctly refuses. See `tests/sum_tree_sync_probe.rs` for the minimal upstream -//! reproducer. The full happy-path test below is therefore `#[ignore]`d until the -//! grovedb pin includes the sum-tree restore fix (dashpay/grovedb#840), and an active -//! test pins today's refusal behavior instead. #[cfg(test)] pub(crate) mod tests { @@ -173,12 +164,10 @@ pub(crate) mod tests { /// hash) and keep requesting whatever the target asks for next. /// /// When `tamper_with_first_chunk` is set, the first served chunk is corrupted to - /// prove the target answers RETRY with a refetch of exactly that chunk (banning - /// the sender) instead of killing the session. At the current grovedb revision the - /// refetched chunk cannot be re-applied within the session (grovedb removes a - /// chunk id from its pending set before processing), so the target then answers - /// RETRY_SNAPSHOT; the driver handles that the way Tenderdash would, by - /// re-offering the same snapshot and restarting the transfer. + /// prove the target answers RETRY_SNAPSHOT (banning the sender) instead of killing + /// the session: grovedb invalidates its session on a failed chunk, so the driver + /// handles that the way Tenderdash would, by re-offering the same snapshot and + /// restarting the transfer. /// How a snapshot transfer ended. /// /// `Rejected` is not an error: the target restored the snapshot, found it unusable, @@ -237,14 +226,10 @@ pub(crate) mod tests { })?; assert_eq!( response.result, - i32::from(response_apply_snapshot_chunk::Result::Retry), - "a tampered chunk must be answered with a retry, not kill the session" - ); - assert_eq!( - response.refetch_chunks, - vec![chunk_id.clone()], - "the tampered chunk must be refetched" + i32::from(response_apply_snapshot_chunk::Result::RetrySnapshot), + "a tampered chunk must be answered with a snapshot restart, not an error" ); + assert!(response.refetch_chunks.is_empty()); assert_eq!(response.reject_senders, vec!["malicious-peer".to_string()]); assert!( target_app @@ -252,8 +237,10 @@ pub(crate) mod tests { .read() .unwrap() .is_some(), - "the session must survive a tampered chunk" + "the session stays in place until the re-offer replaces it" ); + restarts += 1; + continue 'snapshot_attempt; } let response = @@ -373,9 +360,8 @@ pub(crate) mod tests { /// along the way to prove refetch/restart recovery), reconstruct the target /// platform state, and verify the target matches the source checkpoint exactly. #[tokio::test] - #[ignore = "the pinned grovedb (6c882c3) cannot faithfully restore sum trees; un-ignore \ - when the pin includes the sum-tree restore fix (dashpay/grovedb#840) — see \ - tests/sum_tree_sync_probe.rs and state_sync_transfer_detects_sum_tree_restore_defect"] + #[ignore = "needs the grovedb sum-tree restore fix (dashpay/grovedb#840), which reaches \ + this workspace with the GroveDB 6.0.0 bump in #4635; un-ignore at that re-pin"] async fn run_state_sync_between_two_platforms() { let config = state_sync_platform_config(); let mut source_platform = TestPlatformBuilder::new() @@ -513,70 +499,8 @@ pub(crate) mod tests { assert_eq!(info.last_block_app_hash, snapshot.hash); } - /// Pins today's behavior at the pinned grovedb revision: the transfer itself - /// completes (including recovery from a tampered chunk via RETRY and a snapshot - /// restart), but the strict post-restore verification detects that grovedb did not - /// faithfully restore the sum trees and refuses the snapshot instead of accepting - /// latent corruption. When this test starts failing because the sync SUCCEEDS, - /// grovedb has been fixed: un-ignore `run_state_sync_between_two_platforms` and - /// drop this pin. - #[tokio::test] - async fn state_sync_transfer_detects_sum_tree_restore_defect() { - let config = state_sync_platform_config(); - let mut source_platform = TestPlatformBuilder::new() - .with_config(config.clone()) - .build_with_mock_rpc(); - let source = run_source_chain(&mut source_platform, &config).await; - - let mut target_platform = TestPlatformBuilder::new() - .with_config(config.clone()) - .build_with_mock_rpc(); - install_reconstruction_core_mocks( - &mut target_platform.platform, - source.proposers.clone(), - &source.validator_quorums, - ); - let target_app = FullAbciApplication::new(&target_platform); - - let outcome = sync_snapshot(&source.source_app, &target_app, &source.snapshot, true) - .expect("a refused snapshot is answered, not errored"); - assert_eq!( - outcome, - SnapshotSyncOutcome::Rejected, - "at grovedb rev 6c882c3 the restored sum trees must fail verification — if this \ - now completes, grovedb is fixed: un-ignore run_state_sync_between_two_platforms \ - and remove this pin" - ); - - // The target refused the snapshot: it never advanced past genesis, and — since the - // refusal happens after the session was already committed — it wiped itself back to - // a clean slate rather than keeping the unusable state. - assert_eq!( - target_platform.state.load().last_committed_block_height(), - 0 - ); - assert_eq!( - target_platform - .committed_block_height_guard - .load(std::sync::atomic::Ordering::Relaxed), - 0, - "a rejected restore must not open the query height gate" - ); - assert_ne!( - target_platform - .drive - .grove - .root_hash(None, &PlatformVersion::latest().drive.grove_version) - .unwrap() - .expect("target root hash") - .to_vec(), - source.snapshot.hash, - "a refused snapshot must not be left on disk" - ); - } - /// Exercises the platform state reconstruction end to end without going through - /// the (currently defective, see above) grovedb chunk restore: the source chain's + /// the grovedb chunk restore: the source chain's /// own grovedb IS a faithfully "restored" snapshot of itself, so reconstructing /// on it must (a) not change the grovedb root hash — the proof that re-deriving /// masternode identities from Core is byte-idempotent — and (b) reproduce the @@ -826,10 +750,8 @@ pub(crate) mod tests { }; // Even if a peer maliciously offers such a snapshot — lying in the metadata that - // it is restorable — the target must refuse to restore it. (At the current grovedb - // revision the refusal comes from the post-restore verification; once grovedb - // faithfully restores sum trees it comes from the missing reduced platform state - // at the reconstruction step. Either way the snapshot must not be accepted.) + // it is restorable — the target must refuse to restore it: the chunk transfer + // completes, but the reconstruction step finds no reduced platform state. let (height, checkpoint) = { let checkpoints = source_app.platform.drive.checkpoints.load(); let (height, info) = checkpoints diff --git a/packages/rs-drive-abci/tests/sum_tree_sync_probe.rs b/packages/rs-drive-abci/tests/sum_tree_sync_probe.rs deleted file mode 100644 index 3a747a69d7c..00000000000 --- a/packages/rs-drive-abci/tests/sum_tree_sync_probe.rs +++ /dev/null @@ -1,125 +0,0 @@ -//! Minimal reproducer / tripwire for a grovedb state sync limitation at the pinned -//! revision (6c882c3): the replication protocol at this revision does not faithfully -//! restore SumTree subtrees. The chunk transfer copies the source's node hashes, so the restored -//! database reproduces the source ROOT hash — but re-opening the restored sum tree -//! and recomputing its root yields a different hash, i.e. the corruption is latent -//! and `verify_grovedb` detects it. -//! -//! This is why `apply_snapshot_chunk` runs the strict `verify_grovedb` check after -//! committing a state sync session, and why the full two-instance state sync -//! integration test (`run_state_sync_between_two_platforms`) is `#[ignore]`d. -//! -//! WHEN THIS TEST STARTS FAILING because no verification issues are reported, the -//! grovedb pin has been fixed: delete this tripwire and un-ignore the full -//! integration test. - -use drive::grovedb::{Element, GroveDb}; -use drive::grovedb_path::SubtreePath; -use platform_version::version::PlatformVersion; -use std::collections::VecDeque; - -#[test] -fn sum_tree_state_sync_restore_is_latently_corrupt_at_pinned_grovedb() { - let grove_version = &PlatformVersion::latest().drive.grove_version; - let source_dir = tempfile::tempdir().unwrap(); - let source = GroveDb::open(source_dir.path()).unwrap(); - - let root: SubtreePath<[u8; 0]> = SubtreePath::empty(); - - source - .insert( - root.clone(), - b"s", - Element::empty_sum_tree(), - None, - None, - grove_version, - ) - .unwrap() - .unwrap(); - let sum_path: &[&[u8]] = &[b"s"]; - for (key, value) in [(b"a", 5i64), (b"b", 7i64)] { - source - .insert( - sum_path, - key, - Element::new_sum_item(value), - None, - None, - grove_version, - ) - .unwrap() - .unwrap(); - } - // A normal tree with an item, for contrast: it restores cleanly. - source - .insert( - root.clone(), - b"n", - Element::empty_tree(), - None, - None, - grove_version, - ) - .unwrap() - .unwrap(); - let normal_path: &[&[u8]] = &[b"n"]; - source - .insert( - normal_path, - b"k", - Element::new_item(b"v".to_vec()), - None, - None, - grove_version, - ) - .unwrap() - .unwrap(); - - let app_hash = source.root_hash(None, grove_version).unwrap().unwrap(); - - let target_dir = tempfile::tempdir().unwrap(); - let target = GroveDb::open(target_dir.path()).unwrap(); - let mut session = target - .start_snapshot_syncing(app_hash, 64, 1, grove_version) - .unwrap(); - - let mut queue: VecDeque> = VecDeque::from([app_hash.to_vec()]); - while let Some(chunk_id) = queue.pop_front() { - let chunk = source - .fetch_chunk(&chunk_id, None, 1, grove_version) - .unwrap(); - let next = session - .apply_chunk(&chunk_id, &chunk, 1, grove_version) - .unwrap(); - queue.extend(next); - if session.is_sync_completed() { - break; - } - } - assert!(session.is_sync_completed()); - target.commit_session(session, grove_version).unwrap(); - - // The copied node hashes reproduce the source root hash exactly... - let target_root = target.root_hash(None, grove_version).unwrap().unwrap(); - assert_eq!( - target_root, app_hash, - "restored root hash must match the source" - ); - - // ...but recomputing the restored sum tree exposes the latent corruption. - let issues = target - .verify_grovedb(None, true, false, grove_version) - .unwrap(); - let paths: Vec = issues - .keys() - .map(|path| path.iter().map(hex::encode).collect::>().join("/")) - .collect(); - assert_eq!( - paths, - vec!["73".to_string()], // hex of b"s", the sum tree - "expected exactly the sum tree to fail verification — if no issues are \ - reported, grovedb has been fixed: delete this tripwire and un-ignore \ - run_state_sync_between_two_platforms" - ); -} From 3f6ddb2f004220dad0055e91b51505abd611c564 Mon Sep 17 00:00:00 2001 From: pasta Date: Mon, 7 Sep 2026 23:46:13 -0500 Subject: [PATCH 28/34] fix(drive-abci): restore update_core_info_v0 and update_masternode_list_v0 to their v4.2-dev bodies The PR changed both v0 implementations in place so that is_init_chain also bypassed their same-core-height short circuits for state sync reconstruction. That edit was a no-op: reconstruct_platform_state builds its state with last_committed_block_info = None, so last_committed_core_height() is 0 and neither short circuit can fire for any real snapshot; is_init_chain = true already selects the from-scratch build in update_state_masternode_list_v0. Both files go back to their exact v4.2-dev bodies and no method version is bumped. Co-Authored-By: Claude Fable 5.1 --- .../update_core_info/v0/mod.rs | 9 +------ .../update_masternode_list/v0/mod.rs | 24 ++++++++----------- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/v0/mod.rs index e4075cb173e..91564d2db1c 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/v0/mod.rs @@ -59,18 +59,11 @@ where platform_version, )?; - // `is_init_chain` doubles as `start_from_scratch`: on init chain and on state - // sync reconstruction the quorums must be built even if the (freshly - // constructed) block state happens to already report the requested core height. - // The flag's only effect inside update_quorum_info is to skip that - // same-core-height short-circuit; on the normal block path (`is_init_chain = - // false`) behavior is unchanged. The previous hardcoded `false` only worked - // because those flows start from a state whose derived core height is 0. self.update_quorum_info( platform_state, block_platform_state, core_block_height, - is_init_chain, + false, platform_version, ) } diff --git a/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_masternode_list/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_masternode_list/v0/mod.rs index e7842b5fbb9..3bc37499fa7 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_masternode_list/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_masternode_list/v0/mod.rs @@ -41,20 +41,16 @@ where transaction: &Transaction, platform_version: &PlatformVersion, ) -> Result<(), Error> { - // On init chain and on state sync reconstruction the masternode list must be - // built from scratch even if the block state already reports this core height. - if !is_init_chain { - if let Some(last_committed_block_info) = - block_platform_state.last_committed_block_info().as_ref() - { - if core_block_height == last_committed_block_info.basic_info().core_height { - tracing::debug!( - method = "update_masternode_list_v0", - "no update mnl at height {}", - core_block_height, - ); - return Ok(()); // no need to do anything - } + if let Some(last_committed_block_info) = + block_platform_state.last_committed_block_info().as_ref() + { + if core_block_height == last_committed_block_info.basic_info().core_height { + tracing::debug!( + method = "update_masternode_list_v0", + "no update mnl at height {}", + core_block_height, + ); + return Ok(()); // no need to do anything } } tracing::debug!( From 809bb4f36bd46310c9c23a27a87ee4c26dccd7ed Mon Sep 17 00:00:00 2001 From: pasta Date: Mon, 7 Sep 2026 23:50:31 -0500 Subject: [PATCH 29/34] test(drive-abci): give block fixtures a non-zero signature and verify contestant vote proofs in their query direction The proof-metadata guard treats an all-zero block signature as a state restored via state sync and refuses proofs until the next block finalizes. The hand-built ExtendedBlockInfo fixtures in fast_forward_to_block, the masternode vote tests and the document query v1 tests all used an all-zero signature and started failing on that guard; they now share a TEST_BLOCK_SIGNATURE placeholder. Also: get_proved_contestant_votes verified every proof with an ascending query even when the request was descending; the re-pinned grovedb enforces that a layer proof is encoded in its walk direction's family, so the verifier now uses the same order_ascending as the request. offer_snapshot drops a duplicated 'db bound clippy flagged. Stale doc comments that described the old grovedb pin and the pre-#840 refetch ladder are cleaned up, and sync_snapshot's doc comment is moved back onto the function. Co-Authored-By: Claude Fable 5.1 --- .../src/abci/handler/apply_snapshot_chunk.rs | 5 +- .../src/abci/handler/offer_snapshot.rs | 4 +- .../state_transitions/masternode_vote/mod.rs | 75 ++++++++++--------- .../src/query/document_query/v1/tests.rs | 3 +- .../src/test/helpers/fast_forward_to_block.rs | 8 +- .../test_cases/state_sync_sentinel_tests.rs | 11 +-- .../test_cases/state_sync_tests.rs | 18 ++--- 7 files changed, 64 insertions(+), 60 deletions(-) diff --git a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs index 07ddcff8a48..3496e2eae33 100644 --- a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs +++ b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs @@ -68,8 +68,9 @@ where // These are TRANSFER faults, not reasons to abort state sync: an application // error here would reach Tenderdash as an ABCI exception, killing the whole // restore and leaving the node on the wiped database the offer created (with the - // restore sentinel still set). Both caps are therefore answered with the same - // recoverable ladder the other malformed-chunk paths use. + // restore sentinel still set). Both caps are therefore answered with a + // recoverable response; they run before grovedb sees the chunk, so unlike a + // chunk grovedb rejects (below) they leave the session usable. if request.chunk.len() > MAX_STATE_SYNC_CHUNK_SIZE { // Oversized chunk DATA: the chunk id itself is still fine, so ban the sender // and have Tenderdash refetch exactly this chunk from someone else. diff --git a/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs b/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs index 7b9c2b8dc83..d012da3ff90 100644 --- a/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs +++ b/packages/rs-drive-abci/src/abci/handler/offer_snapshot.rs @@ -17,13 +17,13 @@ use tenderdash_abci::proto::abci::response_offer_snapshot; /// Accepting an offer wipes the local grovedb and opens a grovedb state sync session /// targeting the light-client-verified app hash. Any accepted-format offer replaces a /// session already in progress (also answered with Accept), whatever height it carries. -pub fn offer_snapshot<'a, 'db: 'a, A, C: 'db>( +pub fn offer_snapshot<'a, 'db: 'a, A, C>( app: &'a A, request: proto::RequestOfferSnapshot, ) -> Result where A: StateSyncApplication<'db, C> + 'db, - C: CoreRPCLike, + C: CoreRPCLike + 'db, { let request_app_hash: [u8; 32] = request.app_hash.try_into().map_err(|_| { AbciError::StateSyncBadRequest("offer_snapshot invalid app_hash length".to_string()) diff --git a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/masternode_vote/mod.rs b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/masternode_vote/mod.rs index 3707d37c101..619e3b5120b 100644 --- a/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/masternode_vote/mod.rs +++ b/packages/rs-drive-abci/src/execution/validation/state_transition/state_transitions/masternode_vote/mod.rs @@ -94,6 +94,7 @@ impl StateTransitionStateValidation for MasternodeVoteTransition { #[cfg(test)] mod tests { + use crate::test::helpers::fast_forward_to_block::TEST_BLOCK_SIGNATURE; use crate::test::helpers::setup::TestPlatformBuilder; use dpp::block::block_info::BlockInfo; use dpp::dash_to_credits; @@ -3065,7 +3066,7 @@ mod tests { offset: None, limit: None, start_at: None, - order_ascending: true, + order_ascending, }; let (_, voters) = resolved_contested_document_vote_poll_drive_query @@ -4214,7 +4215,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -4406,7 +4407,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -4561,7 +4562,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -4604,7 +4605,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -4823,7 +4824,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -4866,7 +4867,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -4987,7 +4988,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -5030,7 +5031,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -5148,7 +5149,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -5191,7 +5192,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -5325,7 +5326,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -5995,7 +5996,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -6218,7 +6219,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -6505,7 +6506,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -6951,7 +6952,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -7074,7 +7075,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -7434,7 +7435,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -7652,7 +7653,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -7865,7 +7866,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -8067,7 +8068,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -8285,7 +8286,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -8499,7 +8500,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -8694,7 +8695,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -8889,7 +8890,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -9193,7 +9194,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -9383,7 +9384,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -9448,7 +9449,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -9718,7 +9719,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -9777,7 +9778,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -9971,7 +9972,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -10289,7 +10290,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -10481,7 +10482,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -10558,7 +10559,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -10757,7 +10758,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -10984,7 +10985,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), @@ -11502,7 +11503,7 @@ mod tests { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), diff --git a/packages/rs-drive-abci/src/query/document_query/v1/tests.rs b/packages/rs-drive-abci/src/query/document_query/v1/tests.rs index e6aa26b0fce..7d7b75bc9c9 100644 --- a/packages/rs-drive-abci/src/query/document_query/v1/tests.rs +++ b/packages/rs-drive-abci/src/query/document_query/v1/tests.rs @@ -11,6 +11,7 @@ use super::*; use crate::query::tests::{setup_platform, store_data_contract, store_document}; +use crate::test::helpers::fast_forward_to_block::TEST_BLOCK_SIGNATURE; use dapi_grpc::platform::v0::get_documents_request::get_documents_request_v1::{ select as v1_select, Select as V1Select, Start as V1Start, }; @@ -4549,7 +4550,7 @@ mod time_range_proof_verification { quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), diff --git a/packages/rs-drive-abci/src/test/helpers/fast_forward_to_block.rs b/packages/rs-drive-abci/src/test/helpers/fast_forward_to_block.rs index e991b965860..06d2880df84 100644 --- a/packages/rs-drive-abci/src/test/helpers/fast_forward_to_block.rs +++ b/packages/rs-drive-abci/src/test/helpers/fast_forward_to_block.rs @@ -17,6 +17,12 @@ use drive::drive::credit_pools::operations::update_unpaid_epoch_index_operation; use platform_version::version::PlatformVersion; use std::sync::Arc; +/// Placeholder block signature for hand-built `ExtendedBlockInfo` test fixtures. +/// +/// Any non-zero value works: an all-zero signature marks a state restored via state +/// sync, from which proofs are refused until the next block finalizes. +pub(crate) const TEST_BLOCK_SIGNATURE: [u8; 96] = [1u8; 96]; + pub(crate) fn fast_forward_to_block( platform: &TempPlatform, time_ms: u64, @@ -51,7 +57,7 @@ pub(crate) fn fast_forward_to_block( quorum_hash: [0u8; 32], block_id_hash: [0u8; 32], proposer_pro_tx_hash: [0u8; 32], - signature: [0u8; 96], + signature: TEST_BLOCK_SIGNATURE, round: 0, } .into(), diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs index 0682e311204..888ad5ea606 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs @@ -14,14 +14,9 @@ //! the node is self-consistent again, plus a rejection path that wipes back to a clean //! slate instead of returning an error. //! -//! # These tests do not need the patched grovedb -//! -//! Everything here holds at BOTH grovedb pins. Nothing asserts that a restore SUCCEEDS — -//! the tests that do live in `state_sync_equivalence_tests` and need dashpay/grovedb#840, -//! because Dash Platform state always contains sum trees. What is asserted here is that a -//! restore which does not succeed leaves a recoverable node, and at the unpinned revision -//! the sum-tree defect simply supplies the failure for free: the transfer commits, the -//! post-restore verification fails, and the same rejection path runs. +//! Nothing here asserts that a restore SUCCEEDS — the tests that do live in +//! `state_sync_tests`. What is asserted is that a restore which does not succeed leaves +//! a recoverable node. #[cfg(test)] mod tests { diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs index aad81a2c5e6..9e2a1fc011f 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs @@ -159,15 +159,6 @@ pub(crate) mod tests { ); } - /// Drives the chunk transfer loop between a serving app and a restoring app, - /// modeled on grovedb's run_sync driver: start from the root chunk (id == app - /// hash) and keep requesting whatever the target asks for next. - /// - /// When `tamper_with_first_chunk` is set, the first served chunk is corrupted to - /// prove the target answers RETRY_SNAPSHOT (banning the sender) instead of killing - /// the session: grovedb invalidates its session on a failed chunk, so the driver - /// handles that the way Tenderdash would, by re-offering the same snapshot and - /// restarting the transfer. /// How a snapshot transfer ended. /// /// `Rejected` is not an error: the target restored the snapshot, found it unusable, @@ -181,6 +172,15 @@ pub(crate) mod tests { Rejected, } + /// Drives the chunk transfer loop between a serving app and a restoring app, + /// modeled on grovedb's run_sync driver: start from the root chunk (id == app + /// hash) and keep requesting whatever the target asks for next. + /// + /// When `tamper_with_first_chunk` is set, the first served chunk is corrupted to + /// prove the target answers RETRY_SNAPSHOT (banning the sender) instead of killing + /// the session: grovedb invalidates its session on a failed chunk, so the driver + /// handles that the way Tenderdash would, by re-offering the same snapshot and + /// restarting the transfer. pub(crate) fn sync_snapshot( source_app: &FullAbciApplication, target_app: &FullAbciApplication, From e90a96bd4bb496077c0a6150b3ea62398531cf6d Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 9 Sep 2026 14:58:23 -0500 Subject: [PATCH 30/34] fix(drive-abci)!: keep the consensus round out of the reduced platform state run_block_proposal v1 wrote block_proposal.round into the reduced platform state, which sits in the Misc tree under the app hash. Tenderdash re-proposes a block that reached a prevote majority but did not commit with the same header at later rounds, and re-runs ProcessProposal for every round while requiring the returned app hash to equal the header's. With the round hashed in, every validator rejects the re-proposal and the chain halts at that height. The reduced block info now carries only header-fixed fields; the Option app hash, block id hash and signature (only ever Some in transition_to_version_15, whose write v1 overwrote in the same block) go with it, as does that transition. Reconstruction zero-fills them until the next finalized block, which the proof metadata guard already handles. ReducedPlatformState also moves to the platform serialization derive so it encodes big-endian like every other versioned platform type; a test pins the encoding and another pins that the app hash is independent of the round. Co-Authored-By: Claude Fable 5.1 --- .../rs-dpp/src/reduced_platform_state/mod.rs | 79 +++++++-------- .../src/reduced_platform_state/v0/mod.rs | 18 ++-- .../engine/run_block_proposal/v1/mod.rs | 9 +- .../block_end/validator_set_update/mod.rs | 4 - .../v0/mod.rs | 95 ------------------- .../reconstruct_platform_state/mod.rs | 76 ++++----------- .../fetch_reduced_platform_state/v0/mod.rs | 5 +- .../process_proposal_collision_tests.rs | 48 ++++++++++ .../rs-platform-version/src/version/v15.rs | 4 +- 9 files changed, 114 insertions(+), 224 deletions(-) diff --git a/packages/rs-dpp/src/reduced_platform_state/mod.rs b/packages/rs-dpp/src/reduced_platform_state/mod.rs index 62b3c2444d1..ea17ecfc71c 100644 --- a/packages/rs-dpp/src/reduced_platform_state/mod.rs +++ b/packages/rs-dpp/src/reduced_platform_state/mod.rs @@ -5,58 +5,27 @@ //! reconstruct the full Platform state. The full Platform state itself is only persisted //! to GroveDB aux storage, which is not replicated by GroveDB state sync. -use crate::serialization::{PlatformDeserializableFromVersionedStructure, PlatformSerializable}; use crate::ProtocolError; use bincode::{Decode, Encode}; -use platform_version::version::PlatformVersion; +use derive_more::From; +use platform_serialization_derive::{PlatformDeserialize, PlatformSerialize}; pub mod v0; use v0::ReducedPlatformStateV0; /// Reduced Platform State (platform-versioned wrapper) -#[derive(Clone, Debug, PartialEq, Encode, Decode, derive_more::From)] +/// +/// The structure version is the enum discriminant, so it serializes `unversioned` (big +/// endian, no limit) exactly like the other versioned platform types. These bytes are +/// covered by the app hash, so the encoding is consensus-fixed. +#[derive(Clone, Debug, PartialEq, Encode, Decode, PlatformSerialize, PlatformDeserialize, From)] +#[platform_serialize(unversioned)] pub enum ReducedPlatformState { /// Version 0 V0(ReducedPlatformStateV0), } -impl PlatformSerializable for ReducedPlatformState { - type Error = ProtocolError; - - fn serialize_to_bytes(&self) -> Result, Self::Error> { - let config = bincode::config::standard(); - bincode::encode_to_vec(self, config).map_err(|e| { - ProtocolError::PlatformSerializationError(format!( - "cannot serialize ReducedPlatformState: {}", - e - )) - }) - } -} - -impl PlatformDeserializableFromVersionedStructure for ReducedPlatformState { - fn versioned_deserialize( - data: &[u8], - _platform_version: &PlatformVersion, - ) -> Result - where - Self: Sized, - { - // The version of the structure is encoded in the enum discriminant, so the - // platform version is not needed to pick the variant. - let config = bincode::config::standard(); - bincode::decode_from_slice(data, config) - .map_err(|e| { - ProtocolError::PlatformDeserializationError(format!( - "cannot deserialize ReducedPlatformState: {}", - e - )) - }) - .map(|(object, _)| object) - } -} - #[cfg(test)] mod tests { use super::v0::{ @@ -65,18 +34,15 @@ mod tests { }; use super::*; use crate::block::block_info::BlockInfo; + use crate::serialization::{PlatformDeserializable, PlatformSerializable}; #[test] fn should_roundtrip_reduced_platform_state_serialization() { let state = ReducedPlatformState::V0(ReducedPlatformStateV0 { last_committed_block_info: Some(ReducedBlockInfoV0 { basic_info: BlockInfo::default_with_time(1_700_000_000_000), - app_hash: None, quorum_hash: [1u8; 32].into(), - block_id_hash: None, proposer_pro_tx_hash: [2u8; 32].into(), - signature: None, - round: 3, }), current_protocol_version_in_consensus: 15, next_epoch_protocol_version: 15, @@ -109,9 +75,32 @@ mod tests { let bytes = state.serialize_to_bytes().expect("should serialize"); let restored = - ReducedPlatformState::versioned_deserialize(&bytes, PlatformVersion::latest()) - .expect("should deserialize"); + ReducedPlatformState::deserialize_from_bytes(&bytes).expect("should deserialize"); assert_eq!(state, restored); } + + /// The reduced state is encoded like every other versioned platform type: big + /// endian. Pin it so the app-hash-covered encoding cannot drift silently. + #[test] + fn should_encode_big_endian_like_other_platform_types() { + let state = ReducedPlatformState::V0(ReducedPlatformStateV0 { + last_committed_block_info: None, + current_protocol_version_in_consensus: 0x0102_0304, + next_epoch_protocol_version: 0, + current_validator_set_quorum_hash: [0u8; 32].into(), + next_validator_set_quorum_hash: None, + previous_fee_versions: Default::default(), + quorum_positions: vec![], + proposed_core_chain_locked_height: 0, + previous_chain_lock_quorums: None, + previous_instant_lock_quorums: None, + }); + + let bytes = state.serialize_to_bytes().expect("should serialize"); + // discriminant 0, then `None`, then the protocol version as a big-endian varint + // (bincode's varint marker 0xfc precedes a u32 payload) + assert_eq!(&bytes[..2], &[0u8, 0u8]); + assert_eq!(&bytes[2..7], &[0xfc, 0x01, 0x02, 0x03, 0x04]); + } } diff --git a/packages/rs-dpp/src/reduced_platform_state/v0/mod.rs b/packages/rs-dpp/src/reduced_platform_state/v0/mod.rs index 60814fe3160..8017b1cab31 100644 --- a/packages/rs-dpp/src/reduced_platform_state/v0/mod.rs +++ b/packages/rs-dpp/src/reduced_platform_state/v0/mod.rs @@ -6,26 +6,20 @@ use platform_value::Bytes32; /// Block information persisted as part of the reduced platform state. /// -/// The reduced state is written while the block is still being executed, before it is -/// signed and before the resulting app hash is known, so `app_hash`, `block_id_hash` and -/// `signature` are `Option`s rather than zero-filled placeholders. They are `None` when -/// stored and are filled in (where possible) during state reconstruction. +/// Only what the block header fixes goes in here. The reduced state is written during +/// block execution and covered by the app hash, so anything that can differ between two +/// proposals of the same block (the consensus round, the app hash itself, the block id +/// hash and the signature) must stay out: a re-proposal of the same header at a later +/// round has to produce the same app hash. A state-synced node takes the app hash from +/// the snapshot and learns the rest with the next finalized block. #[derive(Clone, Debug, PartialEq, Encode, Decode)] pub struct ReducedBlockInfoV0 { /// Basic block info (height, core height, time, epoch) pub basic_info: BlockInfo, - /// The app hash resulting from this block; unknown at store time - pub app_hash: Option, /// The quorum that signed (or will sign) this block pub quorum_hash: Bytes32, - /// The block id hash; unknown at store time - pub block_id_hash: Option, /// The block proposer's pro tx hash pub proposer_pro_tx_hash: Bytes32, - /// The block signature; unknown at store time - pub signature: Option<[u8; 96]>, - /// The consensus round that produced this block - pub round: u32, } /// One quorum of a signature-verification quorum set, as persisted in the reduced diff --git a/packages/rs-drive-abci/src/execution/engine/run_block_proposal/v1/mod.rs b/packages/rs-drive-abci/src/execution/engine/run_block_proposal/v1/mod.rs index cdaf3180ae3..9ad0bf3f071 100644 --- a/packages/rs-drive-abci/src/execution/engine/run_block_proposal/v1/mod.rs +++ b/packages/rs-drive-abci/src/execution/engine/run_block_proposal/v1/mod.rs @@ -449,19 +449,16 @@ where // Write the reduced platform state into the replicated grovedb state, immediately // before the root hash so it is covered by this block's app hash. A state-synced // node reads it back to reconstruct the full platform state, which otherwise only - // exists in non-replicated aux storage. The app hash, block id hash and signature - // of this block are unknown at this point and are stored as `None`. + // exists in non-replicated aux storage. Only header-fixed fields go in: the same + // block re-proposed at a later round must hash to the same app hash, so the round + // (and the not-yet-known app hash, block id hash and signature) stay out. let reduced_platform_state = block_execution_context .block_platform_state() .to_reduced_platform_state( Some(ReducedBlockInfoV0 { basic_info: block_info, - app_hash: None, quorum_hash: validator_set_quorum_hash.into(), - block_id_hash: None, proposer_pro_tx_hash: proposer_pro_tx_hash.into(), - signature: None, - round: block_proposal.round, }), core_chain_locked_height, ); diff --git a/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs index fcb2237a59a..e40f0adfafe 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/block_end/validator_set_update/mod.rs @@ -1046,12 +1046,8 @@ mod tests { let reduced_platform_state = platform_state.to_reduced_platform_state( Some(ReducedBlockInfoV0 { basic_info: BlockInfo::default(), - app_hash: None, quorum_hash: (*qh1.as_byte_array()).into(), - block_id_hash: None, proposer_pro_tx_hash: proposer.into(), - signature: None, - round: 0, }), 1, ); diff --git a/packages/rs-drive-abci/src/execution/platform_events/protocol_upgrade/perform_events_on_first_block_of_protocol_change/v0/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/protocol_upgrade/perform_events_on_first_block_of_protocol_change/v0/mod.rs index ab03451de62..4f66d59dce8 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/protocol_upgrade/perform_events_on_first_block_of_protocol_change/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/protocol_upgrade/perform_events_on_first_block_of_protocol_change/v0/mod.rs @@ -4,12 +4,10 @@ use crate::platform_types::platform::Platform; use crate::platform_types::platform_state::PlatformState; use crate::platform_types::platform_state::PlatformStateV0Methods; use dpp::block::block_info::BlockInfo; -use dpp::block::extended_block_info::v0::ExtendedBlockInfoV0Getters; use dpp::dashcore::hashes::Hash; use dpp::data_contracts::SystemDataContract; use dpp::fee::Credits; use dpp::platform_value::Identifier; -use dpp::reduced_platform_state::v0::ReducedBlockInfoV0; use dpp::serialization::PlatformDeserializable; use dpp::system_data_contracts::load_system_data_contract; use dpp::version::PlatformVersion; @@ -121,10 +119,6 @@ impl Platform { self.transition_to_version_14(block_info, transaction, platform_version)?; } - if previous_protocol_version < 15 && platform_version.protocol_version >= 15 { - self.transition_to_version_15(platform_state, transaction, platform_version)?; - } - Ok(()) } @@ -744,44 +738,6 @@ impl Platform { Ok(()) } - - /// When transitioning to version 15 we write the initial reduced platform state (built - /// from the last committed platform state) under `Misc/reduced_saved_state`, so the key - /// exists in the replicated state from the fork block onward. `run_block_proposal` v1 - /// overwrites it later in this same block with the state of the block being processed; - /// this initial write guarantees no v15 block ever commits without the key, which is - /// what makes every snapshot taken at or after activation restorable via state sync. - fn transition_to_version_15( - &self, - platform_state: &PlatformState, - transaction: &Transaction, - platform_version: &PlatformVersion, - ) -> Result<(), Error> { - let last_committed_block_info = - platform_state - .last_committed_block_info() - .as_ref() - .map(|extended_block_info| ReducedBlockInfoV0 { - basic_info: *extended_block_info.basic_info(), - app_hash: Some((*extended_block_info.app_hash()).into()), - quorum_hash: (*extended_block_info.quorum_hash()).into(), - block_id_hash: Some((*extended_block_info.block_id_hash()).into()), - proposer_pro_tx_hash: (*extended_block_info.proposer_pro_tx_hash()).into(), - signature: Some(*extended_block_info.signature()), - round: extended_block_info.round(), - }); - - let reduced_platform_state = platform_state.to_reduced_platform_state( - last_committed_block_info, - platform_state.last_committed_core_height(), - ); - - self.store_reduced_platform_state( - &reduced_platform_state, - Some(transaction), - platform_version, - ) - } } #[cfg(test)] @@ -2686,55 +2642,4 @@ mod tests { diffs.join("\n"), ); } - - #[test] - fn test_transition_to_version_15_writes_initial_reduced_platform_state() { - use dpp::reduced_platform_state::ReducedPlatformState; - - let platform = TestPlatformBuilder::new() - .build_with_mock_rpc() - .set_genesis_state(); - let platform_version = PlatformVersion::latest(); - - let transaction = platform.drive.grove.start_transaction(); - - let platform_state = platform.state.load(); - - // Before the transition, the replicated state must not carry the reduced state key. - let pre_transition = platform - .fetch_reduced_platform_state(Some(&transaction), platform_version) - .expect("fetching an absent reduced platform state should not error"); - assert!( - pre_transition.is_none(), - "reduced platform state must not exist before transition_to_version_15" - ); - - let result = - platform.transition_to_version_15(&platform_state, &transaction, platform_version); - assert!(result.is_ok(), "transition failed: {:?}", result.err()); - - let reduced = platform - .fetch_reduced_platform_state(Some(&transaction), platform_version) - .expect("expected to fetch reduced platform state") - .expect("reduced platform state must exist after transition_to_version_15"); - - let ReducedPlatformState::V0(reduced) = reduced; - assert_eq!( - reduced.current_protocol_version_in_consensus, - platform_state.current_protocol_version_in_consensus() - ); - assert_eq!( - reduced.next_epoch_protocol_version, - platform_state.next_epoch_protocol_version() - ); - assert_eq!( - reduced.quorum_positions.len(), - platform_state.validator_sets().len(), - "quorum positions must mirror the validator set order" - ); - assert_eq!( - reduced.proposed_core_chain_locked_height, - platform_state.last_committed_core_height() - ); - } } diff --git a/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs index cf05dc7a596..7da6ce5e249 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs @@ -8,7 +8,7 @@ use crate::platform_types::signature_verification_quorum_set::{ }; use crate::platform_types::validator_set::ValidatorSet; use crate::rpc::core::CoreRPCLike; -use dpp::block::extended_block_info::v0::{ExtendedBlockInfoV0, ExtendedBlockInfoV0Getters}; +use dpp::block::extended_block_info::v0::ExtendedBlockInfoV0; use dpp::block::extended_block_info::ExtendedBlockInfo; use dpp::bls_signatures::PublicKey as BlsPublicKey; use dpp::dashcore::hashes::Hash; @@ -38,11 +38,9 @@ where /// /// 1. restores the scalar fields (protocol versions, quorum hashes, fee versions) /// directly from the reduced state; - /// 2. re-derives the masternode lists, masternode identities and quorums from Core - /// via `update_core_info` with `start_from_scratch = true` — the identity writes - /// are re-derivations of data already present in the restored state, so the - /// grovedb root hash MUST NOT change (the caller's root-hash equality check is - /// the proof of that idempotence); + /// 2. re-derives the masternode lists and quorums from Core, in memory only, via + /// `rebuild_core_info_in_memory`; the restored grovedb already holds every + /// masternode identity, so nothing is written and the root hash cannot change; /// 3. restores the validator set order recorded by the source (`quorum_positions`), /// which cannot be recovered from Core RPC; /// 4. advances the state to the snapshot block via `update_state_cache`, which @@ -114,49 +112,29 @@ where .to_string(), ))?; - // The reduced state is written before the block's root hash exists, so its app - // hash is normally None and the snapshot app hash fills it in. If it does carry - // one, it must agree with the snapshot. - if let Some(saved_app_hash) = saved_block_info.app_hash { - if saved_app_hash.to_buffer() != *app_hash { - return Err(AbciError::StateSyncInternalError(format!( - "reconstruct_platform_state reduced platform state app hash {} does not \ - match snapshot app hash {}", - hex::encode(saved_app_hash.to_buffer()), - hex::encode(app_hash), - )) - .into()); - } - } - let current_block_info: ExtendedBlockInfo = ExtendedBlockInfoV0 { basic_info: saved_block_info.basic_info, app_hash: *app_hash, quorum_hash: saved_block_info.quorum_hash.to_buffer(), - // Not known during proposal processing, and not needed for consensus after - // a restore; restored as zeroes. - block_id_hash: saved_block_info - .block_id_hash - .map(|hash| hash.to_buffer()) - .unwrap_or_default(), proposer_pro_tx_hash: saved_block_info.proposer_pro_tx_hash.to_buffer(), - // Same: unknown at store time, restored as zeroes when absent. - signature: saved_block_info.signature.unwrap_or([0u8; 96]), - round: saved_block_info.round, + // The block id hash, signature and round are not part of the reduced state + // (they are unknown while the block executes, and the round must not affect + // the app hash). They are zero until the next block is finalized; proofs are + // refused until then, see `ensure_block_proof_metadata_is_available`. + block_id_hash: [0u8; 32], + signature: [0u8; 96], + round: 0, } .into(); - // Re-derive masternode lists, masternode identities and quorums from Core, from - // scratch, at the core height the snapshot block ran with. The identity writes - // must be byte-identical to what is already in the restored state. - let transaction = self.drive.grove.start_transaction(); - self.update_core_info( - None, + // Rebuild the Core-derived state in memory only, from scratch, at the core height + // the snapshot block ran with. The restored grovedb already + // holds every masternode identity as the source chain wrote it; rewriting them + // here would be thousands of no-op writes at best and a root-hash mismatch at + // worst, and the caller compares the root hash against the snapshot afterwards. + self.rebuild_core_info_in_memory( &mut platform_state, saved.proposed_core_chain_locked_height, - true, - current_block_info.basic_info(), - &transaction, state_platform_version, )?; @@ -200,8 +178,8 @@ where &saved.quorum_positions, ); - // Reinstate the signature-verification quorum HISTORY. `update_core_info` above - // rebuilt the current sets from Core — which is exact, the quorums of a type at a + // Reinstate the signature-verification quorum HISTORY. The rebuild above + // derived the current sets from Core — which is exact, the quorums of a type at a // core height are whatever Core reports — but it was given `platform_state = None` // and so could not produce any previous set. That history is consensus-relevant: // `select_quorums` uses the previous set for locks signed within `SIGN_OFFSET` @@ -218,22 +196,6 @@ where let block_height = platform_state.last_committed_block_height(); - // Commit the re-derivation BEFORE the in-memory state is published: if this - // commit fails, nothing has been published and the error propagates with the - // node's observable state unchanged. (Publishing first, as normal block - // finalization does, would leave the info handler reporting a snapshot height - // that grovedb never persisted.) - self.drive - .grove - .commit_transaction(transaction) - .unwrap() - .map_err(|e| { - AbciError::StateSyncInternalError(format!( - "reconstruct_platform_state unable to commit transaction: {}", - e - )) - })?; - // Advance the state to the snapshot block: rotates next-into-current exactly as // the source did on finalization, persists to aux storage and publishes the // state for the info handler. Aux writes are not part of the root hash, so diff --git a/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/v0/mod.rs b/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/v0/mod.rs index 29ac4392488..253c299e055 100644 --- a/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/v0/mod.rs +++ b/packages/rs-drive-abci/src/execution/storage/fetch_reduced_platform_state/v0/mod.rs @@ -1,7 +1,7 @@ use crate::error::Error; use crate::platform_types::platform::Platform; use dpp::reduced_platform_state::ReducedPlatformState; -use dpp::serialization::PlatformDeserializableFromVersionedStructure; +use dpp::serialization::PlatformDeserializable; use dpp::version::PlatformVersion; use drive::query::TransactionArg; @@ -15,8 +15,7 @@ impl Platform { .fetch_reduced_platform_state_bytes(transaction, platform_version) .map_err(Error::Drive)? .map(|bytes| { - ReducedPlatformState::versioned_deserialize(&bytes, platform_version) - .map_err(Error::Protocol) + ReducedPlatformState::deserialize_from_bytes(&bytes).map_err(Error::Protocol) }) .transpose() } diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/process_proposal_collision_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/process_proposal_collision_tests.rs index b2fe58a7222..327ca11e15c 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/test_cases/process_proposal_collision_tests.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/process_proposal_collision_tests.rs @@ -566,4 +566,52 @@ mod tests { "the cached context must be the one prepare proposal built" ); } + + /// CONSENSUS PIN: the app hash must not depend on the consensus round. + /// + /// Tenderdash re-proposes a block that reached a prevote majority but did not commit + /// with the SAME header at a later round, and re-runs ProcessProposal for every round, + /// requiring the returned app hash to equal the header's. If anything round-specific + /// reached the replicated state (as the reduced platform state written by + /// `run_block_proposal` v1 once did), every validator would reject the re-proposal + /// and the chain would halt at that height. + #[tokio::test] + async fn process_proposal_of_the_same_block_at_a_later_round_must_return_the_same_app_hash() { + let config = config(); + let mut platform = TestPlatformBuilder::new() + .with_config(config.clone()) + .build_with_mock_rpc(); + + let outcome = run_chain_for_strategy( + &mut platform, + 5, + strategy(), + config, + 7, + &mut None, + &mut None, + ) + .await; + + let at_round_0 = next_block_request(&outcome, 1, [0x42u8; 32], 0); + let mut at_round_3 = at_round_0.clone(); + at_round_3.round = 3; + + let response_round_0 = outcome + .abci_app + .process_proposal(at_round_0) + .expect("the block processes at round 0"); + assert_eq!(response_round_0.status, ProposalStatus::Accept as i32); + + let response_round_3 = outcome + .abci_app + .process_proposal(at_round_3) + .expect("the same block processes again at round 3"); + assert_eq!(response_round_3.status, ProposalStatus::Accept as i32); + + assert_eq!( + response_round_0.app_hash, response_round_3.app_hash, + "the same block re-proposed at a later round must produce the same app hash" + ); + } } diff --git a/packages/rs-platform-version/src/version/v15.rs b/packages/rs-platform-version/src/version/v15.rs index 8b84f8d8757..3080cb5ba83 100644 --- a/packages/rs-platform-version/src/version/v15.rs +++ b/packages/rs-platform-version/src/version/v15.rs @@ -44,8 +44,8 @@ pub const PROTOCOL_VERSION_15: ProtocolVersion = 15; /// would have no way to rebuild its in-memory state. /// * `consensus_params_update` 1 -> 2: the first block of v15 also emits evidence /// params sized for state-synced nodes that do not hold full history (issue #2512). -/// * `perform_events_on_first_block_of_protocol_change` writes the initial reduced state -/// at the v15 activation block, so every snapshot taken at or after activation is +/// * The activation block already runs `run_block_proposal` v1, so the reduced state +/// exists from that block on and every snapshot taken at or after activation is /// restorable. Snapshots from before activation lack the key and are not served. /// /// Everything else matches v14. The grovedb state sync protocol version used for From 8bfad7eb77e251ce1ef59f339692340716d079c8 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 9 Sep 2026 15:05:33 -0500 Subject: [PATCH 31/34] perf(drive-abci): rebuild Core-derived state in memory only after a snapshot restore reconstruct_platform_state routed through update_core_info, which re-issued AddNewIdentity for every masternode; each hit the re-enable branch and rewrote every key of an identity the restored grovedb already held. Thousands of no-op writes on the consensus thread, with correctness resting on byte-idempotence that only the final root-hash compare could catch. The new rebuild_core_info_in_memory helper rebuilds the masternode lists and quorum sets from Core without touching grovedb, and reconstruction no longer opens a write transaction. Co-Authored-By: Claude Fable 5.1 --- .../src/abci/handler/apply_snapshot_chunk.rs | 4 ++-- .../core_based_updates/update_core_info/mod.rs | 18 ++++++++++++++++++ .../reconstruct_platform_state/mod.rs | 2 +- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs index 3496e2eae33..74120b1db2d 100644 --- a/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs +++ b/packages/rs-drive-abci/src/abci/handler/apply_snapshot_chunk.rs @@ -215,8 +215,8 @@ where } // Rebuild the in-memory platform state from the reduced platform state contained in - // the restored snapshot. This re-derives masternode lists and quorums from Core and - // must leave the grovedb root hash untouched; the equality check below proves it. + // the restored snapshot. This re-derives masternode lists and quorums from Core in + // memory only; the root hash equality check below is the restore's integrity backstop. // // This is also where a snapshot taken before the reduced platform state existed // (pre-v15) is refused. Refusing earlier would be better, but grovedb does not expose diff --git a/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/mod.rs index 1196d4ebe92..55a16d6118e 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/core_based_updates/update_core_info/mod.rs @@ -66,4 +66,22 @@ where })), } } + + /// Rebuilds the in-memory Core-derived state (masternode lists and every quorum + /// set) from scratch at `core_block_height`, without touching GroveDB. + /// + /// State sync reconstruction uses this: the restored GroveDB already holds every + /// masternode identity exactly as the source chain wrote it, so the identity writes + /// `update_core_info` would issue are at best no-ops and at worst a root-hash + /// mismatch. Only the platform state, which is not replicated, has to be rebuilt. + /// Not consensus code, hence unversioned. + pub(crate) fn rebuild_core_info_in_memory( + &self, + state: &mut PlatformState, + core_block_height: u32, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + self.update_state_masternode_list_v0(state, core_block_height, true)?; + self.update_quorum_info(None, state, core_block_height, true, platform_version) + } } diff --git a/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs index 7da6ce5e249..c2eb2e961a4 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/state_sync/reconstruct_platform_state/mod.rs @@ -194,7 +194,7 @@ where saved.previous_instant_lock_quorums.as_ref(), )?; - let block_height = platform_state.last_committed_block_height(); + let block_height = saved_block_info.basic_info.height; // Advance the state to the snapshot block: rotates next-into-current exactly as // the source did on finalization, persists to aux storage and publishes the From d0d9d4bf21f4b72348d48879a78140642746a439 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 9 Sep 2026 15:05:33 -0500 Subject: [PATCH 32/34] docs(drive-abci): evidence params expire on the larger bound, not the smaller Tenderdash treats evidence as expired only when both max_age_num_blocks and max_age_duration are exceeded, and backfill likewise stops only when both are satisfied. The note claimed the smaller bound wins; the larger one does, so the effective window is 20 days and the 15 000 block bound never binds. Values unchanged pending the #2512 decision. Co-Authored-By: Claude Fable 5.1 --- .../engine/consensus_params_update/v2/mod.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/rs-drive-abci/src/execution/engine/consensus_params_update/v2/mod.rs b/packages/rs-drive-abci/src/execution/engine/consensus_params_update/v2/mod.rs index 8f1c4663a04..b456b293593 100644 --- a/packages/rs-drive-abci/src/execution/engine/consensus_params_update/v2/mod.rs +++ b/packages/rs-drive-abci/src/execution/engine/consensus_params_update/v2/mod.rs @@ -10,10 +10,13 @@ use tenderdash_abci::proto::types::{ConsensusParams, EvidenceParams}; /// version 15 (state sync). Value proposed in issue #2512 for nodes that bootstrap /// from snapshots and do not hold full history. /// -/// REVIEW BEFORE RELEASE: at ~6s blocks, 15_000 blocks is roughly one day, while -/// [`V15_EVIDENCE_MAX_AGE_DURATION_SECONDS`] below is 20 days. Evidence expires when -/// EITHER bound is exceeded, so the effective window is the smaller (~1 day) — the two -/// values from #2512 look inconsistent and need to be confirmed before this ships. +/// REVIEW BEFORE RELEASE. Tenderdash treats evidence as expired only when BOTH bounds +/// are exceeded (`evidence/pool.go` `isExpired`), and the state sync backfill likewise +/// stops only once BOTH are satisfied (`statesync/reactor.go` `Backfill`). So the +/// effective window is the LARGER of the two: at ~6s blocks 15_000 blocks is about one +/// day and never binds, and a state-synced node backfills the full 20 days of +/// [`V15_EVIDENCE_MAX_AGE_DURATION_SECONDS`] below. If a ~1 day window was the intent +/// of #2512, the duration is the value to lower. Confirm before this ships. const V15_EVIDENCE_MAX_AGE_NUM_BLOCKS: i64 = 15_000; /// Maximum evidence age in time: 20 days, per issue #2512. See the review note on From 34ef7f27578f701ce2577c4a35c517ee7ff503b8 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 9 Sep 2026 15:05:33 -0500 Subject: [PATCH 33/34] docs(platform-version): shorten the FEE_VERSION2 number-collision notes Co-Authored-By: Claude Fable 5.1 --- .../src/version/fee/mod.rs | 28 +++-------- .../rs-platform-version/src/version/fee/v2.rs | 46 ++++--------------- 2 files changed, 14 insertions(+), 60 deletions(-) diff --git a/packages/rs-platform-version/src/version/fee/mod.rs b/packages/rs-platform-version/src/version/fee/mod.rs index 1df42b6beb0..a196f3972ce 100644 --- a/packages/rs-platform-version/src/version/fee/mod.rs +++ b/packages/rs-platform-version/src/version/fee/mod.rs @@ -31,14 +31,10 @@ pub type FeeVersionNumber = u32; /// The fee schedules [`FeeVersion::get`] can resolve, indexed by `fee_version_number - 1`. /// -/// # This list is INCOMPLETE, and that is a known defect +/// Every `FeeVersion` needs a unique `fee_version_number` and an entry here at the index +/// that number implies, because only the number is persisted. /// -/// `FEE_VERSION2` — what protocol versions 9 and later actually run with — is missing, and -/// declares `fee_version_number: 1`, colliding with `FEE_VERSION1`. Since the fee version -/// NUMBER is the only thing persisted (`PlatformStateForSavingV1` and -/// `ReducedPlatformStateV0` both store `epoch index -> number`), every node that restarts -/// or state-syncs rehydrates previous epochs' fees as `FEE_VERSION1`. See the doc comment -/// on [`v2::FEE_VERSION2`] for why that is currently latent and what fixing it requires. +/// BUG(#4647): incomplete. `FEE_VERSION2` is missing and reuses number 1; see its doc comment. pub const FEE_VERSIONS: &[FeeVersion] = &[FEE_VERSION1]; #[derive(Clone, Debug, Encode, Decode, Default, PartialEq, Eq)] @@ -132,21 +128,9 @@ mod tests { use super::*; use crate::version::fee::v2::FEE_VERSION2; - /// Every `FeeVersion` constant must carry a distinct `fee_version_number`, and - /// `FEE_VERSIONS` must contain all of them, because the number is the ONLY thing - /// persisted: `PlatformStateForSavingV1` and `ReducedPlatformStateV0` both store - /// `(epoch index -> fee version number)` and rehydrate through `FeeVersion::get`. A - /// number that does not resolve back to the constant it came from silently substitutes - /// a different fee schedule on any node that restarts or state-syncs. - /// - /// This test FAILS today, which is why it is ignored: `FEE_VERSION2` declares - /// `fee_version_number: 1`, the same as `FEE_VERSION1`, and is absent from - /// `FEE_VERSIONS`, so `FeeVersion::get(1)` returns `FEE_VERSION1` even for the epochs - /// that ran on `FEE_VERSION2`. See the doc comment on `FEE_VERSION2`. - /// - /// Un-ignore it as part of giving `FEE_VERSION2` its own number and adding it to - /// `FEE_VERSIONS`. That is protocol-visible and needs a migration, which is why the - /// defect is pinned here rather than fixed in place. + /// Every `FeeVersion` constant needs a distinct `fee_version_number` that resolves + /// back to it through `FEE_VERSIONS`, because only the number is persisted. Fails + /// today (`FEE_VERSION2` reuses number 1, see its doc comment); un-ignore with the fix. #[test] #[ignore = "known defect: FEE_VERSION2 reuses fee_version_number 1 and is absent from \ FEE_VERSIONS; fixing it is protocol-visible - see the FEE_VERSION2 docs"] diff --git a/packages/rs-platform-version/src/version/fee/v2.rs b/packages/rs-platform-version/src/version/fee/v2.rs index 24f09b06a49..a690d8e864c 100644 --- a/packages/rs-platform-version/src/version/fee/v2.rs +++ b/packages/rs-platform-version/src/version/fee/v2.rs @@ -10,45 +10,15 @@ use crate::version::fee::FeeVersion; /// Introduced in protocol version 9 (2.0) /// -/// # WARNING: `fee_version_number` collides with [`FEE_VERSION1`], and this one is not -/// reachable by number -/// -/// [`FeeVersion::get`] resolves a number through [`FEE_VERSIONS`], which contains only -/// `FEE_VERSION1`. This constant declares the SAME `fee_version_number: 1`, so -/// `FeeVersion::get(1)` can only ever return `FEE_VERSION1` — never this one, even though -/// this is what protocol versions 9 and later actually run with, and the two differ in -/// `data_contract_registration`. -/// -/// That makes every number-only round trip of a fee version silently lossy. Two exist: -/// -/// * `PlatformStateForSavingV1` stores `previous_fee_versions` as -/// `(epoch index -> fee version number)`, so a node that RESTARTS rehydrates previous -/// epochs' fees as `FEE_VERSION1`; -/// * `ReducedPlatformStateV0` does the same, so a node that STATE-SYNCS gets the same -/// substitution without even restarting. -/// -/// It is latent rather than a live consensus fork only because `previous_fee_versions` is -/// consulted solely to price storage refunds (`rs-drive/src/fees/op.rs`), and -/// `FEE_VERSION1` and `FEE_VERSION2` have IDENTICAL `storage` fees. It becomes a fork the -/// moment a future `FeeVersion` changes a storage or processing fee without also taking a -/// distinct number. -/// -/// ## The rule -/// -/// **Every `FeeVersion` constant must have a unique `fee_version_number`, and must be -/// listed in [`FEE_VERSIONS`] at the index its number implies.** Fixing this constant to -/// `fee_version_number: 2` and adding it to `FEE_VERSIONS` is protocol-visible (it changes -/// what a restarted or state-synced node computes for old epochs), so it needs a -/// versioned migration rather than an in-place edit — which is why this is documented here -/// instead of changed. `fee_version_numbers_are_unique` in `super` is the enforcement, and -/// is `#[ignore]`d until then. -/// -/// [`FEE_VERSION1`]: crate::version::fee::v1::FEE_VERSION1 -/// [`FEE_VERSIONS`]: crate::version::fee::FEE_VERSIONS -/// [`FeeVersion::get`]: crate::version::fee::FeeVersion::get +/// BUG(#4647): `fee_version_number` collides with `FEE_VERSION1` and this constant is missing +/// from `FEE_VERSIONS`, so `FeeVersion::get(1)` never resolves to it. The number is the +/// only thing persisted (`PlatformStateForSavingV1`, `ReducedPlatformStateV0`), so a +/// node that restarts or state-syncs rehydrates previous epochs' fees as `FEE_VERSION1`. +/// Latent only because the two share identical storage fees, which is all +/// `previous_fee_versions` is consulted for. Giving it number 2 is protocol-visible and +/// needs a versioned migration; see `fee_version_numbers_are_unique_and_resolvable`. pub const FEE_VERSION2: FeeVersion = FeeVersion { - // BUG: must be 2. See the doc comment above — changing it is protocol-visible. - fee_version_number: 1, + fee_version_number: 1, // BUG: must be 2, see the doc comment above uses_version_fee_multiplier_permille: Some(1000), //No action storage: FEE_STORAGE_VERSION1, signature: FEE_SIGNATURE_VERSION1, From 64a872960258c11914412352fbcf5e1c6c598697 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 9 Sep 2026 15:08:09 -0500 Subject: [PATCH 34/34] test(drive-abci): start state sync source chains near the wall clock Since #4570 checkpoints are only taken for blocks younger than ten minutes, so a source chain starting at the fixed 2023 genesis time never produced a snapshot and every state sync integration test failed on an empty checkpoint registry. Co-Authored-By: Claude Fable 5.1 --- .../test_cases/state_sync_sentinel_tests.rs | 3 ++- .../test_cases/state_sync_tests.rs | 20 +++++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs index 888ad5ea606..a0a43f84789 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_sentinel_tests.rs @@ -23,7 +23,7 @@ mod tests { use crate::execution::run_chain_for_strategy; use crate::strategy::{ChainExecutionOutcome, NetworkStrategy}; use crate::test_cases::state_sync_tests::tests::{ - install_reconstruction_core_mocks, sync_snapshot, SnapshotSyncOutcome, + install_reconstruction_core_mocks, recent_start_time_ms, sync_snapshot, SnapshotSyncOutcome, }; use dpp::version::v15::PROTOCOL_VERSION_15; use dpp::version::PlatformVersion; @@ -99,6 +99,7 @@ mod tests { failure_testing: None, query_testing: None, verify_state_transition_results: false, + start_time_ms: recent_start_time_ms(), ..Default::default() } } diff --git a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs index 9e2a1fc011f..0480ee8b379 100644 --- a/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs +++ b/packages/rs-drive-abci/tests/strategy_tests/test_cases/state_sync_tests.rs @@ -32,6 +32,16 @@ pub(crate) mod tests { use tenderdash_abci::proto::abci::{response_apply_snapshot_chunk, response_offer_snapshot}; use tenderdash_abci::Application; + /// A first-block time near the wall clock. Checkpoints are only taken for blocks + /// younger than ten minutes (`is_historical_block`), so a source chain that starts + /// at the fixed 2023 genesis time never produces a snapshot. + pub(crate) fn recent_start_time_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock is before the unix epoch") + .as_millis() as u64 + } + /// A quiet chain with a trickle of identity inserts, no masternode churn and no /// quorum rotation, so the target's from-scratch Core re-derivation sees exactly /// the same masternodes and quorums the source chain ran with. @@ -62,6 +72,7 @@ pub(crate) mod tests { failure_testing: None, query_testing: None, verify_state_transition_results: false, + start_time_ms: recent_start_time_ms(), ..Default::default() } } @@ -502,9 +513,10 @@ pub(crate) mod tests { /// Exercises the platform state reconstruction end to end without going through /// the grovedb chunk restore: the source chain's /// own grovedb IS a faithfully "restored" snapshot of itself, so reconstructing - /// on it must (a) not change the grovedb root hash — the proof that re-deriving - /// masternode identities from Core is byte-idempotent — and (b) reproduce the - /// source's in-memory platform state from the reduced platform state alone. + /// on it must (a) not change the grovedb root hash — reconstruction rebuilds the + /// Core-derived state in memory and writes nothing to the replicated tree — and + /// (b) reproduce the source's in-memory platform state from the reduced platform + /// state alone. #[tokio::test] async fn platform_state_reconstruction_is_idempotent_and_matches_source_state() { let config = state_sync_platform_config(); @@ -536,7 +548,7 @@ pub(crate) mod tests { .reconstruct_platform_state(&tip_app_hash, platform_version) .expect("platform state reconstruction must succeed"); - // (a) idempotence: re-deriving masternode identities wrote nothing new + // (a) reconstruction wrote nothing to the replicated tree let root_hash_after = platform .drive .grove