From 71955817ef57fc1a44abfbcfc803857b2371ae07 Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 15 Jul 2026 15:04:11 +1200 Subject: [PATCH 01/35] Initial apply_block lock-free refactor --- Cargo.lock | 1 + Cargo.toml | 1 + bin/node/src/commands/modes.rs | 1 - bin/node/src/commands/recover.rs | 4 +- bin/stress-test/src/store/mod.rs | 12 +- crates/block-producer/src/mempool/tests.rs | 29 +- crates/block-producer/src/proof_scheduler.rs | 4 +- crates/block-producer/src/rpc_sync.rs | 6 +- crates/block-producer/src/server/mod.rs | 6 +- crates/block-producer/src/server/tests.rs | 1 - crates/block-producer/src/store/mod.rs | 2 +- crates/rpc/src/server/api.rs | 4 +- crates/rpc/src/server/api/status.rs | 2 +- .../server/api/sync_account_storage_maps.rs | 2 +- .../rpc/src/server/api/sync_account_vault.rs | 2 +- crates/rpc/src/server/api/sync_chain_mmr.rs | 4 +- crates/rpc/src/server/api/sync_notes.rs | 2 +- crates/rpc/src/server/api/sync_nullifiers.rs | 2 +- .../rpc/src/server/api/sync_transactions.rs | 2 +- crates/store/Cargo.toml | 1 + crates/store/src/account_state_forest/mod.rs | 127 +++--- crates/store/src/accounts/mod.rs | 17 +- crates/store/src/db/mod.rs | 22 +- crates/store/src/errors.rs | 6 +- crates/store/src/state/account.rs | 23 +- crates/store/src/state/apply_block.rs | 341 +-------------- crates/store/src/state/apply_proof.rs | 2 +- crates/store/src/state/loader.rs | 3 + crates/store/src/state/mod.rs | 194 ++++----- crates/store/src/state/sync_state.rs | 11 +- crates/store/src/state/writer.rs | 390 ++++++++++++++++++ 31 files changed, 658 insertions(+), 566 deletions(-) create mode 100644 crates/store/src/state/writer.rs diff --git a/Cargo.lock b/Cargo.lock index 0fc736ccf..23c2d367d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3365,6 +3365,7 @@ name = "miden-node-store" version = "0.16.0-alpha.1" dependencies = [ "anyhow", + "arc-swap", "assert_matches", "build-rs", "criterion", diff --git a/Cargo.toml b/Cargo.toml index c9d87fff7..17f0823d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,7 @@ miden-crypto = { version = "0.28" } # External dependencies anyhow = { version = "1.0" } +arc-swap = { version = "1.7" } assert_matches = { version = "1.5" } backon = { version = "1.6" } build-rs = { version = "0.3" } diff --git a/bin/node/src/commands/modes.rs b/bin/node/src/commands/modes.rs index 91b62f1dc..b1c90acce 100644 --- a/bin/node/src/commands/modes.rs +++ b/bin/node/src/commands/modes.rs @@ -73,7 +73,6 @@ impl SequencerCommand { batch_workers: self.block_producer.batch.workers, } .spawn(shutdown.clone()) - .await .context("failed to spawn sequencer")?; let block_producer = sequencer.api(); diff --git a/bin/node/src/commands/recover.rs b/bin/node/src/commands/recover.rs index 343850f99..7504ee364 100644 --- a/bin/node/src/commands/recover.rs +++ b/bin/node/src/commands/recover.rs @@ -81,7 +81,7 @@ async fn recover_from_validator( .chain_tip, ); - let local_tip = state.chain_tip(Finality::Committed).await; + let local_tip = state.chain_tip(Finality::Committed); if local_tip >= validator_tip { info!( target: LOG_TARGET, @@ -121,7 +121,7 @@ async fn recover_from_validator( } // The stream can end before reaching the tip if the validator restarts or drops the connection. - let final_tip = state.chain_tip(Finality::Committed).await; + let final_tip = state.chain_tip(Finality::Committed); anyhow::ensure!( final_tip >= validator_tip, "validator block stream ended at block {} before reaching the chain tip {}", diff --git a/bin/stress-test/src/store/mod.rs b/bin/stress-test/src/store/mod.rs index 9a89e28a8..1f2d63fd4 100644 --- a/bin/stress-test/src/store/mod.rs +++ b/bin/stress-test/src/store/mod.rs @@ -232,7 +232,7 @@ pub async fn bench_sync_notes(data_directory: PathBuf, iterations: usize, concur let store_state = start_store(data_directory).await; // Get the latest block number to determine the range - let chain_tip = store_state.chain_tip(Finality::Committed).await.as_u32(); + let chain_tip = store_state.chain_tip(Finality::Committed).as_u32(); // each request will have `ACCOUNTS_PER_SYNC_NOTES` note tags and will be sent with block number // 0. @@ -304,7 +304,7 @@ pub async fn bench_sync_nullifiers( .collect(); // Get the latest block number to determine the range - let chain_tip = store_state.chain_tip(Finality::Committed).await.as_u32(); + let chain_tip = store_state.chain_tip(Finality::Committed).as_u32(); // Get all nullifier prefixes from the store using sync_notes let mut nullifier_prefixes: Vec = vec![]; @@ -438,7 +438,7 @@ pub async fn bench_sync_transactions( let store_state = start_store(data_directory).await; // Get the latest block number to determine the range - let chain_tip = store_state.chain_tip(Finality::Committed).await.as_u32(); + let chain_tip = store_state.chain_tip(Finality::Committed).as_u32(); // each request will have `accounts_per_request` account ids and will query a range of blocks let request = |_| { @@ -516,7 +516,7 @@ pub async fn sync_transactions( .sync_transactions(account_ids, BlockNumber::from(block_from)..=BlockNumber::from(block_to)) .await .unwrap(); - let chain_tip = state.chain_tip(Finality::Committed).await; + let chain_tip = state.chain_tip(Finality::Committed); let response = proto::rpc::SyncTransactionsResponse { pagination_info: Some(proto::rpc::PaginationInfo { chain_tip: chain_tip.as_u32(), @@ -599,7 +599,7 @@ async fn sync_transactions_paginated( pub async fn bench_sync_chain_mmr(data_directory: PathBuf, iterations: usize, concurrency: usize) { let store_state = start_store(data_directory).await; - let chain_tip = store_state.chain_tip(Finality::Committed).await.as_u32(); + let chain_tip = store_state.chain_tip(Finality::Committed).as_u32(); let request = |_| { let state = Arc::clone(&store_state); @@ -628,7 +628,7 @@ pub async fn bench_sync_chain_mmr(data_directory: PathBuf, iterations: usize, co /// - the response. async fn sync_chain_mmr(state: &Arc, current_client_block_height: u32) -> SyncChainMmrRun { let start = Instant::now(); - let chain_tip = state.chain_tip(Finality::Committed).await; + let chain_tip = state.chain_tip(Finality::Committed); state .sync_chain_mmr(BlockNumber::from(current_client_block_height)..=chain_tip) .await diff --git a/crates/block-producer/src/mempool/tests.rs b/crates/block-producer/src/mempool/tests.rs index 160a1fe69..c5322b0a6 100644 --- a/crates/block-producer/src/mempool/tests.rs +++ b/crates/block-producer/src/mempool/tests.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::time::Duration; use miden_protocol::Word; use miden_protocol::block::{BlockHeader, BlockNumber}; @@ -143,18 +144,30 @@ async fn add_transaction_traces_are_correct() { let (mut uut, _) = Mempool::for_tests(); let txs = MockProvenTxBuilder::sequential(); + let tx_id = txs[0].id().to_string(); uut.add_transaction(txs[0].clone()).unwrap(); - let span_data = rx_export.recv().await.unwrap(); - assert_eq!(span_data.name, "mempool.add_transaction"); + // The exporter is global, so skip spans leaked by tests running concurrently on other threads + // by matching on this test's transaction ID. + let span_data = tokio::time::timeout(Duration::from_secs(30), async { + loop { + let span_data = rx_export.recv().await.unwrap(); + if span_data.name == "mempool.add_transaction" + && span_data + .attributes + .iter() + .any(|kv| kv.key == "transaction.id".into() && kv.value.to_string() == tx_id) + { + break span_data; + } + } + }) + .await + .expect("span for the added transaction should be exported"); + assert!(span_data.attributes.iter().any(|kv| kv.key == "code.module.name".into() && kv.value == "miden_node_block_producer::mempool".into())); - assert!( - span_data - .attributes - .iter() - .any(|kv| kv.key == "transaction.id".into() && kv.value.to_string().starts_with("0x")) - ); + assert!(tx_id.starts_with("0x")); } // BATCH FAILED TESTS diff --git a/crates/block-producer/src/proof_scheduler.rs b/crates/block-producer/src/proof_scheduler.rs index c050dc0a3..c455e92d4 100644 --- a/crates/block-producer/src/proof_scheduler.rs +++ b/crates/block-producer/src/proof_scheduler.rs @@ -119,7 +119,7 @@ pub(crate) async fn run( // Next block number to schedule. Initialized from the proven tip's child so we skip // already-proven blocks on restart. - let mut next_to_prove = state.chain_tip(Finality::Proven).await.child(); + let mut next_to_prove = state.chain_tip(Finality::Proven).child(); // Completed proofs waiting to be committed in order. let mut pending: BTreeMap> = BTreeMap::new(); @@ -145,7 +145,7 @@ pub(crate) async fn run( // Drain completed proofs in ascending order so the proven tip advances without // gaps. - let mut next = state.chain_tip(Finality::Proven).await.child(); + let mut next = state.chain_tip(Finality::Proven).child(); while let Some(proof_bytes) = pending.remove(&next) { state.apply_proof(next, proof_bytes).await?; next = next.child(); diff --git a/crates/block-producer/src/rpc_sync.rs b/crates/block-producer/src/rpc_sync.rs index c36c83b97..a67e9c4bd 100644 --- a/crates/block-producer/src/rpc_sync.rs +++ b/crates/block-producer/src/rpc_sync.rs @@ -120,7 +120,7 @@ impl BlockSync { err, )] async fn sync(&self, shutdown: CancellationToken) -> anyhow::Result<()> { - let block_from = self.state.chain_tip(Finality::Committed).await.child().as_u32(); + let block_from = self.state.chain_tip(Finality::Committed).child().as_u32(); info!(target: LOG_TARGET, block_from, "Connecting to upstream RPC for blocks"); let mut client = self.source_rpc.clone(); @@ -143,7 +143,7 @@ impl BlockSync { .context("failed to deserialize block from upstream")?; self.state.apply_block(block).await?; - let local_tip = self.state.chain_tip(Finality::Committed).await; + let local_tip = self.state.chain_tip(Finality::Committed); self.readiness.update(upstream_tip, local_tip).await; } } @@ -185,7 +185,7 @@ impl ProofSync { async fn sync(&self, shutdown: CancellationToken) -> anyhow::Result<()> { // Subscribe from next proven tip. - let starting_block = self.state.chain_tip(Finality::Proven).await.child(); + let starting_block = self.state.chain_tip(Finality::Proven).child(); info!( target: LOG_TARGET, block_from = %starting_block, diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index 284664c94..340c2d7d9 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -102,12 +102,12 @@ pub struct Sequencer { impl Sequencer { /// Spawns the sequencer tasks and returns its in-process API. - pub async fn spawn(self, shutdown: CancellationToken) -> Result { + pub fn spawn(self, shutdown: CancellationToken) -> Result { tracing::info!(target: LOG_TARGET, "Initializing sequencer"); let store = self.store; let validator = BlockProducerValidatorClient::new(self.validator_url.clone(), self.validator_timeout)?; - let chain_tip = store.chain_tip(Finality::Committed).await; + let chain_tip = store.chain_tip(Finality::Committed); tracing::info!(target: LOG_TARGET, "Sequencer initialized"); @@ -174,7 +174,7 @@ impl Sequencer { /// Executes in place (i.e. not spawned) and will run indefinitely until a fatal error is /// encountered. pub async fn serve(self) -> anyhow::Result<()> { - self.spawn(CancellationToken::new()).await?.wait().await + self.spawn(CancellationToken::new())?.wait().await } } diff --git a/crates/block-producer/src/server/tests.rs b/crates/block-producer/src/server/tests.rs index 75aabdeb8..6e376268c 100644 --- a/crates/block-producer/src/server/tests.rs +++ b/crates/block-producer/src/server/tests.rs @@ -40,7 +40,6 @@ async fn block_producer_starts_with_store_state() { batch_workers: DEFAULT_BATCH_WORKERS, } .spawn(miden_node_utils::shutdown::CancellationToken::new()) - .await .unwrap(); let status = block_producer.api().status().await; diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index 7884250a3..0f00769af 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -206,7 +206,7 @@ pub async fn get_tx_inputs( return Err(StoreError::DuplicateAccountIdPrefix(proven_tx.account_id())); } - let current_block_height = state.chain_tip(Finality::Committed).await; + let current_block_height = state.chain_tip(Finality::Committed); let tx_inputs = TransactionInputs::from_store_inputs( proven_tx.account_id(), store_inputs, diff --git a/crates/rpc/src/server/api.rs b/crates/rpc/src/server/api.rs index a45de8d94..f8b505f52 100644 --- a/crates/rpc/src/server/api.rs +++ b/crates/rpc/src/server/api.rs @@ -200,11 +200,11 @@ impl RpcService { /// /// Returns the chain tip so callers can reuse it (e.g. in the response's pagination info) /// without issuing a second query. - async fn range_bounds_check( + fn range_bounds_check( &self, range: &RangeInclusive, ) -> Result { - let chain_tip = self.store.chain_tip(Finality::Committed).await; + let chain_tip = self.store.chain_tip(Finality::Committed); if *range.end() > chain_tip { return Err(Status::invalid_argument(format!( "block_to ({}) is greater than chain tip ({chain_tip})", diff --git a/crates/rpc/src/server/api/status.rs b/crates/rpc/src/server/api/status.rs index 026e9c622..02ce3b9c6 100644 --- a/crates/rpc/src/server/api/status.rs +++ b/crates/rpc/src/server/api/status.rs @@ -48,7 +48,7 @@ impl proto::server::rpc_api::Status for RpcService { Ok(proto::rpc::RpcStatus { version: env!("CARGO_PKG_VERSION").to_string(), - chain_tip: self.store.chain_tip(Finality::Committed).await.as_u32(), + chain_tip: self.store.chain_tip(Finality::Committed).as_u32(), block_producer: block_producer_status.or(Some(proto::rpc::BlockProducerStatus { status: "unreachable".to_string(), version: "-".to_string(), diff --git a/crates/rpc/src/server/api/sync_account_storage_maps.rs b/crates/rpc/src/server/api/sync_account_storage_maps.rs index 623fa7ff5..8c1a9b184 100644 --- a/crates/rpc/src/server/api/sync_account_storage_maps.rs +++ b/crates/rpc/src/server/api/sync_account_storage_maps.rs @@ -58,7 +58,7 @@ impl proto::server::rpc_api::SyncAccountStorageMaps for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range).await?; + let chain_tip = self.range_bounds_check(&block_range)?; let storage_maps_page = self .store .sync_account_storage_maps(account_id, block_range) diff --git a/crates/rpc/src/server/api/sync_account_vault.rs b/crates/rpc/src/server/api/sync_account_vault.rs index 389f3a47e..7cd103ea4 100644 --- a/crates/rpc/src/server/api/sync_account_vault.rs +++ b/crates/rpc/src/server/api/sync_account_vault.rs @@ -58,7 +58,7 @@ impl proto::server::rpc_api::SyncAccountVault for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range).await?; + let chain_tip = self.range_bounds_check(&block_range)?; let (last_included_block, updates) = self .store .sync_account_vault(account_id, block_range) diff --git a/crates/rpc/src/server/api/sync_chain_mmr.rs b/crates/rpc/src/server/api/sync_chain_mmr.rs index 130e0d8ab..4477c5473 100644 --- a/crates/rpc/src/server/api/sync_chain_mmr.rs +++ b/crates/rpc/src/server/api/sync_chain_mmr.rs @@ -41,9 +41,9 @@ impl proto::server::rpc_api::SyncChainMmr for RpcService { let current_client_block_height = BlockNumber::from(request.current_client_block_height); let sync_target = match request.finality_level() { proto::rpc::FinalityLevel::Committed | proto::rpc::FinalityLevel::Unspecified => { - self.store.chain_tip(Finality::Committed).await + self.store.chain_tip(Finality::Committed) }, - proto::rpc::FinalityLevel::Proven => self.store.chain_tip(Finality::Proven).await, + proto::rpc::FinalityLevel::Proven => self.store.chain_tip(Finality::Proven), }; if current_client_block_height > sync_target { diff --git a/crates/rpc/src/server/api/sync_notes.rs b/crates/rpc/src/server/api/sync_notes.rs index 411d653b7..11ff99a76 100644 --- a/crates/rpc/src/server/api/sync_notes.rs +++ b/crates/rpc/src/server/api/sync_notes.rs @@ -47,7 +47,7 @@ impl proto::server::rpc_api::SyncNotes for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range).await?; + let chain_tip = self.range_bounds_check(&block_range)?; let (results, last_block_checked) = self .store diff --git a/crates/rpc/src/server/api/sync_nullifiers.rs b/crates/rpc/src/server/api/sync_nullifiers.rs index bcd05eaab..0097aecf2 100644 --- a/crates/rpc/src/server/api/sync_nullifiers.rs +++ b/crates/rpc/src/server/api/sync_nullifiers.rs @@ -58,7 +58,7 @@ impl proto::server::rpc_api::SyncNullifiers for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range).await?; + let chain_tip = self.range_bounds_check(&block_range)?; let (nullifiers, block_num) = self .store diff --git a/crates/rpc/src/server/api/sync_transactions.rs b/crates/rpc/src/server/api/sync_transactions.rs index 489e2a483..26e103a53 100644 --- a/crates/rpc/src/server/api/sync_transactions.rs +++ b/crates/rpc/src/server/api/sync_transactions.rs @@ -61,7 +61,7 @@ impl proto::server::rpc_api::SyncTransactions for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range).await?; + let chain_tip = self.range_bounds_check(&block_range)?; let account_ids = read_account_ids::(request.account_ids)?; let (last_block_included, transaction_records_db) = self .store diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 8cc1a9c05..1aa2509a4 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -19,6 +19,7 @@ doctest = false [dependencies] anyhow = { workspace = true } +arc-swap = { workspace = true } deadpool = { features = ["managed", "rt_tokio_1"], workspace = true } deadpool-diesel = { features = ["sqlite"], workspace = true } diesel = { features = ["numeric", "sqlite"], workspace = true } diff --git a/crates/store/src/account_state_forest/mod.rs b/crates/store/src/account_state_forest/mod.rs index 89a911cd2..17073f996 100644 --- a/crates/store/src/account_state_forest/mod.rs +++ b/crates/store/src/account_state_forest/mod.rs @@ -3,7 +3,7 @@ use std::num::NonZeroUsize; use miden_crypto::hash::rpo::Rpo256; #[cfg(feature = "rocksdb")] use miden_crypto::merkle::smt::ForestPersistentBackend; -use miden_crypto::merkle::smt::{Backend, ForestInMemoryBackend}; +use miden_crypto::merkle::smt::{Backend, BackendReader, ForestInMemoryBackend}; use miden_node_proto::domain::account::{ AccountStorageMapDetails, AccountVaultDetails, @@ -69,6 +69,9 @@ pub(crate) type AccountStateForestBackend = ForestPersistentBackend; #[cfg(not(feature = "rocksdb"))] pub(crate) type AccountStateForestBackend = ForestInMemoryBackend; +/// The read-only snapshot backend for the forest, used by in-memory state snapshots. +pub(crate) type AccountStateForestBackendReader = ::Reader; + const fn empty_smt_root() -> Word { *EmptySubtreeRoots::entry(SMT_DEPTH, 0) } @@ -85,7 +88,7 @@ pub enum AccountStorageMapResult { } /// Container for forest-related state that needs to be updated atomically. -pub(crate) struct AccountStateForest { +pub(crate) struct AccountStateForest { /// `LargeSmtForest` for efficient account storage reconstruction. Populated during block import /// with storage and vault SMTs. forest: LargeSmtForest, @@ -124,7 +127,7 @@ impl AccountStateForest { } } -impl AccountStateForest { +impl AccountStateForest { pub(crate) fn from_backend(backend: B) -> Result { Ok(Self { forest: LargeSmtForest::new(backend)?, @@ -195,17 +198,6 @@ impl AccountStateForest { .collect() } - fn cache_hashed_keys_from_patch(&mut self, patch: &AccountPatch) { - let raw_keys = patch.storage().maps().flat_map(|(_slot_name, map_patch)| { - map_patch.entries().into_iter().flat_map(|e| e.as_map().keys().copied()) - }); - self.cache_storage_map_keys(raw_keys); - - let raw_keys = patch.vault().iter().map(|(vault_key, _)| *vault_key); - self.vault_key_cache - .put_many(raw_keys.into_iter().map(|raw_key| (raw_key.hash(), raw_key))); - } - pub(crate) fn cache_storage_map_keys(&self, raw_keys: impl IntoIterator) { self.storage_map_key_cache .put_many(raw_keys.into_iter().map(|raw_key| (raw_key.hash(), raw_key))); @@ -216,30 +208,6 @@ impl AccountStateForest { self.storage_map_key_cache.clear(); } - fn apply_forest_updates( - &mut self, - lineage: LineageId, - block_num: BlockNumber, - operations: Vec, - ) -> Word { - let updates = if operations.is_empty() { - SmtUpdateBatch::empty() - } else { - SmtUpdateBatch::new(operations.into_iter()) - }; - let version = block_num.as_u64(); - let tree = if self.forest.latest_version(lineage).is_some() { - self.forest - .update_tree(lineage, version, updates) - .expect("forest update should succeed") - } else { - self.forest - .add_lineage(lineage, version, updates) - .expect("forest update should succeed") - }; - tree.root() - } - fn map_forest_error(error: LargeSmtForestError) -> MerkleError { match error { LargeSmtForestError::Merkle(merkle) => merkle, @@ -481,6 +449,72 @@ impl AccountStateForest { ))) } + /// Retrieves the most recent vault SMT root for an account. If no vault root is found for the + /// account, returns an empty SMT root. + pub(crate) fn get_latest_vault_root(&self, account_id: AccountId) -> Word { + let lineage = Self::vault_lineage_id(account_id); + self.forest.latest_root(lineage).unwrap_or_else(empty_smt_root) + } + + /// Retrieves the most recent storage map SMT root for an account slot. + pub(crate) fn get_latest_storage_map_root( + &self, + account_id: AccountId, + slot_name: &StorageSlotName, + ) -> Word { + let lineage = Self::storage_lineage_id(account_id, slot_name); + self.forest.latest_root(lineage).unwrap_or_else(empty_smt_root) + } +} + +impl AccountStateForest { + /// Returns a read-only snapshot of this forest backed by a reader view of the backend. + /// + /// The reverse-key caches are shallow clones sharing the same underlying storage, so cache + /// entries added by the writer are visible to snapshot readers and vice versa. + pub(crate) fn reader(&self) -> Result, LargeSmtForestError> { + Ok(AccountStateForest { + forest: self.forest.reader()?, + storage_map_key_cache: self.storage_map_key_cache.clone(), + vault_key_cache: self.vault_key_cache.clone(), + }) + } + + fn cache_hashed_keys_from_patch(&mut self, patch: &AccountPatch) { + let raw_keys = patch.storage().maps().flat_map(|(_slot_name, map_patch)| { + map_patch.entries().into_iter().flat_map(|e| e.as_map().keys().copied()) + }); + self.cache_storage_map_keys(raw_keys); + + let raw_keys = patch.vault().iter().map(|(vault_key, _)| *vault_key); + self.vault_key_cache + .put_many(raw_keys.into_iter().map(|raw_key| (raw_key.hash(), raw_key))); + } + + fn apply_forest_updates( + &mut self, + lineage: LineageId, + block_num: BlockNumber, + operations: Vec, + ) -> Word { + let updates = if operations.is_empty() { + SmtUpdateBatch::empty() + } else { + SmtUpdateBatch::new(operations.into_iter()) + }; + let version = block_num.as_u64(); + let tree = if self.forest.latest_version(lineage).is_some() { + self.forest + .update_tree(lineage, version, updates) + .expect("forest update should succeed") + } else { + self.forest + .add_lineage(lineage, version, updates) + .expect("forest update should succeed") + }; + tree.root() + } + // PUBLIC INTERFACE // -------------------------------------------------------------------------------------------- @@ -554,13 +588,6 @@ impl AccountStateForest { // ASSET VAULT DELTA PROCESSING // -------------------------------------------------------------------------------------------- - /// Retrieves the most recent vault SMT root for an account. If no vault root is found for the - /// account, returns an empty SMT root. - pub(crate) fn get_latest_vault_root(&self, account_id: AccountId) -> Word { - let lineage = Self::vault_lineage_id(account_id); - self.forest.latest_root(lineage).unwrap_or_else(empty_smt_root) - } - /// Inserts asset vault data into the forest for the specified account. Assumes that asset vault /// for this account does not yet exist in the forest. /// @@ -717,16 +744,6 @@ impl AccountStateForest { // STORAGE MAP DELTA PROCESSING // -------------------------------------------------------------------------------------------- - /// Retrieves the most recent storage map SMT root for an account slot. - pub(crate) fn get_latest_storage_map_root( - &self, - account_id: AccountId, - slot_name: &StorageSlotName, - ) -> Word { - let lineage = Self::storage_lineage_id(account_id, slot_name); - self.forest.latest_root(lineage).unwrap_or_else(empty_smt_root) - } - /// Updates the forest with storage map changes from a patch. /// /// # Returns diff --git a/crates/store/src/accounts/mod.rs b/crates/store/src/accounts/mod.rs index 560e208bc..9b4e10dbd 100644 --- a/crates/store/src/accounts/mod.rs +++ b/crates/store/src/accounts/mod.rs @@ -16,6 +16,7 @@ use miden_protocol::crypto::merkle::smt::{ SMT_DEPTH, SmtLeaf, SmtStorage, + SmtStorageReader, }; use miden_protocol::crypto::merkle::{ EmptySubtreeRoots, @@ -118,7 +119,7 @@ impl HistoricalOverlay { /// reversion data (mutations that undo changes). Historical witnesses are reconstructed /// by starting from the latest state and applying reversion overlays backwards in time. #[derive(Debug)] -pub struct AccountTreeWithHistory { +pub struct AccountTreeWithHistory { /// The current block number (latest state). block_number: BlockNumber, /// The latest account tree state. @@ -127,7 +128,7 @@ pub struct AccountTreeWithHistory { overlays: BTreeMap, } -impl AccountTreeWithHistory { +impl AccountTreeWithHistory { /// Maximum number of historical blocks to maintain. pub const MAX_HISTORY: usize = 50; @@ -355,7 +356,9 @@ impl AccountTreeWithHistory { let path = SparseMerklePath::try_from(path).ok()?; Some((path, leaf)) } +} +impl AccountTreeWithHistory { // PUBLIC MUTATORS // -------------------------------------------------------------------------------------------- @@ -409,4 +412,14 @@ impl AccountTreeWithHistory { Ok(()) } + + /// Returns a read-only snapshot of this tree backed by a reader view of the storage. + pub fn reader(&self) -> AccountTreeWithHistory { + let latest = self.latest.reader().expect("snapshot creation should not fail"); + AccountTreeWithHistory { + block_number: self.block_number, + latest, + overlays: self.overlays.clone(), + } + } } diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 745a8855b..ac8cbdff9 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -27,7 +27,6 @@ use miden_protocol::note::{ }; use miden_protocol::transaction::TransactionHeader; use miden_protocol::utils::serde::Deserializable; -use tokio::sync::oneshot; use tracing::info; use crate::db::migrations::{migrate_database, verify_latest_schema}; @@ -567,8 +566,8 @@ impl Db { /// Inserts the data of a new block into the DB. /// - /// `allow_acquire` and `acquire_done` are used to synchronize writes to the DB with writes to - /// the in-memory trees. Further details available on [`super::state::State::apply_block`]. + /// The transaction is committed when this method returns. Synchronization with the in-memory + /// trees is handled by the block writer task; see [`super::state::State::apply_block`]. // TODO: This span is logged in a root span, we should connect it to the parent one. #[miden_instrument( target = COMPONENT, @@ -577,29 +576,12 @@ impl Db { )] pub async fn apply_block( &self, - allow_acquire: oneshot::Sender<()>, - acquire_done: oneshot::Receiver<()>, signed_block: SignedBlock, notes: Vec<(NoteRecord, Option)>, ) -> Result<()> { self.transact("apply block", move |conn| -> Result<()> { models::queries::apply_block(conn, &signed_block, ¬es)?; - - // XXX FIXME TODO free floating mutex MUST NOT exist it doesn't bind it properly to the - // data locked! - { - let _span = tracing::info_span!(target: COMPONENT, "acquire_write_lock").entered(); - if allow_acquire.send(()).is_err() { - tracing::warn!(target: COMPONENT, "failed to send notification for successful block application, potential deadlock"); - } - } - models::queries::prune_history(conn, signed_block.header().block_num())?; - - let _span = - tracing::info_span!(target: COMPONENT, "acquire_done_lock").entered(); - acquire_done.blocking_recv()?; - Ok(()) }) .await diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index c74c1298f..ecff93422 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -205,14 +205,14 @@ pub enum ApplyBlockError { // OTHER ERRORS // --------------------------------------------------------------------------------------------- - #[error("block applying was cancelled because of closed channel on database side")] + #[error("block applying was cancelled because the writer task dropped the result channel")] ClosedChannel(#[from] RecvError), - #[error("concurrent write detected")] - ConcurrentWrite, #[error("database doesn't have any block header data")] DbBlockHeaderEmpty, #[error("database update failed: {0}")] DbUpdateTaskFailed(String), + #[error("failed to send block to the writer task: {0}")] + WriterTaskSendFailed(String), } #[derive(Error, Debug)] diff --git a/crates/store/src/state/account.rs b/crates/store/src/state/account.rs index 347c43937..490dcd70b 100644 --- a/crates/store/src/state/account.rs +++ b/crates/store/src/state/account.rs @@ -25,7 +25,6 @@ use miden_protocol::account::{ }; use miden_protocol::block::BlockNumber; use miden_protocol::block::account_tree::AccountWitness; -use tracing::Instrument; use super::State; use crate::COMPONENT; @@ -156,11 +155,13 @@ impl State { let keys = assets.iter().map(miden_protocol::asset::Asset::id); - let forest = self.forest.write().await; - - forest - .vault_key_cache - .put_many(keys.into_iter().map(|raw_key| (raw_key.hash(), raw_key))); + // The reverse-key caches are shared between the writer and all snapshots, so caching via + // the current snapshot's forest is visible everywhere. + self.with_forest_read_blocking(|forest| { + forest + .vault_key_cache + .put_many(keys.into_iter().map(|raw_key| (raw_key.hash(), raw_key))); + }); Ok(AccountVaultDetails::from_assets(assets)) } @@ -183,10 +184,9 @@ impl State { .await?; if let StorageMapEntries::AllEntries(entries) = &details.entries { - self.forest - .write() - .await - .cache_storage_map_keys(entries.iter().map(|(raw_key, _)| *raw_key)); + self.with_forest_read_blocking(|forest| { + forest.cache_storage_map_keys(entries.iter().map(|(raw_key, _)| *raw_key)); + }); } Ok(details) @@ -225,8 +225,7 @@ impl State { // Validate block exists in the blockchain before querying the database { - let inner = self.inner.read().instrument(tracing::info_span!("acquire_inner")).await; - let latest_block_num = inner.latest_block_num(); + let latest_block_num = self.snapshot().latest_block_num(); if block_num > latest_block_num { return Err(GetAccountError::UnknownBlock(block_num)); diff --git a/crates/store/src/state/apply_block.rs b/crates/store/src/state/apply_block.rs index e2adf4e70..3e63bd199 100644 --- a/crates/store/src/state/apply_block.rs +++ b/crates/store/src/state/apply_block.rs @@ -1,24 +1,12 @@ -use std::sync::Arc; - use miden_node_proto::domain::proof_request::BlockProofRequest; -use miden_node_utils::ErrorReport; -use miden_node_utils::tracing::{miden_instrument, miden_span_record}; -use miden_protocol::Word; -use miden_protocol::account::AccountUpdateDetails; +use miden_node_utils::tracing::miden_instrument; use miden_protocol::batch::OrderedBatches; -use miden_protocol::block::account_tree::AccountMutationSet; -use miden_protocol::block::nullifier_tree::NullifierMutationSet; -use miden_protocol::block::{BlockBody, BlockHeader, BlockInputs, BlockNumber, SignedBlock}; -use miden_protocol::note::{NoteDetails, Nullifier}; -use miden_protocol::transaction::OutputNote; +use miden_protocol::block::{BlockInputs, BlockNumber, SignedBlock}; use miden_protocol::utils::serde::Serializable; -use tokio::sync::oneshot; -use tracing::{Instrument, info_span}; -use crate::db::NoteRecord; -use crate::errors::{ApplyBlockError, ApplyBlockWithProvingInputsError, InvalidBlockError}; -use crate::state::{BlockNotification, State}; -use crate::{COMPONENT, HistoricalError, LOG_TARGET}; +use crate::COMPONENT; +use crate::errors::ApplyBlockWithProvingInputsError; +use crate::state::State; impl State { /// Saves proving inputs for a signed block and applies it to the state. @@ -53,160 +41,6 @@ impl State { .map_err(ApplyBlockWithProvingInputsError::ApplyBlock) } - /// Apply changes of a new block to the DB and in-memory data structures. - /// - /// ## Note on state consistency - /// - /// The server contains in-memory representations of the existing trees, the in-memory - /// representation must be kept consistent with the committed data, this is necessary so to - /// provide consistent results for all endpoints. In order to achieve consistency, the - /// following steps are used: - /// - /// - the request data is validated, prior to starting any modifications. - /// - block is being saved into the store in parallel with updating the DB, but before - /// committing. This block is considered as candidate and not yet available for reading - /// because the latest block pointer is not updated yet. - /// - a transaction is open in the DB and the writes are started. - /// - while the transaction is not committed, concurrent reads are allowed, both the DB and the - /// in-memory representations, which are consistent at this stage. - /// - prior to committing the changes to the DB, an exclusive lock to the in-memory data is - /// acquired, preventing concurrent reads to the in-memory data, since that will be - /// out-of-sync w.r.t. the DB. - /// - the DB transaction is committed, and requests that read only from the DB can proceed to - /// use the fresh data. - /// - the in-memory structures are updated, including the latest block pointer and the lock is - /// released. - // TODO: This span is logged in a root span, we should connect it to the parent span. - #[miden_instrument( - target = COMPONENT, - skip_all, - err, - )] - pub async fn apply_block(&self, signed_block: SignedBlock) -> Result<(), ApplyBlockError> { - let _lock = self.writer.try_lock().map_err(|_| ApplyBlockError::ConcurrentWrite)?; - - let header = signed_block.header(); - let body = signed_block.body(); - - let block_num = header.block_num(); - let block_commitment = header.commitment(); - let num_transactions = body.transactions().as_slice().len(); - - miden_span_record!( - block.number = %block_num, - block.commitment = %block_commitment, - block.transactions.count = num_transactions, - ); - - self.validate_block_header(header, body).await?; - - // Save the block to the block store. In a case of a rolled-back DB transaction, the - // in-memory state will be unchanged, but the file might still be written. Such blocks - // should be considered candidates, not finalized blocks. - let signed_block_bytes = signed_block.to_bytes(); - // Clone before moving into the block-save task so we can cache for replicas at commit. - let cache_bytes = signed_block_bytes.clone(); - let store = Arc::clone(&self.block_store); - let block_save_task = tokio::spawn( - async move { store.save_block(block_num, &signed_block_bytes).await }.in_current_span(), - ); - - let ( - nullifier_tree_old_root, - nullifier_tree_update, - account_tree_old_root, - account_tree_update, - ) = self.compute_tree_mutations(header, body).await?; - - let notes = Self::build_note_records(header, body)?; - - // Signals the transaction is ready to be committed, and the write lock can be acquired. - let (allow_acquire, acquired_allowed) = oneshot::channel::<()>(); - // Signals the write lock has been acquired, and the transaction can be committed. - let (inform_acquire_done, acquire_done) = oneshot::channel::<()>(); - - // Extract public account updates with patches before block is moved into async task. - // Private accounts are filtered out since they don't expose their state changes. - let account_patches = - Vec::from_iter(body.updated_accounts().iter().filter_map( - |update| match update.details() { - AccountUpdateDetails::Public(patch) => Some(patch.clone()), - AccountUpdateDetails::Private => None, - }, - )); - - // The DB and in-memory state updates need to be synchronized and are partially overlapping. - // Namely, the DB transaction only proceeds after this task acquires the in-memory write - // lock. This requires the DB update to run concurrently, so a new task is spawned. - let db = Arc::clone(&self.db); - let db_update_task = tokio::spawn( - async move { db.apply_block(allow_acquire, acquire_done, signed_block, notes).await } - .in_current_span(), - ); - - // Wait for the message from the DB update task, that we ready to commit the DB transaction. - acquired_allowed - .instrument(info_span!(target: COMPONENT, "await_db_readiness")) - .await - .map_err(ApplyBlockError::ClosedChannel)?; - - // Awaiting the block saving task to complete without errors. - block_save_task.await??; - - self.with_inner_write_blocking(|inner| { - // We need to check that neither the nullifier tree nor the account tree have changed - // while we were waiting for the DB preparation task to complete. If either of them did - // change, we do not proceed with in-memory and database updates, since it may lead to - // an inconsistent state. - if inner.nullifier_tree.root() != nullifier_tree_old_root - || inner.account_tree.root_latest() != account_tree_old_root - { - return Err(ApplyBlockError::ConcurrentWrite); - } - - // Notify the DB update task that the write lock has been acquired, so it can commit the - // DB transaction. - inform_acquire_done - .send(()) - .map_err(|_| ApplyBlockError::DbUpdateTaskFailed("Receiver was dropped".into()))?; - - // TODO: shutdown #91 Await for successful commit of the DB transaction. If the commit - // fails, we mustn't change in-memory state, so we return a block applying error and - // don't proceed with in-memory updates. - tokio::runtime::Handle::current() - .block_on(db_update_task)? - .map_err(|err| ApplyBlockError::DbUpdateTaskFailed(err.as_report()))?; - - // Update the in-memory data structures after successful commit of the DB transaction - inner - .nullifier_tree - .apply_mutations(nullifier_tree_update) - .expect("Unreachable: old nullifier tree root must be checked before this step"); - inner - .account_tree - .apply_mutations(account_tree_update) - .expect("Unreachable: old account tree root must be checked before this step"); - - inner.blockchain.push(block_commitment); - - Ok(()) - })?; - - self.with_forest_write_blocking(|forest| { - forest.apply_block_updates(block_num, account_patches); - }); - - // Push to cache and notify replica subscribers. - self.block_cache - .push(block_num, BlockNotification::new(block_num, cache_bytes)) - .expect("block cache receives sequential block numbers"); - let _ = self.committed_tip_tx.send(block_num); - - tracing::debug!(target: LOG_TARGET, "Block applied"); - - Ok(()) - } - /// Saves the proving inputs for the given block to the block store. pub async fn save_proving_inputs( &self, @@ -217,169 +51,4 @@ impl State { .save_proving_inputs(block_num, &proving_inputs.to_bytes()) .await } - - /// Validates that the block header is consistent with the block body and the current state. - #[miden_instrument( - target = COMPONENT, - skip_all, - err, - )] - async fn validate_block_header( - &self, - header: &BlockHeader, - body: &BlockBody, - ) -> Result<(), ApplyBlockError> { - // Validate that header and body match. - let tx_commitment = body.transactions().commitment(); - if header.tx_commitment() != tx_commitment { - return Err(InvalidBlockError::InvalidBlockTxCommitment { - expected: tx_commitment, - actual: header.tx_commitment(), - } - .into()); - } - - let block_num = header.block_num(); - - // Validate that the applied block is the next block in sequence. - let prev_block = self - .db - .select_block_header_by_block_num(None) - .await? - .ok_or(ApplyBlockError::DbBlockHeaderEmpty)?; - let expected_block_num = prev_block.block_num().child(); - if block_num != expected_block_num { - return Err(InvalidBlockError::NewBlockInvalidBlockNum { - expected: expected_block_num, - submitted: block_num, - } - .into()); - } - if header.prev_block_commitment() != prev_block.commitment() { - return Err(InvalidBlockError::NewBlockInvalidPrevCommitment.into()); - } - - Ok(()) - } - - /// Computes nullifier and account tree mutations, validating roots against the block header. - #[miden_instrument( - target = COMPONENT, - skip_all, - err, - )] - async fn compute_tree_mutations( - &self, - header: &BlockHeader, - body: &BlockBody, - ) -> Result<(Word, NullifierMutationSet, Word, AccountMutationSet), ApplyBlockError> { - self.with_inner_read_blocking(|inner| { - let block_num = header.block_num(); - - // nullifiers can be produced only once - let duplicate_nullifiers: Vec<_> = body - .created_nullifiers() - .iter() - .filter(|&nullifier| inner.nullifier_tree.get_block_num(nullifier).is_some()) - .copied() - .collect(); - if !duplicate_nullifiers.is_empty() { - return Err(InvalidBlockError::DuplicatedNullifiers(duplicate_nullifiers).into()); - } - - // new_block.chain_root must be equal to the chain MMR root prior to the update - let peaks = inner.blockchain.peaks(); - if peaks.hash_peaks() != header.chain_commitment() { - return Err(InvalidBlockError::NewBlockInvalidChainCommitment.into()); - } - - // compute update for nullifier tree - let nullifier_tree_update = inner - .nullifier_tree - .compute_mutations( - body.created_nullifiers().iter().map(|nullifier| (*nullifier, block_num)), - ) - .map_err(InvalidBlockError::NewBlockNullifierAlreadySpent)?; - - if nullifier_tree_update.as_mutation_set().root() != header.nullifier_root() { - return Err(InvalidBlockError::NewBlockInvalidNullifierRoot.into()); - } - - // compute update for account tree - let account_tree_update = inner - .account_tree - .compute_mutations( - body.updated_accounts() - .iter() - .map(|update| (update.account_id(), update.final_state_commitment())), - ) - .map_err(|e| match e { - HistoricalError::AccountTreeError(err) => { - InvalidBlockError::NewBlockDuplicateAccountIdPrefix(err) - }, - HistoricalError::MerkleError(_) => { - panic!("Unexpected MerkleError during account tree mutation computation") - }, - })?; - - if account_tree_update.as_mutation_set().root() != header.account_root() { - return Err(InvalidBlockError::NewBlockInvalidAccountRoot.into()); - } - - Ok(( - inner.nullifier_tree.root(), - nullifier_tree_update, - inner.account_tree.root_latest(), - account_tree_update, - )) - }) - } - - /// Builds note records with inclusion proofs from the block body. - #[miden_instrument( - target = COMPONENT, - skip_all, - err, - )] - fn build_note_records( - header: &BlockHeader, - body: &BlockBody, - ) -> Result)>, ApplyBlockError> { - let block_num = header.block_num(); - - let note_tree = body.compute_block_note_tree(); - if note_tree.root() != header.note_root() { - return Err(InvalidBlockError::NewBlockInvalidNoteRoot.into()); - } - - let notes = body - .output_notes() - .map(|(note_index, note)| { - let (details, attachments, nullifier) = match note { - OutputNote::Public(public) => ( - Some(NoteDetails::from(public.as_note())), - public.as_note().attachments().clone(), - Some(public.as_note().nullifier()), - ), - OutputNote::Private(private) => (None, private.attachments().clone(), None), - }; - - let inclusion_path = note_tree.open(note_index); - - let note_record = NoteRecord { - block_num, - note_index, - note_id: note.id().as_word(), - metadata: *note.metadata(), - details, - attachments, - inclusion_path, - }; - - Ok((note_record, nullifier)) - }) - .collect::, InvalidBlockError>>()?; - - Ok(notes) - } } diff --git a/crates/store/src/state/apply_proof.rs b/crates/store/src/state/apply_proof.rs index e6db3e0dd..5f8f53260 100644 --- a/crates/store/src/state/apply_proof.rs +++ b/crates/store/src/state/apply_proof.rs @@ -32,7 +32,7 @@ impl State { "out-of-sequence proof: expected block {expected}, got {block_num}", ); - let committed_tip = self.chain_tip(Finality::Committed).await; + let committed_tip = self.chain_tip(Finality::Committed); ensure!( block_num <= committed_tip, "proof for uncommitted block {block_num} exceeds committed tip {committed_tip}", diff --git a/crates/store/src/state/loader.rs b/crates/store/src/state/loader.rs index d573c9b4a..dd669706c 100644 --- a/crates/store/src/state/loader.rs +++ b/crates/store/src/state/loader.rs @@ -74,6 +74,9 @@ pub type TreeStorage = RocksDbStorage; #[cfg(not(feature = "rocksdb"))] pub type TreeStorage = MemoryStorage; +/// The read-only snapshot storage backend for trees, used by in-memory state snapshots. +pub type TreeStorageReader = ::Reader; + // ERROR CONVERSION // ================================================================================================ diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index b7f39fde8..fb70b5cc0 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -8,7 +8,9 @@ use std::num::NonZeroUsize; use std::path::{Path, PathBuf}; use std::sync::Arc; +use arc_swap::ArcSwap; use miden_node_proto::domain::batch::BatchInputs; +use miden_node_utils::ErrorReport; use miden_node_utils::clap::StorageOptions; use miden_node_utils::formatting::format_array; use miden_node_utils::tracing::miden_instrument; @@ -18,13 +20,17 @@ use miden_protocol::block::account_tree::AccountWitness; use miden_protocol::block::nullifier_tree::{NullifierTree, NullifierWitness}; use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain}; use miden_protocol::crypto::merkle::mmr::{MmrProof, PartialMmr}; -use miden_protocol::crypto::merkle::smt::{LargeSmt, SmtStorage}; +use miden_protocol::crypto::merkle::smt::LargeSmt; use miden_protocol::note::{NoteId, NoteScript, Nullifier}; use miden_protocol::transaction::PartialBlockchain; -use tokio::sync::{Mutex, RwLock, watch}; -use tracing::{Instrument, Span}; +use tokio::sync::watch; +use tracing::Span; -use crate::account_state_forest::{AccountStateForest, AccountStateForestBackend}; +use crate::account_state_forest::{ + AccountStateForest, + AccountStateForestBackend, + AccountStateForestBackendReader, +}; use crate::accounts::AccountTreeWithHistory; use crate::blocks::BlockStore; use crate::db::{Db, NoteRecord, NullifierInfo}; @@ -52,6 +58,7 @@ use loader::{ NULLIFIER_TREE_STORAGE_DIR, TreeStorage, TreeStorageLoader, + TreeStorageReader, load_mmr, verify_account_state_forest_consistency, verify_tree_consistency, @@ -67,6 +74,8 @@ mod apply_proof; mod bootstrap; mod disk_monitor; mod sync_state; +mod writer; +use writer::{BlockWriter, WriteHandle}; // FINALITY // ================================================================================================ @@ -98,17 +107,19 @@ type BlockInputWitnesses = ( PartialMmr, ); -/// Container for state that needs to be updated atomically. -struct InnerState -where - S: SmtStorage, -{ - nullifier_tree: NullifierTree>, +/// Immutable snapshot of the in-memory tree state published after each committed block. +/// +/// The trees are backed by read-only snapshot storage ([`TreeStorageReader`] / +/// [`AccountStateForestBackendReader`]), so any number of readers can access the data concurrently +/// without holding a lock and without blocking the writer. +pub(crate) struct InMemoryState { + nullifier_tree: NullifierTree>, blockchain: Blockchain, - account_tree: AccountTreeWithHistory, + account_tree: AccountTreeWithHistory, + forest: AccountStateForest, } -impl InnerState { +impl InMemoryState { /// Returns the latest block number. fn latest_block_num(&self) -> BlockNumber { self.blockchain @@ -132,24 +143,22 @@ pub struct State { /// The block store which stores full block contents for all blocks. block_store: Arc, - /// Read-write lock used to prevent writing to a structure while it is being used. + /// Atomically swappable pointer to the latest in-memory state snapshot. /// - /// The lock is writer-preferring, meaning the writer won't be starved. - inner: RwLock>, - - /// Forest-related state `(SmtForest, storage_map_roots, vault_roots)` with its own lock. - forest: RwLock>, + /// Readers load the snapshot wait-free via [`ArcSwap::load_full`]; the [`BlockWriter`] task + /// atomically replaces the pointer after each committed block. Readers holding an old snapshot + /// are unaffected by the swap. + in_memory: Arc>, - /// To allow readers to access the tree data while an update in being performed, and prevent - /// TOCTOU issues, there must be no concurrent writers. This locks to serialize the writers. - writer: Mutex<()>, + /// Handle for sending block-write requests to the [`BlockWriter`] task. + write_handle: WriteHandle, /// The latest proven-in-sequence block number, updated by the proof scheduler or `apply_proof`. proven_tip: ProvenTipWriter, /// Watch sender fired after each block is committed. Replicas subscribe via /// `subscribe_committed_tip()` to be woken when new blocks arrive. - committed_tip_tx: watch::Sender, + committed_tip_tx: Arc>, /// FIFO cache of recent committed blocks for replica subscriptions. When a subscriber needs a /// block that has been evicted, it falls back to loading from the block store. @@ -246,10 +255,6 @@ impl State { let forest = forest_backend.load_account_state_forest(&mut db, latest_block_num).await?; verify_account_state_forest_consistency(&forest, &mut db).await?; - let inner = RwLock::new(InnerState { nullifier_tree, blockchain, account_tree }); - - let forest = RwLock::new(forest); - let writer = Mutex::new(()); let db = Arc::new(db); // Initialize the proven tip from the block store. @@ -260,18 +265,53 @@ impl State { // Committed-tip watch: fires after each successful apply_block. let (committed_tip_tx, _rx) = watch::channel(latest_block_num); + let committed_tip_tx = Arc::new(committed_tip_tx); + + let block_cache = BlockCache::new(BLOCK_CACHE_CAPACITY); + let proof_cache = ProofCache::new(PROOF_CACHE_CAPACITY); + + // Create the initial snapshot from reader views of the just-loaded trees. + let initial_snapshot = Arc::new(InMemoryState { + nullifier_tree: nullifier_tree + .reader() + .map_err(|e| StateInitializationError::NullifierTreeIoError(e.as_report()))?, + account_tree: account_tree.reader(), + forest: forest + .reader() + .map_err(|e| StateInitializationError::AccountStateForestIoError(e.as_report()))?, + blockchain: blockchain.clone(), + }); + let in_memory = Arc::new(ArcSwap::from(initial_snapshot)); + + // Spawn the block writer task. It owns the writable trees and processes write requests + // serially, publishing a new snapshot after each committed block. The task exits when all + // write handles are dropped. + let (write_tx, write_rx) = tokio::sync::mpsc::channel(1); + let write_handle = WriteHandle::new(write_tx); + let block_writer = BlockWriter { + db: Arc::clone(&db), + block_store: Arc::clone(&block_store), + in_memory: Arc::clone(&in_memory), + committed_tip_tx: Arc::clone(&committed_tip_tx), + block_cache: block_cache.clone(), + rx: write_rx, + nullifier_tree, + account_tree, + blockchain, + forest, + }; + tokio::spawn(block_writer.run()); Ok(Self { data_directory: data_path.to_path_buf(), db, block_store, - inner, - forest, - writer, + in_memory, + write_handle, proven_tip, committed_tip_tx, - block_cache: BlockCache::new(BLOCK_CACHE_CAPACITY), - proof_cache: ProofCache::new(PROOF_CACHE_CAPACITY), + block_cache, + proof_cache, }) } @@ -293,69 +333,42 @@ impl State { self.proven_tip.subscribe() } - // HELPER FUNCTIONS TO AVOID BLOCKING CALLS IN ASYNC CONTEXT + // SNAPSHOT HELPERS // -------------------------------------------------------------------------------------------- - /// Runs a synchronous read-only operation over the inner state on Tokio's blocking path. + /// Returns the current in-memory state snapshot (wait-free, no lock required). /// - /// The account and nullifier trees may be backed by `RocksDB`, so tree access must not run on - /// an async worker thread directly. This helper preserves the current tracing span while - /// moving the blocking lock acquisition and closure body into `block_in_place`. - fn with_inner_read_blocking(&self, f: impl FnOnce(&InnerState) -> R) -> R { - let span = Span::current(); - tokio::task::block_in_place(|| { - span.in_scope(|| { - let inner = self.inner.blocking_read(); - f(&inner) - }) - }) + /// The returned snapshot is a frozen view: it is unaffected if the writer publishes a new + /// snapshot while it is held. + fn snapshot(&self) -> Arc { + self.in_memory.load_full() } - /// Runs a synchronous mutable operation over the inner state on Tokio's blocking path. + /// Runs a synchronous read-only operation over the current in-memory state snapshot on Tokio's + /// blocking path. /// - /// See [`Self::with_inner_read_blocking`] for why this uses `block_in_place`. - fn with_inner_write_blocking(&self, f: impl FnOnce(&mut InnerState) -> R) -> R { + /// The account and nullifier trees may be backed by `RocksDB`, so tree access must not run on + /// an async worker thread directly. This helper preserves the current tracing span while + /// moving the closure body into `block_in_place`. + fn with_inner_read_blocking(&self, f: impl FnOnce(&InMemoryState) -> R) -> R { let span = Span::current(); tokio::task::block_in_place(|| { span.in_scope(|| { - let mut inner = self.inner.blocking_write(); - f(&mut inner) + let snapshot = self.snapshot(); + f(&snapshot) }) }) } - /// Runs a synchronous read-only operation over the account state forest on Tokio's blocking - /// path. + /// Runs a synchronous read-only operation over the account state forest snapshot on Tokio's + /// blocking path. /// - /// The forest may be backed by `RocksDB`, so accesses to the underlying `LargeSmtForest` must - /// not run directly on an async worker thread. + /// See [`Self::with_inner_read_blocking`] for why this uses `block_in_place`. fn with_forest_read_blocking( &self, - f: impl FnOnce(&AccountStateForest) -> R, - ) -> R { - let span = Span::current(); - tokio::task::block_in_place(|| { - span.in_scope(|| { - let forest = self.forest.blocking_read(); - f(&forest) - }) - }) - } - - /// Runs a synchronous mutable operation over the account state forest on Tokio's blocking path. - /// - /// See [`Self::with_forest_read_blocking`] for why this uses `block_in_place`. - fn with_forest_write_blocking( - &self, - f: impl FnOnce(&mut AccountStateForest) -> R, + f: impl FnOnce(&AccountStateForest) -> R, ) -> R { - let span = Span::current(); - tokio::task::block_in_place(|| { - span.in_scope(|| { - let mut forest = self.forest.blocking_write(); - f(&mut forest) - }) - }) + self.with_inner_read_blocking(|snapshot| f(&snapshot.forest)) } // STATE ACCESSORS @@ -379,8 +392,7 @@ impl State { let block_header = self.db.select_block_header_by_block_num(block_num).await?; if let Some(header) = block_header { let mmr_proof = if include_mmr_proof { - let inner = self.inner.read().await; - let mmr_proof = inner.blockchain.open(header.block_num())?; + let mmr_proof = self.snapshot().blockchain.open(header.block_num())?; Some(mmr_proof) } else { None @@ -447,12 +459,10 @@ impl State { let mut blocks: BTreeSet = tx_reference_blocks; blocks.extend(note_blocks); - // Scoped block to automatically drop the read lock guard as soon as we're done. We also - // avoid accessing the db in the block as this would delay dropping the guard. let (batch_reference_block, partial_mmr) = { - let inner_state = self.inner.read().await; + let snapshot = self.snapshot(); - let latest_block_num = inner_state.latest_block_num(); + let latest_block_num = snapshot.latest_block_num(); let highest_block_num = *blocks.last().expect("we should have checked for empty block references"); @@ -475,7 +485,7 @@ impl State { // number *and* latest block num was removed from the set. Therefore only block // numbers smaller than latest block num remain in the set. Therefore all the block // numbers are guaranteed to exist in the chain state at latest block num. - let partial_mmr = inner_state + let partial_mmr = snapshot .blockchain .partial_mmr_from_blocks(&blocks, latest_block_num) .expect("latest block num should exist and all blocks in set should be < than latest block"); @@ -722,17 +732,13 @@ impl State { /// Returns the effective chain tip for the given finality level. /// - /// - [`Finality::Committed`]: returns the latest committed block number (from in-memory MMR). + /// - [`Finality::Committed`]: returns the latest committed block number (from the in-memory + /// snapshot). /// - [`Finality::Proven`]: returns the latest proven-in-sequence block number (cached via watch /// channel, updated by the proof scheduler). - pub async fn chain_tip(&self, finality: Finality) -> BlockNumber { + pub fn chain_tip(&self, finality: Finality) -> BlockNumber { match finality { - Finality::Committed => self - .inner - .read() - .instrument(tracing::info_span!("acquire_inner")) - .await - .latest_block_num(), + Finality::Committed => self.snapshot().latest_block_num(), Finality::Proven => self.proven_tip.read(), } } @@ -743,7 +749,7 @@ impl State { &self, block_num: BlockNumber, ) -> Result>, DatabaseError> { - if block_num > self.chain_tip(Finality::Committed).await { + if block_num > self.chain_tip(Finality::Committed) { return Ok(None); } if let Some(block) = self.block_cache.get(block_num) { @@ -758,7 +764,7 @@ impl State { &self, block_num: BlockNumber, ) -> Result>, DatabaseError> { - if block_num > self.chain_tip(Finality::Proven).await { + if block_num > self.chain_tip(Finality::Proven) { return Ok(None); } if let Some(proof) = self.proof_cache.get(block_num) { diff --git a/crates/store/src/state/sync_state.rs b/crates/store/src/state/sync_state.rs index 7dc234fee..05617930b 100644 --- a/crates/store/src/state/sync_state.rs +++ b/crates/store/src/state/sync_state.rs @@ -72,9 +72,7 @@ impl State { let to_forest = block_to.as_usize(); let mmr_delta = self - .inner - .read() - .await + .snapshot() .blockchain .as_mmr() .get_delta( @@ -116,11 +114,12 @@ impl State { let mut results = Vec::new(); { - let inner = self.inner.read().await; + let snapshot = self.snapshot(); for note_sync in note_syncs { - let mmr_proof = - inner.blockchain.open_at(note_sync.block_header.block_num(), mmr_checkpoint)?; + let mmr_proof = snapshot + .blockchain + .open_at(note_sync.block_header.block_num(), mmr_checkpoint)?; results.push((note_sync, mmr_proof)); } } diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs new file mode 100644 index 000000000..a44afd398 --- /dev/null +++ b/crates/store/src/state/writer.rs @@ -0,0 +1,390 @@ +//! Serialized block-write path for the store state. +//! +//! A single [`BlockWriter`] task owns the mutable trees and processes incoming [`WriteRequest`]s +//! one at a time via an mpsc channel. After each successful commit it publishes a new +//! [`InMemoryState`] snapshot via an [`ArcSwap`], making the updated trees immediately visible to +//! wait-free readers. + +use std::sync::Arc; + +use arc_swap::ArcSwap; +use miden_node_utils::ErrorReport; +use miden_node_utils::tracing::{miden_instrument, miden_span_record}; +use miden_protocol::account::AccountUpdateDetails; +use miden_protocol::block::account_tree::AccountMutationSet; +use miden_protocol::block::nullifier_tree::{NullifierMutationSet, NullifierTree}; +use miden_protocol::block::{BlockBody, BlockHeader, BlockNumber, Blockchain, SignedBlock}; +use miden_protocol::crypto::merkle::smt::LargeSmt; +use miden_protocol::note::{NoteDetails, Nullifier}; +use miden_protocol::transaction::OutputNote; +use miden_protocol::utils::serde::Serializable; +use tokio::sync::{mpsc, oneshot, watch}; + +use crate::account_state_forest::{AccountStateForest, AccountStateForestBackend}; +use crate::accounts::AccountTreeWithHistory; +use crate::blocks::BlockStore; +use crate::db::{Db, NoteRecord}; +use crate::errors::{ApplyBlockError, InvalidBlockError}; +use crate::state::loader::TreeStorage; +use crate::state::{BlockCache, BlockNotification, InMemoryState, State}; +use crate::{COMPONENT, HistoricalError, LOG_TARGET}; + +// WRITE REQUEST +// ================================================================================================ + +/// A request to apply a block, paired with a one-shot channel for the result. +pub(super) struct WriteRequest { + signed_block: SignedBlock, + result_tx: oneshot::Sender>, +} + +// WRITE HANDLE +// ================================================================================================ + +/// Handle for sending block-write requests to the [`BlockWriter`] task. +pub(super) struct WriteHandle { + tx: mpsc::Sender, +} + +impl WriteHandle { + pub(super) fn new(tx: mpsc::Sender) -> Self { + Self { tx } + } + + /// Sends a block to the writer task and awaits its result. + pub(super) async fn apply_block( + &self, + signed_block: SignedBlock, + ) -> Result<(), ApplyBlockError> { + let (result_tx, result_rx) = oneshot::channel(); + self.tx + .send(WriteRequest { signed_block, result_tx }) + .await + .map_err(|e| ApplyBlockError::WriterTaskSendFailed(e.as_report()))?; + result_rx.await? + } +} + +impl State { + /// Apply changes of a new block to the DB and in-memory data structures. + /// + /// Blocks are forwarded to the [`BlockWriter`] task, which processes them one at a time. + /// Readers are unaffected while a block is being applied: they keep reading from the previous + /// in-memory snapshot until the writer atomically publishes the new one. + #[miden_instrument( + target = COMPONENT, + skip_all, + err, + )] + pub async fn apply_block(&self, signed_block: SignedBlock) -> Result<(), ApplyBlockError> { + self.write_handle.apply_block(signed_block).await + } +} + +// BLOCK WRITER +// ================================================================================================ + +/// Single-task owner of the mutable trees. Processes [`WriteRequest`]s serially. +/// +/// The writer owns the writable trees directly, so no locks are held at any point: validation and +/// mutation-computation read the owned trees, the DB commit runs without touching them, and the +/// new [`InMemoryState`] snapshot is published atomically at the end. +pub(super) struct BlockWriter { + pub db: Arc, + pub block_store: Arc, + /// Atomically swappable pointer through which new snapshots are published. + pub in_memory: Arc>, + pub committed_tip_tx: Arc>, + pub block_cache: BlockCache, + pub rx: mpsc::Receiver, + /// The mutable nullifier tree owned by this writer. + pub nullifier_tree: NullifierTree>, + /// The mutable account tree owned by this writer. + pub account_tree: AccountTreeWithHistory, + /// The blockchain MMR owned by this writer. + pub blockchain: Blockchain, + /// The mutable account state forest owned by this writer. + pub forest: AccountStateForest, +} + +impl BlockWriter { + /// Runs the writer loop, processing requests until all write handles are dropped. + pub async fn run(mut self) { + while let Some(req) = self.rx.recv().await { + let result = self.write_block(req.signed_block).await; + let _ = req.result_tx.send(result); + } + } + + /// Validates and commits a signed block to all persistent and in-memory stores. + /// + /// ## Note on state consistency + /// + /// Readers access the in-memory state through frozen snapshots, so consistency is maintained + /// by ordering the commit steps rather than by locking: + /// + /// - the block is validated against the writer-owned trees and the DB prior to starting any + /// modifications. + /// - the block is saved to the block store. Such blocks are considered candidates and are not + /// yet available for reading because the latest block pointer is not updated yet. + /// - the DB transaction is committed. Concurrent readers still see the previous in-memory + /// snapshot; queries that combine DB and in-memory data are scoped by block number. + /// - the in-memory structures owned by the writer are updated. On a crash in between, the trees + /// lag the DB by one block, which is detected by the consistency checks at startup (the same + /// crash semantics as the previous lock-based implementation). + /// - the new snapshot is published atomically, making the block visible to readers. + #[miden_instrument( + target = COMPONENT, + skip_all, + err, + )] + async fn write_block(&mut self, signed_block: SignedBlock) -> Result<(), ApplyBlockError> { + let header = signed_block.header(); + let body = signed_block.body(); + + let block_num = header.block_num(); + let block_commitment = header.commitment(); + let num_transactions = body.transactions().as_slice().len(); + + miden_span_record!( + block.number = %block_num, + block.commitment = %block_commitment, + block.transactions.count = num_transactions, + ); + + self.validate_block_header(header, body).await?; + + // Compute the tree mutations and note records upfront, before any modifications. + let (notes, nullifier_tree_update, account_tree_update) = + tokio::task::block_in_place(|| { + let notes = Self::build_note_records(header, body)?; + let (nullifier_tree_update, account_tree_update) = + self.compute_tree_mutations(header, body)?; + Ok::<_, ApplyBlockError>((notes, nullifier_tree_update, account_tree_update)) + })?; + + // Extract public account updates with patches before the block is moved into the DB + // commit. Private accounts are filtered out since they don't expose their state changes. + let account_patches = + Vec::from_iter(body.updated_accounts().iter().filter_map( + |update| match update.details() { + AccountUpdateDetails::Public(patch) => Some(patch.clone()), + AccountUpdateDetails::Private => None, + }, + )); + + // Save the block to the block store. In a case of a failed DB transaction, the in-memory + // state will be unchanged, but the file might still be written. Such blocks should be + // considered candidates, not finalized blocks. + let signed_block_bytes = signed_block.to_bytes(); + // Clone before moving into the block-save call so we can cache for replicas at commit. + let cache_bytes = signed_block_bytes.clone(); + self.block_store.save_block(block_num, &signed_block_bytes).await?; + + // Commit to the DB. Readers continue to see the previous in-memory snapshot while the DB + // commits; queries that combine DB and in-memory data are scoped by block number. + self.db + .apply_block(signed_block, notes) + .await + .map_err(|err| ApplyBlockError::DbUpdateTaskFailed(err.as_report()))?; + + // Apply the mutations to the writer-owned trees and build the new snapshot from reader + // views of them. The reader views are point-in-time storage snapshots, so no tree data is + // copied. + let snapshot = tokio::task::block_in_place(|| { + self.nullifier_tree + .apply_mutations(nullifier_tree_update) + .expect("nullifier tree mutation should succeed after validation"); + + self.account_tree + .apply_mutations(account_tree_update) + .expect("account tree mutation should succeed after validation"); + + self.blockchain.push(block_commitment); + + self.forest.apply_block_updates(block_num, account_patches); + + Arc::new(InMemoryState { + nullifier_tree: self + .nullifier_tree + .reader() + .expect("nullifier tree snapshot creation should not fail"), + account_tree: self.account_tree.reader(), + blockchain: self.blockchain.clone(), + forest: self.forest.reader().expect("forest snapshot creation should not fail"), + }) + }); + + // Atomically publish the new state. Readers that call `snapshot()` after this point will + // see the updated state. Readers holding the old snapshot continue unaffected. + self.in_memory.store(snapshot); + + // Push to cache and notify replica subscribers. + self.block_cache + .push(block_num, BlockNotification::new(block_num, cache_bytes)) + .expect("block cache receives sequential block numbers"); + let _ = self.committed_tip_tx.send(block_num); + + tracing::debug!(target: LOG_TARGET, "Block applied"); + + Ok(()) + } + + /// Validates that the block header is consistent with the block body and the current state. + #[miden_instrument( + target = COMPONENT, + skip_all, + err, + )] + async fn validate_block_header( + &self, + header: &BlockHeader, + body: &BlockBody, + ) -> Result<(), ApplyBlockError> { + // Validate that header and body match. + let tx_commitment = body.transactions().commitment(); + if header.tx_commitment() != tx_commitment { + return Err(InvalidBlockError::InvalidBlockTxCommitment { + expected: tx_commitment, + actual: header.tx_commitment(), + } + .into()); + } + + let block_num = header.block_num(); + + // Validate that the applied block is the next block in sequence. + let prev_block = self + .db + .select_block_header_by_block_num(None) + .await? + .ok_or(ApplyBlockError::DbBlockHeaderEmpty)?; + let expected_block_num = prev_block.block_num().child(); + if block_num != expected_block_num { + return Err(InvalidBlockError::NewBlockInvalidBlockNum { + expected: expected_block_num, + submitted: block_num, + } + .into()); + } + if header.prev_block_commitment() != prev_block.commitment() { + return Err(InvalidBlockError::NewBlockInvalidPrevCommitment.into()); + } + + Ok(()) + } + + /// Computes nullifier and account tree mutations, validating roots against the block header. + #[miden_instrument( + target = COMPONENT, + skip_all, + err, + )] + fn compute_tree_mutations( + &self, + header: &BlockHeader, + body: &BlockBody, + ) -> Result<(NullifierMutationSet, AccountMutationSet), ApplyBlockError> { + let block_num = header.block_num(); + + // nullifiers can be produced only once + let duplicate_nullifiers: Vec<_> = body + .created_nullifiers() + .iter() + .filter(|&nullifier| self.nullifier_tree.get_block_num(nullifier).is_some()) + .copied() + .collect(); + if !duplicate_nullifiers.is_empty() { + return Err(InvalidBlockError::DuplicatedNullifiers(duplicate_nullifiers).into()); + } + + // new_block.chain_root must be equal to the chain MMR root prior to the update + let peaks = self.blockchain.peaks(); + if peaks.hash_peaks() != header.chain_commitment() { + return Err(InvalidBlockError::NewBlockInvalidChainCommitment.into()); + } + + // compute update for nullifier tree + let nullifier_tree_update = self + .nullifier_tree + .compute_mutations( + body.created_nullifiers().iter().map(|nullifier| (*nullifier, block_num)), + ) + .map_err(InvalidBlockError::NewBlockNullifierAlreadySpent)?; + + if nullifier_tree_update.as_mutation_set().root() != header.nullifier_root() { + return Err(InvalidBlockError::NewBlockInvalidNullifierRoot.into()); + } + + // compute update for account tree + let account_tree_update = self + .account_tree + .compute_mutations( + body.updated_accounts() + .iter() + .map(|update| (update.account_id(), update.final_state_commitment())), + ) + .map_err(|e| match e { + HistoricalError::AccountTreeError(err) => { + InvalidBlockError::NewBlockDuplicateAccountIdPrefix(err) + }, + HistoricalError::MerkleError(_) => { + panic!("Unexpected MerkleError during account tree mutation computation") + }, + })?; + + if account_tree_update.as_mutation_set().root() != header.account_root() { + return Err(InvalidBlockError::NewBlockInvalidAccountRoot.into()); + } + + Ok((nullifier_tree_update, account_tree_update)) + } + + /// Builds note records with inclusion proofs from the block body. + #[miden_instrument( + target = COMPONENT, + skip_all, + err, + )] + fn build_note_records( + header: &BlockHeader, + body: &BlockBody, + ) -> Result)>, ApplyBlockError> { + let block_num = header.block_num(); + + let note_tree = body.compute_block_note_tree(); + if note_tree.root() != header.note_root() { + return Err(InvalidBlockError::NewBlockInvalidNoteRoot.into()); + } + + let notes = body + .output_notes() + .map(|(note_index, note)| { + let (details, attachments, nullifier) = match note { + OutputNote::Public(public) => ( + Some(NoteDetails::from(public.as_note())), + public.as_note().attachments().clone(), + Some(public.as_note().nullifier()), + ), + OutputNote::Private(private) => (None, private.attachments().clone(), None), + }; + + let inclusion_path = note_tree.open(note_index); + + let note_record = NoteRecord { + block_num, + note_index, + note_id: note.id().as_word(), + metadata: *note.metadata(), + details, + attachments, + inclusion_path, + }; + + Ok((note_record, nullifier)) + }) + .collect::, InvalidBlockError>>()?; + + Ok(notes) + } +} From 5c7dbebc98922b096523ca1d5c6dbc75ac3a6a4c Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 15 Jul 2026 15:48:57 +1200 Subject: [PATCH 02/35] Add snapshot count observability --- crates/store/src/state/mod.rs | 47 ++++++++++++++++++++++++++++++++ crates/store/src/state/writer.rs | 8 +++++- crates/tracing-macro/src/lib.rs | 3 ++ 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index fb70b5cc0..6a678337f 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -7,6 +7,8 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::num::NonZeroUsize; use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Instant; use arc_swap::ArcSwap; use miden_node_proto::domain::batch::BatchInputs; @@ -107,6 +109,44 @@ type BlockInputWitnesses = ( PartialMmr, ); +/// RAII member of [`InMemoryState`] that tracks the number of live snapshot generations. +/// +/// [`InMemoryState`] is dropped exactly when the last [`Arc`] reference to it is released, so the +/// shared counter reports how many distinct snapshot generations are currently pinned by readers. +/// A sustained count above 1-2 means slow readers are holding old generations alive, which also +/// pins the corresponding `RocksDB` snapshots and blocks compaction. +pub(crate) struct SnapshotGuard { + live: Arc, + created_at: Instant, + block_num: BlockNumber, +} + +impl SnapshotGuard { + pub(super) fn new(live: Arc, block_num: BlockNumber) -> Self { + live.fetch_add(1, Ordering::Relaxed); + Self { + live, + created_at: Instant::now(), + block_num, + } + } +} + +impl Drop for SnapshotGuard { + fn drop(&mut self) { + let remaining = self.live.fetch_sub(1, Ordering::Relaxed) - 1; + let lifetime_ms = u64::try_from(self.created_at.elapsed().as_millis()).unwrap_or(u64::MAX); + let block_num = self.block_num.as_u32(); + tracing::debug!( + target: COMPONENT, + block_num, + snapshot.lifetime_ms = lifetime_ms, + snapshots.live = remaining, + "state snapshot released", + ); + } +} + /// Immutable snapshot of the in-memory tree state published after each committed block. /// /// The trees are backed by read-only snapshot storage ([`TreeStorageReader`] / @@ -117,6 +157,8 @@ pub(crate) struct InMemoryState { blockchain: Blockchain, account_tree: AccountTreeWithHistory, forest: AccountStateForest, + /// Keeps the live-snapshot count accurate; see [`SnapshotGuard`]. + _guard: SnapshotGuard, } impl InMemoryState { @@ -270,6 +312,9 @@ impl State { let block_cache = BlockCache::new(BLOCK_CACHE_CAPACITY); let proof_cache = ProofCache::new(PROOF_CACHE_CAPACITY); + // Shared counter of live snapshot generations, for observability. + let snapshots_live = Arc::new(AtomicUsize::new(0)); + // Create the initial snapshot from reader views of the just-loaded trees. let initial_snapshot = Arc::new(InMemoryState { nullifier_tree: nullifier_tree @@ -280,6 +325,7 @@ impl State { .reader() .map_err(|e| StateInitializationError::AccountStateForestIoError(e.as_report()))?, blockchain: blockchain.clone(), + _guard: SnapshotGuard::new(Arc::clone(&snapshots_live), latest_block_num), }); let in_memory = Arc::new(ArcSwap::from(initial_snapshot)); @@ -299,6 +345,7 @@ impl State { account_tree, blockchain, forest, + snapshots_live, }; tokio::spawn(block_writer.run()); diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index a44afd398..5acc4270b 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -6,6 +6,7 @@ //! wait-free readers. use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use arc_swap::ArcSwap; use miden_node_utils::ErrorReport; @@ -26,7 +27,7 @@ use crate::blocks::BlockStore; use crate::db::{Db, NoteRecord}; use crate::errors::{ApplyBlockError, InvalidBlockError}; use crate::state::loader::TreeStorage; -use crate::state::{BlockCache, BlockNotification, InMemoryState, State}; +use crate::state::{BlockCache, BlockNotification, InMemoryState, SnapshotGuard, State}; use crate::{COMPONENT, HistoricalError, LOG_TARGET}; // WRITE REQUEST @@ -105,6 +106,8 @@ pub(super) struct BlockWriter { pub blockchain: Blockchain, /// The mutable account state forest owned by this writer. pub forest: AccountStateForest, + /// Shared counter of live snapshot generations, for observability. + pub snapshots_live: Arc, } impl BlockWriter { @@ -212,12 +215,15 @@ impl BlockWriter { account_tree: self.account_tree.reader(), blockchain: self.blockchain.clone(), forest: self.forest.reader().expect("forest snapshot creation should not fail"), + _guard: SnapshotGuard::new(Arc::clone(&self.snapshots_live), block_num), }) }); // Atomically publish the new state. Readers that call `snapshot()` after this point will // see the updated state. Readers holding the old snapshot continue unaffected. self.in_memory.store(snapshot); + let snapshots_live = self.snapshots_live.load(Ordering::Relaxed) as u64; + miden_span_record!(snapshots.live = snapshots_live); // Push to cache and notify replica subscribers. self.block_cache diff --git a/crates/tracing-macro/src/lib.rs b/crates/tracing-macro/src/lib.rs index 583a1b25a..3d9b5707a 100644 --- a/crates/tracing-macro/src/lib.rs +++ b/crates/tracing-macro/src/lib.rs @@ -69,6 +69,9 @@ const ALLOWED_FIELD_NAMES: &[&str] = &[ "reference_block.number", "request.kind", "script.root", + "snapshot.block_num", + "snapshot.lifetime_ms", + "snapshots.live", "transaction.id", "transaction.expires_at", "transaction.input_notes.count", From 46b8df6cd587c977e2d2290bb8b250d6fadb31ce Mon Sep 17 00:00:00 2001 From: sergerad Date: Mon, 20 Jul 2026 11:22:32 +0800 Subject: [PATCH 03/35] Fix ui tests --- crates/utils/tests/ui/tracing_macros/invalid_field_name.stderr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/utils/tests/ui/tracing_macros/invalid_field_name.stderr b/crates/utils/tests/ui/tracing_macros/invalid_field_name.stderr index e4ac03ffd..05e495e04 100644 --- a/crates/utils/tests/ui/tracing_macros/invalid_field_name.stderr +++ b/crates/utils/tests/ui/tracing_macros/invalid_field_name.stderr @@ -1,4 +1,4 @@ -error: unsupported tracing field `tx_id`; use one of: account.id, account.id.network_prefix, account.ids, account.ids.count, account.updated, batch.id, batch.account_updates.count, batch.expires_at, batch.expiration_height, batch.input_notes.count, batch.output_notes.count, batch.reference_block.commitment, batch.reference_block.number, block.batch.ids, block.batches.count, block.batches.output_notes.count, block.commitment, block.commitments.account, block.commitments.chain, block.commitments.kernel, block.commitments.note, block.commitments.nullifier, block.commitments.transaction, block.erased_note_proofs.count, block.erased_notes.count, block.from, block.nullifiers.count, block.number, block.output_notes.count, block.prev_block_commitment, block.protocol.version, block.sub_commitment, block.timestamp, block.transactions.ids, block.transactions.count, block.updated_accounts.count, block_range.from, block_range.to, db.account_state_forest.size, db.account_tree.size, db.block_store.size, db.nullifier_tree.size, db.sqlite.size, db.sqlite.wal.size, dice_roll, failure_rate, mempool.accounts, mempool.batches.proposed, mempool.batches.proven, mempool.nullifiers, mempool.output_notes, mempool.transactions.unbatched, mempool.transactions.uncommitted, note.id, notes.count, prover.kind, reference_block.number, request.kind, script.root, transaction.id, transaction.expires_at, transaction.input_notes.count, transaction.output_notes.count, transaction.reference_block.commitment, transaction.reference_block.number, tip.number, transactions.count, transactions.ids, transactions.input_notes.count, transactions.output_notes.count, transactions.unauthenticated_notes.count, workers.active, workers.capacity, workers.count +error: unsupported tracing field `tx_id`; use one of: account.id, account.id.network_prefix, account.ids, account.ids.count, account.updated, batch.id, batch.account_updates.count, batch.expires_at, batch.expiration_height, batch.input_notes.count, batch.output_notes.count, batch.reference_block.commitment, batch.reference_block.number, block.batch.ids, block.batches.count, block.batches.output_notes.count, block.commitment, block.commitments.account, block.commitments.chain, block.commitments.kernel, block.commitments.note, block.commitments.nullifier, block.commitments.transaction, block.erased_note_proofs.count, block.erased_notes.count, block.from, block.nullifiers.count, block.number, block.output_notes.count, block.prev_block_commitment, block.protocol.version, block.sub_commitment, block.timestamp, block.transactions.ids, block.transactions.count, block.updated_accounts.count, block_range.from, block_range.to, db.account_state_forest.size, db.account_tree.size, db.block_store.size, db.nullifier_tree.size, db.sqlite.size, db.sqlite.wal.size, dice_roll, failure_rate, mempool.accounts, mempool.batches.proposed, mempool.batches.proven, mempool.nullifiers, mempool.output_notes, mempool.transactions.unbatched, mempool.transactions.uncommitted, note.id, notes.count, prover.kind, reference_block.number, request.kind, script.root, snapshot.block_num, snapshot.lifetime_ms, snapshots.live, transaction.id, transaction.expires_at, transaction.input_notes.count, transaction.output_notes.count, transaction.reference_block.commitment, transaction.reference_block.number, tip.number, transactions.count, transactions.ids, transactions.input_notes.count, transactions.output_notes.count, transactions.unauthenticated_notes.count, workers.active, workers.capacity, workers.count --> tests/ui/tracing_macros/invalid_field_name.rs:10:9 | 10 | tx_id = %tx_id, From bb3cf85fe40bd91a96060876b06d5d8451683da4 Mon Sep 17 00:00:00 2001 From: sergerad Date: Mon, 20 Jul 2026 11:32:10 +0800 Subject: [PATCH 04/35] Lint --- crates/store/src/state/writer.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index 5acc4270b..9399727b1 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -166,8 +166,8 @@ impl BlockWriter { Ok::<_, ApplyBlockError>((notes, nullifier_tree_update, account_tree_update)) })?; - // Extract public account updates with patches before the block is moved into the DB - // commit. Private accounts are filtered out since they don't expose their state changes. + // Extract public account updates with patches before the block is moved into the DB commit. + // Private accounts are filtered out since they don't expose their state changes. let account_patches = Vec::from_iter(body.updated_accounts().iter().filter_map( |update| match update.details() { From 891d7026734bc05aa8ca8fc444401bd4d514d4b4 Mon Sep 17 00:00:00 2001 From: sergerad Date: Mon, 20 Jul 2026 12:10:20 +0800 Subject: [PATCH 05/35] Warn about long lived snapshots --- crates/store/src/state/mod.rs | 44 ++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 6a678337f..994432262 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -8,7 +8,7 @@ use std::num::NonZeroUsize; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::time::Instant; +use std::time::{Duration, Instant}; use arc_swap::ArcSwap; use miden_node_proto::domain::batch::BatchInputs; @@ -52,6 +52,12 @@ const BLOCK_CACHE_CAPACITY: NonZeroUsize = NonZeroUsize::new(512).unwrap(); /// Number of recent block proofs held in the in-memory cache for replica subscriptions. const PROOF_CACHE_CAPACITY: NonZeroUsize = NonZeroUsize::new(512).unwrap(); +/// Snapshot lifetime above which [`SnapshotGuard`] logs a warning on release. +/// +/// Readers are expected to be request-scoped, so a snapshot outliving several block intervals +/// indicates a slow or leaked reader pinning a `RocksDB` snapshot (see [`SnapshotGuard`]). +const SNAPSHOT_LIFETIME_WARN_THRESHOLD: Duration = Duration::from_secs(10); + mod loader; use loader::{ ACCOUNT_STATE_FOREST_STORAGE_DIR, @@ -113,8 +119,13 @@ type BlockInputWitnesses = ( /// /// [`InMemoryState`] is dropped exactly when the last [`Arc`] reference to it is released, so the /// shared counter reports how many distinct snapshot generations are currently pinned by readers. -/// A sustained count above 1-2 means slow readers are holding old generations alive, which also -/// pins the corresponding `RocksDB` snapshots and blocks compaction. +/// A sustained count above 1-2 means slow readers are holding old generations alive. Each +/// generation pins a `RocksDB` snapshot, which delays garbage collection of superseded key +/// versions during compaction (compaction itself keeps running); the retained garbage grows with +/// write churn for as long as the snapshot is held and is reclaimed once it is released. +/// +/// Readers are expected to be request-scoped, so snapshot lifetimes should be well under a block +/// interval. A lifetime exceeding [`SNAPSHOT_LIFETIME_WARN_THRESHOLD`] is logged at warn level. pub(crate) struct SnapshotGuard { live: Arc, created_at: Instant, @@ -135,15 +146,26 @@ impl SnapshotGuard { impl Drop for SnapshotGuard { fn drop(&mut self) { let remaining = self.live.fetch_sub(1, Ordering::Relaxed) - 1; - let lifetime_ms = u64::try_from(self.created_at.elapsed().as_millis()).unwrap_or(u64::MAX); + let lifetime = self.created_at.elapsed(); + let lifetime_ms = u64::try_from(lifetime.as_millis()).unwrap_or(u64::MAX); let block_num = self.block_num.as_u32(); - tracing::debug!( - target: COMPONENT, - block_num, - snapshot.lifetime_ms = lifetime_ms, - snapshots.live = remaining, - "state snapshot released", - ); + if lifetime > SNAPSHOT_LIFETIME_WARN_THRESHOLD { + tracing::warn!( + target: COMPONENT, + block_num, + snapshot.lifetime_ms = lifetime_ms, + snapshots.live = remaining, + "state snapshot held for excessive time", + ); + } else { + tracing::debug!( + target: COMPONENT, + block_num, + snapshot.lifetime_ms = lifetime_ms, + snapshots.live = remaining, + "state snapshot released", + ); + } } } From 8c2bc02e9ed8c5beab58e7a18257ebad5b430304 Mon Sep 17 00:00:00 2001 From: sergerad Date: Mon, 20 Jul 2026 12:16:22 +0800 Subject: [PATCH 06/35] Warn on snapshot count --- crates/store/src/state/mod.rs | 8 ++++++++ crates/store/src/state/writer.rs | 17 ++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 994432262..ac035912a 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -58,6 +58,14 @@ const PROOF_CACHE_CAPACITY: NonZeroUsize = NonZeroUsize::new(512).unwrap(); /// indicates a slow or leaked reader pinning a `RocksDB` snapshot (see [`SnapshotGuard`]). const SNAPSHOT_LIFETIME_WARN_THRESHOLD: Duration = Duration::from_secs(10); +/// Number of live snapshot generations above which the [`BlockWriter`] logs a warning after +/// publishing a new snapshot. +/// +/// Steady state is 1-2 generations: the just-published snapshot plus predecessors briefly pinned +/// by in-flight requests. A sustained higher count means slow or leaked readers are holding old +/// generations alive (see [`SnapshotGuard`]). +const SNAPSHOTS_LIVE_WARN_THRESHOLD: u64 = 4; + mod loader; use loader::{ ACCOUNT_STATE_FOREST_STORAGE_DIR, diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index 9399727b1..c347a87db 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -27,7 +27,14 @@ use crate::blocks::BlockStore; use crate::db::{Db, NoteRecord}; use crate::errors::{ApplyBlockError, InvalidBlockError}; use crate::state::loader::TreeStorage; -use crate::state::{BlockCache, BlockNotification, InMemoryState, SnapshotGuard, State}; +use crate::state::{ + BlockCache, + BlockNotification, + InMemoryState, + SNAPSHOTS_LIVE_WARN_THRESHOLD, + SnapshotGuard, + State, +}; use crate::{COMPONENT, HistoricalError, LOG_TARGET}; // WRITE REQUEST @@ -224,6 +231,14 @@ impl BlockWriter { self.in_memory.store(snapshot); let snapshots_live = self.snapshots_live.load(Ordering::Relaxed) as u64; miden_span_record!(snapshots.live = snapshots_live); + if snapshots_live > SNAPSHOTS_LIVE_WARN_THRESHOLD { + tracing::warn!( + target: COMPONENT, + block_num = block_num.as_u32(), + snapshots.live = snapshots_live, + "too many live state snapshots; slow readers are pinning old generations", + ); + } // Push to cache and notify replica subscribers. self.block_cache From 8373d9bd0656663e62e215518b587753002f8c7f Mon Sep 17 00:00:00 2001 From: sergerad Date: Mon, 20 Jul 2026 12:59:45 +0800 Subject: [PATCH 07/35] Scape all requests --- crates/store/src/db/mod.rs | 10 ++++-- crates/store/src/db/models/queries/notes.rs | 7 +++-- crates/store/src/state/mod.rs | 35 ++++++++++++++++----- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index ac8cbdff9..43330a996 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -530,7 +530,8 @@ impl Db { .await } - /// Returns all note commitments from the DB that match the provided ones. + /// Returns all note commitments from the DB that match the provided ones and were committed + /// at or before `up_to_block`. #[miden_instrument( level = "debug", target = COMPONENT, @@ -540,9 +541,14 @@ impl Db { pub async fn select_existing_note_commitments( &self, note_commitments: Vec, + up_to_block: BlockNumber, ) -> Result> { self.transact("note by commitment", move |conn| { - queries::select_existing_note_commitments(conn, note_commitments.as_slice()) + queries::select_existing_note_commitments( + conn, + note_commitments.as_slice(), + up_to_block, + ) }) .await } diff --git a/crates/store/src/db/models/queries/notes.rs b/crates/store/src/db/models/queries/notes.rs index da9b78508..e0ee31720 100644 --- a/crates/store/src/db/models/queries/notes.rs +++ b/crates/store/src/db/models/queries/notes.rs @@ -221,7 +221,8 @@ pub(crate) fn select_notes_by_id( Ok(records) } -/// Select the subset of note commitments that already exist in the notes table +/// Select the subset of note commitments that already exist in the notes table and were +/// committed at or before `up_to_block`. /// /// # Raw SQL /// @@ -229,11 +230,12 @@ pub(crate) fn select_notes_by_id( /// SELECT /// notes.note_commitment /// FROM notes -/// WHERE note_commitment IN (?1) +/// WHERE note_commitment IN (?1) AND committed_at <= ?2 /// ``` pub(crate) fn select_existing_note_commitments( conn: &mut SqliteConnection, note_commitments: &[Word], + up_to_block: BlockNumber, ) -> Result, DatabaseError> { QueryParamNoteCommitmentLimit::check(note_commitments.len())?; @@ -241,6 +243,7 @@ pub(crate) fn select_existing_note_commitments( let raw_commitments = SelectDsl::select(schema::notes::table, schema::notes::note_id) .filter(schema::notes::note_id.eq_any(¬e_commitments)) + .filter(schema::notes::committed_at.le(up_to_block.to_raw_sql())) .load::>(conn)?; let commitments = raw_commitments diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index ac035912a..0b8d7f886 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -466,10 +466,20 @@ impl State { block_num: Option, include_mmr_proof: bool, ) -> Result<(Option, Option), GetBlockHeaderError> { - let block_header = self.db.select_block_header_by_block_num(block_num).await?; + // Resolve "latest" against the in-memory snapshot rather than the DB: mid-apply, the DB + // may already contain a block that the snapshot's blockchain cannot prove yet. Scoping the + // DB query by the snapshot's tip keeps the header and MMR proof consistent. + let snapshot = self.snapshot(); + let latest_block_num = snapshot.latest_block_num(); + let block_num = block_num.unwrap_or(latest_block_num); + if block_num > latest_block_num { + return Ok((None, None)); + } + + let block_header = self.db.select_block_header_by_block_num(Some(block_num)).await?; if let Some(header) = block_header { let mmr_proof = if include_mmr_proof { - let mmr_proof = self.snapshot().blockchain.open(header.block_num())?; + let mmr_proof = snapshot.blockchain.open(header.block_num())?; Some(mmr_proof) } else { None @@ -779,16 +789,25 @@ impl State { }) .collect(); - Ok((account_commitment, nullifiers, new_account_id_prefix_is_unique)) + Ok(( + account_commitment, + nullifiers, + new_account_id_prefix_is_unique, + inner.latest_block_num(), + )) }); - let (account_commitment, nullifiers, new_account_id_prefix_is_unique) = match tree_inputs { - Ok(inputs) => inputs, - Err(inputs) => return Ok(inputs), - }; + let (account_commitment, nullifiers, new_account_id_prefix_is_unique, latest_block_num) = + match tree_inputs { + Ok(inputs) => inputs, + Err(inputs) => return Ok(inputs), + }; + // Scope the note lookup by the snapshot's tip so the result is consistent with the tree + // reads above: mid-apply, the DB may already contain notes from a block the snapshot does + // not include yet. let found_unauthenticated_notes = self .db - .select_existing_note_commitments(unauthenticated_note_commitments) + .select_existing_note_commitments(unauthenticated_note_commitments, latest_block_num) .await?; Ok(TransactionInputs { From 3a274317dcd9309c79a150d15cc547682b6eb610 Mon Sep 17 00:00:00 2001 From: sergerad Date: Mon, 20 Jul 2026 18:58:35 +0800 Subject: [PATCH 08/35] Lint --- crates/store/src/db/mod.rs | 4 ++-- crates/store/src/state/mod.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 43330a996..d117f4c74 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -530,8 +530,8 @@ impl Db { .await } - /// Returns all note commitments from the DB that match the provided ones and were committed - /// at or before `up_to_block`. + /// Returns all note commitments from the DB that match the provided ones and were committed at + /// or before `up_to_block`. #[miden_instrument( level = "debug", target = COMPONENT, diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 0b8d7f886..7986ed3f3 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -466,9 +466,9 @@ impl State { block_num: Option, include_mmr_proof: bool, ) -> Result<(Option, Option), GetBlockHeaderError> { - // Resolve "latest" against the in-memory snapshot rather than the DB: mid-apply, the DB - // may already contain a block that the snapshot's blockchain cannot prove yet. Scoping the - // DB query by the snapshot's tip keeps the header and MMR proof consistent. + // Resolve "latest" against the in-memory snapshot rather than the DB: mid-apply, the DB may + // already contain a block that the snapshot's blockchain cannot prove yet. Scoping the DB + // query by the snapshot's tip keeps the header and MMR proof consistent. let snapshot = self.snapshot(); let latest_block_num = snapshot.latest_block_num(); let block_num = block_num.unwrap_or(latest_block_num); From c21a95009c10b4a367c032d4de9a762f06be0297 Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 22 Jul 2026 11:10:08 +0800 Subject: [PATCH 09/35] Replace result with control flow --- crates/store/src/state/mod.rs | 12 +++++++----- crates/store/src/state/writer.rs | 6 +++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 7986ed3f3..781bf6e3b 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -5,6 +5,7 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::num::NonZeroUsize; +use std::ops::ControlFlow; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -773,9 +774,10 @@ impl State { None }; - // Non-unique account Id prefixes for new accounts are not allowed. + // Non-unique account Id prefixes for new accounts are not allowed, so the transaction + // cannot be valid and the response is already complete. if let Some(false) = new_account_id_prefix_is_unique { - return Err(TransactionInputs { + return ControlFlow::Break(TransactionInputs { new_account_id_prefix_is_unique, ..Default::default() }); @@ -789,7 +791,7 @@ impl State { }) .collect(); - Ok(( + ControlFlow::Continue(( account_commitment, nullifiers, new_account_id_prefix_is_unique, @@ -798,8 +800,8 @@ impl State { }); let (account_commitment, nullifiers, new_account_id_prefix_is_unique, latest_block_num) = match tree_inputs { - Ok(inputs) => inputs, - Err(inputs) => return Ok(inputs), + ControlFlow::Continue(inputs) => inputs, + ControlFlow::Break(response) => return Ok(response), }; // Scope the note lookup by the snapshot's tip so the result is consistent with the tree diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index 3c23044ba..cf5f6144d 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -165,9 +165,9 @@ impl BlockWriter { self.validate_block_header(header, body).await?; - // Compute the tree and forest mutations and note records upfront, before any - // modifications. The writer is the sole forest mutator, so the precomputed forest update - // stays valid until it is applied after the DB commit below. + // Compute the tree and forest mutations and note records upfront, before any modifications. + // The writer is the sole forest mutator, so the precomputed forest update stays valid until + // it is applied after the DB commit below. let (notes, nullifier_tree_update, account_tree_update, account_forest_update) = tokio::task::block_in_place(|| { let notes = Self::build_note_records(header, body)?; From 0b562a89984fd2151f763d5a10a09da56f8b0622 Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 22 Jul 2026 11:35:38 +0800 Subject: [PATCH 10/35] Compartmentalize apply_block --- crates/store/src/state/writer.rs | 170 ++++++++++++++++++++----------- 1 file changed, 111 insertions(+), 59 deletions(-) diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index cf5f6144d..baf2191ac 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -12,6 +12,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use arc_swap::ArcSwap; use miden_node_utils::ErrorReport; use miden_node_utils::tracing::{miden_instrument, miden_span_record}; +use miden_protocol::Word; use miden_protocol::account::AccountUpdateDetails; use miden_protocol::block::account_tree::AccountMutationSet; use miden_protocol::block::nullifier_tree::{NullifierMutationSet, NullifierTree}; @@ -22,7 +23,11 @@ use miden_protocol::transaction::OutputNote; use miden_protocol::utils::serde::Serializable; use tokio::sync::{mpsc, oneshot, watch}; -use crate::account_state_forest::{AccountStateForest, AccountStateForestBackend}; +use crate::account_state_forest::{ + AccountStateForest, + AccountStateForestBackend, + PreparedAccountStateForestBlockUpdate, +}; use crate::accounts::AccountTreeWithHistory; use crate::blocks::BlockStore; use crate::db::{Db, NoteRecord}; @@ -118,6 +123,14 @@ pub(super) struct BlockWriter { pub snapshots_live: Arc, } +/// Note records and state mutations computed from a validated block, before any modifications. +struct PreparedBlockUpdate { + notes: Vec<(NoteRecord, Option)>, + nullifier_tree_update: NullifierMutationSet, + account_tree_update: AccountMutationSet, + account_forest_update: PreparedAccountStateForestBlockUpdate, +} + impl BlockWriter { /// Runs the writer loop, processing requests until all write handles are dropped. pub async fn run(mut self) { @@ -168,31 +181,12 @@ impl BlockWriter { // Compute the tree and forest mutations and note records upfront, before any modifications. // The writer is the sole forest mutator, so the precomputed forest update stays valid until // it is applied after the DB commit below. - let (notes, nullifier_tree_update, account_tree_update, account_forest_update) = - tokio::task::block_in_place(|| { - let notes = Self::build_note_records(header, body)?; - let (nullifier_tree_update, account_tree_update) = - self.compute_tree_mutations(header, body)?; - - // Public account updates carry patches; private accounts are filtered out since - // they don't expose their state changes. - let account_patches = - body.updated_accounts().iter().filter_map(|update| match update.details() { - AccountUpdateDetails::Public(patch) => Some(patch.clone()), - AccountUpdateDetails::Private => None, - }); - let account_forest_update = self - .forest - .compute_block_update_mutations(block_num, account_patches) - .map_err(ApplyBlockError::AccountStateForestPreparation)?; - - Ok::<_, ApplyBlockError>(( - notes, - nullifier_tree_update, - account_tree_update, - account_forest_update, - )) - })?; + let PreparedBlockUpdate { + notes, + nullifier_tree_update, + account_tree_update, + account_forest_update, + } = tokio::task::block_in_place(|| self.prepare_block_update(header, body))?; let precomputed_public_states = account_forest_update.account_states.clone(); // Save the block to the block store. In a case of a failed DB transaction, the in-memory @@ -210,39 +204,17 @@ impl BlockWriter { .await .map_err(|err| ApplyBlockError::DbUpdateTaskFailed(err.as_report()))?; - // Apply the mutations to the writer-owned trees and build the new snapshot from reader - // views of them. The reader views are point-in-time storage snapshots, so no tree data is - // copied. The DB is committed at this point, so a failed mutation would leave the trees - // divergent from canonical state and the process must abort. - let snapshot = - tokio::task::block_in_place(|| { - self.nullifier_tree.apply_mutations(nullifier_tree_update).unwrap_or_else( - |error| Self::abort_after_post_commit_failure("nullifier tree", &error), - ); - - self.account_tree.apply_mutations(account_tree_update).unwrap_or_else(|error| { - Self::abort_after_post_commit_failure("account tree", &error) - }); - - self.blockchain.push(block_commitment); - - self.forest - .apply_precomputed_block_update(block_num, account_forest_update) - .unwrap_or_else(|error| { - Self::abort_after_post_commit_failure("account-state forest", &error) - }); - - Arc::new(InMemoryState { - nullifier_tree: self - .nullifier_tree - .reader() - .expect("nullifier tree snapshot creation should not fail"), - account_tree: self.account_tree.reader(), - blockchain: self.blockchain.clone(), - forest: self.forest.reader().expect("forest snapshot creation should not fail"), - _guard: SnapshotGuard::new(Arc::clone(&self.snapshots_live), block_num), - }) - }); + // The DB is committed at this point, so the prepared mutations must be applied and any + // failure to do so aborts the process. + let snapshot = tokio::task::block_in_place(|| { + self.apply_prepared_mutations( + block_num, + block_commitment, + nullifier_tree_update, + account_tree_update, + account_forest_update, + ) + }); // Atomically publish the new state. Readers that call `snapshot()` after this point will // see the updated state. Readers holding the old snapshot continue unaffected. @@ -269,6 +241,86 @@ impl BlockWriter { Ok(()) } + /// Computes the note records and all tree and forest mutations for a block, without mutating + /// any state. + /// + /// May block on backend I/O, so it must run on Tokio's blocking path. The returned forest + /// update is bound to the forest state observed here; it remains valid until applied because + /// the writer is the sole forest mutator. + fn prepare_block_update( + &self, + header: &BlockHeader, + body: &BlockBody, + ) -> Result { + let notes = Self::build_note_records(header, body)?; + let (nullifier_tree_update, account_tree_update) = + self.compute_tree_mutations(header, body)?; + + // Public account updates carry patches; private accounts are filtered out since they don't + // expose their state changes. + let account_patches = + body.updated_accounts().iter().filter_map(|update| match update.details() { + AccountUpdateDetails::Public(patch) => Some(patch.clone()), + AccountUpdateDetails::Private => None, + }); + let account_forest_update = self + .forest + .compute_block_update_mutations(header.block_num(), account_patches) + .map_err(ApplyBlockError::AccountStateForestPreparation)?; + + Ok(PreparedBlockUpdate { + notes, + nullifier_tree_update, + account_tree_update, + account_forest_update, + }) + } + + /// Applies the prepared mutations to the writer-owned trees and builds the new snapshot from + /// reader views of them. The reader views are point-in-time storage snapshots, so no tree data + /// is copied. + /// + /// Must only be called after the corresponding DB commit: at that point the mutations are part + /// of canonical state, so a failure to apply them leaves the trees divergent and aborts the + /// process. May block on backend I/O, so it must run on Tokio's blocking path. + fn apply_prepared_mutations( + &mut self, + block_num: BlockNumber, + block_commitment: Word, + nullifier_tree_update: NullifierMutationSet, + account_tree_update: AccountMutationSet, + account_forest_update: PreparedAccountStateForestBlockUpdate, + ) -> Arc { + self.nullifier_tree + .apply_mutations(nullifier_tree_update) + .unwrap_or_else(|error| { + Self::abort_after_post_commit_failure("nullifier tree", &error) + }); + + self.account_tree + .apply_mutations(account_tree_update) + .unwrap_or_else(|error| Self::abort_after_post_commit_failure("account tree", &error)); + + self.blockchain.push(block_commitment); + + self.forest + .apply_precomputed_block_update(block_num, account_forest_update) + .unwrap_or_else(|error| { + Self::abort_after_post_commit_failure("account-state forest", &error) + }); + + Arc::new(InMemoryState { + nullifier_tree: self + .nullifier_tree + .reader() + .expect("nullifier tree snapshot creation should not fail"), + account_tree: self.account_tree.reader(), + blockchain: self.blockchain.clone(), + forest: self.forest.reader().expect("forest snapshot creation should not fail"), + _guard: SnapshotGuard::new(Arc::clone(&self.snapshots_live), block_num), + }) + } + /// Terminates after a persistent state failure that occurred after the canonical DB commit. /// /// Returning would expose components at different block heights. Tests panic so the fatal path From ddf4f158686c54846cf20e2c3bd2fd590bf0aaa2 Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 22 Jul 2026 11:40:53 +0800 Subject: [PATCH 11/35] snapshots_live fn --- crates/store/src/state/writer.rs | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index baf2191ac..2f3840592 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -219,16 +219,9 @@ impl BlockWriter { // Atomically publish the new state. Readers that call `snapshot()` after this point will // see the updated state. Readers holding the old snapshot continue unaffected. self.in_memory.store(snapshot); - let snapshots_live = self.snapshots_live.load(Ordering::Relaxed) as u64; + + let snapshots_live = self.check_live_snapshots(block_num); miden_span_record!(snapshots.live = snapshots_live); - if snapshots_live > SNAPSHOTS_LIVE_WARN_THRESHOLD { - tracing::warn!( - target: COMPONENT, - block_num = block_num.as_u32(), - snapshots.live = snapshots_live, - "too many live state snapshots; slow readers are pinning old generations", - ); - } // Push to cache and notify replica subscribers. self.block_cache @@ -241,6 +234,24 @@ impl BlockWriter { Ok(()) } + /// Returns the number of live snapshot generations, warning when slow readers are pinning too + /// many old generations in memory. + /// + /// The count is returned rather than recorded here because `miden_span_record!` must be used + /// within a `#[miden_instrument]` function. + fn check_live_snapshots(&self, block_num: BlockNumber) -> u64 { + let snapshots_live = self.snapshots_live.load(Ordering::Relaxed) as u64; + if snapshots_live > SNAPSHOTS_LIVE_WARN_THRESHOLD { + tracing::warn!( + target: COMPONENT, + block_num = block_num.as_u32(), + snapshots.live = snapshots_live, + "too many live state snapshots; slow readers are pinning old generations", + ); + } + snapshots_live + } + /// Computes the note records and all tree and forest mutations for a block, without mutating /// any state. /// From 6556067fa943140cb7ed11c8966e35fb6b2b3ddd Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 22 Jul 2026 13:11:11 +0800 Subject: [PATCH 12/35] Shutdown writer --- bin/stress-test/src/seeding/mod.rs | 4 ++++ bin/stress-test/src/seeding/tests.rs | 6 ++++++ crates/store/src/state/mod.rs | 28 +++++++++++++++++++++++++++- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/bin/stress-test/src/seeding/mod.rs b/bin/stress-test/src/seeding/mod.rs index 0b9c817cb..15f1929e9 100644 --- a/bin/stress-test/src/seeding/mod.rs +++ b/bin/stress-test/src/seeding/mod.rs @@ -243,6 +243,10 @@ pub async fn seed_store( )) .await; + // Wait for the store to release its backing storage so callers can immediately re-load the + // state from the same data directory. + assert!(store_state.shutdown().await.is_ok(), "no other references to the store state remain"); + println!("Total time: {:.3} seconds", start.elapsed().as_secs_f64()); println!("{metrics}"); } diff --git a/bin/stress-test/src/seeding/tests.rs b/bin/stress-test/src/seeding/tests.rs index e0be9a7b2..8d4dff1ae 100644 --- a/bin/stress-test/src/seeding/tests.rs +++ b/bin/stress-test/src/seeding/tests.rs @@ -198,6 +198,9 @@ async fn seed_store_persists_one_public_account_and_applies_one_map_update() { .map(|(_, value)| *value), Some(benchmark_storage_map_update_value(0, 0, 1)) ); + + // Release the backing storage before the temporary directory is deleted. + assert!(state.shutdown().await.is_ok(), "no other references to the store state remain"); } #[tokio::test(flavor = "multi_thread")] @@ -223,4 +226,7 @@ async fn seed_store_handles_map_larger_than_transaction_account_update_limit() { .await .unwrap(); assert_ne!(response.witness.state_commitment(), Word::empty()); + + // Release the backing storage before the temporary directory is deleted. + assert!(state.shutdown().await.is_ok(), "no other references to the store state remain"); } diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 781bf6e3b..14b0ad7c1 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -226,6 +226,10 @@ pub struct State { /// Handle for sending block-write requests to the [`BlockWriter`] task. write_handle: WriteHandle, + /// Join handle of the [`BlockWriter`] task, used by [`Self::shutdown`] to wait for the writer + /// to release the tree storage it owns. + writer_task: tokio::task::JoinHandle<()>, + /// The latest proven-in-sequence block number, updated by the proof scheduler or `apply_proof`. proven_tip: ProvenTipWriter, @@ -378,7 +382,7 @@ impl State { forest, snapshots_live, }; - tokio::spawn(block_writer.run()); + let writer_task = tokio::spawn(block_writer.run()); Ok(Self { data_directory: data_path.to_path_buf(), @@ -386,6 +390,7 @@ impl State { block_store, in_memory, write_handle, + writer_task, proven_tip, committed_tip_tx, block_cache, @@ -393,6 +398,27 @@ impl State { }) } + /// Shuts down the block writer task, waiting until it has released the tree storage it owns. + /// + /// Dropping the state also stops the writer, but only asynchronously: the task exits once it + /// observes the closed request channel and may briefly outlive the state, keeping the backing + /// storage open. Callers that need the storage released deterministically — e.g. to re-load + /// the state from the same data directory, or to delete it — must use this method instead of + /// dropping. + /// + /// # Errors + /// + /// Returns `self` unchanged if other references to the state are still alive. + pub async fn shutdown(self: Arc) -> Result<(), Arc> { + let state = Arc::try_unwrap(self)?; + // Destructuring drops every unbound field right here, including the write handle — the + // only sender of the writer's request channel. The writer drains any in-flight requests, + // observes the closed channel, and exits, so the join below is a graceful drain. + let Self { writer_task, .. } = state; + writer_task.await.expect("block writer task should not panic"); + Ok(()) + } + /// Returns a watch receiver that wakes every time a new block is committed. pub fn subscribe_committed_tip(&self) -> watch::Receiver { self.committed_tip_tx.subscribe() From 2ad2bbfdc8fe8adca5327aa1b43a05fffcc0249d Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 22 Jul 2026 17:14:13 +0800 Subject: [PATCH 13/35] Lint --- bin/stress-test/src/seeding/mod.rs | 5 ++++- crates/store/src/state/mod.rs | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bin/stress-test/src/seeding/mod.rs b/bin/stress-test/src/seeding/mod.rs index 15f1929e9..96e11b7af 100644 --- a/bin/stress-test/src/seeding/mod.rs +++ b/bin/stress-test/src/seeding/mod.rs @@ -245,7 +245,10 @@ pub async fn seed_store( // Wait for the store to release its backing storage so callers can immediately re-load the // state from the same data directory. - assert!(store_state.shutdown().await.is_ok(), "no other references to the store state remain"); + assert!( + store_state.shutdown().await.is_ok(), + "no other references to the store state remain" + ); println!("Total time: {:.3} seconds", start.elapsed().as_secs_f64()); println!("{metrics}"); diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 14b0ad7c1..92077eb3b 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -411,8 +411,8 @@ impl State { /// Returns `self` unchanged if other references to the state are still alive. pub async fn shutdown(self: Arc) -> Result<(), Arc> { let state = Arc::try_unwrap(self)?; - // Destructuring drops every unbound field right here, including the write handle — the - // only sender of the writer's request channel. The writer drains any in-flight requests, + // Destructuring drops every unbound field right here, including the write handle — the only + // sender of the writer's request channel. The writer drains any in-flight requests, // observes the closed channel, and exits, so the join below is a graceful drain. let Self { writer_task, .. } = state; writer_task.await.expect("block writer task should not panic"); From dc1e08a4f117fe7a17baec18b8c45f1f19868d60 Mon Sep 17 00:00:00 2001 From: sergerad Date: Wed, 22 Jul 2026 17:18:15 +0800 Subject: [PATCH 14/35] Fix comments --- crates/store/src/state/writer.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index 2f3840592..99943321d 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -408,7 +408,8 @@ impl BlockWriter { ) -> Result<(NullifierMutationSet, AccountMutationSet), ApplyBlockError> { let block_num = header.block_num(); - // nullifiers can be produced only once + // A nullifier can only ever be created once, so the block is invalid if any of its + // nullifiers are already recorded in the tree. let duplicate_nullifiers: Vec<_> = body .created_nullifiers() .iter() @@ -419,13 +420,14 @@ impl BlockWriter { return Err(InvalidBlockError::DuplicatedNullifiers(duplicate_nullifiers).into()); } - // new_block.chain_root must be equal to the chain MMR root prior to the update + // The header's chain commitment must equal the chain MMR root prior to this block. let peaks = self.blockchain.peaks(); if peaks.hash_peaks() != header.chain_commitment() { return Err(InvalidBlockError::NewBlockInvalidChainCommitment.into()); } - // compute update for nullifier tree + // Compute the nullifier tree mutations and verify that they produce the nullifier root + // claimed in the header. let nullifier_tree_update = self .nullifier_tree .compute_mutations( @@ -437,7 +439,8 @@ impl BlockWriter { return Err(InvalidBlockError::NewBlockInvalidNullifierRoot.into()); } - // compute update for account tree + // Compute the account tree mutations and verify that they produce the account root + // claimed in the header. let account_tree_update = self .account_tree .compute_mutations( From d7e6c523587c933f690172f9fa678241e9665b16 Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 23 Jul 2026 09:06:48 +0800 Subject: [PATCH 15/35] Lint --- crates/store/src/state/writer.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index 99943321d..4250f0dfa 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -439,8 +439,8 @@ impl BlockWriter { return Err(InvalidBlockError::NewBlockInvalidNullifierRoot.into()); } - // Compute the account tree mutations and verify that they produce the account root - // claimed in the header. + // Compute the account tree mutations and verify that they produce the account root claimed + // in the header. let account_tree_update = self .account_tree .compute_mutations( From d14362edc018308497e7e2b669729776d19fb5c2 Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 23 Jul 2026 09:31:08 +0800 Subject: [PATCH 16/35] Add branch comment --- crates/store/src/state/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 92077eb3b..20423c275 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -824,6 +824,9 @@ impl State { inner.latest_block_num(), )) }); + // `Break` carries a complete response (duplicate account ID prefix), so it is returned + // as-is without the note lookup below; `Continue` carries the tree reads needed to build + // the full response. let (account_commitment, nullifiers, new_account_id_prefix_is_unique, latest_block_num) = match tree_inputs { ControlFlow::Continue(inputs) => inputs, From d6bf7bd6c88bfcb409d2d3f0720089012735d967 Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 23 Jul 2026 10:49:08 +0800 Subject: [PATCH 17/35] Fix shutdown --- crates/store/src/state/mod.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 20423c275..1ca2319aa 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -411,10 +411,12 @@ impl State { /// Returns `self` unchanged if other references to the state are still alive. pub async fn shutdown(self: Arc) -> Result<(), Arc> { let state = Arc::try_unwrap(self)?; - // Destructuring drops every unbound field right here, including the write handle — the only - // sender of the writer's request channel. The writer drains any in-flight requests, - // observes the closed channel, and exits, so the join below is a graceful drain. - let Self { writer_task, .. } = state; + let Self { writer_task, write_handle, .. } = state; + // The unbound `..` fields above are dropped at the end of this scope, i.e. after the join + // below, so the write handle — the only sender of the writer's request channel — must be + // dropped explicitly to close the channel. The writer drains any in-flight requests, + // observes the closed channel, and exits, so the join is a graceful drain. + drop(write_handle); writer_task.await.expect("block writer task should not panic"); Ok(()) } From 220f911a97278be4a3a1b784aab4c4d36b8d47eb Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 23 Jul 2026 11:13:02 +0800 Subject: [PATCH 18/35] Shutdown integration without token --- bin/node/src/commands/mod.rs | 25 +++++++++++++++++++++++++ bin/node/src/commands/modes.rs | 14 ++++++++++---- bin/node/src/commands/recover.rs | 4 +++- crates/store/src/state/mod.rs | 4 ++++ 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/bin/node/src/commands/mod.rs b/bin/node/src/commands/mod.rs index 158a9fe03..9246d2ed6 100644 --- a/bin/node/src/commands/mod.rs +++ b/bin/node/src/commands/mod.rs @@ -7,8 +7,12 @@ mod runtime; pub(crate) mod section; mod store; +use std::sync::Arc; +use std::time::Duration; + use clap::Subcommand; pub use lifecycle::{BootstrapCommand, MigrateCommand}; +use miden_node_store::State; use miden_node_utils::logging::OpenTelemetry; use miden_node_utils::shutdown::CancellationToken; pub use modes::{FullNodeCommand, SequencerCommand}; @@ -16,6 +20,27 @@ pub use recover::RecoverCommand; const ENV_DATA_DIRECTORY: &str = "MIDEN_NODE_DATA_DIRECTORY"; +/// Best-effort drain of the store's block writer so any in-flight block write commits fully before +/// the process exits. +/// +/// Must be called after every other holder of the state has dropped its reference. Failure is +/// logged rather than returned: an unclean stop is recovered by the store's startup consistency +/// checks, so refusing to exit would be worse. +async fn shutdown_state(state: Arc) { + const DRAIN_TIMEOUT: Duration = Duration::from_secs(10); + match tokio::time::timeout(DRAIN_TIMEOUT, state.shutdown()).await { + Ok(Ok(())) => {}, + Ok(Err(_state)) => tracing::error!( + target: crate::LOG_TARGET, + "store state still referenced at shutdown; skipping block writer drain" + ), + Err(_elapsed) => tracing::error!( + target: crate::LOG_TARGET, + "store block writer drain timed out" + ), + } +} + #[derive(Subcommand, Debug)] pub enum Command { /// Start the node in sequencer mode. diff --git a/bin/node/src/commands/modes.rs b/bin/node/src/commands/modes.rs index b1c90acce..75197bd97 100644 --- a/bin/node/src/commands/modes.rs +++ b/bin/node/src/commands/modes.rs @@ -78,7 +78,7 @@ impl SequencerCommand { let rpc = Rpc { listener: bind_rpc(runtime.rpc_listen).await?, - store: state, + store: Arc::clone(&state), mode: RpcMode::sequencer( block_producer.clone(), self.external_services.validator_client()?, @@ -99,7 +99,10 @@ impl SequencerCommand { tasks.spawn("sequencer internal server", sequencer_internal.serve(shutdown.clone())); } - tasks.join_next_or_cancelled(shutdown).await + let result = tasks.join_next_or_cancelled(shutdown).await; + // All tasks have exited and dropped their state clones, so the writer can be drained. + super::shutdown_state(state).await; + result } } @@ -191,7 +194,7 @@ impl FullNodeCommand { let rpc = Rpc { listener: bind_rpc(runtime.rpc_listen).await?, - store: state, + store: Arc::clone(&state), mode: RpcMode::full_node( source_rpc, self.sync.readiness_threshold, @@ -205,7 +208,10 @@ impl FullNodeCommand { let mut tasks = Tasks::new(); tasks.spawn("RPC server", rpc.serve(shutdown.clone())); - tasks.join_next_or_cancelled(shutdown).await + let result = tasks.join_next_or_cancelled(shutdown).await; + // All tasks have exited and dropped their state clones, so the writer can be drained. + super::shutdown_state(state).await; + result } fn sequencer_client(&self) -> Option { diff --git a/bin/node/src/commands/recover.rs b/bin/node/src/commands/recover.rs index 7504ee364..e64959998 100644 --- a/bin/node/src/commands/recover.rs +++ b/bin/node/src/commands/recover.rs @@ -38,7 +38,9 @@ impl RecoverCommand { pub async fn handle(self) -> anyhow::Result<()> { let state = self.load_state().await?; let validator = self.validator_client()?; - recover_from_validator(&state, validator).await + let result = recover_from_validator(&state, validator).await; + super::shutdown_state(state).await; + result } async fn load_state(&self) -> anyhow::Result> { diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 1ca2319aa..3123e1b42 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -406,6 +406,10 @@ impl State { /// the state from the same data directory, or to delete it — must use this method instead of /// dropping. /// + /// This is also the node's graceful-shutdown drain: any in-flight block write commits fully + /// (database, trees, and published snapshot) before this returns, so a subsequent startup + /// skips crash recovery. + /// /// # Errors /// /// Returns `self` unchanged if other references to the state are still alive. From 8a531c28a40b7ce23f3c669199709e70bcf0baac Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 23 Jul 2026 12:03:48 +0800 Subject: [PATCH 19/35] integrate cancellation token --- bin/node/src/commands/modes.rs | 32 +++++++++------ bin/node/src/commands/recover.rs | 4 ++ bin/stress-test/src/seeding/mod.rs | 3 +- bin/stress-test/src/store/mod.rs | 5 ++- crates/block-producer/src/server/tests.rs | 5 ++- crates/rpc/src/tests.rs | 5 ++- crates/store/src/state/mod.rs | 48 +++++++++++++++++++---- crates/store/src/state/writer.rs | 20 +++++++++- 8 files changed, 97 insertions(+), 25 deletions(-) diff --git a/bin/node/src/commands/modes.rs b/bin/node/src/commands/modes.rs index 75197bd97..5d2f578d9 100644 --- a/bin/node/src/commands/modes.rs +++ b/bin/node/src/commands/modes.rs @@ -55,7 +55,7 @@ impl SequencerCommand { let runtime = self.runtime.runtime_config(&self.store); self.block_producer.validate()?; let network_tx_auth = self.runtime.rpc.network_tx_auth()?; - let state = load_state(&runtime).await?; + let state = load_state(&runtime, shutdown.clone()).await?; let _disk_monitor = state.spawn_disk_monitor(shutdown.clone()); let sequencer = Sequencer { @@ -90,6 +90,7 @@ impl SequencerCommand { let mut tasks = Tasks::new(); tasks.spawn("sequencer", sequencer.wait()); tasks.spawn("RPC server", rpc.serve(shutdown.clone())); + tasks.spawn("store block writer", store_writer_drain(state)); if let Some(internal_listen) = self.internal { let sequencer_internal = SequencerInternal { listener: bind_rpc(internal_listen).await?, @@ -99,10 +100,7 @@ impl SequencerCommand { tasks.spawn("sequencer internal server", sequencer_internal.serve(shutdown.clone())); } - let result = tasks.join_next_or_cancelled(shutdown).await; - // All tasks have exited and dropped their state clones, so the writer can be drained. - super::shutdown_state(state).await; - result + tasks.join_next_or_cancelled(shutdown).await } } @@ -189,7 +187,7 @@ impl FullNodeCommand { let validator_client = self.validator_client(); let sequencer_client = self.sequencer_client(); let network_tx_auth = self.runtime.rpc.network_tx_auth()?; - let state = load_state(&runtime).await?; + let state = load_state(&runtime, shutdown.clone()).await?; let _disk_monitor = state.spawn_disk_monitor(shutdown.clone()); let rpc = Rpc { @@ -207,11 +205,9 @@ impl FullNodeCommand { }; let mut tasks = Tasks::new(); tasks.spawn("RPC server", rpc.serve(shutdown.clone())); + tasks.spawn("store block writer", store_writer_drain(state)); - let result = tasks.join_next_or_cancelled(shutdown).await; - // All tasks have exited and dropped their state clones, so the writer can be drained. - super::shutdown_state(state).await; - result + tasks.join_next_or_cancelled(shutdown).await } fn sequencer_client(&self) -> Option { @@ -253,11 +249,15 @@ impl SyncOptions { } } -async fn load_state(runtime: &RuntimeConfig) -> anyhow::Result> { +async fn load_state( + runtime: &RuntimeConfig, + shutdown: CancellationToken, +) -> anyhow::Result> { let state = State::load_with_database_options( &runtime.data_directory, runtime.storage_options.clone(), runtime.database_options, + shutdown, ) .await .context("failed to load state")?; @@ -265,6 +265,16 @@ async fn load_state(runtime: &RuntimeConfig) -> anyhow::Result> { Ok(Arc::new(state)) } +/// Waits for the store's block writer to exit and release its storage. +/// +/// Spawned into the node's task set: on shutdown the task-drain loop waits for the writer to +/// finish any in-flight block write and close its storage, and an unexpected writer exit takes +/// the node down like any other failed task. +async fn store_writer_drain(state: Arc) -> anyhow::Result<()> { + state.writer_done().await; + Ok(()) +} + async fn bind_rpc(listen: SocketAddr) -> anyhow::Result { TcpListener::bind(listen) .await diff --git a/bin/node/src/commands/recover.rs b/bin/node/src/commands/recover.rs index e64959998..af691d73b 100644 --- a/bin/node/src/commands/recover.rs +++ b/bin/node/src/commands/recover.rs @@ -7,6 +7,7 @@ use miden_node_proto::clients::{Builder, ValidatorClient}; use miden_node_proto::generated::validator::BlockSubscriptionRequest; use miden_node_store::State; use miden_node_store::state::Finality; +use miden_node_utils::shutdown::CancellationToken; use miden_protocol::block::{BlockNumber, SignedBlock}; use miden_protocol::utils::serde::Deserializable; use tokio_stream::StreamExt; @@ -44,10 +45,13 @@ impl RecoverCommand { } async fn load_state(&self) -> anyhow::Result> { + // Recovery is not wired into the node's shutdown token; the writer is drained explicitly + // via `shutdown_state` once recovery completes. let state = State::load_with_database_options( &self.data_directory, self.store.storage.clone().into(), self.store.sqlite.database_options(), + CancellationToken::new(), ) .await .context("failed to load state")?; diff --git a/bin/stress-test/src/seeding/mod.rs b/bin/stress-test/src/seeding/mod.rs index 96e11b7af..22c30c487 100644 --- a/bin/stress-test/src/seeding/mod.rs +++ b/bin/stress-test/src/seeding/mod.rs @@ -7,6 +7,7 @@ use metrics::SeedingMetrics; use miden_node_proto::domain::batch::BatchInputs; use miden_node_store::{DataDirectory, GenesisState, State}; use miden_node_utils::clap::StorageOptions; +use miden_node_utils::shutdown::CancellationToken; use miden_protocol::account::auth::AuthScheme; use miden_protocol::account::{ Account, @@ -1026,7 +1027,7 @@ pub async fn start_store(data_directory: PathBuf) -> Arc { } async fn load_state(data_directory: PathBuf) -> Arc { - let state = State::load(&data_directory, StorageOptions::bench()) + let state = State::load(&data_directory, StorageOptions::bench(), CancellationToken::new()) .await .expect("store state should load"); Arc::new(state) diff --git a/bin/stress-test/src/store/mod.rs b/bin/stress-test/src/store/mod.rs index 1f2d63fd4..b4e57f67b 100644 --- a/bin/stress-test/src/store/mod.rs +++ b/bin/stress-test/src/store/mod.rs @@ -7,6 +7,7 @@ use miden_node_proto::domain::account::AccountRequest; use miden_node_proto::generated::{self as proto}; use miden_node_store::state::{Finality, State}; use miden_node_utils::clap::StorageOptions; +use miden_node_utils::shutdown::CancellationToken; use miden_protocol::Word; use miden_protocol::account::AccountId; use miden_protocol::block::BlockNumber; @@ -685,7 +686,9 @@ fn transaction_record_to_proto( pub async fn load_state(data_directory: &Path) { let start = Instant::now(); - let _state = State::load(data_directory, StorageOptions::default()).await.unwrap(); + let _state = State::load(data_directory, StorageOptions::default(), CancellationToken::new()) + .await + .unwrap(); let elapsed = start.elapsed(); // Get database path and run SQL commands to count records diff --git a/crates/block-producer/src/server/tests.rs b/crates/block-producer/src/server/tests.rs index 6e376268c..69902aca8 100644 --- a/crates/block-producer/src/server/tests.rs +++ b/crates/block-producer/src/server/tests.rs @@ -6,6 +6,7 @@ use miden_node_store::GenesisState; use miden_node_store::state::State; use miden_node_utils::clap::StorageOptions; use miden_node_utils::fee::test_fee_params; +use miden_node_utils::shutdown::CancellationToken; use miden_protocol::block::BlockNumber; use miden_protocol::testing::random_secret_key::random_secret_key; use url::Url; @@ -56,6 +57,8 @@ fn bootstrap_store(path: &std::path::Path) { } async fn load_state(path: &std::path::Path) -> Arc { - let state = State::load(path, StorageOptions::default()).await.expect("state should load"); + let state = State::load(path, StorageOptions::default(), CancellationToken::new()) + .await + .expect("state should load"); Arc::new(state) } diff --git a/crates/rpc/src/tests.rs b/crates/rpc/src/tests.rs index af93ea1b1..93e2e7204 100644 --- a/crates/rpc/src/tests.rs +++ b/crates/rpc/src/tests.rs @@ -31,6 +31,7 @@ use miden_node_utils::limiter::{ QueryParamStorageMapKeyTotalLimit, QueryParamStorageMapSlotLimit, }; +use miden_node_utils::shutdown::CancellationToken; use miden_protocol::Word; use miden_protocol::account::{ Account, @@ -119,7 +120,9 @@ impl TestStore { } async fn load_state(path: &std::path::Path) -> Arc { - let state = State::load(path, StorageOptions::default()).await.expect("state should load"); + let state = State::load(path, StorageOptions::default(), CancellationToken::new()) + .await + .expect("state should load"); Arc::new(state) } diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 3123e1b42..991bd0b28 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -16,6 +16,7 @@ use miden_node_proto::domain::batch::BatchInputs; use miden_node_utils::ErrorReport; use miden_node_utils::clap::StorageOptions; use miden_node_utils::formatting::format_array; +use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::tracing::miden_instrument; use miden_protocol::Word; use miden_protocol::account::AccountId; @@ -230,6 +231,10 @@ pub struct State { /// to release the tree storage it owns. writer_task: tokio::task::JoinHandle<()>, + /// Watch receiver whose channel is closed when the writer task exits; see + /// [`Self::writer_done`]. + writer_done_rx: watch::Receiver<()>, + /// The latest proven-in-sequence block number, updated by the proof scheduler or `apply_proof`. proven_tip: ProvenTipWriter, @@ -261,9 +266,15 @@ impl State { pub async fn load( data_path: &Path, storage_options: StorageOptions, + shutdown: CancellationToken, ) -> Result { - Self::load_with_database_options(data_path, storage_options, DatabaseOptions::default()) - .await + Self::load_with_database_options( + data_path, + storage_options, + DatabaseOptions::default(), + shutdown, + ) + .await } /// Loads the state from the data directory using explicit database options. @@ -278,6 +289,7 @@ impl State { data_path: &Path, storage_options: StorageOptions, database_options: DatabaseOptions, + shutdown: CancellationToken, ) -> Result { let data_directory = DataDirectory::load(data_path.to_path_buf()) .map_err(StateInitializationError::DataDirectoryLoadError)?; @@ -365,8 +377,8 @@ impl State { let in_memory = Arc::new(ArcSwap::from(initial_snapshot)); // Spawn the block writer task. It owns the writable trees and processes write requests - // serially, publishing a new snapshot after each committed block. The task exits when all - // write handles are dropped. + // serially, publishing a new snapshot after each committed block. The task exits when the + // shutdown token is cancelled or all write handles are dropped. let (write_tx, write_rx) = tokio::sync::mpsc::channel(1); let write_handle = WriteHandle::new(write_tx); let block_writer = BlockWriter { @@ -376,13 +388,20 @@ impl State { committed_tip_tx: Arc::clone(&committed_tip_tx), block_cache: block_cache.clone(), rx: write_rx, + shutdown, nullifier_tree, account_tree, blockchain, forest, snapshots_live, }; - let writer_task = tokio::spawn(block_writer.run()); + let (writer_done_tx, writer_done_rx) = watch::channel(()); + let writer_task = tokio::spawn(async move { + // `run` consumes the writer, so the tree storage it owns is closed by the time the call + // returns; only then signal completion by dropping the watch sender. + block_writer.run().await; + drop(writer_done_tx); + }); Ok(Self { data_directory: data_path.to_path_buf(), @@ -391,6 +410,7 @@ impl State { in_memory, write_handle, writer_task, + writer_done_rx, proven_tip, committed_tip_tx, block_cache, @@ -398,6 +418,18 @@ impl State { }) } + /// Resolves once the block writer task has exited and released the tree storage it owns. + /// + /// The writer exits after the shutdown token passed to [`Self::load`] is cancelled, once any + /// in-flight block write has committed fully — or unexpectedly if it fails. Intended for + /// spawning into the node's task set so shutdown waits for the drain and an unexpected writer + /// exit takes the node down. + pub async fn writer_done(&self) { + let mut rx = self.writer_done_rx.clone(); + // `changed` errors once the writer task drops its end of the channel. + while rx.changed().await.is_ok() {} + } + /// Shuts down the block writer task, waiting until it has released the tree storage it owns. /// /// Dropping the state also stops the writer, but only asynchronously: the task exits once it @@ -406,9 +438,9 @@ impl State { /// the state from the same data directory, or to delete it — must use this method instead of /// dropping. /// - /// This is also the node's graceful-shutdown drain: any in-flight block write commits fully - /// (database, trees, and published snapshot) before this returns, so a subsequent startup - /// skips crash recovery. + /// The running node does not use this method: its writer exits via the shutdown token passed + /// to [`Self::load`], and the drain is awaited through [`Self::writer_done`]. This method + /// serves callers without a token to cancel, such as tests and the recover command. /// /// # Errors /// diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index 4250f0dfa..2ea5c12e3 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -11,6 +11,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use arc_swap::ArcSwap; use miden_node_utils::ErrorReport; +use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::tracing::{miden_instrument, miden_span_record}; use miden_protocol::Word; use miden_protocol::account::AccountUpdateDetails; @@ -111,6 +112,8 @@ pub(super) struct BlockWriter { pub committed_tip_tx: Arc>, pub block_cache: BlockCache, pub rx: mpsc::Receiver, + /// Token signalling node shutdown; the writer stops accepting new requests once cancelled. + pub shutdown: CancellationToken, /// The mutable nullifier tree owned by this writer. pub nullifier_tree: NullifierTree>, /// The mutable account tree owned by this writer. @@ -132,9 +135,22 @@ struct PreparedBlockUpdate { } impl BlockWriter { - /// Runs the writer loop, processing requests until all write handles are dropped. + /// Runs the writer loop, processing requests until shutdown is signalled or all write handles + /// are dropped. + /// + /// Cancellation is only observed between requests: an in-flight block write always runs to + /// completion, so shutdown never leaves the trees lagging the committed database state. + /// Requests still queued when cancellation fires are dropped, failing their senders. pub async fn run(mut self) { - while let Some(req) = self.rx.recv().await { + loop { + let req = tokio::select! { + biased; + () = self.shutdown.cancelled() => break, + req = self.rx.recv() => match req { + Some(req) => req, + None => break, + }, + }; let result = self.write_block(req.signed_block).await; let _ = req.result_tx.send(result); } From 3b648ff0381b7b2ca5831474fccb8e0e080f7d5d Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 23 Jul 2026 12:59:15 +0800 Subject: [PATCH 20/35] Refactor writer task management --- bin/node/src/commands/mod.rs | 25 ---- bin/node/src/commands/modes.rs | 32 ++--- bin/node/src/commands/recover.rs | 18 +-- bin/stress-test/src/seeding/mod.rs | 26 ++-- bin/stress-test/src/seeding/tests.rs | 14 ++- bin/stress-test/src/store/mod.rs | 4 +- crates/block-producer/src/server/tests.rs | 10 +- crates/rpc/src/tests.rs | 10 +- crates/store/src/lib.rs | 2 +- crates/store/src/state/mod.rs | 142 ++++++++++++---------- 10 files changed, 147 insertions(+), 136 deletions(-) diff --git a/bin/node/src/commands/mod.rs b/bin/node/src/commands/mod.rs index 9246d2ed6..158a9fe03 100644 --- a/bin/node/src/commands/mod.rs +++ b/bin/node/src/commands/mod.rs @@ -7,12 +7,8 @@ mod runtime; pub(crate) mod section; mod store; -use std::sync::Arc; -use std::time::Duration; - use clap::Subcommand; pub use lifecycle::{BootstrapCommand, MigrateCommand}; -use miden_node_store::State; use miden_node_utils::logging::OpenTelemetry; use miden_node_utils::shutdown::CancellationToken; pub use modes::{FullNodeCommand, SequencerCommand}; @@ -20,27 +16,6 @@ pub use recover::RecoverCommand; const ENV_DATA_DIRECTORY: &str = "MIDEN_NODE_DATA_DIRECTORY"; -/// Best-effort drain of the store's block writer so any in-flight block write commits fully before -/// the process exits. -/// -/// Must be called after every other holder of the state has dropped its reference. Failure is -/// logged rather than returned: an unclean stop is recovered by the store's startup consistency -/// checks, so refusing to exit would be worse. -async fn shutdown_state(state: Arc) { - const DRAIN_TIMEOUT: Duration = Duration::from_secs(10); - match tokio::time::timeout(DRAIN_TIMEOUT, state.shutdown()).await { - Ok(Ok(())) => {}, - Ok(Err(_state)) => tracing::error!( - target: crate::LOG_TARGET, - "store state still referenced at shutdown; skipping block writer drain" - ), - Err(_elapsed) => tracing::error!( - target: crate::LOG_TARGET, - "store block writer drain timed out" - ), - } -} - #[derive(Subcommand, Debug)] pub enum Command { /// Start the node in sequencer mode. diff --git a/bin/node/src/commands/modes.rs b/bin/node/src/commands/modes.rs index 5d2f578d9..e0d8ae8d3 100644 --- a/bin/node/src/commands/modes.rs +++ b/bin/node/src/commands/modes.rs @@ -17,6 +17,7 @@ use miden_node_utils::clap::{GrpcOptionsInternal, duration_to_human_readable_str use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::tasks::Tasks; use tokio::net::TcpListener; +use tokio::task::JoinHandle; use url::Url; use super::block_producer::BlockProducerOptions; @@ -55,7 +56,7 @@ impl SequencerCommand { let runtime = self.runtime.runtime_config(&self.store); self.block_producer.validate()?; let network_tx_auth = self.runtime.rpc.network_tx_auth()?; - let state = load_state(&runtime, shutdown.clone()).await?; + let (state, writer_task) = load_state(&runtime, shutdown.clone()).await?; let _disk_monitor = state.spawn_disk_monitor(shutdown.clone()); let sequencer = Sequencer { @@ -78,7 +79,7 @@ impl SequencerCommand { let rpc = Rpc { listener: bind_rpc(runtime.rpc_listen).await?, - store: Arc::clone(&state), + store: state, mode: RpcMode::sequencer( block_producer.clone(), self.external_services.validator_client()?, @@ -90,7 +91,7 @@ impl SequencerCommand { let mut tasks = Tasks::new(); tasks.spawn("sequencer", sequencer.wait()); tasks.spawn("RPC server", rpc.serve(shutdown.clone())); - tasks.spawn("store block writer", store_writer_drain(state)); + tasks.spawn("store block writer", join_store_writer(writer_task)); if let Some(internal_listen) = self.internal { let sequencer_internal = SequencerInternal { listener: bind_rpc(internal_listen).await?, @@ -187,12 +188,12 @@ impl FullNodeCommand { let validator_client = self.validator_client(); let sequencer_client = self.sequencer_client(); let network_tx_auth = self.runtime.rpc.network_tx_auth()?; - let state = load_state(&runtime, shutdown.clone()).await?; + let (state, writer_task) = load_state(&runtime, shutdown.clone()).await?; let _disk_monitor = state.spawn_disk_monitor(shutdown.clone()); let rpc = Rpc { listener: bind_rpc(runtime.rpc_listen).await?, - store: Arc::clone(&state), + store: state, mode: RpcMode::full_node( source_rpc, self.sync.readiness_threshold, @@ -205,7 +206,7 @@ impl FullNodeCommand { }; let mut tasks = Tasks::new(); tasks.spawn("RPC server", rpc.serve(shutdown.clone())); - tasks.spawn("store block writer", store_writer_drain(state)); + tasks.spawn("store block writer", join_store_writer(writer_task)); tasks.join_next_or_cancelled(shutdown).await } @@ -252,8 +253,8 @@ impl SyncOptions { async fn load_state( runtime: &RuntimeConfig, shutdown: CancellationToken, -) -> anyhow::Result> { - let state = State::load_with_database_options( +) -> anyhow::Result<(Arc, JoinHandle<()>)> { + let loaded = State::load_with_database_options( &runtime.data_directory, runtime.storage_options.clone(), runtime.database_options, @@ -262,17 +263,16 @@ async fn load_state( .await .context("failed to load state")?; - Ok(Arc::new(state)) + Ok(loaded.start()) } -/// Waits for the store's block writer to exit and release its storage. +/// Supervises the store's block writer task. /// -/// Spawned into the node's task set: on shutdown the task-drain loop waits for the writer to -/// finish any in-flight block write and close its storage, and an unexpected writer exit takes -/// the node down like any other failed task. -async fn store_writer_drain(state: Arc) -> anyhow::Result<()> { - state.writer_done().await; - Ok(()) +/// On shutdown the task-drain loop waits for the writer to finish any in-flight block write and +/// close its storage; an early exit or panic surfaces through the task set like any other task +/// failure. +async fn join_store_writer(writer_task: JoinHandle<()>) -> anyhow::Result<()> { + writer_task.await.map_err(anyhow::Error::from) } async fn bind_rpc(listen: SocketAddr) -> anyhow::Result { diff --git a/bin/node/src/commands/recover.rs b/bin/node/src/commands/recover.rs index af691d73b..be362d62e 100644 --- a/bin/node/src/commands/recover.rs +++ b/bin/node/src/commands/recover.rs @@ -37,17 +37,21 @@ pub struct RecoverCommand { impl RecoverCommand { pub async fn handle(self) -> anyhow::Result<()> { - let state = self.load_state().await?; + let (state, writer_task) = self.load_state().await?; let validator = self.validator_client()?; let result = recover_from_validator(&state, validator).await; - super::shutdown_state(state).await; + // Wait for the writer to drain and release the backing storage before the process exits. + state + .stop(writer_task) + .await + .map_err(|_| anyhow::anyhow!("store state still referenced; cannot stop the store"))?; result } - async fn load_state(&self) -> anyhow::Result> { - // Recovery is not wired into the node's shutdown token; the writer is drained explicitly - // via `shutdown_state` once recovery completes. - let state = State::load_with_database_options( + async fn load_state(&self) -> anyhow::Result<(Arc, tokio::task::JoinHandle<()>)> { + // Recovery is not wired into the node's shutdown token; the writer exits once the state + // (holding the only write handle) is dropped after recovery completes. + let loaded = State::load_with_database_options( &self.data_directory, self.store.storage.clone().into(), self.store.sqlite.database_options(), @@ -56,7 +60,7 @@ impl RecoverCommand { .await .context("failed to load state")?; - Ok(Arc::new(state)) + Ok(loaded.start()) } fn validator_client(&self) -> anyhow::Result { diff --git a/bin/stress-test/src/seeding/mod.rs b/bin/stress-test/src/seeding/mod.rs index 22c30c487..81159c388 100644 --- a/bin/stress-test/src/seeding/mod.rs +++ b/bin/stress-test/src/seeding/mod.rs @@ -200,7 +200,7 @@ pub async fn seed_store( let genesis_header = genesis_block.inner().header().clone(); State::bootstrap(genesis_block, &data_directory).expect("store should bootstrap"); - let store_state = load_state(data_directory.clone()).await; + let (store_state, writer_task) = load_state(data_directory.clone()).await; // Recreate the deterministic genesis benchmark accounts after bootstrapping instead of keeping // another copy of their potentially very large maps alive while the genesis block is built. @@ -247,8 +247,8 @@ pub async fn seed_store( // Wait for the store to release its backing storage so callers can immediately re-load the // state from the same data directory. assert!( - store_state.shutdown().await.is_ok(), - "no other references to the store state remain" + store_state.stop(writer_task).await.is_ok(), + "no other references to the store state should remain" ); println!("Total time: {:.3} seconds", start.elapsed().as_secs_f64()); @@ -1021,14 +1021,22 @@ async fn get_block_inputs( inputs } -/// Loads the store state from the given data directory. +/// Loads the store state from the given data directory, detaching the block writer task. +/// +/// Intended for benches that run until process exit and never need the storage released +/// deterministically; use [`load_state`] when the writer must be joined. pub async fn start_store(data_directory: PathBuf) -> Arc { - load_state(data_directory).await + let (state, _writer_task) = load_state(data_directory).await; + state } -async fn load_state(data_directory: PathBuf) -> Arc { - let state = State::load(&data_directory, StorageOptions::bench(), CancellationToken::new()) +/// Loads the store state and spawns its block writer, returning the writer's join handle. +/// +/// The writer exits once the last reference to the returned state is dropped; awaiting the handle +/// after that guarantees the backing storage has been released. +async fn load_state(data_directory: PathBuf) -> (Arc, tokio::task::JoinHandle<()>) { + State::load(&data_directory, StorageOptions::bench(), CancellationToken::new()) .await - .expect("store state should load"); - Arc::new(state) + .expect("store state should load") + .start() } diff --git a/bin/stress-test/src/seeding/tests.rs b/bin/stress-test/src/seeding/tests.rs index 8d4dff1ae..e5c6c6e3b 100644 --- a/bin/stress-test/src/seeding/tests.rs +++ b/bin/stress-test/src/seeding/tests.rs @@ -170,7 +170,7 @@ async fn seed_store_persists_one_public_account_and_applies_one_map_update() { assert_eq!(account_ids.len(), 1); let account_id = AccountId::from_hex(account_ids[0]).unwrap(); - let state = load_state(data_directory).await; + let (state, writer_task) = load_state(data_directory).await; let response = state .get_account(AccountRequest { account_id, @@ -200,7 +200,10 @@ async fn seed_store_persists_one_public_account_and_applies_one_map_update() { ); // Release the backing storage before the temporary directory is deleted. - assert!(state.shutdown().await.is_ok(), "no other references to the store state remain"); + assert!( + state.stop(writer_task).await.is_ok(), + "no other references to the store state should remain" + ); } #[tokio::test(flavor = "multi_thread")] @@ -216,7 +219,7 @@ async fn seed_store_handles_map_larger_than_transaction_account_update_limit() { assert_eq!(account_ids.len(), 1); let account_id = AccountId::from_hex(account_ids[0]).unwrap(); - let state = load_state(data_directory).await; + let (state, writer_task) = load_state(data_directory).await; let response = state .get_account(AccountRequest { account_id, @@ -228,5 +231,8 @@ async fn seed_store_handles_map_larger_than_transaction_account_update_limit() { assert_ne!(response.witness.state_commitment(), Word::empty()); // Release the backing storage before the temporary directory is deleted. - assert!(state.shutdown().await.is_ok(), "no other references to the store state remain"); + assert!( + state.stop(writer_task).await.is_ok(), + "no other references to the store state should remain" + ); } diff --git a/bin/stress-test/src/store/mod.rs b/bin/stress-test/src/store/mod.rs index b4e57f67b..d79ce09a6 100644 --- a/bin/stress-test/src/store/mod.rs +++ b/bin/stress-test/src/store/mod.rs @@ -686,7 +686,9 @@ fn transaction_record_to_proto( pub async fn load_state(data_directory: &Path) { let start = Instant::now(); - let _state = State::load(data_directory, StorageOptions::default(), CancellationToken::new()) + // The writer is never started: this bench only measures load time, and dropping the un-started + // state releases the tree storage the writer owns. + let _loaded = State::load(data_directory, StorageOptions::default(), CancellationToken::new()) .await .unwrap(); let elapsed = start.elapsed(); diff --git a/crates/block-producer/src/server/tests.rs b/crates/block-producer/src/server/tests.rs index 69902aca8..e6143e45b 100644 --- a/crates/block-producer/src/server/tests.rs +++ b/crates/block-producer/src/server/tests.rs @@ -57,8 +57,10 @@ fn bootstrap_store(path: &std::path::Path) { } async fn load_state(path: &std::path::Path) -> Arc { - let state = State::load(path, StorageOptions::default(), CancellationToken::new()) - .await - .expect("state should load"); - Arc::new(state) + let (state, _writer_task) = + State::load(path, StorageOptions::default(), CancellationToken::new()) + .await + .expect("state should load") + .start(); + state } diff --git a/crates/rpc/src/tests.rs b/crates/rpc/src/tests.rs index 93e2e7204..1d2081275 100644 --- a/crates/rpc/src/tests.rs +++ b/crates/rpc/src/tests.rs @@ -120,10 +120,12 @@ impl TestStore { } async fn load_state(path: &std::path::Path) -> Arc { - let state = State::load(path, StorageOptions::default(), CancellationToken::new()) - .await - .expect("state should load"); - Arc::new(state) + let (state, _writer_task) = + State::load(path, StorageOptions::default(), CancellationToken::new()) + .await + .expect("state should load") + .start(); + state } /// Byte offset of the account delta commitment in serialized `ProvenTransaction`. Layout: diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 416e08736..17fdba636 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -38,7 +38,7 @@ pub use errors::{ StateSyncError, }; pub use genesis::GenesisState; -pub use state::State; +pub use state::{LoadedState, State}; /// Returns the store crate version. pub fn version() -> &'static str { diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 991bd0b28..46647f9d0 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -227,14 +227,6 @@ pub struct State { /// Handle for sending block-write requests to the [`BlockWriter`] task. write_handle: WriteHandle, - /// Join handle of the [`BlockWriter`] task, used by [`Self::shutdown`] to wait for the writer - /// to release the tree storage it owns. - writer_task: tokio::task::JoinHandle<()>, - - /// Watch receiver whose channel is closed when the writer task exits; see - /// [`Self::writer_done`]. - writer_done_rx: watch::Receiver<()>, - /// The latest proven-in-sequence block number, updated by the proof scheduler or `apply_proof`. proven_tip: ProvenTipWriter, @@ -251,6 +243,71 @@ pub struct State { pub(crate) proof_cache: ProofCache, } +/// A loaded store state whose block writer has not been started yet. +/// +/// Returned by [`State::load`]. [`Self::start`] spawns the writer and yields the usable +/// [`State`]; since this is the only way to obtain one, [`State::apply_block`] can always make +/// progress. +#[must_use = "call `start` to spawn the block writer and obtain the state"] +pub struct LoadedState { + state: State, + writer: BlockWriter, +} + +impl LoadedState { + /// Spawns the block writer onto the current runtime and returns the state together with the + /// writer task's join handle. + /// + /// The writer exits once the shutdown token passed to [`State::load`] is cancelled or the last + /// state reference (holding the only write handle) is dropped — an in-flight block write + /// always completes first. Awaiting the returned handle after either event guarantees the + /// writer has released the tree storage it owns; a join error carries a writer panic. + /// + /// Callers without a token to cancel should stop the store via [`State::stop`] rather than + /// dropping and joining by hand. + pub fn start(self) -> (Arc, tokio::task::JoinHandle<()>) { + let writer_task = tokio::spawn(self.writer.run()); + (Arc::new(self.state), writer_task) + } +} + +impl State { + /// Stops the store, waiting until the block writer has released the tree storage it owns. + /// + /// Consumes the last state reference — closing the write channel the writer listens on — and + /// then joins the writer task returned by [`LoadedState::start`]. The drop must precede the + /// join or the writer never observes the closed channel; doing both here keeps that ordering + /// out of caller hands. + /// + /// Callers that need the storage released deterministically must use this method instead of + /// dropping: the node's `recover` command stops the store before the process exits, and the + /// stress-test's store seeding stops it so the same data directory can be re-loaded (or its + /// temporary directory deleted) immediately afterwards. The running node does not use this + /// method — its writer exits via the shutdown token passed to [`State::load`] and is joined + /// through the node's task set. + /// + /// # Errors + /// + /// Returns both pieces unchanged if other references to the state are still alive. + /// + /// # Panics + /// + /// Panics if the writer task panicked. + pub async fn stop( + self: Arc, + writer_task: tokio::task::JoinHandle<()>, + ) -> Result<(), (Arc, tokio::task::JoinHandle<()>)> { + match Arc::try_unwrap(self) { + Ok(state) => { + drop(state); + writer_task.await.expect("block writer task should not panic"); + Ok(()) + }, + Err(state) => Err((state, writer_task)), + } + } +} + impl State { // CONSTRUCTOR // -------------------------------------------------------------------------------------------- @@ -258,7 +315,8 @@ impl State { /// Loads the state from the data directory. /// /// The loaded state owns all store data structures and exposes subscription methods for - /// sequencer and replica tasks. + /// sequencer and replica tasks. Call [`LoadedState::start`] on the result to spawn the block + /// writer and obtain the usable [`State`]. #[miden_instrument( target = COMPONENT, skip_all, @@ -267,7 +325,7 @@ impl State { data_path: &Path, storage_options: StorageOptions, shutdown: CancellationToken, - ) -> Result { + ) -> Result { Self::load_with_database_options( data_path, storage_options, @@ -280,7 +338,8 @@ impl State { /// Loads the state from the data directory using explicit database options. /// /// The loaded state owns all store data structures and exposes subscription methods for - /// sequencer and replica tasks. + /// sequencer and replica tasks. Call [`LoadedState::start`] on the result to spawn the block + /// writer and obtain the usable [`State`]. #[miden_instrument( target = COMPONENT, skip_all, @@ -290,7 +349,7 @@ impl State { storage_options: StorageOptions, database_options: DatabaseOptions, shutdown: CancellationToken, - ) -> Result { + ) -> Result { let data_directory = DataDirectory::load(data_path.to_path_buf()) .map_err(StateInitializationError::DataDirectoryLoadError)?; @@ -376,9 +435,9 @@ impl State { }); let in_memory = Arc::new(ArcSwap::from(initial_snapshot)); - // Spawn the block writer task. It owns the writable trees and processes write requests - // serially, publishing a new snapshot after each committed block. The task exits when the - // shutdown token is cancelled or all write handles are dropped. + // Assemble the block writer. It owns the writable trees and processes write requests + // serially, publishing a new snapshot after each committed block. The caller runs it; it + // exits when the shutdown token is cancelled or all write handles are dropped. let (write_tx, write_rx) = tokio::sync::mpsc::channel(1); let write_handle = WriteHandle::new(write_tx); let block_writer = BlockWriter { @@ -395,66 +454,19 @@ impl State { forest, snapshots_live, }; - let (writer_done_tx, writer_done_rx) = watch::channel(()); - let writer_task = tokio::spawn(async move { - // `run` consumes the writer, so the tree storage it owns is closed by the time the call - // returns; only then signal completion by dropping the watch sender. - block_writer.run().await; - drop(writer_done_tx); - }); - - Ok(Self { + let state = Self { data_directory: data_path.to_path_buf(), db, block_store, in_memory, write_handle, - writer_task, - writer_done_rx, proven_tip, committed_tip_tx, block_cache, proof_cache, - }) - } - - /// Resolves once the block writer task has exited and released the tree storage it owns. - /// - /// The writer exits after the shutdown token passed to [`Self::load`] is cancelled, once any - /// in-flight block write has committed fully — or unexpectedly if it fails. Intended for - /// spawning into the node's task set so shutdown waits for the drain and an unexpected writer - /// exit takes the node down. - pub async fn writer_done(&self) { - let mut rx = self.writer_done_rx.clone(); - // `changed` errors once the writer task drops its end of the channel. - while rx.changed().await.is_ok() {} - } + }; - /// Shuts down the block writer task, waiting until it has released the tree storage it owns. - /// - /// Dropping the state also stops the writer, but only asynchronously: the task exits once it - /// observes the closed request channel and may briefly outlive the state, keeping the backing - /// storage open. Callers that need the storage released deterministically — e.g. to re-load - /// the state from the same data directory, or to delete it — must use this method instead of - /// dropping. - /// - /// The running node does not use this method: its writer exits via the shutdown token passed - /// to [`Self::load`], and the drain is awaited through [`Self::writer_done`]. This method - /// serves callers without a token to cancel, such as tests and the recover command. - /// - /// # Errors - /// - /// Returns `self` unchanged if other references to the state are still alive. - pub async fn shutdown(self: Arc) -> Result<(), Arc> { - let state = Arc::try_unwrap(self)?; - let Self { writer_task, write_handle, .. } = state; - // The unbound `..` fields above are dropped at the end of this scope, i.e. after the join - // below, so the write handle — the only sender of the writer's request channel — must be - // dropped explicitly to close the channel. The writer drains any in-flight requests, - // observes the closed channel, and exits, so the join is a graceful drain. - drop(write_handle); - writer_task.await.expect("block writer task should not panic"); - Ok(()) + Ok(LoadedState { state, writer: block_writer }) } /// Returns a watch receiver that wakes every time a new block is committed. From aea62cefb0f1f41bcba592e0f024b807132c37fb Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 23 Jul 2026 13:11:00 +0800 Subject: [PATCH 21/35] Split module --- crates/store/src/state/inputs.rs | 369 ++++++++++++ crates/store/src/state/lifecycle.rs | 269 +++++++++ crates/store/src/state/mod.rs | 905 +--------------------------- crates/store/src/state/queries.rs | 96 +++ crates/store/src/state/replica.rs | 57 ++ crates/store/src/state/snapshot.rs | 157 +++++ 6 files changed, 968 insertions(+), 885 deletions(-) create mode 100644 crates/store/src/state/inputs.rs create mode 100644 crates/store/src/state/lifecycle.rs create mode 100644 crates/store/src/state/queries.rs create mode 100644 crates/store/src/state/snapshot.rs diff --git a/crates/store/src/state/inputs.rs b/crates/store/src/state/inputs.rs new file mode 100644 index 000000000..cc03f4191 --- /dev/null +++ b/crates/store/src/state/inputs.rs @@ -0,0 +1,369 @@ +//! Block-producer input queries: batch, block, and transaction inputs. +//! +//! These read paths combine in-memory snapshot data (tree witnesses, partial MMRs) with database +//! lookups, scoping the latter by the snapshot's tip so both sources describe the same block +//! height. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::ops::ControlFlow; + +use miden_node_proto::domain::batch::BatchInputs; +use miden_node_utils::formatting::format_array; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::Word; +use miden_protocol::account::AccountId; +use miden_protocol::block::account_tree::AccountWitness; +use miden_protocol::block::nullifier_tree::NullifierWitness; +use miden_protocol::block::{BlockInputs, BlockNumber}; +use miden_protocol::crypto::merkle::mmr::PartialMmr; +use miden_protocol::note::Nullifier; +use miden_protocol::transaction::PartialBlockchain; + +use crate::COMPONENT; +use crate::db::NullifierInfo; +use crate::errors::{DatabaseError, GetBatchInputsError, GetBlockInputsError}; +use crate::state::State; + +// STRUCTURES +// ================================================================================================ + +#[derive(Debug, Default)] +pub struct TransactionInputs { + pub account_commitment: Word, + pub nullifiers: Vec, + pub found_unauthenticated_notes: HashSet, + pub new_account_id_prefix_is_unique: Option, +} + +type BlockInputWitnesses = ( + BlockNumber, + BTreeMap, + BTreeMap, + PartialMmr, +); + +// INPUT QUERIES +// ================================================================================================ + +impl State { + /// Fetches the inputs for a transaction batch from the database. + /// + /// ## Inputs + /// + /// The function takes as input: + /// - The tx reference blocks are the set of blocks referenced by transactions in the batch. + /// - The unauthenticated note commitments are the set of commitments of unauthenticated notes + /// consumed by all transactions in the batch. For these notes, we attempt to find inclusion + /// proofs. Not all notes will exist in the DB necessarily, as some notes can be created and + /// consumed within the same batch. + /// + /// ## Outputs + /// + /// The function will return: + /// - A block inclusion proof for all tx reference blocks and for all blocks which are + /// referenced by a note inclusion proof. + /// - Note inclusion proofs for all notes that were found in the DB. + /// - The block header that the batch should reference, i.e. the latest known block. + pub async fn get_batch_inputs( + &self, + tx_reference_blocks: BTreeSet, + unauthenticated_note_commitments: BTreeSet, + ) -> Result { + if tx_reference_blocks.is_empty() { + return Err(GetBatchInputsError::TransactionBlockReferencesEmpty); + } + + // First we grab note inclusion proofs for the known notes. These proofs only prove that the + // note was included in a given block. We then also need to prove that each of those blocks + // is included in the chain. + let note_proofs = self + .db + .select_note_inclusion_proofs(unauthenticated_note_commitments) + .await + .map_err(GetBatchInputsError::SelectNoteInclusionProofError)?; + + // The set of blocks that the notes are included in. + let note_blocks = note_proofs.values().map(|proof| proof.location().block_num()); + + // Collect all blocks we need to query without duplicates, which is: + // - all blocks for which we need to prove note inclusion. + // - all blocks referenced by transactions in the batch. + let mut blocks: BTreeSet = tx_reference_blocks; + blocks.extend(note_blocks); + + let (batch_reference_block, partial_mmr) = { + let snapshot = self.snapshot(); + + let latest_block_num = snapshot.latest_block_num(); + + let highest_block_num = + *blocks.last().expect("we should have checked for empty block references"); + if highest_block_num > latest_block_num { + return Err(GetBatchInputsError::UnknownTransactionBlockReference { + highest_block_num, + latest_block_num, + }); + } + + // Remove the latest block from the to-be-tracked blocks as it will be the reference + // block for the batch itself and thus added to the MMR within the batch kernel, so + // there is no need to prove its inclusion. + blocks.remove(&latest_block_num); + + // SAFETY: + // - The latest block num was retrieved from the inner blockchain from which we will + // also retrieve the proofs, so it is guaranteed to exist in that chain. + // - We have checked that no block number in the blocks set is greater than latest block + // number *and* latest block num was removed from the set. Therefore only block + // numbers smaller than latest block num remain in the set. Therefore all the block + // numbers are guaranteed to exist in the chain state at latest block num. + let partial_mmr = snapshot + .blockchain + .partial_mmr_from_blocks(&blocks, latest_block_num) + .expect("latest block num should exist and all blocks in set should be < than latest block"); + + (latest_block_num, partial_mmr) + }; + + // Fetch the reference block of the batch as part of this query, so we can avoid looking it + // up in a separate DB access. + let mut headers = self + .db + .select_block_headers(blocks.into_iter().chain(std::iter::once(batch_reference_block))) + .await + .map_err(GetBatchInputsError::SelectBlockHeaderError)?; + + // Find and remove the batch reference block as we don't want to add it to the chain MMR. + let header_index = headers + .iter() + .enumerate() + .find_map(|(index, header)| { + (header.block_num() == batch_reference_block).then_some(index) + }) + .expect("DB should have returned the header of the batch reference block"); + + // The order doesn't matter for PartialBlockchain::new, so swap remove is fine. + let batch_reference_block_header = headers.swap_remove(header_index); + + // SAFETY: This should not error because: + // - we're passing exactly the block headers that we've added to the partial MMR, + // - so none of the block headers block numbers should exceed the chain length of the + // partial MMR, + // - and we've added blocks to a BTreeSet, so there can be no duplicates. + // + // We construct headers and partial MMR in concert, so they are consistent. This is why we + // can call the unchecked constructor. + let partial_block_chain = PartialBlockchain::new_unchecked(partial_mmr, headers) + .expect("partial mmr and block headers should be consistent"); + + Ok(BatchInputs { + batch_reference_block_header, + note_proofs, + partial_block_chain, + }) + } + + /// Returns data needed by the block producer to construct and prove the next block. + pub async fn get_block_inputs( + &self, + account_ids: Vec, + nullifiers: Vec, + unauthenticated_note_commitments: BTreeSet, + reference_blocks: BTreeSet, + ) -> Result { + // Get the note inclusion proofs from the DB. We do this first so we have to acquire the + // lock to the state just once. There we need the reference blocks of the note proofs to get + // their authentication paths in the chain MMR. + let unauthenticated_note_proofs = self + .db + .select_note_inclusion_proofs(unauthenticated_note_commitments) + .await + .map_err(GetBlockInputsError::SelectNoteInclusionProofError)?; + + // The set of blocks that the notes are included in. + let note_proof_reference_blocks = + unauthenticated_note_proofs.values().map(|proof| proof.location().block_num()); + + // Collect all blocks we need to prove inclusion for, without duplicates. + let mut blocks = reference_blocks; + blocks.extend(note_proof_reference_blocks); + + let (latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr) = + self.get_block_inputs_witnesses(&mut blocks, &account_ids, &nullifiers)?; + + // Fetch the block headers for all blocks in the partial MMR plus the latest one which will + // be used as the previous block header of the block being built. + let mut headers = self + .db + .select_block_headers(blocks.into_iter().chain(std::iter::once(latest_block_number))) + .await + .map_err(GetBlockInputsError::SelectBlockHeaderError)?; + + // Find and remove the latest block as we must not add it to the chain MMR, since it is not + // yet in the chain. + let latest_block_header_index = headers + .iter() + .enumerate() + .find_map(|(index, header)| { + (header.block_num() == latest_block_number).then_some(index) + }) + .expect("DB should have returned the header of the latest block header"); + + // The order doesn't matter for PartialBlockchain::new, so swap remove is fine. + let latest_block_header = headers.swap_remove(latest_block_header_index); + + // SAFETY: This should not error because: + // - we're passing exactly the block headers that we've added to the partial MMR, + // - so none of the block header's block numbers should exceed the chain length of the + // partial MMR, + // - and we've added blocks to a BTreeSet, so there can be no duplicates. + // + // We construct headers and partial MMR in concert, so they are consistent. This is why we + // can call the unchecked constructor. + let partial_block_chain = PartialBlockchain::new_unchecked(partial_mmr, headers) + .expect("partial mmr and block headers should be consistent"); + + Ok(BlockInputs::new( + latest_block_header, + partial_block_chain, + account_witnesses, + nullifier_witnesses, + unauthenticated_note_proofs, + )) + } + + /// Get account and nullifier witnesses for the requested account IDs and nullifier as well as + /// the [`PartialMmr`] for the given blocks. The MMR won't contain the latest block and its + /// number is removed from `blocks` and returned separately. + /// + /// This method acquires the lock to the inner state and does not access the DB so we release + /// the lock asap. + fn get_block_inputs_witnesses( + &self, + blocks: &mut BTreeSet, + account_ids: &[AccountId], + nullifiers: &[Nullifier], + ) -> Result { + self.with_inner_read_blocking(|inner| { + let latest_block_number = inner.latest_block_num(); + + // If `blocks` is empty, use the latest block number which will never trigger the error. + let highest_block_number = blocks.last().copied().unwrap_or(latest_block_number); + if highest_block_number > latest_block_number { + return Err(GetBlockInputsError::UnknownBatchBlockReference { + highest_block_number, + latest_block_number, + }); + } + + // The latest block is not yet in the chain MMR, so we can't (and don't need to) prove + // its inclusion in the chain. + blocks.remove(&latest_block_number); + + // Fetch the partial MMR at the state of the latest block with authentication paths for + // the provided set of blocks. + // + // SAFETY: + // - The latest block num was retrieved from the inner blockchain from which we will + // also retrieve the proofs, so it is guaranteed to exist in that chain. + // - We have checked that no block number in the blocks set is greater than latest block + // number *and* latest block num was removed from the set. Therefore only block + // numbers smaller than latest block num remain in the set. Therefore all the block + // numbers are guaranteed to exist in the chain state at latest block num. + let partial_mmr = + inner.blockchain.partial_mmr_from_blocks(blocks, latest_block_number).expect( + "latest block num should exist and all blocks in set should be < than latest block", + ); + + // Fetch witnesses for all accounts. + let account_witnesses = account_ids + .iter() + .copied() + .map(|account_id| (account_id, inner.account_tree.open_latest(account_id))) + .collect::>(); + + // Fetch witnesses for all nullifiers. We don't check whether the nullifiers are spent + // or not as this is done as part of proposing the block. + let nullifier_witnesses: BTreeMap = nullifiers + .iter() + .copied() + .map(|nullifier| (nullifier, inner.nullifier_tree.open(&nullifier))) + .collect(); + + Ok((latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr)) + }) + } + + /// Returns data needed by the block producer to verify transactions validity. + #[miden_instrument( + target = COMPONENT, + skip_all, + fields( + account.id=%account_id, + nullifiers = %format_array(nullifiers), + ), + )] + pub async fn get_transaction_inputs( + &self, + account_id: AccountId, + nullifiers: &[Nullifier], + unauthenticated_note_commitments: Vec, + ) -> Result { + let tree_inputs = self.with_inner_read_blocking(|inner| { + let account_commitment = inner.account_tree.get_latest_commitment(account_id); + + let new_account_id_prefix_is_unique = if account_commitment.is_empty() { + Some(!inner.account_tree.contains_account_id_prefix_in_latest(account_id.prefix())) + } else { + None + }; + + // Non-unique account Id prefixes for new accounts are not allowed, so the transaction + // cannot be valid and the response is already complete. + if let Some(false) = new_account_id_prefix_is_unique { + return ControlFlow::Break(TransactionInputs { + new_account_id_prefix_is_unique, + ..Default::default() + }); + } + + let nullifiers = nullifiers + .iter() + .map(|nullifier| NullifierInfo { + nullifier: *nullifier, + block_num: inner.nullifier_tree.get_block_num(nullifier).unwrap_or_default(), + }) + .collect(); + + ControlFlow::Continue(( + account_commitment, + nullifiers, + new_account_id_prefix_is_unique, + inner.latest_block_num(), + )) + }); + // `Break` carries a complete response (duplicate account ID prefix), so it is returned + // as-is without the note lookup below; `Continue` carries the tree reads needed to build + // the full response. + let (account_commitment, nullifiers, new_account_id_prefix_is_unique, latest_block_num) = + match tree_inputs { + ControlFlow::Continue(inputs) => inputs, + ControlFlow::Break(response) => return Ok(response), + }; + + // Scope the note lookup by the snapshot's tip so the result is consistent with the tree + // reads above: mid-apply, the DB may already contain notes from a block the snapshot does + // not include yet. + let found_unauthenticated_notes = self + .db + .select_existing_note_commitments(unauthenticated_note_commitments, latest_block_num) + .await?; + + Ok(TransactionInputs { + account_commitment, + nullifiers, + found_unauthenticated_notes, + new_account_id_prefix_is_unique, + }) + } +} diff --git a/crates/store/src/state/lifecycle.rs b/crates/store/src/state/lifecycle.rs new file mode 100644 index 000000000..57db356fa --- /dev/null +++ b/crates/store/src/state/lifecycle.rs @@ -0,0 +1,269 @@ +//! Store lifecycle: loading the state, starting its block writer, and stopping the store. + +use std::num::NonZeroUsize; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::AtomicUsize; + +use arc_swap::ArcSwap; +use miden_node_utils::ErrorReport; +use miden_node_utils::clap::StorageOptions; +use miden_node_utils::shutdown::CancellationToken; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::block::BlockNumber; +use tokio::sync::watch; + +use crate::account_state_forest::AccountStateForestBackend; +use crate::accounts::AccountTreeWithHistory; +use crate::blocks::BlockStore; +use crate::db::Db; +use crate::errors::StateInitializationError; +use crate::proven_tip::ProvenTipWriter; +use crate::state::loader::{ + ACCOUNT_STATE_FOREST_STORAGE_DIR, + ACCOUNT_TREE_STORAGE_DIR, + AccountForestLoader, + NULLIFIER_TREE_STORAGE_DIR, + TreeStorage, + TreeStorageLoader, + load_mmr, + verify_account_state_forest_consistency, + verify_tree_consistency, +}; +use crate::state::writer::{BlockWriter, WriteHandle}; +use crate::state::{BlockCache, InMemoryState, ProofCache, SnapshotGuard, State}; +use crate::{COMPONENT, DataDirectory, DatabaseOptions}; + +/// Number of recent committed blocks held in the in-memory cache for replica subscriptions. +const BLOCK_CACHE_CAPACITY: NonZeroUsize = NonZeroUsize::new(512).unwrap(); + +/// Number of recent block proofs held in the in-memory cache for replica subscriptions. +const PROOF_CACHE_CAPACITY: NonZeroUsize = NonZeroUsize::new(512).unwrap(); + +// LOADED STATE +// ================================================================================================ + +/// A loaded store state whose block writer has not been started yet. +/// +/// Returned by [`State::load`]. [`Self::start`] spawns the writer and yields the usable +/// [`State`]; since this is the only way to obtain one, [`State::apply_block`] can always make +/// progress. +#[must_use = "call `start` to spawn the block writer and obtain the state"] +pub struct LoadedState { + state: State, + writer: BlockWriter, +} + +impl LoadedState { + /// Spawns the block writer onto the current runtime and returns the state together with the + /// writer task's join handle. + /// + /// The writer exits once the shutdown token passed to [`State::load`] is cancelled or the last + /// state reference (holding the only write handle) is dropped — an in-flight block write + /// always completes first. Awaiting the returned handle after either event guarantees the + /// writer has released the tree storage it owns; a join error carries a writer panic. + /// + /// Callers without a token to cancel should stop the store via [`State::stop`] rather than + /// dropping and joining by hand. + pub fn start(self) -> (Arc, tokio::task::JoinHandle<()>) { + let writer_task = tokio::spawn(self.writer.run()); + (Arc::new(self.state), writer_task) + } +} + +// LOAD & STOP +// ================================================================================================ + +impl State { + /// Loads the state from the data directory. + /// + /// The loaded state owns all store data structures and exposes subscription methods for + /// sequencer and replica tasks. Call [`LoadedState::start`] on the result to spawn the block + /// writer and obtain the usable [`State`]. + #[miden_instrument( + target = COMPONENT, + skip_all, + )] + pub async fn load( + data_path: &Path, + storage_options: StorageOptions, + shutdown: CancellationToken, + ) -> Result { + Self::load_with_database_options( + data_path, + storage_options, + DatabaseOptions::default(), + shutdown, + ) + .await + } + + /// Loads the state from the data directory using explicit database options. + /// + /// The loaded state owns all store data structures and exposes subscription methods for + /// sequencer and replica tasks. Call [`LoadedState::start`] on the result to spawn the block + /// writer and obtain the usable [`State`]. + #[miden_instrument( + target = COMPONENT, + skip_all, + )] + pub async fn load_with_database_options( + data_path: &Path, + storage_options: StorageOptions, + database_options: DatabaseOptions, + shutdown: CancellationToken, + ) -> Result { + let data_directory = DataDirectory::load(data_path.to_path_buf()) + .map_err(StateInitializationError::DataDirectoryLoadError)?; + + let block_store = Arc::new( + BlockStore::load(data_directory.block_store_dir()) + .map_err(StateInitializationError::BlockStoreLoadError)?, + ); + + let database_filepath = data_directory.database_path(); + let mut db = Db::load_with_pool_size( + database_filepath.clone(), + database_options.connection_pool_size, + ) + .await + .map_err(StateInitializationError::DatabaseLoadError)?; + + let blockchain = load_mmr(&mut db).await?; + let latest_block_num = blockchain.chain_tip().unwrap_or(BlockNumber::GENESIS); + + #[cfg(feature = "rocksdb")] + let (account_storage_config, nullifier_storage_config, forest_storage_config) = ( + storage_options.account_tree.into(), + storage_options.nullifier_tree.into(), + storage_options.account_state_forest.into(), + ); + #[cfg(not(feature = "rocksdb"))] + let (account_storage_config, nullifier_storage_config, forest_storage_config) = { + let _ = &storage_options; + ((), (), ()) + }; + let account_storage = + TreeStorage::create(data_path, &account_storage_config, ACCOUNT_TREE_STORAGE_DIR)?; + let account_tree = account_storage.load_account_tree(&mut db).await?; + + let nullifier_storage = + TreeStorage::create(data_path, &nullifier_storage_config, NULLIFIER_TREE_STORAGE_DIR)?; + let nullifier_tree = nullifier_storage.load_nullifier_tree(&mut db).await?; + + // Verify that tree roots match the expected roots from the database. This catches any + // divergence between persistent storage and the database caused by corruption or incomplete + // shutdown. + verify_tree_consistency(account_tree.root(), nullifier_tree.root(), &mut db).await?; + + let account_tree = AccountTreeWithHistory::new(account_tree, latest_block_num); + + let forest_backend = AccountStateForestBackend::create( + data_path, + &forest_storage_config, + ACCOUNT_STATE_FOREST_STORAGE_DIR, + )?; + let forest = forest_backend.load_account_state_forest(&mut db, latest_block_num).await?; + verify_account_state_forest_consistency(&forest, &mut db).await?; + + let db = Arc::new(db); + + // Initialize the proven tip from the block store. + let proven_tip_init = block_store + .load_proven_tip() + .map_err(StateInitializationError::ProvenTipLoadError)?; + let (proven_tip, _rx) = ProvenTipWriter::new(proven_tip_init); + + // Committed-tip watch: fires after each successful apply_block. + let (committed_tip_tx, _rx) = watch::channel(latest_block_num); + let committed_tip_tx = Arc::new(committed_tip_tx); + + let block_cache = BlockCache::new(BLOCK_CACHE_CAPACITY); + let proof_cache = ProofCache::new(PROOF_CACHE_CAPACITY); + + // Shared counter of live snapshot generations, for observability. + let snapshots_live = Arc::new(AtomicUsize::new(0)); + + // Create the initial snapshot from reader views of the just-loaded trees. + let initial_snapshot = Arc::new(InMemoryState { + nullifier_tree: nullifier_tree + .reader() + .map_err(|e| StateInitializationError::NullifierTreeIoError(e.as_report()))?, + account_tree: account_tree.reader(), + forest: forest + .reader() + .map_err(|e| StateInitializationError::AccountStateForestIoError(e.as_report()))?, + blockchain: blockchain.clone(), + _guard: SnapshotGuard::new(Arc::clone(&snapshots_live), latest_block_num), + }); + let in_memory = Arc::new(ArcSwap::from(initial_snapshot)); + + // Assemble the block writer. It owns the writable trees and processes write requests + // serially, publishing a new snapshot after each committed block. The caller runs it; it + // exits when the shutdown token is cancelled or all write handles are dropped. + let (write_tx, write_rx) = tokio::sync::mpsc::channel(1); + let write_handle = WriteHandle::new(write_tx); + let block_writer = BlockWriter { + db: Arc::clone(&db), + block_store: Arc::clone(&block_store), + in_memory: Arc::clone(&in_memory), + committed_tip_tx: Arc::clone(&committed_tip_tx), + block_cache: block_cache.clone(), + rx: write_rx, + shutdown, + nullifier_tree, + account_tree, + blockchain, + forest, + snapshots_live, + }; + let state = Self { + data_directory: data_path.to_path_buf(), + db, + block_store, + in_memory, + write_handle, + proven_tip, + committed_tip_tx, + block_cache, + proof_cache, + }; + + Ok(LoadedState { state, writer: block_writer }) + } + + /// Stops the store, waiting until the block writer has released the tree storage it owns. + /// + /// Consumes the last state reference — closing the write channel the writer listens on — and + /// then joins the writer task returned by [`LoadedState::start`]. The drop must precede the + /// join or the writer never observes the closed channel; doing both here keeps that ordering + /// out of caller hands. + /// + /// Callers that need the storage released deterministically must use this method instead of + /// dropping: the node's `recover` command stops the store before the process exits, and the + /// stress-test's store seeding stops it so the same data directory can be re-loaded (or its + /// temporary directory deleted) immediately afterwards. The running node does not use this + /// method — its writer exits via the shutdown token passed to [`State::load`] and is joined + /// through the node's task set. + /// + /// # Errors + /// + /// Returns both pieces unchanged if other references to the state are still alive. + /// + /// # Panics + /// + /// Panics if the writer task panicked. + pub async fn stop( + self: Arc, + writer_task: tokio::task::JoinHandle<()>, + ) -> Result<(), (Arc, tokio::task::JoinHandle<()>)> { + match Arc::try_unwrap(self) { + Ok(state) => { + drop(state); + writer_task.await.expect("block writer task should not panic"); + Ok(()) + }, + Err(state) => Err((state, writer_task)), + } + } +} diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 46647f9d0..b0725ddc1 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -3,84 +3,18 @@ //! The [State] provides data access and modifications methods, its main purpose is to ensure that //! data is atomically written, and that reads are consistent. -use std::collections::{BTreeMap, BTreeSet, HashSet}; -use std::num::NonZeroUsize; -use std::ops::ControlFlow; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::time::{Duration, Instant}; use arc_swap::ArcSwap; -use miden_node_proto::domain::batch::BatchInputs; -use miden_node_utils::ErrorReport; -use miden_node_utils::clap::StorageOptions; -use miden_node_utils::formatting::format_array; -use miden_node_utils::shutdown::CancellationToken; -use miden_node_utils::tracing::miden_instrument; -use miden_protocol::Word; -use miden_protocol::account::AccountId; -use miden_protocol::block::account_tree::AccountWitness; -use miden_protocol::block::nullifier_tree::{NullifierTree, NullifierWitness}; -use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain}; -use miden_protocol::crypto::merkle::mmr::{MmrProof, PartialMmr}; -use miden_protocol::crypto::merkle::smt::LargeSmt; -use miden_protocol::note::{NoteId, NoteScript, Nullifier}; -use miden_protocol::transaction::PartialBlockchain; +use miden_protocol::block::BlockNumber; use tokio::sync::watch; -use tracing::Span; -use crate::account_state_forest::{ - AccountStateForest, - AccountStateForestBackend, - AccountStateForestBackendReader, -}; -use crate::accounts::AccountTreeWithHistory; use crate::blocks::BlockStore; -use crate::db::{Db, NoteRecord, NullifierInfo}; -use crate::errors::{ - DatabaseError, - GetBatchInputsError, - GetBlockHeaderError, - GetBlockInputsError, - StateInitializationError, -}; +use crate::db::Db; use crate::proven_tip::ProvenTipWriter; -use crate::{COMPONENT, DataDirectory, DatabaseOptions}; - -/// Number of recent committed blocks held in the in-memory cache for replica subscriptions. -const BLOCK_CACHE_CAPACITY: NonZeroUsize = NonZeroUsize::new(512).unwrap(); - -/// Number of recent block proofs held in the in-memory cache for replica subscriptions. -const PROOF_CACHE_CAPACITY: NonZeroUsize = NonZeroUsize::new(512).unwrap(); - -/// Snapshot lifetime above which [`SnapshotGuard`] logs a warning on release. -/// -/// Readers are expected to be request-scoped, so a snapshot outliving several block intervals -/// indicates a slow or leaked reader pinning a `RocksDB` snapshot (see [`SnapshotGuard`]). -const SNAPSHOT_LIFETIME_WARN_THRESHOLD: Duration = Duration::from_secs(10); - -/// Number of live snapshot generations above which the [`BlockWriter`] logs a warning after -/// publishing a new snapshot. -/// -/// Steady state is 1-2 generations: the just-published snapshot plus predecessors briefly pinned -/// by in-flight requests. A sustained higher count means slow or leaked readers are holding old -/// generations alive (see [`SnapshotGuard`]). -const SNAPSHOTS_LIVE_WARN_THRESHOLD: u64 = 4; mod loader; -use loader::{ - ACCOUNT_STATE_FOREST_STORAGE_DIR, - ACCOUNT_TREE_STORAGE_DIR, - AccountForestLoader, - NULLIFIER_TREE_STORAGE_DIR, - TreeStorage, - TreeStorageLoader, - TreeStorageReader, - load_mmr, - verify_account_state_forest_consistency, - verify_tree_consistency, -}; mod replica; pub use replica::{BlockCache, BlockNotification, ProofCache, ProofNotification}; @@ -92,8 +26,20 @@ mod apply_proof; mod bootstrap; mod disk_monitor; mod sync_state; + +mod inputs; +pub use inputs::TransactionInputs; + +mod lifecycle; +pub use lifecycle::LoadedState; + +mod queries; + +mod snapshot; +pub(crate) use snapshot::{InMemoryState, SNAPSHOTS_LIVE_WARN_THRESHOLD, SnapshotGuard}; + mod writer; -use writer::{BlockWriter, WriteHandle}; +use writer::WriteHandle; // FINALITY // ================================================================================================ @@ -107,101 +53,6 @@ pub enum Finality { Proven, } -// STRUCTURES -// ================================================================================================ - -#[derive(Debug, Default)] -pub struct TransactionInputs { - pub account_commitment: Word, - pub nullifiers: Vec, - pub found_unauthenticated_notes: HashSet, - pub new_account_id_prefix_is_unique: Option, -} - -type BlockInputWitnesses = ( - BlockNumber, - BTreeMap, - BTreeMap, - PartialMmr, -); - -/// RAII member of [`InMemoryState`] that tracks the number of live snapshot generations. -/// -/// [`InMemoryState`] is dropped exactly when the last [`Arc`] reference to it is released, so the -/// shared counter reports how many distinct snapshot generations are currently pinned by readers. -/// A sustained count above 1-2 means slow readers are holding old generations alive. Each -/// generation pins a `RocksDB` snapshot, which delays garbage collection of superseded key -/// versions during compaction (compaction itself keeps running); the retained garbage grows with -/// write churn for as long as the snapshot is held and is reclaimed once it is released. -/// -/// Readers are expected to be request-scoped, so snapshot lifetimes should be well under a block -/// interval. A lifetime exceeding [`SNAPSHOT_LIFETIME_WARN_THRESHOLD`] is logged at warn level. -pub(crate) struct SnapshotGuard { - live: Arc, - created_at: Instant, - block_num: BlockNumber, -} - -impl SnapshotGuard { - pub(super) fn new(live: Arc, block_num: BlockNumber) -> Self { - live.fetch_add(1, Ordering::Relaxed); - Self { - live, - created_at: Instant::now(), - block_num, - } - } -} - -impl Drop for SnapshotGuard { - fn drop(&mut self) { - let remaining = self.live.fetch_sub(1, Ordering::Relaxed) - 1; - let lifetime = self.created_at.elapsed(); - let lifetime_ms = u64::try_from(lifetime.as_millis()).unwrap_or(u64::MAX); - let block_num = self.block_num.as_u32(); - if lifetime > SNAPSHOT_LIFETIME_WARN_THRESHOLD { - tracing::warn!( - target: COMPONENT, - block_num, - snapshot.lifetime_ms = lifetime_ms, - snapshots.live = remaining, - "state snapshot held for excessive time", - ); - } else { - tracing::debug!( - target: COMPONENT, - block_num, - snapshot.lifetime_ms = lifetime_ms, - snapshots.live = remaining, - "state snapshot released", - ); - } - } -} - -/// Immutable snapshot of the in-memory tree state published after each committed block. -/// -/// The trees are backed by read-only snapshot storage ([`TreeStorageReader`] / -/// [`AccountStateForestBackendReader`]), so any number of readers can access the data concurrently -/// without holding a lock and without blocking the writer. -pub(crate) struct InMemoryState { - nullifier_tree: NullifierTree>, - blockchain: Blockchain, - account_tree: AccountTreeWithHistory, - forest: AccountStateForest, - /// Keeps the live-snapshot count accurate; see [`SnapshotGuard`]. - _guard: SnapshotGuard, -} - -impl InMemoryState { - /// Returns the latest block number. - fn latest_block_num(&self) -> BlockNumber { - self.blockchain - .chain_tip() - .expect("chain should always have at least the genesis block") - } -} - // CHAIN STATE // ================================================================================================ @@ -219,12 +70,12 @@ pub struct State { /// Atomically swappable pointer to the latest in-memory state snapshot. /// - /// Readers load the snapshot wait-free via [`ArcSwap::load_full`]; the [`BlockWriter`] task - /// atomically replaces the pointer after each committed block. Readers holding an old snapshot - /// are unaffected by the swap. + /// Readers load the snapshot wait-free via [`ArcSwap::load_full`]; the + /// [`BlockWriter`](writer::BlockWriter) task atomically replaces the pointer after each + /// committed block. Readers holding an old snapshot are unaffected by the swap. in_memory: Arc>, - /// Handle for sending block-write requests to the [`BlockWriter`] task. + /// Handle for sending block-write requests to the [`BlockWriter`](writer::BlockWriter) task. write_handle: WriteHandle, /// The latest proven-in-sequence block number, updated by the proof scheduler or `apply_proof`. @@ -242,719 +93,3 @@ pub struct State { /// that has been evicted, it falls back to loading from the block store. pub(crate) proof_cache: ProofCache, } - -/// A loaded store state whose block writer has not been started yet. -/// -/// Returned by [`State::load`]. [`Self::start`] spawns the writer and yields the usable -/// [`State`]; since this is the only way to obtain one, [`State::apply_block`] can always make -/// progress. -#[must_use = "call `start` to spawn the block writer and obtain the state"] -pub struct LoadedState { - state: State, - writer: BlockWriter, -} - -impl LoadedState { - /// Spawns the block writer onto the current runtime and returns the state together with the - /// writer task's join handle. - /// - /// The writer exits once the shutdown token passed to [`State::load`] is cancelled or the last - /// state reference (holding the only write handle) is dropped — an in-flight block write - /// always completes first. Awaiting the returned handle after either event guarantees the - /// writer has released the tree storage it owns; a join error carries a writer panic. - /// - /// Callers without a token to cancel should stop the store via [`State::stop`] rather than - /// dropping and joining by hand. - pub fn start(self) -> (Arc, tokio::task::JoinHandle<()>) { - let writer_task = tokio::spawn(self.writer.run()); - (Arc::new(self.state), writer_task) - } -} - -impl State { - /// Stops the store, waiting until the block writer has released the tree storage it owns. - /// - /// Consumes the last state reference — closing the write channel the writer listens on — and - /// then joins the writer task returned by [`LoadedState::start`]. The drop must precede the - /// join or the writer never observes the closed channel; doing both here keeps that ordering - /// out of caller hands. - /// - /// Callers that need the storage released deterministically must use this method instead of - /// dropping: the node's `recover` command stops the store before the process exits, and the - /// stress-test's store seeding stops it so the same data directory can be re-loaded (or its - /// temporary directory deleted) immediately afterwards. The running node does not use this - /// method — its writer exits via the shutdown token passed to [`State::load`] and is joined - /// through the node's task set. - /// - /// # Errors - /// - /// Returns both pieces unchanged if other references to the state are still alive. - /// - /// # Panics - /// - /// Panics if the writer task panicked. - pub async fn stop( - self: Arc, - writer_task: tokio::task::JoinHandle<()>, - ) -> Result<(), (Arc, tokio::task::JoinHandle<()>)> { - match Arc::try_unwrap(self) { - Ok(state) => { - drop(state); - writer_task.await.expect("block writer task should not panic"); - Ok(()) - }, - Err(state) => Err((state, writer_task)), - } - } -} - -impl State { - // CONSTRUCTOR - // -------------------------------------------------------------------------------------------- - - /// Loads the state from the data directory. - /// - /// The loaded state owns all store data structures and exposes subscription methods for - /// sequencer and replica tasks. Call [`LoadedState::start`] on the result to spawn the block - /// writer and obtain the usable [`State`]. - #[miden_instrument( - target = COMPONENT, - skip_all, - )] - pub async fn load( - data_path: &Path, - storage_options: StorageOptions, - shutdown: CancellationToken, - ) -> Result { - Self::load_with_database_options( - data_path, - storage_options, - DatabaseOptions::default(), - shutdown, - ) - .await - } - - /// Loads the state from the data directory using explicit database options. - /// - /// The loaded state owns all store data structures and exposes subscription methods for - /// sequencer and replica tasks. Call [`LoadedState::start`] on the result to spawn the block - /// writer and obtain the usable [`State`]. - #[miden_instrument( - target = COMPONENT, - skip_all, - )] - pub async fn load_with_database_options( - data_path: &Path, - storage_options: StorageOptions, - database_options: DatabaseOptions, - shutdown: CancellationToken, - ) -> Result { - let data_directory = DataDirectory::load(data_path.to_path_buf()) - .map_err(StateInitializationError::DataDirectoryLoadError)?; - - let block_store = Arc::new( - BlockStore::load(data_directory.block_store_dir()) - .map_err(StateInitializationError::BlockStoreLoadError)?, - ); - - let database_filepath = data_directory.database_path(); - let mut db = Db::load_with_pool_size( - database_filepath.clone(), - database_options.connection_pool_size, - ) - .await - .map_err(StateInitializationError::DatabaseLoadError)?; - - let blockchain = load_mmr(&mut db).await?; - let latest_block_num = blockchain.chain_tip().unwrap_or(BlockNumber::GENESIS); - - #[cfg(feature = "rocksdb")] - let (account_storage_config, nullifier_storage_config, forest_storage_config) = ( - storage_options.account_tree.into(), - storage_options.nullifier_tree.into(), - storage_options.account_state_forest.into(), - ); - #[cfg(not(feature = "rocksdb"))] - let (account_storage_config, nullifier_storage_config, forest_storage_config) = { - let _ = &storage_options; - ((), (), ()) - }; - let account_storage = - TreeStorage::create(data_path, &account_storage_config, ACCOUNT_TREE_STORAGE_DIR)?; - let account_tree = account_storage.load_account_tree(&mut db).await?; - - let nullifier_storage = - TreeStorage::create(data_path, &nullifier_storage_config, NULLIFIER_TREE_STORAGE_DIR)?; - let nullifier_tree = nullifier_storage.load_nullifier_tree(&mut db).await?; - - // Verify that tree roots match the expected roots from the database. This catches any - // divergence between persistent storage and the database caused by corruption or incomplete - // shutdown. - verify_tree_consistency(account_tree.root(), nullifier_tree.root(), &mut db).await?; - - let account_tree = AccountTreeWithHistory::new(account_tree, latest_block_num); - - let forest_backend = AccountStateForestBackend::create( - data_path, - &forest_storage_config, - ACCOUNT_STATE_FOREST_STORAGE_DIR, - )?; - let forest = forest_backend.load_account_state_forest(&mut db, latest_block_num).await?; - verify_account_state_forest_consistency(&forest, &mut db).await?; - - let db = Arc::new(db); - - // Initialize the proven tip from the block store. - let proven_tip_init = block_store - .load_proven_tip() - .map_err(StateInitializationError::ProvenTipLoadError)?; - let (proven_tip, _rx) = ProvenTipWriter::new(proven_tip_init); - - // Committed-tip watch: fires after each successful apply_block. - let (committed_tip_tx, _rx) = watch::channel(latest_block_num); - let committed_tip_tx = Arc::new(committed_tip_tx); - - let block_cache = BlockCache::new(BLOCK_CACHE_CAPACITY); - let proof_cache = ProofCache::new(PROOF_CACHE_CAPACITY); - - // Shared counter of live snapshot generations, for observability. - let snapshots_live = Arc::new(AtomicUsize::new(0)); - - // Create the initial snapshot from reader views of the just-loaded trees. - let initial_snapshot = Arc::new(InMemoryState { - nullifier_tree: nullifier_tree - .reader() - .map_err(|e| StateInitializationError::NullifierTreeIoError(e.as_report()))?, - account_tree: account_tree.reader(), - forest: forest - .reader() - .map_err(|e| StateInitializationError::AccountStateForestIoError(e.as_report()))?, - blockchain: blockchain.clone(), - _guard: SnapshotGuard::new(Arc::clone(&snapshots_live), latest_block_num), - }); - let in_memory = Arc::new(ArcSwap::from(initial_snapshot)); - - // Assemble the block writer. It owns the writable trees and processes write requests - // serially, publishing a new snapshot after each committed block. The caller runs it; it - // exits when the shutdown token is cancelled or all write handles are dropped. - let (write_tx, write_rx) = tokio::sync::mpsc::channel(1); - let write_handle = WriteHandle::new(write_tx); - let block_writer = BlockWriter { - db: Arc::clone(&db), - block_store: Arc::clone(&block_store), - in_memory: Arc::clone(&in_memory), - committed_tip_tx: Arc::clone(&committed_tip_tx), - block_cache: block_cache.clone(), - rx: write_rx, - shutdown, - nullifier_tree, - account_tree, - blockchain, - forest, - snapshots_live, - }; - let state = Self { - data_directory: data_path.to_path_buf(), - db, - block_store, - in_memory, - write_handle, - proven_tip, - committed_tip_tx, - block_cache, - proof_cache, - }; - - Ok(LoadedState { state, writer: block_writer }) - } - - /// Returns a watch receiver that wakes every time a new block is committed. - pub fn subscribe_committed_tip(&self) -> watch::Receiver { - self.committed_tip_tx.subscribe() - } - - /// Loads serialized block proving inputs from the block store. - pub async fn load_proving_inputs( - &self, - block_num: BlockNumber, - ) -> std::io::Result>> { - self.block_store.load_proving_inputs(block_num).await - } - - /// Returns a watch receiver that wakes every time the proven-in-sequence tip advances. - pub fn subscribe_proven_tip(&self) -> watch::Receiver { - self.proven_tip.subscribe() - } - - // SNAPSHOT HELPERS - // -------------------------------------------------------------------------------------------- - - /// Returns the current in-memory state snapshot (wait-free, no lock required). - /// - /// The returned snapshot is a frozen view: it is unaffected if the writer publishes a new - /// snapshot while it is held. - fn snapshot(&self) -> Arc { - self.in_memory.load_full() - } - - /// Runs a synchronous read-only operation over the current in-memory state snapshot on Tokio's - /// blocking path. - /// - /// The account and nullifier trees may be backed by `RocksDB`, so tree access must not run on - /// an async worker thread directly. This helper preserves the current tracing span while - /// moving the closure body into `block_in_place`. - fn with_inner_read_blocking(&self, f: impl FnOnce(&InMemoryState) -> R) -> R { - let span = Span::current(); - tokio::task::block_in_place(|| { - span.in_scope(|| { - let snapshot = self.snapshot(); - f(&snapshot) - }) - }) - } - - /// Runs a synchronous read-only operation over the account state forest snapshot on Tokio's - /// blocking path. - /// - /// See [`Self::with_inner_read_blocking`] for why this uses `block_in_place`. - fn with_forest_read_blocking( - &self, - f: impl FnOnce(&AccountStateForest) -> R, - ) -> R { - self.with_inner_read_blocking(|snapshot| f(&snapshot.forest)) - } - - // STATE ACCESSORS - // -------------------------------------------------------------------------------------------- - - /// Queries a [BlockHeader] from the database, and returns it alongside its inclusion proof. - /// - /// If [None] is given as the value of `block_num`, the data for the latest [BlockHeader] is - /// returned. - #[miden_instrument( - level = "debug", - target = COMPONENT, - skip_all, - err, - )] - pub async fn get_block_header( - &self, - block_num: Option, - include_mmr_proof: bool, - ) -> Result<(Option, Option), GetBlockHeaderError> { - // Resolve "latest" against the in-memory snapshot rather than the DB: mid-apply, the DB may - // already contain a block that the snapshot's blockchain cannot prove yet. Scoping the DB - // query by the snapshot's tip keeps the header and MMR proof consistent. - let snapshot = self.snapshot(); - let latest_block_num = snapshot.latest_block_num(); - let block_num = block_num.unwrap_or(latest_block_num); - if block_num > latest_block_num { - return Ok((None, None)); - } - - let block_header = self.db.select_block_header_by_block_num(Some(block_num)).await?; - if let Some(header) = block_header { - let mmr_proof = if include_mmr_proof { - let mmr_proof = snapshot.blockchain.open(header.block_num())?; - Some(mmr_proof) - } else { - None - }; - Ok((Some(header), mmr_proof)) - } else { - Ok((None, None)) - } - } - - /// Queries a list of notes from the database. - /// - /// If the provided list of [`NoteId`] given is empty or no note matches the provided - /// [`NoteId`] an empty list is returned. - pub async fn get_notes_by_id( - &self, - note_ids: Vec, - ) -> Result, DatabaseError> { - self.db.select_notes_by_id(note_ids).await - } - - /// Fetches the inputs for a transaction batch from the database. - /// - /// ## Inputs - /// - /// The function takes as input: - /// - The tx reference blocks are the set of blocks referenced by transactions in the batch. - /// - The unauthenticated note commitments are the set of commitments of unauthenticated notes - /// consumed by all transactions in the batch. For these notes, we attempt to find inclusion - /// proofs. Not all notes will exist in the DB necessarily, as some notes can be created and - /// consumed within the same batch. - /// - /// ## Outputs - /// - /// The function will return: - /// - A block inclusion proof for all tx reference blocks and for all blocks which are - /// referenced by a note inclusion proof. - /// - Note inclusion proofs for all notes that were found in the DB. - /// - The block header that the batch should reference, i.e. the latest known block. - pub async fn get_batch_inputs( - &self, - tx_reference_blocks: BTreeSet, - unauthenticated_note_commitments: BTreeSet, - ) -> Result { - if tx_reference_blocks.is_empty() { - return Err(GetBatchInputsError::TransactionBlockReferencesEmpty); - } - - // First we grab note inclusion proofs for the known notes. These proofs only prove that the - // note was included in a given block. We then also need to prove that each of those blocks - // is included in the chain. - let note_proofs = self - .db - .select_note_inclusion_proofs(unauthenticated_note_commitments) - .await - .map_err(GetBatchInputsError::SelectNoteInclusionProofError)?; - - // The set of blocks that the notes are included in. - let note_blocks = note_proofs.values().map(|proof| proof.location().block_num()); - - // Collect all blocks we need to query without duplicates, which is: - // - all blocks for which we need to prove note inclusion. - // - all blocks referenced by transactions in the batch. - let mut blocks: BTreeSet = tx_reference_blocks; - blocks.extend(note_blocks); - - let (batch_reference_block, partial_mmr) = { - let snapshot = self.snapshot(); - - let latest_block_num = snapshot.latest_block_num(); - - let highest_block_num = - *blocks.last().expect("we should have checked for empty block references"); - if highest_block_num > latest_block_num { - return Err(GetBatchInputsError::UnknownTransactionBlockReference { - highest_block_num, - latest_block_num, - }); - } - - // Remove the latest block from the to-be-tracked blocks as it will be the reference - // block for the batch itself and thus added to the MMR within the batch kernel, so - // there is no need to prove its inclusion. - blocks.remove(&latest_block_num); - - // SAFETY: - // - The latest block num was retrieved from the inner blockchain from which we will - // also retrieve the proofs, so it is guaranteed to exist in that chain. - // - We have checked that no block number in the blocks set is greater than latest block - // number *and* latest block num was removed from the set. Therefore only block - // numbers smaller than latest block num remain in the set. Therefore all the block - // numbers are guaranteed to exist in the chain state at latest block num. - let partial_mmr = snapshot - .blockchain - .partial_mmr_from_blocks(&blocks, latest_block_num) - .expect("latest block num should exist and all blocks in set should be < than latest block"); - - (latest_block_num, partial_mmr) - }; - - // Fetch the reference block of the batch as part of this query, so we can avoid looking it - // up in a separate DB access. - let mut headers = self - .db - .select_block_headers(blocks.into_iter().chain(std::iter::once(batch_reference_block))) - .await - .map_err(GetBatchInputsError::SelectBlockHeaderError)?; - - // Find and remove the batch reference block as we don't want to add it to the chain MMR. - let header_index = headers - .iter() - .enumerate() - .find_map(|(index, header)| { - (header.block_num() == batch_reference_block).then_some(index) - }) - .expect("DB should have returned the header of the batch reference block"); - - // The order doesn't matter for PartialBlockchain::new, so swap remove is fine. - let batch_reference_block_header = headers.swap_remove(header_index); - - // SAFETY: This should not error because: - // - we're passing exactly the block headers that we've added to the partial MMR, - // - so none of the block headers block numbers should exceed the chain length of the - // partial MMR, - // - and we've added blocks to a BTreeSet, so there can be no duplicates. - // - // We construct headers and partial MMR in concert, so they are consistent. This is why we - // can call the unchecked constructor. - let partial_block_chain = PartialBlockchain::new_unchecked(partial_mmr, headers) - .expect("partial mmr and block headers should be consistent"); - - Ok(BatchInputs { - batch_reference_block_header, - note_proofs, - partial_block_chain, - }) - } - - /// Returns data needed by the block producer to construct and prove the next block. - pub async fn get_block_inputs( - &self, - account_ids: Vec, - nullifiers: Vec, - unauthenticated_note_commitments: BTreeSet, - reference_blocks: BTreeSet, - ) -> Result { - // Get the note inclusion proofs from the DB. We do this first so we have to acquire the - // lock to the state just once. There we need the reference blocks of the note proofs to get - // their authentication paths in the chain MMR. - let unauthenticated_note_proofs = self - .db - .select_note_inclusion_proofs(unauthenticated_note_commitments) - .await - .map_err(GetBlockInputsError::SelectNoteInclusionProofError)?; - - // The set of blocks that the notes are included in. - let note_proof_reference_blocks = - unauthenticated_note_proofs.values().map(|proof| proof.location().block_num()); - - // Collect all blocks we need to prove inclusion for, without duplicates. - let mut blocks = reference_blocks; - blocks.extend(note_proof_reference_blocks); - - let (latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr) = - self.get_block_inputs_witnesses(&mut blocks, &account_ids, &nullifiers)?; - - // Fetch the block headers for all blocks in the partial MMR plus the latest one which will - // be used as the previous block header of the block being built. - let mut headers = self - .db - .select_block_headers(blocks.into_iter().chain(std::iter::once(latest_block_number))) - .await - .map_err(GetBlockInputsError::SelectBlockHeaderError)?; - - // Find and remove the latest block as we must not add it to the chain MMR, since it is not - // yet in the chain. - let latest_block_header_index = headers - .iter() - .enumerate() - .find_map(|(index, header)| { - (header.block_num() == latest_block_number).then_some(index) - }) - .expect("DB should have returned the header of the latest block header"); - - // The order doesn't matter for PartialBlockchain::new, so swap remove is fine. - let latest_block_header = headers.swap_remove(latest_block_header_index); - - // SAFETY: This should not error because: - // - we're passing exactly the block headers that we've added to the partial MMR, - // - so none of the block header's block numbers should exceed the chain length of the - // partial MMR, - // - and we've added blocks to a BTreeSet, so there can be no duplicates. - // - // We construct headers and partial MMR in concert, so they are consistent. This is why we - // can call the unchecked constructor. - let partial_block_chain = PartialBlockchain::new_unchecked(partial_mmr, headers) - .expect("partial mmr and block headers should be consistent"); - - Ok(BlockInputs::new( - latest_block_header, - partial_block_chain, - account_witnesses, - nullifier_witnesses, - unauthenticated_note_proofs, - )) - } - - /// Get account and nullifier witnesses for the requested account IDs and nullifier as well as - /// the [`PartialMmr`] for the given blocks. The MMR won't contain the latest block and its - /// number is removed from `blocks` and returned separately. - /// - /// This method acquires the lock to the inner state and does not access the DB so we release - /// the lock asap. - fn get_block_inputs_witnesses( - &self, - blocks: &mut BTreeSet, - account_ids: &[AccountId], - nullifiers: &[Nullifier], - ) -> Result { - self.with_inner_read_blocking(|inner| { - let latest_block_number = inner.latest_block_num(); - - // If `blocks` is empty, use the latest block number which will never trigger the error. - let highest_block_number = blocks.last().copied().unwrap_or(latest_block_number); - if highest_block_number > latest_block_number { - return Err(GetBlockInputsError::UnknownBatchBlockReference { - highest_block_number, - latest_block_number, - }); - } - - // The latest block is not yet in the chain MMR, so we can't (and don't need to) prove - // its inclusion in the chain. - blocks.remove(&latest_block_number); - - // Fetch the partial MMR at the state of the latest block with authentication paths for - // the provided set of blocks. - // - // SAFETY: - // - The latest block num was retrieved from the inner blockchain from which we will - // also retrieve the proofs, so it is guaranteed to exist in that chain. - // - We have checked that no block number in the blocks set is greater than latest block - // number *and* latest block num was removed from the set. Therefore only block - // numbers smaller than latest block num remain in the set. Therefore all the block - // numbers are guaranteed to exist in the chain state at latest block num. - let partial_mmr = - inner.blockchain.partial_mmr_from_blocks(blocks, latest_block_number).expect( - "latest block num should exist and all blocks in set should be < than latest block", - ); - - // Fetch witnesses for all accounts. - let account_witnesses = account_ids - .iter() - .copied() - .map(|account_id| (account_id, inner.account_tree.open_latest(account_id))) - .collect::>(); - - // Fetch witnesses for all nullifiers. We don't check whether the nullifiers are spent - // or not as this is done as part of proposing the block. - let nullifier_witnesses: BTreeMap = nullifiers - .iter() - .copied() - .map(|nullifier| (nullifier, inner.nullifier_tree.open(&nullifier))) - .collect(); - - Ok((latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr)) - }) - } - - /// Returns data needed by the block producer to verify transactions validity. - #[miden_instrument( - target = COMPONENT, - skip_all, - fields( - account.id=%account_id, - nullifiers = %format_array(nullifiers), - ), - )] - pub async fn get_transaction_inputs( - &self, - account_id: AccountId, - nullifiers: &[Nullifier], - unauthenticated_note_commitments: Vec, - ) -> Result { - let tree_inputs = self.with_inner_read_blocking(|inner| { - let account_commitment = inner.account_tree.get_latest_commitment(account_id); - - let new_account_id_prefix_is_unique = if account_commitment.is_empty() { - Some(!inner.account_tree.contains_account_id_prefix_in_latest(account_id.prefix())) - } else { - None - }; - - // Non-unique account Id prefixes for new accounts are not allowed, so the transaction - // cannot be valid and the response is already complete. - if let Some(false) = new_account_id_prefix_is_unique { - return ControlFlow::Break(TransactionInputs { - new_account_id_prefix_is_unique, - ..Default::default() - }); - } - - let nullifiers = nullifiers - .iter() - .map(|nullifier| NullifierInfo { - nullifier: *nullifier, - block_num: inner.nullifier_tree.get_block_num(nullifier).unwrap_or_default(), - }) - .collect(); - - ControlFlow::Continue(( - account_commitment, - nullifiers, - new_account_id_prefix_is_unique, - inner.latest_block_num(), - )) - }); - // `Break` carries a complete response (duplicate account ID prefix), so it is returned - // as-is without the note lookup below; `Continue` carries the tree reads needed to build - // the full response. - let (account_commitment, nullifiers, new_account_id_prefix_is_unique, latest_block_num) = - match tree_inputs { - ControlFlow::Continue(inputs) => inputs, - ControlFlow::Break(response) => return Ok(response), - }; - - // Scope the note lookup by the snapshot's tip so the result is consistent with the tree - // reads above: mid-apply, the DB may already contain notes from a block the snapshot does - // not include yet. - let found_unauthenticated_notes = self - .db - .select_existing_note_commitments(unauthenticated_note_commitments, latest_block_num) - .await?; - - Ok(TransactionInputs { - account_commitment, - nullifiers, - found_unauthenticated_notes, - new_account_id_prefix_is_unique, - }) - } - - /// Filters `account_ids` down to the subset classified as network accounts. - pub async fn filter_network_accounts( - &self, - account_ids: &[AccountId], - ) -> Result, DatabaseError> { - self.db.select_network_accounts_subset(account_ids.to_vec()).await - } - - /// Returns the effective chain tip for the given finality level. - /// - /// - [`Finality::Committed`]: returns the latest committed block number (from the in-memory - /// snapshot). - /// - [`Finality::Proven`]: returns the latest proven-in-sequence block number (cached via watch - /// channel, updated by the proof scheduler). - pub fn chain_tip(&self, finality: Finality) -> BlockNumber { - match finality { - Finality::Committed => self.snapshot().latest_block_num(), - Finality::Proven => self.proven_tip.read(), - } - } - - /// Loads a block from the in-memory replica cache or block store. Return `Ok(None)` if the - /// block is not found. - pub async fn load_block( - &self, - block_num: BlockNumber, - ) -> Result>, DatabaseError> { - if block_num > self.chain_tip(Finality::Committed) { - return Ok(None); - } - if let Some(block) = self.block_cache.get(block_num) { - return Ok(Some(block.block_bytes().to_vec())); - } - self.block_store.load_block(block_num).await.map_err(Into::into) - } - - /// Loads a block proof from the in-memory replica cache or block store. Returns `Ok(None)` if - /// the proof is not found. - pub async fn load_proof( - &self, - block_num: BlockNumber, - ) -> Result>, DatabaseError> { - if block_num > self.chain_tip(Finality::Proven) { - return Ok(None); - } - if let Some(proof) = self.proof_cache.get(block_num) { - return Ok(Some(proof.proof_bytes().to_vec())); - } - self.block_store.load_proof(block_num).await.map_err(Into::into) - } - - /// Returns the script for a note by its root. - pub async fn get_note_script_by_root( - &self, - root: Word, - ) -> Result, DatabaseError> { - self.db.select_note_script_by_root(root).await - } -} diff --git a/crates/store/src/state/queries.rs b/crates/store/src/state/queries.rs new file mode 100644 index 000000000..d12fc9cb9 --- /dev/null +++ b/crates/store/src/state/queries.rs @@ -0,0 +1,96 @@ +//! General read queries over the store state. + +use std::collections::HashSet; + +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::Word; +use miden_protocol::account::AccountId; +use miden_protocol::block::{BlockHeader, BlockNumber}; +use miden_protocol::crypto::merkle::mmr::MmrProof; +use miden_protocol::note::{NoteId, NoteScript}; + +use crate::COMPONENT; +use crate::db::NoteRecord; +use crate::errors::{DatabaseError, GetBlockHeaderError}; +use crate::state::{Finality, State}; + +impl State { + /// Queries a [BlockHeader] from the database, and returns it alongside its inclusion proof. + /// + /// If [None] is given as the value of `block_num`, the data for the latest [BlockHeader] is + /// returned. + #[miden_instrument( + level = "debug", + target = COMPONENT, + skip_all, + err, + )] + pub async fn get_block_header( + &self, + block_num: Option, + include_mmr_proof: bool, + ) -> Result<(Option, Option), GetBlockHeaderError> { + // Resolve "latest" against the in-memory snapshot rather than the DB: mid-apply, the DB may + // already contain a block that the snapshot's blockchain cannot prove yet. Scoping the DB + // query by the snapshot's tip keeps the header and MMR proof consistent. + let snapshot = self.snapshot(); + let latest_block_num = snapshot.latest_block_num(); + let block_num = block_num.unwrap_or(latest_block_num); + if block_num > latest_block_num { + return Ok((None, None)); + } + + let block_header = self.db.select_block_header_by_block_num(Some(block_num)).await?; + if let Some(header) = block_header { + let mmr_proof = if include_mmr_proof { + let mmr_proof = snapshot.blockchain.open(header.block_num())?; + Some(mmr_proof) + } else { + None + }; + Ok((Some(header), mmr_proof)) + } else { + Ok((None, None)) + } + } + + /// Queries a list of notes from the database. + /// + /// If the provided list of [`NoteId`] given is empty or no note matches the provided + /// [`NoteId`] an empty list is returned. + pub async fn get_notes_by_id( + &self, + note_ids: Vec, + ) -> Result, DatabaseError> { + self.db.select_notes_by_id(note_ids).await + } + + /// Returns the script for a note by its root. + pub async fn get_note_script_by_root( + &self, + root: Word, + ) -> Result, DatabaseError> { + self.db.select_note_script_by_root(root).await + } + + /// Filters `account_ids` down to the subset classified as network accounts. + pub async fn filter_network_accounts( + &self, + account_ids: &[AccountId], + ) -> Result, DatabaseError> { + self.db.select_network_accounts_subset(account_ids.to_vec()).await + } + + /// Returns the effective chain tip for the given finality level. + /// + /// - [`Finality::Committed`]: returns the latest committed block number (from the in-memory + /// snapshot). + /// - [`Finality::Proven`]: returns the latest proven-in-sequence block number (cached via watch + /// channel, updated by the proof scheduler). + pub fn chain_tip(&self, finality: Finality) -> BlockNumber { + match finality { + Finality::Committed => self.snapshot().latest_block_num(), + Finality::Proven => self.proven_tip.read(), + } + } +} diff --git a/crates/store/src/state/replica.rs b/crates/store/src/state/replica.rs index 61533de8b..0399f7e21 100644 --- a/crates/store/src/state/replica.rs +++ b/crates/store/src/state/replica.rs @@ -2,6 +2,10 @@ use std::sync::Arc; use miden_node_utils::block_cache::BlockOrderedCache; use miden_protocol::block::BlockNumber; +use tokio::sync::watch; + +use crate::errors::DatabaseError; +use crate::state::{Finality, State}; // BLOCK NOTIFICATION // ================================================================================================ @@ -65,3 +69,56 @@ pub type BlockCache = BlockOrderedCache; /// FIFO cache of recent block proofs for replica subscriptions. pub type ProofCache = BlockOrderedCache; + +// REPLICA STATE ACCESS +// ================================================================================================ + +impl State { + /// Returns a watch receiver that wakes every time a new block is committed. + pub fn subscribe_committed_tip(&self) -> watch::Receiver { + self.committed_tip_tx.subscribe() + } + + /// Returns a watch receiver that wakes every time the proven-in-sequence tip advances. + pub fn subscribe_proven_tip(&self) -> watch::Receiver { + self.proven_tip.subscribe() + } + + /// Loads a block from the in-memory replica cache or block store. Return `Ok(None)` if the + /// block is not found. + pub async fn load_block( + &self, + block_num: BlockNumber, + ) -> Result>, DatabaseError> { + if block_num > self.chain_tip(Finality::Committed) { + return Ok(None); + } + if let Some(block) = self.block_cache.get(block_num) { + return Ok(Some(block.block_bytes().to_vec())); + } + self.block_store.load_block(block_num).await.map_err(Into::into) + } + + /// Loads a block proof from the in-memory replica cache or block store. Returns `Ok(None)` if + /// the proof is not found. + pub async fn load_proof( + &self, + block_num: BlockNumber, + ) -> Result>, DatabaseError> { + if block_num > self.chain_tip(Finality::Proven) { + return Ok(None); + } + if let Some(proof) = self.proof_cache.get(block_num) { + return Ok(Some(proof.proof_bytes().to_vec())); + } + self.block_store.load_proof(block_num).await.map_err(Into::into) + } + + /// Loads serialized block proving inputs from the block store. + pub async fn load_proving_inputs( + &self, + block_num: BlockNumber, + ) -> std::io::Result>> { + self.block_store.load_proving_inputs(block_num).await + } +} diff --git a/crates/store/src/state/snapshot.rs b/crates/store/src/state/snapshot.rs new file mode 100644 index 000000000..364acb052 --- /dev/null +++ b/crates/store/src/state/snapshot.rs @@ -0,0 +1,157 @@ +//! In-memory snapshot machinery for lock-free reads. +//! +//! Readers access the store's tree state through immutable [`InMemoryState`] snapshots published +//! by the block writer after each committed block. [`SnapshotGuard`] tracks how many snapshot +//! generations are pinned by readers, since each generation pins a `RocksDB` snapshot. + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use miden_protocol::block::nullifier_tree::NullifierTree; +use miden_protocol::block::{BlockNumber, Blockchain}; +use miden_protocol::crypto::merkle::smt::LargeSmt; +use tracing::Span; + +use crate::COMPONENT; +use crate::account_state_forest::{AccountStateForest, AccountStateForestBackendReader}; +use crate::accounts::AccountTreeWithHistory; +use crate::state::State; +use crate::state::loader::TreeStorageReader; + +/// Snapshot lifetime above which [`SnapshotGuard`] logs a warning on release. +/// +/// Readers are expected to be request-scoped, so a snapshot outliving several block intervals +/// indicates a slow or leaked reader pinning a `RocksDB` snapshot (see [`SnapshotGuard`]). +const SNAPSHOT_LIFETIME_WARN_THRESHOLD: Duration = Duration::from_secs(10); + +/// Number of live snapshot generations above which the block writer logs a warning after +/// publishing a new snapshot. +/// +/// Steady state is 1-2 generations: the just-published snapshot plus predecessors briefly pinned +/// by in-flight requests. A sustained higher count means slow or leaked readers are holding old +/// generations alive (see [`SnapshotGuard`]). +pub(crate) const SNAPSHOTS_LIVE_WARN_THRESHOLD: u64 = 4; + +// SNAPSHOT GUARD +// ================================================================================================ + +/// RAII member of [`InMemoryState`] that tracks the number of live snapshot generations. +/// +/// [`InMemoryState`] is dropped exactly when the last [`Arc`] reference to it is released, so the +/// shared counter reports how many distinct snapshot generations are currently pinned by readers. +/// A sustained count above 1-2 means slow readers are holding old generations alive. Each +/// generation pins a `RocksDB` snapshot, which delays garbage collection of superseded key +/// versions during compaction (compaction itself keeps running); the retained garbage grows with +/// write churn for as long as the snapshot is held and is reclaimed once it is released. +/// +/// Readers are expected to be request-scoped, so snapshot lifetimes should be well under a block +/// interval. A lifetime exceeding [`SNAPSHOT_LIFETIME_WARN_THRESHOLD`] is logged at warn level. +pub(crate) struct SnapshotGuard { + live: Arc, + created_at: Instant, + block_num: BlockNumber, +} + +impl SnapshotGuard { + pub(crate) fn new(live: Arc, block_num: BlockNumber) -> Self { + live.fetch_add(1, Ordering::Relaxed); + Self { + live, + created_at: Instant::now(), + block_num, + } + } +} + +impl Drop for SnapshotGuard { + fn drop(&mut self) { + let remaining = self.live.fetch_sub(1, Ordering::Relaxed) - 1; + let lifetime = self.created_at.elapsed(); + let lifetime_ms = u64::try_from(lifetime.as_millis()).unwrap_or(u64::MAX); + let block_num = self.block_num.as_u32(); + if lifetime > SNAPSHOT_LIFETIME_WARN_THRESHOLD { + tracing::warn!( + target: COMPONENT, + block_num, + snapshot.lifetime_ms = lifetime_ms, + snapshots.live = remaining, + "state snapshot held for excessive time", + ); + } else { + tracing::debug!( + target: COMPONENT, + block_num, + snapshot.lifetime_ms = lifetime_ms, + snapshots.live = remaining, + "state snapshot released", + ); + } + } +} + +// IN-MEMORY STATE +// ================================================================================================ + +/// Immutable snapshot of the in-memory tree state published after each committed block. +/// +/// The trees are backed by read-only snapshot storage ([`TreeStorageReader`] / +/// [`AccountStateForestBackendReader`]), so any number of readers can access the data concurrently +/// without holding a lock and without blocking the writer. +pub(crate) struct InMemoryState { + pub(crate) nullifier_tree: NullifierTree>, + pub(crate) blockchain: Blockchain, + pub(crate) account_tree: AccountTreeWithHistory, + pub(crate) forest: AccountStateForest, + /// Keeps the live-snapshot count accurate; see [`SnapshotGuard`]. + pub(crate) _guard: SnapshotGuard, +} + +impl InMemoryState { + /// Returns the latest block number. + pub(crate) fn latest_block_num(&self) -> BlockNumber { + self.blockchain + .chain_tip() + .expect("chain should always have at least the genesis block") + } +} + +// SNAPSHOT HELPERS +// ================================================================================================ + +impl State { + /// Returns the current in-memory state snapshot (wait-free, no lock required). + /// + /// The returned snapshot is a frozen view: it is unaffected if the writer publishes a new + /// snapshot while it is held. + pub(super) fn snapshot(&self) -> Arc { + self.in_memory.load_full() + } + + /// Runs a synchronous read-only operation over the current in-memory state snapshot on Tokio's + /// blocking path. + /// + /// The account and nullifier trees may be backed by `RocksDB`, so tree access must not run on + /// an async worker thread directly. This helper preserves the current tracing span while + /// moving the closure body into `block_in_place`. + pub(super) fn with_inner_read_blocking(&self, f: impl FnOnce(&InMemoryState) -> R) -> R { + let span = Span::current(); + tokio::task::block_in_place(|| { + span.in_scope(|| { + let snapshot = self.snapshot(); + f(&snapshot) + }) + }) + } + + /// Runs a synchronous read-only operation over the account state forest snapshot on Tokio's + /// blocking path. + /// + /// See [`Self::with_inner_read_blocking`] for why this uses `block_in_place`. + pub(super) fn with_forest_read_blocking( + &self, + f: impl FnOnce(&AccountStateForest) -> R, + ) -> R { + self.with_inner_read_blocking(|snapshot| f(&snapshot.forest)) + } +} From 7f8ebc50558073de57c2f2f58625386567f72c31 Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 23 Jul 2026 13:24:50 +0800 Subject: [PATCH 22/35] WriterTask --- bin/node/src/commands/modes.rs | 7 +++--- bin/node/src/commands/recover.rs | 4 ++-- bin/stress-test/src/seeding/mod.rs | 4 ++-- crates/store/src/lib.rs | 2 +- crates/store/src/state/lifecycle.rs | 34 ++++++++++++++++++++++++----- crates/store/src/state/mod.rs | 2 +- 6 files changed, 38 insertions(+), 15 deletions(-) diff --git a/bin/node/src/commands/modes.rs b/bin/node/src/commands/modes.rs index e0d8ae8d3..6cd06a73b 100644 --- a/bin/node/src/commands/modes.rs +++ b/bin/node/src/commands/modes.rs @@ -12,12 +12,11 @@ use miden_node_proto::clients::{ ValidatorClient, }; use miden_node_rpc::{Rpc, RpcMode, SequencerInternal}; -use miden_node_store::State; +use miden_node_store::{State, WriterTask}; use miden_node_utils::clap::{GrpcOptionsInternal, duration_to_human_readable_string}; use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::tasks::Tasks; use tokio::net::TcpListener; -use tokio::task::JoinHandle; use url::Url; use super::block_producer::BlockProducerOptions; @@ -253,7 +252,7 @@ impl SyncOptions { async fn load_state( runtime: &RuntimeConfig, shutdown: CancellationToken, -) -> anyhow::Result<(Arc, JoinHandle<()>)> { +) -> anyhow::Result<(Arc, WriterTask)> { let loaded = State::load_with_database_options( &runtime.data_directory, runtime.storage_options.clone(), @@ -271,7 +270,7 @@ async fn load_state( /// On shutdown the task-drain loop waits for the writer to finish any in-flight block write and /// close its storage; an early exit or panic surfaces through the task set like any other task /// failure. -async fn join_store_writer(writer_task: JoinHandle<()>) -> anyhow::Result<()> { +async fn join_store_writer(writer_task: WriterTask) -> anyhow::Result<()> { writer_task.await.map_err(anyhow::Error::from) } diff --git a/bin/node/src/commands/recover.rs b/bin/node/src/commands/recover.rs index be362d62e..555351bee 100644 --- a/bin/node/src/commands/recover.rs +++ b/bin/node/src/commands/recover.rs @@ -5,8 +5,8 @@ use std::time::Duration; use anyhow::Context; use miden_node_proto::clients::{Builder, ValidatorClient}; use miden_node_proto::generated::validator::BlockSubscriptionRequest; -use miden_node_store::State; use miden_node_store::state::Finality; +use miden_node_store::{State, WriterTask}; use miden_node_utils::shutdown::CancellationToken; use miden_protocol::block::{BlockNumber, SignedBlock}; use miden_protocol::utils::serde::Deserializable; @@ -48,7 +48,7 @@ impl RecoverCommand { result } - async fn load_state(&self) -> anyhow::Result<(Arc, tokio::task::JoinHandle<()>)> { + async fn load_state(&self) -> anyhow::Result<(Arc, WriterTask)> { // Recovery is not wired into the node's shutdown token; the writer exits once the state // (holding the only write handle) is dropped after recovery completes. let loaded = State::load_with_database_options( diff --git a/bin/stress-test/src/seeding/mod.rs b/bin/stress-test/src/seeding/mod.rs index 81159c388..c5e332685 100644 --- a/bin/stress-test/src/seeding/mod.rs +++ b/bin/stress-test/src/seeding/mod.rs @@ -5,7 +5,7 @@ use std::time::Instant; use metrics::SeedingMetrics; use miden_node_proto::domain::batch::BatchInputs; -use miden_node_store::{DataDirectory, GenesisState, State}; +use miden_node_store::{DataDirectory, GenesisState, State, WriterTask}; use miden_node_utils::clap::StorageOptions; use miden_node_utils::shutdown::CancellationToken; use miden_protocol::account::auth::AuthScheme; @@ -1034,7 +1034,7 @@ pub async fn start_store(data_directory: PathBuf) -> Arc { /// /// The writer exits once the last reference to the returned state is dropped; awaiting the handle /// after that guarantees the backing storage has been released. -async fn load_state(data_directory: PathBuf) -> (Arc, tokio::task::JoinHandle<()>) { +async fn load_state(data_directory: PathBuf) -> (Arc, WriterTask) { State::load(&data_directory, StorageOptions::bench(), CancellationToken::new()) .await .expect("store state should load") diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 17fdba636..11487af7b 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -38,7 +38,7 @@ pub use errors::{ StateSyncError, }; pub use genesis::GenesisState; -pub use state::{LoadedState, State}; +pub use state::{LoadedState, State, WriterTask}; /// Returns the store crate version. pub fn version() -> &'static str { diff --git a/crates/store/src/state/lifecycle.rs b/crates/store/src/state/lifecycle.rs index 57db356fa..f34b7582a 100644 --- a/crates/store/src/state/lifecycle.rs +++ b/crates/store/src/state/lifecycle.rs @@ -1,9 +1,12 @@ //! Store lifecycle: loading the state, starting its block writer, and stopping the store. +use std::future::Future; use std::num::NonZeroUsize; use std::path::Path; +use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::AtomicUsize; +use std::task::{Context, Poll}; use arc_swap::ArcSwap; use miden_node_utils::ErrorReport; @@ -56,7 +59,7 @@ pub struct LoadedState { impl LoadedState { /// Spawns the block writer onto the current runtime and returns the state together with the - /// writer task's join handle. + /// writer task's handle. /// /// The writer exits once the shutdown token passed to [`State::load`] is cancelled or the last /// state reference (holding the only write handle) is dropped — an in-flight block write @@ -65,9 +68,30 @@ impl LoadedState { /// /// Callers without a token to cancel should stop the store via [`State::stop`] rather than /// dropping and joining by hand. - pub fn start(self) -> (Arc, tokio::task::JoinHandle<()>) { + pub fn start(self) -> (Arc, WriterTask) { let writer_task = tokio::spawn(self.writer.run()); - (Arc::new(self.state), writer_task) + (Arc::new(self.state), WriterTask(writer_task)) + } +} + +// WRITER TASK +// ================================================================================================ + +/// Handle of the store's block writer task, returned by [`LoadedState::start`]. +/// +/// Awaiting it resolves once the writer has exited and released the tree storage it owns; a join +/// error carries a writer panic. The newtype ensures [`State::stop`] can only be given the store's +/// own writer task, and deliberately does not expose [`tokio::task::JoinHandle::abort`]: aborting +/// the writer mid-write could leave the trees lagging the committed database state, voiding the +/// guarantee that an in-flight block write always completes. +#[must_use = "await the writer task to observe its exit, or pass it to `State::stop`"] +pub struct WriterTask(tokio::task::JoinHandle<()>); + +impl Future for WriterTask { + type Output = Result<(), tokio::task::JoinError>; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + Pin::new(&mut self.0).poll(cx) } } @@ -255,8 +279,8 @@ impl State { /// Panics if the writer task panicked. pub async fn stop( self: Arc, - writer_task: tokio::task::JoinHandle<()>, - ) -> Result<(), (Arc, tokio::task::JoinHandle<()>)> { + writer_task: WriterTask, + ) -> Result<(), (Arc, WriterTask)> { match Arc::try_unwrap(self) { Ok(state) => { drop(state); diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index b0725ddc1..ebf740ede 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -31,7 +31,7 @@ mod inputs; pub use inputs::TransactionInputs; mod lifecycle; -pub use lifecycle::LoadedState; +pub use lifecycle::{LoadedState, WriterTask}; mod queries; From c528f192034f8c263e05a15635536119837cc6fe Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 23 Jul 2026 13:52:17 +0800 Subject: [PATCH 23/35] Add benchmarks --- bin/stress-test/src/main.rs | 64 +++++++++++++++++++++ bin/stress-test/src/seeding/mod.rs | 91 ++++++++++++++++++++++++++++++ bin/stress-test/src/store/mod.rs | 2 +- 3 files changed, 156 insertions(+), 1 deletion(-) diff --git a/bin/stress-test/src/main.rs b/bin/stress-test/src/main.rs index 870055bbc..fe9e5770d 100644 --- a/bin/stress-test/src/main.rs +++ b/bin/stress-test/src/main.rs @@ -61,6 +61,50 @@ pub enum Command { account_update_blocks: usize, }, + /// Benchmark read latency under concurrent write load. + /// + /// Seeds the store like `seed-store` while concurrent reader tasks continuously request the + /// latest block header (with its MMR proof) from the same in-process state. Reports read + /// latency percentiles alongside the usual seeding metrics, capturing how reads behave while + /// blocks are being applied. + BenchmarkMixed { + /// Directory in which to store the database and raw block data. If the directory contains a + /// database dump file, it will be replaced. + #[arg(short, long, value_name = "DATA_DIRECTORY")] + data_directory: PathBuf, + + /// Number of accounts to create. + #[arg(short, long, value_name = "NUM_ACCOUNTS")] + num_accounts: NonZeroUsize, + + /// Percentage of accounts that will be created as public accounts. The rest will be private + /// accounts. + #[arg( + short, + long, + value_name = "PUBLIC_ACCOUNTS_PERCENTAGE", + default_value = "0", + value_parser = clap::value_parser!(u8).range(0..=100) + )] + public_accounts_percentage: u8, + + /// Number of entries to add to a deterministic storage map on every public account. + #[arg(long, value_name = "STORAGE_MAP_ENTRIES", default_value = "0")] + storage_map_entries: u32, + + /// Number of distinct vault assets to add to every public account. + #[arg(long, value_name = "VAULT_ENTRIES", default_value = "1")] + vault_entries: NonZeroUsize, + + /// Number of post-initialization blocks containing public account updates. + #[arg(long, value_name = "ACCOUNT_UPDATE_BLOCKS", default_value = "0")] + account_update_blocks: usize, + + /// Number of concurrent reader tasks measuring read latency during seeding. + #[arg(short, long, value_name = "READERS", default_value = "8")] + readers: NonZeroUsize, + }, + /// Benchmark the performance of the store endpoints. BenchmarkStore { /// Store endpoint to test against. @@ -139,6 +183,26 @@ async fn main() { )) .await; }, + Command::BenchmarkMixed { + data_directory, + num_accounts, + public_accounts_percentage, + storage_map_entries, + vault_entries, + account_update_blocks, + readers, + } => { + Box::pin(seeding::seed_store_with_readers( + data_directory, + num_accounts.get(), + public_accounts_percentage, + storage_map_entries, + vault_entries.get(), + account_update_blocks, + readers.get(), + )) + .await; + }, Command::BenchmarkStore { endpoint, data_directory, diff --git a/bin/stress-test/src/seeding/mod.rs b/bin/stress-test/src/seeding/mod.rs index c5e332685..ce66b8d64 100644 --- a/bin/stress-test/src/seeding/mod.rs +++ b/bin/stress-test/src/seeding/mod.rs @@ -1,5 +1,6 @@ use std::collections::BTreeMap; use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Instant; @@ -142,6 +143,34 @@ pub async fn seed_store( storage_map_entries: u32, vault_entries: usize, account_update_blocks: usize, +) { + seed_store_with_readers( + data_directory, + num_accounts, + public_accounts_percentage, + storage_map_entries, + vault_entries, + account_update_blocks, + 0, + ) + .await; +} + +/// Seeds the store while `readers` concurrent tasks measure read latency against the same state. +/// +/// This is the mixed read/write benchmark: seeding provides a continuous block-application load, +/// and each reader loops `get_block_header` (latest header plus MMR proof) against the live state, +/// so the reported latencies capture how reads behave while blocks are being committed. With +/// `readers == 0` this is plain seeding. +#[expect(clippy::too_many_lines)] +pub async fn seed_store_with_readers( + data_directory: PathBuf, + num_accounts: usize, + public_accounts_percentage: u8, + storage_map_entries: u32, + vault_entries: usize, + account_update_blocks: usize, + readers: usize, ) { let start = Instant::now(); assert!(num_accounts > 0, "--num-accounts must be greater than zero"); @@ -219,6 +248,17 @@ pub async fn seed_store( plan_account_batches(num_accounts, public_accounts_percentage, ACCOUNT_UPDATES_PER_BLOCK) }; + // Spawn the benchmark readers before block generation starts so they observe the store under + // continuous write load for the whole run. + let stop_readers = Arc::new(AtomicBool::new(false)); + let reader_tasks: Vec<_> = (0..readers) + .map(|_| { + let state = Arc::clone(&store_state); + let stop = Arc::clone(&stop_readers); + tokio::spawn(async move { read_latest_header_until(&state, &stop).await }) + }) + .collect(); + // start generating blocks let accounts_filepath = data_directory.join(ACCOUNTS_FILENAME); let data_directory = @@ -244,6 +284,18 @@ pub async fn seed_store( )) .await; + // Stop the readers and report their latencies before stopping the store: they hold state + // references, and the read phase should not include post-write quiescence. + let write_load_elapsed = start.elapsed(); + stop_readers.store(true, Ordering::Relaxed); + let mut read_latencies = Vec::new(); + for task in reader_tasks { + read_latencies.extend(task.await.expect("benchmark reader should not panic")); + } + if readers > 0 { + report_read_latencies(&read_latencies, readers, write_load_elapsed); + } + // Wait for the store to release its backing storage so callers can immediately re-load the // state from the same data directory. assert!( @@ -255,6 +307,45 @@ pub async fn seed_store( println!("{metrics}"); } +/// Loops `get_block_header` (latest header with MMR proof) against the state until `stop` is set, +/// returning the latency of every request. +/// +/// The request combines an in-memory read (opening the latest block's MMR proof against the +/// snapshot's blockchain) with a database header lookup scoped to the snapshot's tip, so it +/// exercises the read path most exposed to concurrent block application. +async fn read_latest_header_until( + state: &Arc, + stop: &AtomicBool, +) -> Vec { + let mut latencies = Vec::new(); + while !stop.load(Ordering::Relaxed) { + let start = Instant::now(); + let (header, proof) = state + .get_block_header(None, true) + .await + .expect("get_block_header should succeed during seeding"); + latencies.push(start.elapsed()); + assert!( + header.is_some() && proof.is_some(), + "latest block header and MMR proof should exist" + ); + } + latencies +} + +/// Prints read-side statistics for the mixed read/write benchmark. +#[expect(clippy::cast_precision_loss)] +fn report_read_latencies( + latencies: &[std::time::Duration], + readers: usize, + elapsed: std::time::Duration, +) { + println!("Concurrent read statistics ({readers} readers during seeding):"); + println!(" Total reads: {}", latencies.len()); + println!(" Reads per second: {:.0}", latencies.len() as f64 / elapsed.as_secs_f64()); + crate::store::metrics::print_summary(latencies); +} + /// Generates batches of transactions to be inserted into the store. /// /// Account-creation notes are emitted in batches of at most 255 and consumed in a subsequent diff --git a/bin/stress-test/src/store/mod.rs b/bin/stress-test/src/store/mod.rs index d79ce09a6..e267f68bc 100644 --- a/bin/stress-test/src/store/mod.rs +++ b/bin/stress-test/src/store/mod.rs @@ -20,7 +20,7 @@ use tokio::fs; use crate::seeding::{ACCOUNTS_FILENAME, start_store}; use crate::store::metrics::print_summary; -mod metrics; +pub(crate) mod metrics; // CONSTANTS // ================================================================================================ From 3ceabd1313908f3c57d06d40c3f98e34dcf043bc Mon Sep 17 00:00:00 2001 From: sergerad Date: Fri, 24 Jul 2026 08:12:41 +0800 Subject: [PATCH 24/35] Revert "Add benchmarks" This reverts commit c528f192034f8c263e05a15635536119837cc6fe. --- bin/stress-test/src/main.rs | 64 --------------------- bin/stress-test/src/seeding/mod.rs | 91 ------------------------------ bin/stress-test/src/store/mod.rs | 2 +- 3 files changed, 1 insertion(+), 156 deletions(-) diff --git a/bin/stress-test/src/main.rs b/bin/stress-test/src/main.rs index fe9e5770d..870055bbc 100644 --- a/bin/stress-test/src/main.rs +++ b/bin/stress-test/src/main.rs @@ -61,50 +61,6 @@ pub enum Command { account_update_blocks: usize, }, - /// Benchmark read latency under concurrent write load. - /// - /// Seeds the store like `seed-store` while concurrent reader tasks continuously request the - /// latest block header (with its MMR proof) from the same in-process state. Reports read - /// latency percentiles alongside the usual seeding metrics, capturing how reads behave while - /// blocks are being applied. - BenchmarkMixed { - /// Directory in which to store the database and raw block data. If the directory contains a - /// database dump file, it will be replaced. - #[arg(short, long, value_name = "DATA_DIRECTORY")] - data_directory: PathBuf, - - /// Number of accounts to create. - #[arg(short, long, value_name = "NUM_ACCOUNTS")] - num_accounts: NonZeroUsize, - - /// Percentage of accounts that will be created as public accounts. The rest will be private - /// accounts. - #[arg( - short, - long, - value_name = "PUBLIC_ACCOUNTS_PERCENTAGE", - default_value = "0", - value_parser = clap::value_parser!(u8).range(0..=100) - )] - public_accounts_percentage: u8, - - /// Number of entries to add to a deterministic storage map on every public account. - #[arg(long, value_name = "STORAGE_MAP_ENTRIES", default_value = "0")] - storage_map_entries: u32, - - /// Number of distinct vault assets to add to every public account. - #[arg(long, value_name = "VAULT_ENTRIES", default_value = "1")] - vault_entries: NonZeroUsize, - - /// Number of post-initialization blocks containing public account updates. - #[arg(long, value_name = "ACCOUNT_UPDATE_BLOCKS", default_value = "0")] - account_update_blocks: usize, - - /// Number of concurrent reader tasks measuring read latency during seeding. - #[arg(short, long, value_name = "READERS", default_value = "8")] - readers: NonZeroUsize, - }, - /// Benchmark the performance of the store endpoints. BenchmarkStore { /// Store endpoint to test against. @@ -183,26 +139,6 @@ async fn main() { )) .await; }, - Command::BenchmarkMixed { - data_directory, - num_accounts, - public_accounts_percentage, - storage_map_entries, - vault_entries, - account_update_blocks, - readers, - } => { - Box::pin(seeding::seed_store_with_readers( - data_directory, - num_accounts.get(), - public_accounts_percentage, - storage_map_entries, - vault_entries.get(), - account_update_blocks, - readers.get(), - )) - .await; - }, Command::BenchmarkStore { endpoint, data_directory, diff --git a/bin/stress-test/src/seeding/mod.rs b/bin/stress-test/src/seeding/mod.rs index ce66b8d64..c5e332685 100644 --- a/bin/stress-test/src/seeding/mod.rs +++ b/bin/stress-test/src/seeding/mod.rs @@ -1,6 +1,5 @@ use std::collections::BTreeMap; use std::path::PathBuf; -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Instant; @@ -143,34 +142,6 @@ pub async fn seed_store( storage_map_entries: u32, vault_entries: usize, account_update_blocks: usize, -) { - seed_store_with_readers( - data_directory, - num_accounts, - public_accounts_percentage, - storage_map_entries, - vault_entries, - account_update_blocks, - 0, - ) - .await; -} - -/// Seeds the store while `readers` concurrent tasks measure read latency against the same state. -/// -/// This is the mixed read/write benchmark: seeding provides a continuous block-application load, -/// and each reader loops `get_block_header` (latest header plus MMR proof) against the live state, -/// so the reported latencies capture how reads behave while blocks are being committed. With -/// `readers == 0` this is plain seeding. -#[expect(clippy::too_many_lines)] -pub async fn seed_store_with_readers( - data_directory: PathBuf, - num_accounts: usize, - public_accounts_percentage: u8, - storage_map_entries: u32, - vault_entries: usize, - account_update_blocks: usize, - readers: usize, ) { let start = Instant::now(); assert!(num_accounts > 0, "--num-accounts must be greater than zero"); @@ -248,17 +219,6 @@ pub async fn seed_store_with_readers( plan_account_batches(num_accounts, public_accounts_percentage, ACCOUNT_UPDATES_PER_BLOCK) }; - // Spawn the benchmark readers before block generation starts so they observe the store under - // continuous write load for the whole run. - let stop_readers = Arc::new(AtomicBool::new(false)); - let reader_tasks: Vec<_> = (0..readers) - .map(|_| { - let state = Arc::clone(&store_state); - let stop = Arc::clone(&stop_readers); - tokio::spawn(async move { read_latest_header_until(&state, &stop).await }) - }) - .collect(); - // start generating blocks let accounts_filepath = data_directory.join(ACCOUNTS_FILENAME); let data_directory = @@ -284,18 +244,6 @@ pub async fn seed_store_with_readers( )) .await; - // Stop the readers and report their latencies before stopping the store: they hold state - // references, and the read phase should not include post-write quiescence. - let write_load_elapsed = start.elapsed(); - stop_readers.store(true, Ordering::Relaxed); - let mut read_latencies = Vec::new(); - for task in reader_tasks { - read_latencies.extend(task.await.expect("benchmark reader should not panic")); - } - if readers > 0 { - report_read_latencies(&read_latencies, readers, write_load_elapsed); - } - // Wait for the store to release its backing storage so callers can immediately re-load the // state from the same data directory. assert!( @@ -307,45 +255,6 @@ pub async fn seed_store_with_readers( println!("{metrics}"); } -/// Loops `get_block_header` (latest header with MMR proof) against the state until `stop` is set, -/// returning the latency of every request. -/// -/// The request combines an in-memory read (opening the latest block's MMR proof against the -/// snapshot's blockchain) with a database header lookup scoped to the snapshot's tip, so it -/// exercises the read path most exposed to concurrent block application. -async fn read_latest_header_until( - state: &Arc, - stop: &AtomicBool, -) -> Vec { - let mut latencies = Vec::new(); - while !stop.load(Ordering::Relaxed) { - let start = Instant::now(); - let (header, proof) = state - .get_block_header(None, true) - .await - .expect("get_block_header should succeed during seeding"); - latencies.push(start.elapsed()); - assert!( - header.is_some() && proof.is_some(), - "latest block header and MMR proof should exist" - ); - } - latencies -} - -/// Prints read-side statistics for the mixed read/write benchmark. -#[expect(clippy::cast_precision_loss)] -fn report_read_latencies( - latencies: &[std::time::Duration], - readers: usize, - elapsed: std::time::Duration, -) { - println!("Concurrent read statistics ({readers} readers during seeding):"); - println!(" Total reads: {}", latencies.len()); - println!(" Reads per second: {:.0}", latencies.len() as f64 / elapsed.as_secs_f64()); - crate::store::metrics::print_summary(latencies); -} - /// Generates batches of transactions to be inserted into the store. /// /// Account-creation notes are emitted in batches of at most 255 and consumed in a subsequent diff --git a/bin/stress-test/src/store/mod.rs b/bin/stress-test/src/store/mod.rs index e267f68bc..d79ce09a6 100644 --- a/bin/stress-test/src/store/mod.rs +++ b/bin/stress-test/src/store/mod.rs @@ -20,7 +20,7 @@ use tokio::fs; use crate::seeding::{ACCOUNTS_FILENAME, start_store}; use crate::store::metrics::print_summary; -pub(crate) mod metrics; +mod metrics; // CONSTANTS // ================================================================================================ From 974aa1210b2b5063cbe2e62a5065762dc56c1605 Mon Sep 17 00:00:00 2001 From: sergerad Date: Tue, 28 Jul 2026 13:20:12 +1200 Subject: [PATCH 25/35] Llint --- crates/store/src/db/mod.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index c758ef906..ea3e5bb6b 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -612,9 +612,7 @@ impl Db { models::queries::prune_history(conn, signed_block.header().block_num())?; let mut resolved_note_ids = BTreeMap::new(); - for chunk in - unresolved_note_nullifiers.chunks(QueryParamNoteCommitmentLimit::LIMIT) - { + for chunk in unresolved_note_nullifiers.chunks(QueryParamNoteCommitmentLimit::LIMIT) { match queries::select_note_ids_by_nullifier(conn, chunk) { Ok(note_ids) => resolved_note_ids.extend(note_ids), Err(err) => { From 5b6f02293c1c45032560e084dce0e4a2a2818fcb Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 30 Jul 2026 11:57:09 +1200 Subject: [PATCH 26/35] RM deadcode --- crates/block-producer/src/server/mod.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index 340c2d7d9..8f072dda5 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -168,14 +168,6 @@ impl Sequencer { Ok(SequencerHandle { api, task }) } - - /// Serves the sequencer tasks. - /// - /// Executes in place (i.e. not spawned) and will run indefinitely until a fatal error is - /// encountered. - pub async fn serve(self) -> anyhow::Result<()> { - self.spawn(CancellationToken::new())?.wait().await - } } /// Running sequencer tasks plus the API used to submit work to them. From 4369f8d1734f681b3373f4f17a23f6742ee3dbc0 Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 30 Jul 2026 12:05:04 +1200 Subject: [PATCH 27/35] State::for_tests --- crates/block-producer/src/server/tests.rs | 14 +------------- crates/rpc/src/tests.rs | 18 ++++-------------- crates/store/src/state/lifecycle.rs | 20 ++++++++++++++++++++ 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/crates/block-producer/src/server/tests.rs b/crates/block-producer/src/server/tests.rs index e6143e45b..e7f0b065c 100644 --- a/crates/block-producer/src/server/tests.rs +++ b/crates/block-producer/src/server/tests.rs @@ -1,12 +1,9 @@ use std::num::NonZeroUsize; -use std::sync::Arc; use std::time::Duration; use miden_node_store::GenesisState; use miden_node_store::state::State; -use miden_node_utils::clap::StorageOptions; use miden_node_utils::fee::test_fee_params; -use miden_node_utils::shutdown::CancellationToken; use miden_protocol::block::BlockNumber; use miden_protocol::testing::random_secret_key::random_secret_key; use url::Url; @@ -24,7 +21,7 @@ use crate::{ async fn block_producer_starts_with_store_state() { let data_directory = tempfile::tempdir().expect("tempdir should be created"); bootstrap_store(data_directory.path()); - let store = load_state(data_directory.path()).await; + let store = State::for_tests(data_directory.path()).await; let block_producer = Sequencer { store, @@ -55,12 +52,3 @@ fn bootstrap_store(path: &std::path::Path) { State::bootstrap(genesis_block, path).expect("store should bootstrap"); } - -async fn load_state(path: &std::path::Path) -> Arc { - let (state, _writer_task) = - State::load(path, StorageOptions::default(), CancellationToken::new()) - .await - .expect("state should load") - .start(); - state -} diff --git a/crates/rpc/src/tests.rs b/crates/rpc/src/tests.rs index 1d2081275..6a92a82a7 100644 --- a/crates/rpc/src/tests.rs +++ b/crates/rpc/src/tests.rs @@ -21,7 +21,7 @@ use miden_node_proto::generated::{self as proto}; use miden_node_proto::server::{ntx_builder_api, rpc_api, validator_api}; use miden_node_store::genesis::config::GenesisConfig; use miden_node_store::state::State; -use miden_node_utils::clap::{GrpcOptionsExternal, StorageOptions}; +use miden_node_utils::clap::GrpcOptionsExternal; use miden_node_utils::limiter::{ QueryParamAccountIdLimit, QueryParamLimiter, @@ -31,7 +31,6 @@ use miden_node_utils::limiter::{ QueryParamStorageMapKeyTotalLimit, QueryParamStorageMapSlotLimit, }; -use miden_node_utils::shutdown::CancellationToken; use miden_protocol::Word; use miden_protocol::account::{ Account, @@ -95,7 +94,7 @@ impl TestStore { async fn start() -> Self { let data_directory = new_tempdir(); let genesis_commitment = Self::bootstrap(&data_directory); - let state = load_state(&data_directory).await; + let state = State::for_tests(&data_directory).await; Self { state, genesis_commitment, @@ -119,15 +118,6 @@ impl TestStore { } } -async fn load_state(path: &std::path::Path) -> Arc { - let (state, _writer_task) = - State::load(path, StorageOptions::default(), CancellationToken::new()) - .await - .expect("state should load") - .start(); - state -} - /// Byte offset of the account delta commitment in serialized `ProvenTransaction`. Layout: /// `AccountId` (15) + `initial_commitment` (32) + `final_commitment` (32) = 79 const DELTA_COMMITMENT_BYTE_OFFSET: usize = 15 + 32 + 32; @@ -639,7 +629,7 @@ async fn start_source_rpc( let store = TestStore::start().await; let block_producer_dir = new_tempdir(); TestStore::bootstrap(&block_producer_dir); - let block_producer_state = load_state(&block_producer_dir).await; + let block_producer_state = State::for_tests(&block_producer_dir).await; let store_state = Arc::clone(&store.state); let listener = TcpListener::bind("127.0.0.1:0").await.expect("Failed to bind source RPC"); @@ -1099,7 +1089,7 @@ async fn start_rpc_with_options( let store = TestStore::start().await; let block_producer_dir = new_tempdir(); TestStore::bootstrap(&block_producer_dir); - let block_producer_state = load_state(&block_producer_dir).await; + let block_producer_state = State::for_tests(&block_producer_dir).await; let store_state = Arc::clone(&store.state); // Start the rpc component. diff --git a/crates/store/src/state/lifecycle.rs b/crates/store/src/state/lifecycle.rs index f34b7582a..f483d4da9 100644 --- a/crates/store/src/state/lifecycle.rs +++ b/crates/store/src/state/lifecycle.rs @@ -256,6 +256,26 @@ impl State { Ok(LoadedState { state, writer: block_writer }) } + /// Loads the state with default options and starts its block writer, detaching the writer + /// task. + /// + /// Test-only helper for tests in sibling crates that don't manage the writer's lifecycle: + /// the detached writer exits once the returned state (holding the only write handle) is + /// dropped. Hidden from public docs and not part of the stable API. + /// + /// # Panics + /// + /// Panics if the state fails to load. + #[doc(hidden)] + pub async fn for_tests(data_path: &Path) -> Arc { + let (state, _writer_task) = + Self::load(data_path, StorageOptions::default(), CancellationToken::new()) + .await + .expect("state should load") + .start(); + state + } + /// Stops the store, waiting until the block writer has released the tree storage it owns. /// /// Consumes the last state reference — closing the write channel the writer listens on — and From 858ee246f45675d3b86ba86efbbd8f623fa0812e Mon Sep 17 00:00:00 2001 From: sergerad Date: Thu, 30 Jul 2026 13:51:19 +1200 Subject: [PATCH 28/35] BlockWriter and WriteWorker --- bin/node/src/commands/modes.rs | 22 +- bin/node/src/commands/recover.rs | 32 +-- bin/stress-test/src/seeding/mod.rs | 73 +++---- bin/stress-test/src/seeding/tests.rs | 16 +- .../block-producer/src/batch_builder/mod.rs | 12 +- .../block-producer/src/block_builder/mod.rs | 17 +- crates/block-producer/src/proof_scheduler.rs | 12 +- crates/block-producer/src/rpc_sync.rs | 27 +-- crates/block-producer/src/server/mod.rs | 40 ++-- crates/block-producer/src/server/tests.rs | 6 +- crates/rpc/src/server/api.rs | 12 +- crates/rpc/src/server/api/get_account.rs | 2 +- .../rpc/src/server/api/get_block_by_number.rs | 4 +- .../server/api/get_block_header_by_number.rs | 2 +- .../src/server/api/get_note_script_by_root.rs | 2 +- crates/rpc/src/server/api/get_notes_by_id.rs | 2 +- crates/rpc/src/server/api/status.rs | 2 +- crates/rpc/src/server/api/submit_proven_tx.rs | 2 +- .../src/server/api/submit_proven_tx_batch.rs | 2 +- .../src/server/api/subscription/stream/mod.rs | 8 +- .../server/api/sync_account_storage_maps.rs | 2 +- .../rpc/src/server/api/sync_account_vault.rs | 2 +- crates/rpc/src/server/api/sync_chain_mmr.rs | 6 +- crates/rpc/src/server/api/sync_notes.rs | 2 +- crates/rpc/src/server/api/sync_nullifiers.rs | 2 +- .../rpc/src/server/api/sync_transactions.rs | 2 +- crates/rpc/src/server/mod.rs | 19 +- crates/rpc/src/tests.rs | 15 +- crates/store/src/lib.rs | 2 +- crates/store/src/state/apply_block.rs | 4 +- crates/store/src/state/apply_proof.rs | 4 +- crates/store/src/state/lifecycle.rs | 196 +++++++++++------- crates/store/src/state/mod.rs | 13 +- crates/store/src/state/writer.rs | 52 ++--- 34 files changed, 336 insertions(+), 280 deletions(-) diff --git a/bin/node/src/commands/modes.rs b/bin/node/src/commands/modes.rs index a827aff64..9070c4221 100644 --- a/bin/node/src/commands/modes.rs +++ b/bin/node/src/commands/modes.rs @@ -12,7 +12,7 @@ use miden_node_proto::clients::{ ValidatorClient, }; use miden_node_rpc::{Rpc, RpcMode, SequencerInternal}; -use miden_node_store::{State, WriterTask}; +use miden_node_store::{BlockWriter, ProofWriter, State, WriterTask}; use miden_node_utils::clap::{GrpcOptionsInternal, duration_to_human_readable_string}; use miden_node_utils::formatting::format_endpoint; use miden_node_utils::shutdown::CancellationToken; @@ -57,11 +57,14 @@ impl SequencerCommand { let runtime = self.runtime.runtime_config(&self.store); self.block_producer.validate()?; let network_tx_auth = self.runtime.rpc.network_tx_auth()?; - let (state, writer_task) = load_state(&runtime, shutdown.clone()).await?; + let (state, block_writer, proof_writer, writer_task) = + load_state(&runtime, shutdown.clone()).await?; let _disk_monitor = state.spawn_disk_monitor(shutdown.clone()); let sequencer = Sequencer { - store: Arc::clone(&state), + state: Arc::clone(&state), + block_writer, + proof_writer, validator_url: self.external_services.validator_url.clone(), validator_timeout: self.external_services.validator_timeout, batch_prover_url: self.block_producer.batch.prover_url, @@ -80,11 +83,12 @@ impl SequencerCommand { let rpc = Rpc { listener: bind_rpc(runtime.rpc_listen).await?, - store: state, + state, mode: RpcMode::sequencer( block_producer.clone(), self.external_services.validator_client()?, ), + sync_writers: None, ntx_builder: Some(self.external_services.ntx_builder_client()?), grpc_options: runtime.external_grpc_options, network_tx_auth, @@ -213,18 +217,20 @@ impl FullNodeCommand { let validator_client = self.validator_client(); let sequencer_client = self.sequencer_client(); let network_tx_auth = self.runtime.rpc.network_tx_auth()?; - let (state, writer_task) = load_state(&runtime, shutdown.clone()).await?; + let (state, block_writer, proof_writer, writer_task) = + load_state(&runtime, shutdown.clone()).await?; let _disk_monitor = state.spawn_disk_monitor(shutdown.clone()); let rpc = Rpc { listener: bind_rpc(runtime.rpc_listen).await?, - store: state, + state, mode: RpcMode::full_node( source_rpc, self.sync.readiness_threshold, validator_client, sequencer_client, ), + sync_writers: Some((block_writer, proof_writer)), ntx_builder: None, grpc_options: runtime.external_grpc_options, network_tx_auth, @@ -303,7 +309,7 @@ impl SyncOptions { async fn load_state( runtime: &RuntimeConfig, shutdown: CancellationToken, -) -> anyhow::Result<(Arc, WriterTask)> { +) -> anyhow::Result<(Arc, BlockWriter, ProofWriter, WriterTask)> { let loaded = State::load_with_database_options( &runtime.data_directory, runtime.storage_options.clone(), @@ -316,7 +322,7 @@ async fn load_state( Ok(loaded.start()) } -/// Supervises the store's block writer task. +/// Supervises the store's write worker task. /// /// On shutdown the task-drain loop waits for the writer to finish any in-flight block write and /// close its storage; an early exit or panic surfaces through the task set like any other task diff --git a/bin/node/src/commands/recover.rs b/bin/node/src/commands/recover.rs index 555351bee..50ee24567 100644 --- a/bin/node/src/commands/recover.rs +++ b/bin/node/src/commands/recover.rs @@ -1,12 +1,11 @@ use std::path::PathBuf; -use std::sync::Arc; use std::time::Duration; use anyhow::Context; use miden_node_proto::clients::{Builder, ValidatorClient}; use miden_node_proto::generated::validator::BlockSubscriptionRequest; use miden_node_store::state::Finality; -use miden_node_store::{State, WriterTask}; +use miden_node_store::{BlockWriter, State, WriterTask}; use miden_node_utils::shutdown::CancellationToken; use miden_protocol::block::{BlockNumber, SignedBlock}; use miden_protocol::utils::serde::Deserializable; @@ -37,20 +36,17 @@ pub struct RecoverCommand { impl RecoverCommand { pub async fn handle(self) -> anyhow::Result<()> { - let (state, writer_task) = self.load_state().await?; + let (block_writer, writer_task) = self.load_state().await?; let validator = self.validator_client()?; - let result = recover_from_validator(&state, validator).await; + let result = recover_from_validator(&block_writer, validator).await; // Wait for the writer to drain and release the backing storage before the process exits. - state - .stop(writer_task) - .await - .map_err(|_| anyhow::anyhow!("store state still referenced; cannot stop the store"))?; + block_writer.stop(writer_task).await; result } - async fn load_state(&self) -> anyhow::Result<(Arc, WriterTask)> { - // Recovery is not wired into the node's shutdown token; the writer exits once the state - // (holding the only write handle) is dropped after recovery completes. + async fn load_state(&self) -> anyhow::Result<(BlockWriter, WriterTask)> { + // Recovery is not wired into the node's shutdown token; the writer exits once the + // `BlockWriter` (holding the only write handle) is dropped after recovery completes. let loaded = State::load_with_database_options( &self.data_directory, self.store.storage.clone().into(), @@ -60,7 +56,8 @@ impl RecoverCommand { .await .context("failed to load state")?; - Ok(loaded.start()) + let (_state, block_writer, _proof_writer, writer_task) = loaded.start(); + Ok((block_writer, writer_task)) } fn validator_client(&self) -> anyhow::Result { @@ -76,7 +73,7 @@ impl RecoverCommand { /// Streams blocks from the validator into the local store until the chain tip is reached. async fn recover_from_validator( - state: &Arc, + block_writer: &BlockWriter, mut validator: ValidatorClient, ) -> anyhow::Result<()> { // Capture the validator's chain tip as the recovery target. The validator's block stream @@ -91,7 +88,7 @@ async fn recover_from_validator( .chain_tip, ); - let local_tip = state.chain_tip(Finality::Committed); + let local_tip = block_writer.chain_tip(Finality::Committed); if local_tip >= validator_tip { info!( target: LOG_TARGET, @@ -121,7 +118,10 @@ async fn recover_from_validator( let block = SignedBlock::read_from_bytes(&event.block) .context("failed to deserialize block from validator")?; let block_num = block.header().block_num(); - state.apply_block(block).await.context("failed to apply recovered block")?; + block_writer + .apply_block(block) + .await + .context("failed to apply recovered block")?; info!(target: LOG_TARGET, block_number = %block_num.as_u32(), "Applied recovered block"); // Stop once we reach the tip captured at the start of recovery. @@ -131,7 +131,7 @@ async fn recover_from_validator( } // The stream can end before reaching the tip if the validator restarts or drops the connection. - let final_tip = state.chain_tip(Finality::Committed); + let final_tip = block_writer.chain_tip(Finality::Committed); anyhow::ensure!( final_tip >= validator_tip, "validator block stream ended at block {} before reaching the chain tip {}", diff --git a/bin/stress-test/src/seeding/mod.rs b/bin/stress-test/src/seeding/mod.rs index ce66b8d64..7b0ca4f6d 100644 --- a/bin/stress-test/src/seeding/mod.rs +++ b/bin/stress-test/src/seeding/mod.rs @@ -6,7 +6,7 @@ use std::time::Instant; use metrics::SeedingMetrics; use miden_node_proto::domain::batch::BatchInputs; -use miden_node_store::{DataDirectory, GenesisState, State, WriterTask}; +use miden_node_store::{BlockWriter, DataDirectory, GenesisState, State, WriterTask}; use miden_node_utils::clap::StorageOptions; use miden_node_utils::shutdown::CancellationToken; use miden_protocol::account::auth::AuthScheme; @@ -229,7 +229,7 @@ pub async fn seed_store_with_readers( let genesis_header = genesis_block.inner().header().clone(); State::bootstrap(genesis_block, &data_directory).expect("store should bootstrap"); - let (store_state, writer_task) = load_state(data_directory.clone()).await; + let (state, block_writer, writer_task) = load_state(data_directory.clone()).await; // Recreate the deterministic genesis benchmark accounts after bootstrapping instead of keeping // another copy of their potentially very large maps alive while the genesis block is built. @@ -253,7 +253,7 @@ pub async fn seed_store_with_readers( let stop_readers = Arc::new(AtomicBool::new(false)); let reader_tasks: Vec<_> = (0..readers) .map(|_| { - let state = Arc::clone(&store_state); + let state = Arc::clone(&state); let stop = Arc::clone(&stop_readers); tokio::spawn(async move { read_latest_header_until(&state, &stop).await }) }) @@ -273,7 +273,7 @@ pub async fn seed_store_with_readers( }, faucet, genesis_header, - &store_state, + &block_writer, data_directory, accounts_filepath, &signer, @@ -298,10 +298,7 @@ pub async fn seed_store_with_readers( // Wait for the store to release its backing storage so callers can immediately re-load the // state from the same data directory. - assert!( - store_state.stop(writer_task).await.is_ok(), - "no other references to the store state should remain" - ); + block_writer.stop(writer_task).await; println!("Total time: {:.3} seconds", start.elapsed().as_secs_f64()); println!("{metrics}"); @@ -359,7 +356,7 @@ async fn generate_blocks( first_account_index: u64, mut faucet: Account, genesis_header: BlockHeader, - store_state: &Arc, + block_writer: &BlockWriter, data_directory: DataDirectory, accounts_filepath: PathBuf, signer: &EcdsaSecretKey, @@ -440,7 +437,7 @@ async fn generate_blocks( .collect(); // create the block and send it to the store - let block_inputs = get_block_inputs(store_state, &batches, &mut metrics).await; + let block_inputs = get_block_inputs(block_writer, &batches, &mut metrics).await; // update blocks let block_kind = if has_pending_account_creations { @@ -451,7 +448,7 @@ async fn generate_blocks( prev_block_header = apply_block( batches, block_inputs, - store_state, + block_writer, &mut metrics, signer, block_kind, @@ -463,7 +460,7 @@ async fn generate_blocks( // create the consume notes txs to be used in the next block let batch_inputs = - get_batch_inputs(store_state, &prev_block_header, ¬es, &mut metrics).await; + get_batch_inputs(block_writer, &prev_block_header, ¬es, &mut metrics).await; (pending_consumed_accounts, consume_notes_txs) = create_consume_note_txs( &prev_block_header, accounts, @@ -487,11 +484,11 @@ async fn generate_blocks( .par_chunks(TRANSACTIONS_PER_BATCH) .map(|txs| create_batch(txs, &prev_block_header)) .collect(); - let block_inputs = get_block_inputs(store_state, &batches, &mut metrics).await; + let block_inputs = get_block_inputs(block_writer, &batches, &mut metrics).await; prev_block_header = apply_block( batches, block_inputs, - store_state, + block_writer, &mut metrics, signer, metrics::BlockKind::AccountCreation, @@ -527,11 +524,11 @@ async fn generate_blocks( let emit_note_tx = create_emit_note_tx(&prev_block_header, &mut faucet, notes.clone()); let batches = vec![create_batch(std::slice::from_ref(&emit_note_tx), &prev_block_header)]; - let block_inputs = get_block_inputs(store_state, &batches, &mut metrics).await; + let block_inputs = get_block_inputs(block_writer, &batches, &mut metrics).await; prev_block_header = apply_block( batches, block_inputs, - store_state, + block_writer, &mut metrics, signer, metrics::BlockKind::UpdateNoteEmission, @@ -540,7 +537,7 @@ async fn generate_blocks( .await; let batch_inputs = - get_batch_inputs(store_state, &prev_block_header, ¬es, &mut metrics).await; + get_batch_inputs(block_writer, &prev_block_header, ¬es, &mut metrics).await; let accounts = selected_account_ids .iter() .map(|account_id| { @@ -563,11 +560,11 @@ async fn generate_blocks( .par_chunks(TRANSACTIONS_PER_BATCH) .map(|txs| create_batch(txs, &prev_block_header)) .collect(); - let block_inputs = get_block_inputs(store_state, &batches, &mut metrics).await; + let block_inputs = get_block_inputs(block_writer, &batches, &mut metrics).await; prev_block_header = apply_block( batches, block_inputs, - store_state, + block_writer, &mut metrics, signer, metrics::BlockKind::AccountUpdate, @@ -596,7 +593,7 @@ async fn generate_blocks( async fn apply_block( batches: Vec, block_inputs: BlockInputs, - store_state: &Arc, + block_writer: &BlockWriter, metrics: &mut SeedingMetrics, signer: &EcdsaSecretKey, block_kind: metrics::BlockKind, @@ -614,7 +611,7 @@ async fn apply_block( let ordered_batches = proposed_block.batches().clone(); let start = Instant::now(); - store_state + block_writer .apply_block_with_proving_inputs(ordered_batches, block_inputs, signed_block) .await .unwrap(); @@ -1064,7 +1061,7 @@ fn create_emit_note_tx( /// Gets the batch inputs from the store and tracks the query time on the metrics. async fn get_batch_inputs( - store_state: &Arc, + state: &State, block_ref: &BlockHeader, notes: &[Note], metrics: &mut SeedingMetrics, @@ -1072,7 +1069,7 @@ async fn get_batch_inputs( let start = Instant::now(); // Mark every note as unauthenticated, so that the store returns the inclusion proofs for all of // them - let batch_inputs = store_state + let batch_inputs = state .get_batch_inputs( [block_ref.block_num()].into_iter().collect(), notes.iter().map(|note| note.id().as_word()).collect(), @@ -1085,12 +1082,12 @@ async fn get_batch_inputs( /// Gets the block inputs from the store and tracks the query time on the metrics. async fn get_block_inputs( - store_state: &Arc, + state: &State, batches: &[ProvenBatch], metrics: &mut SeedingMetrics, ) -> BlockInputs { let start = Instant::now(); - let inputs = store_state + let inputs = state .get_block_inputs( batches.iter().flat_map(ProvenBatch::updated_accounts).collect(), batches.iter().flat_map(ProvenBatch::created_nullifiers).collect(), @@ -1115,19 +1112,25 @@ async fn get_block_inputs( /// Loads the store state from the given data directory, detaching the block writer task. /// /// Intended for benches that run until process exit and never need the storage released -/// deterministically; use [`load_state`] when the writer must be joined. +/// deterministically; use [`load_state`] when the writer must be joined. The write capability is +/// leaked to keep the block writer alive for the process lifetime, as before the read/write +/// split. pub async fn start_store(data_directory: PathBuf) -> Arc { - let (state, _writer_task) = load_state(data_directory).await; + let (state, block_writer, _writer_task) = load_state(data_directory).await; + std::mem::forget(block_writer); state } -/// Loads the store state and spawns its block writer, returning the writer's join handle. +/// Loads the store state and spawns its block writer, returning the write capability and the +/// writer's join handle. /// -/// The writer exits once the last reference to the returned state is dropped; awaiting the handle -/// after that guarantees the backing storage has been released. -async fn load_state(data_directory: PathBuf) -> (Arc, WriterTask) { - State::load(&data_directory, StorageOptions::bench(), CancellationToken::new()) - .await - .expect("store state should load") - .start() +/// The writer exits once the returned [`BlockWriter`] is dropped; awaiting the handle after that +/// guarantees the backing storage has been released. +async fn load_state(data_directory: PathBuf) -> (Arc, BlockWriter, WriterTask) { + let (state, block_writer, _proof_writer, writer_task) = + State::load(&data_directory, StorageOptions::bench(), CancellationToken::new()) + .await + .expect("store state should load") + .start(); + (state, block_writer, writer_task) } diff --git a/bin/stress-test/src/seeding/tests.rs b/bin/stress-test/src/seeding/tests.rs index e5c6c6e3b..6fb7063ec 100644 --- a/bin/stress-test/src/seeding/tests.rs +++ b/bin/stress-test/src/seeding/tests.rs @@ -170,7 +170,7 @@ async fn seed_store_persists_one_public_account_and_applies_one_map_update() { assert_eq!(account_ids.len(), 1); let account_id = AccountId::from_hex(account_ids[0]).unwrap(); - let (state, writer_task) = load_state(data_directory).await; + let (state, block_writer, writer_task) = load_state(data_directory).await; let response = state .get_account(AccountRequest { account_id, @@ -200,10 +200,8 @@ async fn seed_store_persists_one_public_account_and_applies_one_map_update() { ); // Release the backing storage before the temporary directory is deleted. - assert!( - state.stop(writer_task).await.is_ok(), - "no other references to the store state should remain" - ); + drop(state); + block_writer.stop(writer_task).await; } #[tokio::test(flavor = "multi_thread")] @@ -219,7 +217,7 @@ async fn seed_store_handles_map_larger_than_transaction_account_update_limit() { assert_eq!(account_ids.len(), 1); let account_id = AccountId::from_hex(account_ids[0]).unwrap(); - let (state, writer_task) = load_state(data_directory).await; + let (state, block_writer, writer_task) = load_state(data_directory).await; let response = state .get_account(AccountRequest { account_id, @@ -231,8 +229,6 @@ async fn seed_store_handles_map_larger_than_transaction_account_update_limit() { assert_ne!(response.witness.state_commitment(), Word::empty()); // Release the backing storage before the temporary directory is deleted. - assert!( - state.stop(writer_task).await.is_ok(), - "no other references to the store state should remain" - ); + drop(state); + block_writer.stop(writer_task).await; } diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index 5cc4c3693..be7c69258 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -45,7 +45,7 @@ pub struct BatchBuilder { /// /// If not provided, a local batch prover is used. batch_prover: BatchProver, - store: Arc, + state: Arc, } pub(crate) struct BatchIntervals { @@ -79,7 +79,7 @@ impl BatchBuilder { /// /// If no batch prover URL is provided, a local batch prover is used instead. pub fn new( - store: Arc, + state: Arc, num_workers: NonZeroUsize, batch_prover_url: Option, intervals: BatchIntervals, @@ -92,7 +92,7 @@ impl BatchBuilder { num_workers, intervals, batch_prover, - store, + state, }) } @@ -160,7 +160,7 @@ impl BatchBuilder { transactions.unauthenticated_notes.count = telemetry.unauthenticated_notes_count, ); let job = BatchJob { - store: self.store.clone(), + state: self.state.clone(), mempool, batch_prover: self.batch_prover.clone(), }; @@ -240,7 +240,7 @@ impl BatchBuilder { /// /// Recoverable errors are handled internally. Mempool poison is propagated as a fatal error. struct BatchJob { - store: Arc, + state: Arc, batch_prover: BatchProver, mempool: SharedMempool, } @@ -305,7 +305,7 @@ impl BatchJob { .map(Deref::deref) .flat_map(AuthenticatedTransaction::unauthenticated_note_ids); - self.store + self.state .get_batch_inputs( block_references.map(|(block_num, _)| block_num).collect(), unauthenticated_notes.collect(), diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index 39519d510..f4c61c87b 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -2,7 +2,7 @@ use std::ops::Deref; use std::sync::Arc; use anyhow::Context; -use miden_node_store::state::State; +use miden_node_store::state::BlockWriter; use miden_node_utils::formatting::format_array; use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::spawn::spawn_blocking_in_current_span; @@ -31,23 +31,24 @@ pub struct BlockBuilder { /// The frequency at which blocks are produced. pub block_interval: Duration, - /// The store state for committing blocks. - pub store: Arc, + /// The store's block-write capability, used for committing blocks. + pub block_writer: BlockWriter, /// The validator RPC client for validating blocks. pub validator: BlockProducerValidatorClient, } impl BlockBuilder { - /// Creates a new [`BlockBuilder`] with the given store state and optional block prover URL. + /// Creates a new [`BlockBuilder`] with the given block-write capability and optional block + /// prover URL. /// /// If the block prover URL is not set, the block builder will use the local block prover. pub fn new( - store: Arc, + block_writer: BlockWriter, validator: BlockProducerValidatorClient, block_interval: Duration, ) -> Self { - Self { block_interval, store, validator } + Self { block_interval, block_writer, validator } } /// Starts the [`BlockBuilder`], infinitely producing blocks at the configured interval. /// @@ -213,7 +214,7 @@ impl BlockBuilder { batch_iter.map(Deref::deref).flat_map(ProvenBatch::created_nullifiers); let inputs = self - .store + .block_writer .get_block_inputs( account_ids_iter.collect(), created_nullifiers_iter.collect(), @@ -351,7 +352,7 @@ impl BlockBuilder { tracing::debug!(target: LOG_TARGET, transactions = %format_array(transaction_ids), "Included transactions"); } - self.store + self.block_writer .apply_block_with_proving_inputs(ordered_batches, block_inputs, signed_block) .await .map_err(StoreError::ApplyBlockFailed) diff --git a/crates/block-producer/src/proof_scheduler.rs b/crates/block-producer/src/proof_scheduler.rs index c455e92d4..cbbc5bcf9 100644 --- a/crates/block-producer/src/proof_scheduler.rs +++ b/crates/block-producer/src/proof_scheduler.rs @@ -18,7 +18,7 @@ use std::time::Duration; use anyhow::Context; use miden_node_proto::BlockProofRequest; -use miden_node_store::state::{Finality, State}; +use miden_node_store::state::{Finality, ProofWriter, State}; use miden_node_utils::retry::{self, Retryable}; use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::tracing::miden_instrument; @@ -107,7 +107,7 @@ impl ProofTaskJoinSet { /// Transient errors are retried internally. pub(crate) async fn run( block_prover: Arc, - state: Arc, + proof_writer: ProofWriter, mut chain_tip_rx: watch::Receiver, max_concurrent_proofs: NonZeroUsize, shutdown: CancellationToken, @@ -119,7 +119,7 @@ pub(crate) async fn run( // Next block number to schedule. Initialized from the proven tip's child so we skip // already-proven blocks on restart. - let mut next_to_prove = state.chain_tip(Finality::Proven).child(); + let mut next_to_prove = proof_writer.chain_tip(Finality::Proven).child(); // Completed proofs waiting to be committed in order. let mut pending: BTreeMap> = BTreeMap::new(); @@ -128,7 +128,7 @@ pub(crate) async fn run( // Schedule blocks up to chain_tip that haven't been scheduled yet. let chain_tip = *chain_tip_rx.borrow(); while proving_tasks.len() < max_concurrent_proofs.get() && next_to_prove <= chain_tip { - proving_tasks.spawn(&state, &block_prover, next_to_prove); + proving_tasks.spawn(proof_writer.state(), &block_prover, next_to_prove); next_to_prove = next_to_prove.child(); } @@ -145,9 +145,9 @@ pub(crate) async fn run( // Drain completed proofs in ascending order so the proven tip advances without // gaps. - let mut next = state.chain_tip(Finality::Proven).child(); + let mut next = proof_writer.chain_tip(Finality::Proven).child(); while let Some(proof_bytes) = pending.remove(&next) { - state.apply_proof(next, proof_bytes).await?; + proof_writer.apply_proof(next, proof_bytes).await?; next = next.child(); } }, diff --git a/crates/block-producer/src/rpc_sync.rs b/crates/block-producer/src/rpc_sync.rs index b1c0bc833..e7cbe792f 100644 --- a/crates/block-producer/src/rpc_sync.rs +++ b/crates/block-producer/src/rpc_sync.rs @@ -5,7 +5,7 @@ use std::time::Duration; use anyhow::Context; use miden_node_proto::clients::RpcClient; use miden_node_proto::generated::rpc::{BlockSubscriptionRequest, ProofSubscriptionRequest}; -use miden_node_store::state::{Finality, State}; +use miden_node_store::state::{BlockWriter, Finality, ProofWriter}; use miden_node_utils::retry::{self, Retryable}; use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::tasks::Tasks; @@ -130,7 +130,10 @@ impl RpcReadiness { /// Synchronizes local state from an upstream RPC service. pub struct RpcSync { - pub state: Arc, + /// The store's block-write capability, consumed by the block sync loop. + pub block_writer: BlockWriter, + /// The store's proof-write capability, consumed by the proof sync loop. + pub proof_writer: ProofWriter, pub source_rpc: RpcClient, pub readiness: RpcReadiness, } @@ -140,12 +143,12 @@ impl RpcSync { pub async fn run(self, shutdown: CancellationToken) -> anyhow::Result<()> { let mut tasks = Tasks::new(); let block_sync = BlockSync { - state: Arc::clone(&self.state), + writer: self.block_writer, source_rpc: self.source_rpc.clone(), readiness: self.readiness, }; let proof_sync = ProofSync { - state: self.state, + writer: self.proof_writer, source_rpc: self.source_rpc, }; @@ -160,7 +163,7 @@ impl RpcSync { // ================================================================================================ struct BlockSync { - state: Arc, + writer: BlockWriter, source_rpc: RpcClient, readiness: RpcReadiness, } @@ -194,7 +197,7 @@ impl BlockSync { err, )] async fn sync(&self, shutdown: CancellationToken) -> anyhow::Result<()> { - let local_tip = self.state.chain_tip(Finality::Committed); + let local_tip = self.writer.chain_tip(Finality::Committed); let mut client = self.source_rpc.clone(); let upstream_tip = BlockNumber::from(client.status(tonic::Request::new(())).await?.into_inner().chain_tip); @@ -220,16 +223,16 @@ impl BlockSync { let upstream_tip = BlockNumber::from(event.committed_chain_tip); let block = SignedBlock::read_from_bytes(&event.block) .context("failed to deserialize block from upstream")?; - self.state.apply_block(block).await?; + self.writer.apply_block(block).await?; - let local_tip = self.state.chain_tip(Finality::Committed); + let local_tip = self.writer.chain_tip(Finality::Committed); self.readiness.update(upstream_tip, local_tip).await; } } } struct ProofSync { - state: Arc, + writer: ProofWriter, source_rpc: RpcClient, } @@ -264,7 +267,7 @@ impl ProofSync { async fn sync(&self, shutdown: CancellationToken) -> anyhow::Result<()> { // Subscribe from next proven tip. - let starting_block = self.state.chain_tip(Finality::Proven).child(); + let starting_block = self.writer.chain_tip(Finality::Proven).child(); info!( target: LOG_TARGET, block_from = %starting_block, @@ -277,7 +280,7 @@ impl ProofSync { .into_inner(); let mut expected = starting_block; - let mut committed_tip_rx = self.state.subscribe_committed_tip(); + let mut committed_tip_rx = self.writer.subscribe_committed_tip(); loop { let result = tokio::select! { () = shutdown.cancelled() => return Ok(()), @@ -303,7 +306,7 @@ impl ProofSync { }, } - self.state.apply_proof(block_num, event.proof).await?; + self.writer.apply_proof(block_num, event.proof).await?; expected = expected.child(); } diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index 8f072dda5..bed649869 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use std::time::Duration; use anyhow::Result; -use miden_node_store::state::{Finality, State}; +use miden_node_store::state::{BlockWriter, Finality, ProofWriter, State}; use miden_node_utils::formatting::{format_input_notes, format_output_notes}; use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::tasks::Tasks; @@ -69,8 +69,12 @@ impl BlockProducerApiConfig { /// /// Specifies how to connect to the batch prover and block prover components. pub struct Sequencer { - /// The store state shared with the block producer. - pub store: Arc, + /// The read-only store state shared with the block producer. + pub state: Arc, + /// The store's block-write capability, handed to the block builder. + pub block_writer: BlockWriter, + /// The store's proof-write capability, handed to the proof scheduler. + pub proof_writer: ProofWriter, /// The address of the validator component. pub validator_url: Url, /// The request timeout for calls to the validator component. @@ -104,17 +108,17 @@ impl Sequencer { /// Spawns the sequencer tasks and returns its in-process API. pub fn spawn(self, shutdown: CancellationToken) -> Result { tracing::info!(target: LOG_TARGET, "Initializing sequencer"); - let store = self.store; + let state = self.state; let validator = BlockProducerValidatorClient::new(self.validator_url.clone(), self.validator_timeout)?; - let chain_tip = store.chain_tip(Finality::Committed); + let chain_tip = state.chain_tip(Finality::Committed); tracing::info!(target: LOG_TARGET, "Sequencer initialized"); - let block_builder = BlockBuilder::new(Arc::clone(&store), validator, self.block_interval); + let block_builder = BlockBuilder::new(self.block_writer, validator, self.block_interval); let batch_intervals = BatchIntervals::derive_from(self.block_interval, self.batch_interval); let batch_builder = BatchBuilder::new( - Arc::clone(&store), + Arc::clone(&state), self.batch_workers, self.batch_prover_url, batch_intervals, @@ -125,13 +129,13 @@ impl Sequencer { mempool_tx_capacity: self.mempool_tx_capacity, }; let mempool = Mempool::shared(chain_tip, api_config.mempool_config()); - let api = BlockProducerApi::from_shared_mempool(mempool.clone(), store, shutdown.clone()); + let api = BlockProducerApi::from_shared_mempool(mempool.clone(), state, shutdown.clone()); let block_prover = if let Some(url) = self.block_prover_url { Arc::new(BlockProver::remote(url)?) } else { Arc::new(BlockProver::local()) }; - let chain_tip_rx = api.store.subscribe_committed_tip(); + let chain_tip_rx = api.state.subscribe_committed_tip(); // Spawn batch builder, block builder, and proof scheduler. The builders communicate // indirectly via a shared mempool. @@ -151,12 +155,12 @@ impl Sequencer { async { block_builder.run(mempool, shutdown).await } }); tasks.spawn("proof-scheduler", { - let store = Arc::clone(&api.store); + let proof_writer = self.proof_writer; let shutdown = shutdown.clone(); async move { proof_scheduler::run( block_prover, - store, + proof_writer, chain_tip_rx, self.max_concurrent_proofs, shutdown, @@ -202,7 +206,7 @@ pub struct BlockProducerApi { /// the block-producers usage of the mempool. mempool: Arc>, - store: Arc, + state: Arc, /// Cached mempool statistics that are updated periodically to avoid locking the mempool for /// each status request. @@ -230,17 +234,17 @@ pub struct BlockProducerStatus { impl BlockProducerApi { /// Creates an API backed by a fresh mempool. - pub fn new(store: Arc, chain_tip: BlockNumber, config: BlockProducerApiConfig) -> Self { + pub fn new(state: Arc, chain_tip: BlockNumber, config: BlockProducerApiConfig) -> Self { Self::from_shared_mempool( Mempool::shared(chain_tip, config.mempool_config()), - store, + state, CancellationToken::new(), ) } fn from_shared_mempool( mempool: SharedMempool, - store: Arc, + state: Arc, shutdown: CancellationToken, ) -> Self { let cached_mempool_stats = mempool @@ -249,7 +253,7 @@ impl BlockProducerApi { .unwrap_or_default(); let api = Self { mempool: Arc::new(Mutex::new(mempool)), - store, + state, cached_mempool_stats: Arc::new(RwLock::new(cached_mempool_stats)), }; api.spawn_mempool_stats_updater(shutdown); @@ -321,7 +325,7 @@ impl BlockProducerApi { tracing::debug!(target: COMPONENT, proof = ?tx.proof()); // Authenticate against the local store, then add to the mempool. - let inputs = get_tx_inputs(&self.store, &tx) + let inputs = get_tx_inputs(&self.state, &tx) .await .map_err(MempoolSubmissionError::StoreStateReadFailed)?; // SAFETY: we assume that the rpc component has verified the transaction proof already. @@ -368,7 +372,7 @@ impl BlockProducerApi { let mut inputs = Vec::with_capacity(batch.transactions().len()); for tx in batch.transactions() { inputs.push( - get_tx_inputs(&self.store, tx) + get_tx_inputs(&self.state, tx) .await .map_err(MempoolSubmissionError::StoreStateReadFailed)?, ); diff --git a/crates/block-producer/src/server/tests.rs b/crates/block-producer/src/server/tests.rs index e7f0b065c..d5a640cc4 100644 --- a/crates/block-producer/src/server/tests.rs +++ b/crates/block-producer/src/server/tests.rs @@ -21,10 +21,12 @@ use crate::{ async fn block_producer_starts_with_store_state() { let data_directory = tempfile::tempdir().expect("tempdir should be created"); bootstrap_store(data_directory.path()); - let store = State::for_tests(data_directory.path()).await; + let (state, block_writer, proof_writer) = State::for_tests(data_directory.path()).await; let block_producer = Sequencer { - store, + state, + block_writer, + proof_writer, validator_url: Url::parse("http://127.0.0.1:0").unwrap(), validator_timeout: DEFAULT_VALIDATOR_TIMEOUT, batch_prover_url: None, diff --git a/crates/rpc/src/server/api.rs b/crates/rpc/src/server/api.rs index fedb53d9f..78cdd236d 100644 --- a/crates/rpc/src/server/api.rs +++ b/crates/rpc/src/server/api.rs @@ -75,7 +75,7 @@ impl From for RpcInvalidBlockRange { // ================================================================================================ pub struct RpcService { - store: Arc, + state: Arc, mode: RpcMode, ntx_builder: Option, network_tx_auth: Option, @@ -88,14 +88,14 @@ pub struct RpcService { impl RpcService { pub(crate) fn new( - store: Arc, + state: Arc, mode: RpcMode, ntx_builder: Option, commitment_cache_capacity: NonZeroUsize, network_tx_auth: Option, ) -> Self { Self { - store, + state, mode, ntx_builder, network_tx_auth, @@ -167,7 +167,7 @@ impl RpcService { } let header = self - .store + .state .get_block_header(Some(block), false) .await .map_err(get_block_header_error_to_status)? @@ -205,7 +205,7 @@ impl RpcService { &self, range: &RangeInclusive, ) -> Result { - let chain_tip = self.store.chain_tip(Finality::Committed); + let chain_tip = self.state.chain_tip(Finality::Committed); if *range.end() > chain_tip { return Err(Status::invalid_argument(format!( "block_to ({}) is greater than chain tip ({chain_tip})", @@ -228,7 +228,7 @@ impl RpcService { } let network_accounts = - self.store.filter_network_accounts(&account_ids).await.map_err(|err| { + self.state.filter_network_accounts(&account_ids).await.map_err(|err| { Status::internal(format!("network-account classification failed: {err}")) })?; diff --git a/crates/rpc/src/server/api/get_account.rs b/crates/rpc/src/server/api/get_account.rs index 18e5ea0b5..24d01b95d 100644 --- a/crates/rpc/src/server/api/get_account.rs +++ b/crates/rpc/src/server/api/get_account.rs @@ -59,7 +59,7 @@ impl proto::server::rpc_api::GetAccount for RpcService { } let account_data = - self.store.get_account(request).await.map_err(get_account_error_to_status)?; + self.state.get_account(request).await.map_err(get_account_error_to_status)?; Ok(account_data) } } diff --git a/crates/rpc/src/server/api/get_block_by_number.rs b/crates/rpc/src/server/api/get_block_by_number.rs index d59e989a3..b69141854 100644 --- a/crates/rpc/src/server/api/get_block_by_number.rs +++ b/crates/rpc/src/server/api/get_block_by_number.rs @@ -38,12 +38,12 @@ impl proto::server::rpc_api::GetBlockByNumber for RpcService { let block_num = BlockNumber::from(request.block_num); let block = self - .store + .state .load_block(block_num) .await .map_err(|err| database_error_to_status(&err))?; let proof = if request.include_proof.unwrap_or_default() { - self.store + self.state .load_proof(block_num) .await .map_err(|err| database_error_to_status(&err))? diff --git a/crates/rpc/src/server/api/get_block_header_by_number.rs b/crates/rpc/src/server/api/get_block_header_by_number.rs index e39002cb4..27265587f 100644 --- a/crates/rpc/src/server/api/get_block_header_by_number.rs +++ b/crates/rpc/src/server/api/get_block_header_by_number.rs @@ -38,7 +38,7 @@ impl proto::server::rpc_api::GetBlockHeaderByNumber for RpcService { let block_num = request.block_num.map(BlockNumber::from); let (block_header, mmr_proof) = self - .store + .state .get_block_header(block_num, request.include_mmr_proof.unwrap_or(false)) .await .map_err(super::get_block_header_error_to_status)?; diff --git a/crates/rpc/src/server/api/get_note_script_by_root.rs b/crates/rpc/src/server/api/get_note_script_by_root.rs index 4cc123b6c..437baf57e 100644 --- a/crates/rpc/src/server/api/get_note_script_by_root.rs +++ b/crates/rpc/src/server/api/get_note_script_by_root.rs @@ -43,7 +43,7 @@ impl proto::server::rpc_api::GetNoteScriptByRoot for RpcService { debug!(target: LOG_TARGET, "Getting note script by root"); let script = self - .store + .state .get_note_script_by_root(root) .await .map_err(|err| database_error_to_status(&err))?; diff --git a/crates/rpc/src/server/api/get_notes_by_id.rs b/crates/rpc/src/server/api/get_notes_by_id.rs index f27c32d84..8f9ddd1f2 100644 --- a/crates/rpc/src/server/api/get_notes_by_id.rs +++ b/crates/rpc/src/server/api/get_notes_by_id.rs @@ -45,7 +45,7 @@ impl proto::server::rpc_api::GetNotesById for RpcService { let note_ids: Vec = note_ids.into_iter().map(NoteId::from_raw).collect(); let notes = self - .store + .state .get_notes_by_id(note_ids) .await .map_err(|err| database_error_to_status(&err))? diff --git a/crates/rpc/src/server/api/status.rs b/crates/rpc/src/server/api/status.rs index 02ce3b9c6..2b937d670 100644 --- a/crates/rpc/src/server/api/status.rs +++ b/crates/rpc/src/server/api/status.rs @@ -48,7 +48,7 @@ impl proto::server::rpc_api::Status for RpcService { Ok(proto::rpc::RpcStatus { version: env!("CARGO_PKG_VERSION").to_string(), - chain_tip: self.store.chain_tip(Finality::Committed).as_u32(), + chain_tip: self.state.chain_tip(Finality::Committed).as_u32(), block_producer: block_producer_status.or(Some(proto::rpc::BlockProducerStatus { status: "unreachable".to_string(), version: "-".to_string(), diff --git a/crates/rpc/src/server/api/submit_proven_tx.rs b/crates/rpc/src/server/api/submit_proven_tx.rs index 4aebd7e40..10d853106 100644 --- a/crates/rpc/src/server/api/submit_proven_tx.rs +++ b/crates/rpc/src/server/api/submit_proven_tx.rs @@ -179,7 +179,7 @@ impl RpcService { request: proto::transaction::ProvenTransaction, rebuilt_tx: ProvenTransaction, ) -> tonic::Result { - let tx_inputs = get_tx_inputs(&self.store, &rebuilt_tx).await.map_err(|err| { + let tx_inputs = get_tx_inputs(&self.state, &rebuilt_tx).await.map_err(|err| { Status::internal(err.as_report_context("failed to get transaction inputs")) })?; diff --git a/crates/rpc/src/server/api/submit_proven_tx_batch.rs b/crates/rpc/src/server/api/submit_proven_tx_batch.rs index 2dedae98c..578f4829b 100644 --- a/crates/rpc/src/server/api/submit_proven_tx_batch.rs +++ b/crates/rpc/src/server/api/submit_proven_tx_batch.rs @@ -170,7 +170,7 @@ impl RpcService { let mut auth_inputs = Vec::with_capacity(proposed_batch.transactions().len()); for tx in proposed_batch.transactions() { - let inputs = get_tx_inputs(&self.store, tx).await.map_err(|err| { + let inputs = get_tx_inputs(&self.state, tx).await.map_err(|err| { Status::internal(err.as_report_context("failed to authenticate transaction")) })?; auth_inputs.push(inputs.into()); diff --git a/crates/rpc/src/server/api/subscription/stream/mod.rs b/crates/rpc/src/server/api/subscription/stream/mod.rs index 0645a48dc..e1fda85dc 100644 --- a/crates/rpc/src/server/api/subscription/stream/mod.rs +++ b/crates/rpc/src/server/api/subscription/stream/mod.rs @@ -61,14 +61,14 @@ impl SubscriptionStream { from: BlockNumber, client_ip: Option, ) -> tonic::Result { - let store = Arc::clone(&rpc.store); + let store = Arc::clone(&rpc.state); Self::create( from, client_ip, Arc::clone(&rpc.subscription_ban), Arc::clone(&rpc.block_subscription_semaphore), - rpc.store.subscribe_committed_tip(), + rpc.state.subscribe_committed_tip(), move |block| { let store = Arc::clone(&store); async move { store.load_block(block).await } @@ -82,14 +82,14 @@ impl SubscriptionStream { from: BlockNumber, client_ip: Option, ) -> tonic::Result { - let store = Arc::clone(&rpc.store); + let store = Arc::clone(&rpc.state); Self::create( from, client_ip, Arc::clone(&rpc.subscription_ban), Arc::clone(&rpc.proof_subscription_semaphore), - rpc.store.subscribe_proven_tip(), + rpc.state.subscribe_proven_tip(), move |block| { let store = Arc::clone(&store); async move { store.load_proof(block).await } diff --git a/crates/rpc/src/server/api/sync_account_storage_maps.rs b/crates/rpc/src/server/api/sync_account_storage_maps.rs index 8c1a9b184..8fcf75387 100644 --- a/crates/rpc/src/server/api/sync_account_storage_maps.rs +++ b/crates/rpc/src/server/api/sync_account_storage_maps.rs @@ -60,7 +60,7 @@ impl proto::server::rpc_api::SyncAccountStorageMaps for RpcService { .map_err(invalid_block_range_to_status)?; let chain_tip = self.range_bounds_check(&block_range)?; let storage_maps_page = self - .store + .state .sync_account_storage_maps(account_id, block_range) .await .map_err(|err| database_error_to_status(&err))?; diff --git a/crates/rpc/src/server/api/sync_account_vault.rs b/crates/rpc/src/server/api/sync_account_vault.rs index 7cd103ea4..9e965a8c5 100644 --- a/crates/rpc/src/server/api/sync_account_vault.rs +++ b/crates/rpc/src/server/api/sync_account_vault.rs @@ -60,7 +60,7 @@ impl proto::server::rpc_api::SyncAccountVault for RpcService { .map_err(invalid_block_range_to_status)?; let chain_tip = self.range_bounds_check(&block_range)?; let (last_included_block, updates) = self - .store + .state .sync_account_vault(account_id, block_range) .await .map_err(|err| database_error_to_status(&err))?; diff --git a/crates/rpc/src/server/api/sync_chain_mmr.rs b/crates/rpc/src/server/api/sync_chain_mmr.rs index 4477c5473..6116f77b4 100644 --- a/crates/rpc/src/server/api/sync_chain_mmr.rs +++ b/crates/rpc/src/server/api/sync_chain_mmr.rs @@ -41,9 +41,9 @@ impl proto::server::rpc_api::SyncChainMmr for RpcService { let current_client_block_height = BlockNumber::from(request.current_client_block_height); let sync_target = match request.finality_level() { proto::rpc::FinalityLevel::Committed | proto::rpc::FinalityLevel::Unspecified => { - self.store.chain_tip(Finality::Committed) + self.state.chain_tip(Finality::Committed) }, - proto::rpc::FinalityLevel::Proven => self.store.chain_tip(Finality::Proven), + proto::rpc::FinalityLevel::Proven => self.state.chain_tip(Finality::Proven), }; if current_client_block_height > sync_target { @@ -54,7 +54,7 @@ impl proto::server::rpc_api::SyncChainMmr for RpcService { let block_range = current_client_block_height..=sync_target; let (mmr_delta, block_header, block_signature) = self - .store + .state .sync_chain_mmr(block_range.clone()) .await .map_err(|err| Status::internal(err.to_string()))?; diff --git a/crates/rpc/src/server/api/sync_notes.rs b/crates/rpc/src/server/api/sync_notes.rs index 11ff99a76..adeb14b4b 100644 --- a/crates/rpc/src/server/api/sync_notes.rs +++ b/crates/rpc/src/server/api/sync_notes.rs @@ -50,7 +50,7 @@ impl proto::server::rpc_api::SyncNotes for RpcService { let chain_tip = self.range_bounds_check(&block_range)?; let (results, last_block_checked) = self - .store + .state .sync_notes(request.note_tags, block_range) .await .map_err(note_sync_error_to_status)?; diff --git a/crates/rpc/src/server/api/sync_nullifiers.rs b/crates/rpc/src/server/api/sync_nullifiers.rs index 0097aecf2..a6ae3a053 100644 --- a/crates/rpc/src/server/api/sync_nullifiers.rs +++ b/crates/rpc/src/server/api/sync_nullifiers.rs @@ -61,7 +61,7 @@ impl proto::server::rpc_api::SyncNullifiers for RpcService { let chain_tip = self.range_bounds_check(&block_range)?; let (nullifiers, block_num) = self - .store + .state .sync_nullifiers(request.prefix_len, request.nullifiers, block_range) .await .map_err(|err| database_error_to_status(&err))?; diff --git a/crates/rpc/src/server/api/sync_transactions.rs b/crates/rpc/src/server/api/sync_transactions.rs index 26e103a53..ef8441aba 100644 --- a/crates/rpc/src/server/api/sync_transactions.rs +++ b/crates/rpc/src/server/api/sync_transactions.rs @@ -64,7 +64,7 @@ impl proto::server::rpc_api::SyncTransactions for RpcService { let chain_tip = self.range_bounds_check(&block_range)?; let account_ids = read_account_ids::(request.account_ids)?; let (last_block_included, transaction_records_db) = self - .store + .state .sync_transactions(account_ids, block_range) .await .map_err(|err| database_error_to_status(&err))?; diff --git a/crates/rpc/src/server/mod.rs b/crates/rpc/src/server/mod.rs index e9c429d83..1b641b8ae 100644 --- a/crates/rpc/src/server/mod.rs +++ b/crates/rpc/src/server/mod.rs @@ -13,7 +13,7 @@ use miden_node_proto::clients::{ }; use miden_node_proto::server::{rpc_api, sequencer_api}; use miden_node_proto_build::rpc_api_descriptor; -use miden_node_store::state::{Finality, State}; +use miden_node_store::state::{BlockWriter, Finality, ProofWriter, State}; use miden_node_utils::clap::{GrpcOptionsExternal, GrpcOptionsInternal}; use miden_node_utils::cors::cors_for_grpc_web_layer; use miden_node_utils::grpc; @@ -44,8 +44,13 @@ mod health; /// It uses the supplied store state and mode-specific submission handling. pub struct Rpc { pub listener: TcpListener, - pub store: Arc, + pub state: Arc, pub mode: RpcMode, + /// Store write capabilities consumed by the full-node sync loop. + /// + /// Must be provided in full-node mode and omitted in sequencer mode. The RPC service itself + /// only ever reads through `store`; these are passed through untouched to the sync tasks. + pub sync_writers: Option<(BlockWriter, ProofWriter)>, pub ntx_builder: Option, pub grpc_options: GrpcOptionsExternal, pub network_tx_auth: Option, @@ -121,7 +126,7 @@ impl Rpc { let endpoint = self.listener.local_addr().context("failed to read RPC listen address")?; let mode = self.mode.as_str(); let mut api = api::RpcService::new( - self.store.clone(), + self.state.clone(), self.mode.clone(), self.ntx_builder.clone(), NonZeroUsize::new(1_000_000).unwrap(), @@ -149,7 +154,7 @@ impl Rpc { tonic_health::ServingStatus::Serving, ) .await; - let chain_tip = self.store.chain_tip(Finality::Committed); + let chain_tip = self.state.chain_tip(Finality::Committed); log_node_ready(mode, endpoint, chain_tip); }, RpcMode::FullNode { source_rpc, readiness_threshold, .. } => { @@ -160,10 +165,14 @@ impl Rpc { ) .await; let readiness = RpcReadiness::new(health_reporter, readiness_threshold); + let (block_writer, proof_writer) = self.sync_writers.context( + "full-node RPC requires the store write capabilities for its sync loop", + )?; tasks.spawn( "RPC sync", RpcSync { - state: Arc::clone(&self.store), + block_writer, + proof_writer, source_rpc: *source_rpc, readiness, } diff --git a/crates/rpc/src/tests.rs b/crates/rpc/src/tests.rs index 6a92a82a7..dc8fbbda2 100644 --- a/crates/rpc/src/tests.rs +++ b/crates/rpc/src/tests.rs @@ -94,7 +94,7 @@ impl TestStore { async fn start() -> Self { let data_directory = new_tempdir(); let genesis_commitment = Self::bootstrap(&data_directory); - let state = State::for_tests(&data_directory).await; + let (state, ..) = State::for_tests(&data_directory).await; Self { state, genesis_commitment, @@ -629,8 +629,8 @@ async fn start_source_rpc( let store = TestStore::start().await; let block_producer_dir = new_tempdir(); TestStore::bootstrap(&block_producer_dir); - let block_producer_state = State::for_tests(&block_producer_dir).await; - let store_state = Arc::clone(&store.state); + let (block_producer_state, ..) = State::for_tests(&block_producer_dir).await; + let state = Arc::clone(&store.state); let listener = TcpListener::bind("127.0.0.1:0").await.expect("Failed to bind source RPC"); let addr = listener.local_addr().expect("Failed to get source RPC address"); @@ -642,7 +642,7 @@ async fn start_source_rpc( BlockProducerApiConfig::default(), ); let source_rpc = RpcService::new( - store_state, + state, RpcMode::sequencer(block_producer, validator), Some(ntx_builder), NonZeroUsize::new(1_000_000).unwrap(), @@ -1089,8 +1089,8 @@ async fn start_rpc_with_options( let store = TestStore::start().await; let block_producer_dir = new_tempdir(); TestStore::bootstrap(&block_producer_dir); - let block_producer_state = State::for_tests(&block_producer_dir).await; - let store_state = Arc::clone(&store.state); + let (block_producer_state, ..) = State::for_tests(&block_producer_dir).await; + let state = Arc::clone(&store.state); // Start the rpc component. let rpc_listener = TcpListener::bind("127.0.0.1:0").await.expect("Failed to bind rpc"); @@ -1112,8 +1112,9 @@ async fn start_rpc_with_options( .connect_lazy::(); Rpc { listener: rpc_listener, - store: store_state, + state, mode: RpcMode::sequencer(block_producer, validator), + sync_writers: None, ntx_builder: None, grpc_options, network_tx_auth: None, diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 11487af7b..6a11cb39f 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -38,7 +38,7 @@ pub use errors::{ StateSyncError, }; pub use genesis::GenesisState; -pub use state::{LoadedState, State, WriterTask}; +pub use state::{BlockWriter, LoadedState, ProofWriter, State, WriterTask}; /// Returns the store crate version. pub fn version() -> &'static str { diff --git a/crates/store/src/state/apply_block.rs b/crates/store/src/state/apply_block.rs index 3e63bd199..9806316fe 100644 --- a/crates/store/src/state/apply_block.rs +++ b/crates/store/src/state/apply_block.rs @@ -6,9 +6,9 @@ use miden_protocol::utils::serde::Serializable; use crate::COMPONENT; use crate::errors::ApplyBlockWithProvingInputsError; -use crate::state::State; +use crate::state::BlockWriter; -impl State { +impl BlockWriter { /// Saves proving inputs for a signed block and applies it to the state. /// /// Used by the in-process block producer after it has built and signed a block. diff --git a/crates/store/src/state/apply_proof.rs b/crates/store/src/state/apply_proof.rs index 5f8f53260..cd7b87423 100644 --- a/crates/store/src/state/apply_proof.rs +++ b/crates/store/src/state/apply_proof.rs @@ -4,9 +4,9 @@ use miden_protocol::block::{BlockNumber, BlockProof}; use miden_protocol::utils::serde::Deserializable; use crate::COMPONENT; -use crate::state::{Finality, ProofNotification, State}; +use crate::state::{Finality, ProofNotification, ProofWriter}; -impl State { +impl ProofWriter { /// Saves a block proof, advances the proven-in-sequence tip, and notifies replica subscribers. /// /// # Errors diff --git a/crates/store/src/state/lifecycle.rs b/crates/store/src/state/lifecycle.rs index f483d4da9..837311d0d 100644 --- a/crates/store/src/state/lifecycle.rs +++ b/crates/store/src/state/lifecycle.rs @@ -1,4 +1,4 @@ -//! Store lifecycle: loading the state, starting its block writer, and stopping the store. +//! Store lifecycle: loading the state, starting its write worker, and stopping the store. use std::future::Future; use std::num::NonZeroUsize; @@ -14,7 +14,7 @@ use miden_node_utils::clap::StorageOptions; use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::tracing::miden_instrument; use miden_protocol::block::BlockNumber; -use tokio::sync::watch; +use tokio::sync::{mpsc, watch}; use crate::account_state_forest::AccountStateForestBackend; use crate::accounts::AccountTreeWithHistory; @@ -33,7 +33,7 @@ use crate::state::loader::{ verify_account_state_forest_consistency, verify_tree_consistency, }; -use crate::state::writer::{BlockWriter, WriteHandle}; +use crate::state::writer::{WriteRequest, WriteWorker}; use crate::state::{BlockCache, InMemoryState, ProofCache, SnapshotGuard, State}; use crate::{COMPONENT, DataDirectory, DatabaseOptions}; @@ -46,45 +46,135 @@ const PROOF_CACHE_CAPACITY: NonZeroUsize = NonZeroUsize::new(512).unwrap(); // LOADED STATE // ================================================================================================ -/// A loaded store state whose block writer has not been started yet. +/// A loaded store state whose write worker has not been started yet. /// -/// Returned by [`State::load`]. [`Self::start`] spawns the writer and yields the usable -/// [`State`]; since this is the only way to obtain one, [`State::apply_block`] can always make -/// progress. -#[must_use = "call `start` to spawn the block writer and obtain the state"] +/// Returned by [`State::load`]. [`Self::start`] spawns the writer and yields the read-only +/// [`State`] together with the write capabilities; since this is the only way to obtain a +/// [`BlockWriter`], [`BlockWriter::apply_block`] can always make progress. +#[must_use = "call `start` to spawn the write worker and obtain the state"] pub struct LoadedState { state: State, - writer: BlockWriter, + writer: WriteWorker, + write_tx: mpsc::Sender, } impl LoadedState { - /// Spawns the block writer onto the current runtime and returns the state together with the - /// writer task's handle. + /// Spawns the write worker onto the current runtime and returns the read-only state together + /// with the write capabilities and the writer task's handle. /// - /// The writer exits once the shutdown token passed to [`State::load`] is cancelled or the last - /// state reference (holding the only write handle) is dropped — an in-flight block write + /// [`Arc`] is the read-only view shared with every component that queries or + /// subscribes; it exposes no mutating methods. [`BlockWriter`] and [`ProofWriter`] are the + /// only handles able to mutate the store — hand them to the single task driving each write + /// path (block production or sync, and proof scheduling or sync respectively). + /// + /// The writer exits once the shutdown token passed to [`State::load`] is cancelled or the + /// [`BlockWriter`] (holding the only request sender) is dropped — an in-flight block write /// always completes first. Awaiting the returned handle after either event guarantees the /// writer has released the tree storage it owns; a join error carries a writer panic. /// - /// Callers without a token to cancel should stop the store via [`State::stop`] rather than - /// dropping and joining by hand. - pub fn start(self) -> (Arc, WriterTask) { + /// Callers without a token to cancel should stop the store via [`BlockWriter::stop`] rather + /// than dropping and joining by hand. + pub fn start(self) -> (Arc, BlockWriter, ProofWriter, WriterTask) { let writer_task = tokio::spawn(self.writer.run()); - (Arc::new(self.state), WriterTask(writer_task)) + let state = Arc::new(self.state); + let block_writer = BlockWriter { + state: Arc::clone(&state), + write_tx: self.write_tx, + }; + let proof_writer = ProofWriter { state: Arc::clone(&state) }; + (state, block_writer, proof_writer, WriterTask(writer_task)) + } +} + +// WRITE CAPABILITIES +// ================================================================================================ + +/// The store's block-write capability. +/// +/// Only handle able to apply blocks; obtained exactly once from [`LoadedState::start`] and +/// deliberately not cloneable, so granting it to a single task (the block builder in sequencer +/// mode, the block sync loop in full-node mode) statically prevents every other component from +/// writing blocks. Dereferences to [`State`] for read access. +pub struct BlockWriter { + state: Arc, + /// Sender for block-write requests to the [`WriteWorker`](crate::state::writer::WriteWorker) + /// task. Never cloned out of this struct: the writer exits once it is dropped. + pub(super) write_tx: mpsc::Sender, +} + +impl std::ops::Deref for BlockWriter { + type Target = State; + + fn deref(&self) -> &State { + &self.state + } +} + +impl BlockWriter { + /// Returns the shared read-only state. + pub fn state(&self) -> &Arc { + &self.state + } + + /// Stops the store, waiting until the write worker has released the tree storage it owns. + /// + /// Consumes the capability — closing the write channel the write worker listens on — and then + /// joins the writer task returned by [`LoadedState::start`]. The drop must precede the join + /// or the write worker never observes the closed channel; doing both here keeps that ordering + /// out of caller hands. Read-only [`State`] references may outlive the stop. + /// + /// Callers that need the storage released deterministically must use this method instead of + /// dropping: the node's `recover` command stops the store before the process exits, and the + /// stress-test's store seeding stops it so the same data directory can be re-loaded (or its + /// temporary directory deleted) immediately afterwards. The running node does not use this + /// method — its writer exits via the shutdown token passed to [`State::load`] and is joined + /// through the node's task set. + /// + /// # Panics + /// + /// Panics if the writer task panicked. + pub async fn stop(self, writer_task: WriterTask) { + drop(self); + writer_task.await.expect("write worker task should not panic"); + } +} + +/// The store's proof-write capability. +/// +/// Only handle able to commit block proofs and advance the proven tip; obtained exactly once from +/// [`LoadedState::start`] and deliberately not cloneable, so granting it to a single task (the +/// proof scheduler in sequencer mode, the proof sync loop in full-node mode) statically prevents +/// every other component from writing proofs. Dereferences to [`State`] for read access. +pub struct ProofWriter { + state: Arc, +} + +impl std::ops::Deref for ProofWriter { + type Target = State; + + fn deref(&self) -> &State { + &self.state + } +} + +impl ProofWriter { + /// Returns the shared read-only state. + pub fn state(&self) -> &Arc { + &self.state } } // WRITER TASK // ================================================================================================ -/// Handle of the store's block writer task, returned by [`LoadedState::start`]. +/// Handle of the store's write worker task, returned by [`LoadedState::start`]. /// /// Awaiting it resolves once the writer has exited and released the tree storage it owns; a join -/// error carries a writer panic. The newtype ensures [`State::stop`] can only be given the store's -/// own writer task, and deliberately does not expose [`tokio::task::JoinHandle::abort`]: aborting -/// the writer mid-write could leave the trees lagging the committed database state, voiding the -/// guarantee that an in-flight block write always completes. -#[must_use = "await the writer task to observe its exit, or pass it to `State::stop`"] +/// error carries a writer panic. The newtype ensures [`BlockWriter::stop`] can only be given the +/// store's own writer task, and deliberately does not expose [`tokio::task::JoinHandle::abort`]: +/// aborting the writer mid-write could leave the trees lagging the committed database state, +/// voiding the guarantee that an in-flight block write always completes. +#[must_use = "await the writer task to observe its exit, or pass it to `BlockWriter::stop`"] pub struct WriterTask(tokio::task::JoinHandle<()>); impl Future for WriterTask { @@ -222,12 +312,12 @@ impl State { }); let in_memory = Arc::new(ArcSwap::from(initial_snapshot)); - // Assemble the block writer. It owns the writable trees and processes write requests + // Assemble the write worker. It owns the writable trees and processes write requests // serially, publishing a new snapshot after each committed block. The caller runs it; it - // exits when the shutdown token is cancelled or all write handles are dropped. - let (write_tx, write_rx) = tokio::sync::mpsc::channel(1); - let write_handle = WriteHandle::new(write_tx); - let block_writer = BlockWriter { + // exits when the shutdown token is cancelled or the `BlockWriter` (holding the only request + // sender) is dropped. + let (write_tx, write_rx) = mpsc::channel(1); + let block_writer = WriteWorker { db: Arc::clone(&db), block_store: Arc::clone(&block_store), in_memory: Arc::clone(&in_memory), @@ -246,68 +336,32 @@ impl State { db, block_store, in_memory, - write_handle, proven_tip, committed_tip_tx, block_cache, proof_cache, }; - Ok(LoadedState { state, writer: block_writer }) + Ok(LoadedState { state, writer: block_writer, write_tx }) } - /// Loads the state with default options and starts its block writer, detaching the writer + /// Loads the state with default options and starts its write worker, detaching the worker /// task. /// /// Test-only helper for tests in sibling crates that don't manage the writer's lifecycle: - /// the detached writer exits once the returned state (holding the only write handle) is - /// dropped. Hidden from public docs and not part of the stable API. + /// the detached writer exits once the returned [`BlockWriter`] is dropped. Hidden from public + /// docs and not part of the stable API. /// /// # Panics /// /// Panics if the state fails to load. #[doc(hidden)] - pub async fn for_tests(data_path: &Path) -> Arc { - let (state, _writer_task) = + pub async fn for_tests(data_path: &Path) -> (Arc, BlockWriter, ProofWriter) { + let (state, block_writer, proof_writer, _writer_task) = Self::load(data_path, StorageOptions::default(), CancellationToken::new()) .await .expect("state should load") .start(); - state - } - - /// Stops the store, waiting until the block writer has released the tree storage it owns. - /// - /// Consumes the last state reference — closing the write channel the writer listens on — and - /// then joins the writer task returned by [`LoadedState::start`]. The drop must precede the - /// join or the writer never observes the closed channel; doing both here keeps that ordering - /// out of caller hands. - /// - /// Callers that need the storage released deterministically must use this method instead of - /// dropping: the node's `recover` command stops the store before the process exits, and the - /// stress-test's store seeding stops it so the same data directory can be re-loaded (or its - /// temporary directory deleted) immediately afterwards. The running node does not use this - /// method — its writer exits via the shutdown token passed to [`State::load`] and is joined - /// through the node's task set. - /// - /// # Errors - /// - /// Returns both pieces unchanged if other references to the state are still alive. - /// - /// # Panics - /// - /// Panics if the writer task panicked. - pub async fn stop( - self: Arc, - writer_task: WriterTask, - ) -> Result<(), (Arc, WriterTask)> { - match Arc::try_unwrap(self) { - Ok(state) => { - drop(state); - writer_task.await.expect("block writer task should not panic"); - Ok(()) - }, - Err(state) => Err((state, writer_task)), - } + (state, block_writer, proof_writer) } } diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index d11d54534..7f3cd1f1c 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -32,7 +32,7 @@ mod inputs; pub use inputs::TransactionInputs; mod lifecycle; -pub use lifecycle::{LoadedState, WriterTask}; +pub use lifecycle::{BlockWriter, LoadedState, ProofWriter, WriterTask}; mod queries; @@ -40,7 +40,6 @@ mod snapshot; pub(crate) use snapshot::{InMemoryState, SNAPSHOTS_LIVE_WARN_THRESHOLD, SnapshotGuard}; mod writer; -use writer::WriteHandle; // FINALITY // ================================================================================================ @@ -57,7 +56,10 @@ pub enum Finality { // CHAIN STATE // ================================================================================================ -/// The rollup state. +/// The rollup state, read-only. +/// +/// Mutations go through the [`BlockWriter`] and [`ProofWriter`] capabilities returned by +/// [`LoadedState::start`]; every holder of this type can only query and subscribe. pub struct State { /// Root directory containing the store's on-disk data. data_directory: PathBuf, @@ -72,13 +74,10 @@ pub struct State { /// Atomically swappable pointer to the latest in-memory state snapshot. /// /// Readers load the snapshot wait-free via [`ArcSwap::load_full`]; the - /// [`BlockWriter`](writer::BlockWriter) task atomically replaces the pointer after each + /// [`WriteWorker`](writer::WriteWorker) task atomically replaces the pointer after each /// committed block. Readers holding an old snapshot are unaffected by the swap. in_memory: Arc>, - /// Handle for sending block-write requests to the [`BlockWriter`](writer::BlockWriter) task. - write_handle: WriteHandle, - /// The latest proven-in-sequence block number, updated by the proof scheduler or `apply_proof`. proven_tip: ProvenTipWriter, diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer.rs index b86bbb642..168d9a619 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer.rs @@ -1,6 +1,6 @@ //! Serialized block-write path for the store state. //! -//! A single [`BlockWriter`] task owns the mutable trees and processes incoming [`WriteRequest`]s +//! A single [`WriteWorker`] task owns the mutable trees and processes incoming [`WriteRequest`]s //! one at a time via an mpsc channel. After each successful commit it publishes a new //! [`InMemoryState`] snapshot via an [`ArcSwap`], making the updated trees immediately visible to //! wait-free readers. @@ -38,10 +38,10 @@ use crate::state::loader::TreeStorage; use crate::state::{ BlockCache, BlockNotification, + BlockWriter, InMemoryState, SNAPSHOTS_LIVE_WARN_THRESHOLD, SnapshotGuard, - State, }; use crate::{COMPONENT, HistoricalError, LOG_TARGET}; @@ -54,37 +54,10 @@ pub(super) struct WriteRequest { result_tx: oneshot::Sender>, } -// WRITE HANDLE -// ================================================================================================ - -/// Handle for sending block-write requests to the [`BlockWriter`] task. -pub(super) struct WriteHandle { - tx: mpsc::Sender, -} - -impl WriteHandle { - pub(super) fn new(tx: mpsc::Sender) -> Self { - Self { tx } - } - - /// Sends a block to the writer task and awaits its result. - pub(super) async fn apply_block( - &self, - signed_block: SignedBlock, - ) -> Result<(), ApplyBlockError> { - let (result_tx, result_rx) = oneshot::channel(); - self.tx - .send(WriteRequest { signed_block, result_tx }) - .await - .map_err(|e| ApplyBlockError::WriterTaskSendFailed(e.as_report()))?; - result_rx.await? - } -} - -impl State { +impl BlockWriter { /// Apply changes of a new block to the DB and in-memory data structures. /// - /// Blocks are forwarded to the [`BlockWriter`] task, which processes them one at a time. + /// Blocks are forwarded to the [`WriteWorker`] task, which processes them one at a time. /// Readers are unaffected while a block is being applied: they keep reading from the previous /// in-memory snapshot until the writer atomically publishes the new one. #[miden_instrument( @@ -93,11 +66,16 @@ impl State { err, )] pub async fn apply_block(&self, signed_block: SignedBlock) -> Result<(), ApplyBlockError> { - self.write_handle.apply_block(signed_block).await + let (result_tx, result_rx) = oneshot::channel(); + self.write_tx + .send(WriteRequest { signed_block, result_tx }) + .await + .map_err(|e| ApplyBlockError::WriterTaskSendFailed(e.as_report()))?; + result_rx.await? } } -// BLOCK WRITER +// WRITE WORKER // ================================================================================================ /// Single-task owner of the mutable trees. Processes [`WriteRequest`]s serially. @@ -105,7 +83,7 @@ impl State { /// The writer owns the writable trees directly, so no locks are held at any point: validation and /// mutation-computation read the owned trees, the DB commit runs without touching them, and the /// new [`InMemoryState`] snapshot is published atomically at the end. -pub(super) struct BlockWriter { +pub(super) struct WriteWorker { pub db: Arc, pub block_store: Arc, /// Atomically swappable pointer through which new snapshots are published. @@ -135,9 +113,9 @@ struct PreparedBlockUpdate { account_forest_update: PreparedAccountStateForestBlockUpdate, } -impl BlockWriter { - /// Runs the writer loop, processing requests until shutdown is signalled or all write handles - /// are dropped. +impl WriteWorker { + /// Runs the writer loop, processing requests until shutdown is signalled or the + /// [`BlockWriter`] (holding the only request sender) is dropped. /// /// Cancellation is only observed between requests: an in-flight block write always runs to /// completion, so shutdown never leaves the trees lagging the committed database state. From 082ed2106c006bc810741f3260456ab7e58e1fc3 Mon Sep 17 00:00:00 2001 From: Serge Radinovich <47865535+sergerad@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:34:33 +1200 Subject: [PATCH 29/35] StateView (#2415) --- bin/node/src/commands/recover.rs | 17 +- bin/stress-test/src/seeding/mod.rs | 19 +- bin/stress-test/src/seeding/tests.rs | 2 + bin/stress-test/src/store/mod.rs | 28 +- .../block-producer/src/batch_builder/mod.rs | 1 + .../block-producer/src/block_builder/mod.rs | 16 +- crates/block-producer/src/proof_scheduler.rs | 9 +- crates/block-producer/src/rpc_sync.rs | 16 +- crates/block-producer/src/server/mod.rs | 13 +- crates/block-producer/src/store/mod.rs | 8 +- crates/rpc/src/server/api.rs | 26 +- crates/rpc/src/server/api/get_account.rs | 8 +- .../server/api/get_block_header_by_number.rs | 1 + .../src/server/api/get_note_script_by_root.rs | 1 + crates/rpc/src/server/api/get_notes_by_id.rs | 1 + crates/rpc/src/server/api/status.rs | 4 +- .../server/api/sync_account_storage_maps.rs | 6 +- .../rpc/src/server/api/sync_account_vault.rs | 6 +- crates/rpc/src/server/api/sync_chain_mmr.rs | 21 +- crates/rpc/src/server/api/sync_notes.rs | 8 +- crates/rpc/src/server/api/sync_nullifiers.rs | 6 +- .../rpc/src/server/api/sync_transactions.rs | 6 +- crates/rpc/src/server/mod.rs | 5 +- crates/store/src/errors.rs | 19 +- crates/store/src/lib.rs | 3 +- crates/store/src/state/inputs.rs | 369 ---------------- crates/store/src/state/lifecycle.rs | 72 ++- crates/store/src/state/mod.rs | 42 +- crates/store/src/state/queries.rs | 96 ---- crates/store/src/state/replica.rs | 30 +- crates/store/src/state/tip.rs | 45 ++ crates/store/src/state/view/account/mod.rs | 414 ++++++++++++++++++ .../account/response_budget.rs} | 407 +---------------- crates/store/src/state/view/batch_inputs.rs | 129 ++++++ crates/store/src/state/view/block.rs | 49 +++ crates/store/src/state/view/block_inputs.rs | 155 +++++++ crates/store/src/state/view/mod.rs | 123 ++++++ crates/store/src/state/view/note.rs | 32 ++ crates/store/src/state/{ => view}/snapshot.rs | 95 ++-- .../src/state/{sync_state.rs => view/sync.rs} | 67 ++- .../src/state/view/transaction_inputs.rs | 93 ++++ .../src/state/{ => writer}/apply_block.rs | 0 .../src/state/{ => writer}/apply_proof.rs | 13 +- .../src/state/{writer.rs => writer/mod.rs} | 66 +-- 44 files changed, 1376 insertions(+), 1171 deletions(-) delete mode 100644 crates/store/src/state/inputs.rs delete mode 100644 crates/store/src/state/queries.rs create mode 100644 crates/store/src/state/tip.rs create mode 100644 crates/store/src/state/view/account/mod.rs rename crates/store/src/state/{account.rs => view/account/response_budget.rs} (50%) create mode 100644 crates/store/src/state/view/batch_inputs.rs create mode 100644 crates/store/src/state/view/block.rs create mode 100644 crates/store/src/state/view/block_inputs.rs create mode 100644 crates/store/src/state/view/mod.rs create mode 100644 crates/store/src/state/view/note.rs rename crates/store/src/state/{ => view}/snapshot.rs (61%) rename crates/store/src/state/{sync_state.rs => view/sync.rs} (72%) create mode 100644 crates/store/src/state/view/transaction_inputs.rs rename crates/store/src/state/{ => writer}/apply_block.rs (100%) rename crates/store/src/state/{ => writer}/apply_proof.rs (83%) rename crates/store/src/state/{writer.rs => writer/mod.rs} (92%) diff --git a/bin/node/src/commands/recover.rs b/bin/node/src/commands/recover.rs index 50ee24567..94cd00baf 100644 --- a/bin/node/src/commands/recover.rs +++ b/bin/node/src/commands/recover.rs @@ -1,10 +1,10 @@ use std::path::PathBuf; +use std::sync::Arc; use std::time::Duration; use anyhow::Context; use miden_node_proto::clients::{Builder, ValidatorClient}; use miden_node_proto::generated::validator::BlockSubscriptionRequest; -use miden_node_store::state::Finality; use miden_node_store::{BlockWriter, State, WriterTask}; use miden_node_utils::shutdown::CancellationToken; use miden_protocol::block::{BlockNumber, SignedBlock}; @@ -36,15 +36,15 @@ pub struct RecoverCommand { impl RecoverCommand { pub async fn handle(self) -> anyhow::Result<()> { - let (block_writer, writer_task) = self.load_state().await?; + let (state, block_writer, writer_task) = self.load_state().await?; let validator = self.validator_client()?; - let result = recover_from_validator(&block_writer, validator).await; + let result = recover_from_validator(&state, &block_writer, validator).await; // Wait for the writer to drain and release the backing storage before the process exits. block_writer.stop(writer_task).await; result } - async fn load_state(&self) -> anyhow::Result<(BlockWriter, WriterTask)> { + async fn load_state(&self) -> anyhow::Result<(Arc, BlockWriter, WriterTask)> { // Recovery is not wired into the node's shutdown token; the writer exits once the // `BlockWriter` (holding the only write handle) is dropped after recovery completes. let loaded = State::load_with_database_options( @@ -56,8 +56,8 @@ impl RecoverCommand { .await .context("failed to load state")?; - let (_state, block_writer, _proof_writer, writer_task) = loaded.start(); - Ok((block_writer, writer_task)) + let (state, block_writer, _proof_writer, writer_task) = loaded.start(); + Ok((state, block_writer, writer_task)) } fn validator_client(&self) -> anyhow::Result { @@ -73,6 +73,7 @@ impl RecoverCommand { /// Streams blocks from the validator into the local store until the chain tip is reached. async fn recover_from_validator( + state: &State, block_writer: &BlockWriter, mut validator: ValidatorClient, ) -> anyhow::Result<()> { @@ -88,7 +89,7 @@ async fn recover_from_validator( .chain_tip, ); - let local_tip = block_writer.chain_tip(Finality::Committed); + let local_tip = state.committed_tip(); if local_tip >= validator_tip { info!( target: LOG_TARGET, @@ -131,7 +132,7 @@ async fn recover_from_validator( } // The stream can end before reaching the tip if the validator restarts or drops the connection. - let final_tip = block_writer.chain_tip(Finality::Committed); + let final_tip = state.committed_tip(); anyhow::ensure!( final_tip >= validator_tip, "validator block stream ended at block {} before reaching the chain tip {}", diff --git a/bin/stress-test/src/seeding/mod.rs b/bin/stress-test/src/seeding/mod.rs index 7738f66e4..e64bb1d82 100644 --- a/bin/stress-test/src/seeding/mod.rs +++ b/bin/stress-test/src/seeding/mod.rs @@ -271,6 +271,7 @@ pub async fn seed_store_with_readers( let data_directory = miden_node_store::DataDirectory::load(data_directory).expect("data directory should exist"); let metrics = Box::pin(generate_blocks( + &state, account_batches, initial_accounts, if seed_public_accounts_at_genesis { @@ -325,6 +326,7 @@ async fn read_latest_header_until( while !stop.load(Ordering::Relaxed) { let start = Instant::now(); let (header, proof) = state + .view() .get_block_header(None, true) .await .expect("get_block_header should succeed during seeding"); @@ -358,6 +360,7 @@ fn report_read_latencies( #[expect(clippy::too_many_arguments)] #[expect(clippy::too_many_lines)] async fn generate_blocks( + state: &State, account_batches: Vec, initial_accounts: Vec, first_account_index: u64, @@ -444,7 +447,7 @@ async fn generate_blocks( .collect(); // create the block and send it to the store - let block_inputs = get_block_inputs(block_writer, &batches, &mut metrics).await; + let block_inputs = get_block_inputs(state, &batches, &mut metrics).await; // update blocks let block_kind = if has_pending_account_creations { @@ -466,8 +469,7 @@ async fn generate_blocks( .extend(pending_consumed_accounts.into_iter().map(|account| (account.id(), account))); // create the consume notes txs to be used in the next block - let batch_inputs = - get_batch_inputs(block_writer, &prev_block_header, ¬es, &mut metrics).await; + let batch_inputs = get_batch_inputs(state, &prev_block_header, ¬es, &mut metrics).await; (pending_consumed_accounts, consume_notes_txs) = create_consume_note_txs( &prev_block_header, accounts, @@ -491,7 +493,7 @@ async fn generate_blocks( .par_chunks(TRANSACTIONS_PER_BATCH) .map(|txs| create_batch(txs, &prev_block_header)) .collect(); - let block_inputs = get_block_inputs(block_writer, &batches, &mut metrics).await; + let block_inputs = get_block_inputs(state, &batches, &mut metrics).await; prev_block_header = apply_block( batches, block_inputs, @@ -531,7 +533,7 @@ async fn generate_blocks( let emit_note_tx = create_emit_note_tx(&prev_block_header, &mut faucet, notes.clone()); let batches = vec![create_batch(std::slice::from_ref(&emit_note_tx), &prev_block_header)]; - let block_inputs = get_block_inputs(block_writer, &batches, &mut metrics).await; + let block_inputs = get_block_inputs(state, &batches, &mut metrics).await; prev_block_header = apply_block( batches, block_inputs, @@ -543,8 +545,7 @@ async fn generate_blocks( ) .await; - let batch_inputs = - get_batch_inputs(block_writer, &prev_block_header, ¬es, &mut metrics).await; + let batch_inputs = get_batch_inputs(state, &prev_block_header, ¬es, &mut metrics).await; let accounts = selected_account_ids .iter() .map(|account_id| { @@ -567,7 +568,7 @@ async fn generate_blocks( .par_chunks(TRANSACTIONS_PER_BATCH) .map(|txs| create_batch(txs, &prev_block_header)) .collect(); - let block_inputs = get_block_inputs(block_writer, &batches, &mut metrics).await; + let block_inputs = get_block_inputs(state, &batches, &mut metrics).await; prev_block_header = apply_block( batches, block_inputs, @@ -1077,6 +1078,7 @@ async fn get_batch_inputs( // Mark every note as unauthenticated, so that the store returns the inclusion proofs for all of // them let batch_inputs = state + .view() .get_batch_inputs( [block_ref.block_num()].into_iter().collect(), notes.iter().map(|note| note.id().as_word()).collect(), @@ -1095,6 +1097,7 @@ async fn get_block_inputs( ) -> BlockInputs { let start = Instant::now(); let inputs = state + .view() .get_block_inputs( batches.iter().flat_map(ProvenBatch::updated_accounts).collect(), batches.iter().flat_map(ProvenBatch::created_nullifiers).collect(), diff --git a/bin/stress-test/src/seeding/tests.rs b/bin/stress-test/src/seeding/tests.rs index 6fb7063ec..420b96de3 100644 --- a/bin/stress-test/src/seeding/tests.rs +++ b/bin/stress-test/src/seeding/tests.rs @@ -172,6 +172,7 @@ async fn seed_store_persists_one_public_account_and_applies_one_map_update() { let (state, block_writer, writer_task) = load_state(data_directory).await; let response = state + .view() .get_account(AccountRequest { account_id, block_num: None, @@ -219,6 +220,7 @@ async fn seed_store_handles_map_larger_than_transaction_account_update_limit() { let (state, block_writer, writer_task) = load_state(data_directory).await; let response = state + .view() .get_account(AccountRequest { account_id, block_num: None, diff --git a/bin/stress-test/src/store/mod.rs b/bin/stress-test/src/store/mod.rs index e267f68bc..d21af7436 100644 --- a/bin/stress-test/src/store/mod.rs +++ b/bin/stress-test/src/store/mod.rs @@ -5,7 +5,7 @@ use std::time::{Duration, Instant}; use futures::{StreamExt, stream}; use miden_node_proto::domain::account::AccountRequest; use miden_node_proto::generated::{self as proto}; -use miden_node_store::state::{Finality, State}; +use miden_node_store::state::State; use miden_node_utils::clap::StorageOptions; use miden_node_utils::shutdown::CancellationToken; use miden_protocol::Word; @@ -128,7 +128,8 @@ async fn get_account( let start = Instant::now(); let request = AccountRequest::try_from(request).expect("request should be valid"); - let response: proto::rpc::AccountResponse = state.get_account(request).await.unwrap().into(); + let response: proto::rpc::AccountResponse = + state.view().get_account(request).await.unwrap().into(); let duration = start.elapsed(); let details = response.details; @@ -233,7 +234,7 @@ pub async fn bench_sync_notes(data_directory: PathBuf, iterations: usize, concur let store_state = start_store(data_directory).await; // Get the latest block number to determine the range - let chain_tip = store_state.chain_tip(Finality::Committed).as_u32(); + let chain_tip = store_state.committed_tip().as_u32(); // each request will have `ACCOUNTS_PER_SYNC_NOTES` note tags and will be sent with block number // 0. @@ -269,6 +270,7 @@ pub async fn sync_notes( .collect::>(); let start = Instant::now(); state + .view() .sync_notes(note_tags, BlockNumber::from(0)..=BlockNumber::from(chain_tip)) .await .unwrap(); @@ -305,7 +307,7 @@ pub async fn bench_sync_nullifiers( .collect(); // Get the latest block number to determine the range - let chain_tip = store_state.chain_tip(Finality::Committed).as_u32(); + let chain_tip = store_state.committed_tip().as_u32(); // Get all nullifier prefixes from the store using sync_notes let mut nullifier_prefixes: Vec = vec![]; @@ -317,6 +319,7 @@ pub async fn bench_sync_nullifiers( .map(|id| u32::from(NoteTag::with_account_target(*id))) .collect(); let (blocks, last_block_checked) = store_state + .view() .sync_notes( note_tags, BlockNumber::from(current_block_num)..=BlockNumber::from(chain_tip), @@ -339,6 +342,7 @@ pub async fn bench_sync_nullifiers( note_ids.iter().take(NOTE_IDS_PER_NULLIFIERS_CHECK).copied().collect(); if !note_ids_to_fetch.is_empty() { let notes = store_state + .view() .get_notes_by_id(note_ids_to_fetch) .await .unwrap() @@ -394,6 +398,7 @@ async fn sync_nullifiers( ) -> (Duration, usize) { let start = Instant::now(); let (nullifiers, _) = state + .view() .sync_nullifiers( 16, nullifiers_prefixes, @@ -439,7 +444,7 @@ pub async fn bench_sync_transactions( let store_state = start_store(data_directory).await; // Get the latest block number to determine the range - let chain_tip = store_state.chain_tip(Finality::Committed).as_u32(); + let chain_tip = store_state.committed_tip().as_u32(); // each request will have `accounts_per_request` account ids and will query a range of blocks let request = |_| { @@ -513,11 +518,12 @@ pub async fn sync_transactions( block_to: u32, ) -> (Duration, proto::rpc::SyncTransactionsResponse) { let start = Instant::now(); - let (last_block_included, records) = state + let view = state.view(); + let (last_block_included, records) = view .sync_transactions(account_ids, BlockNumber::from(block_from)..=BlockNumber::from(block_to)) .await .unwrap(); - let chain_tip = state.chain_tip(Finality::Committed); + let chain_tip = view.tip(); let response = proto::rpc::SyncTransactionsResponse { pagination_info: Some(proto::rpc::PaginationInfo { chain_tip: chain_tip.as_u32(), @@ -600,7 +606,7 @@ async fn sync_transactions_paginated( pub async fn bench_sync_chain_mmr(data_directory: PathBuf, iterations: usize, concurrency: usize) { let store_state = start_store(data_directory).await; - let chain_tip = store_state.chain_tip(Finality::Committed).as_u32(); + let chain_tip = store_state.committed_tip().as_u32(); let request = |_| { let state = Arc::clone(&store_state); @@ -629,9 +635,9 @@ pub async fn bench_sync_chain_mmr(data_directory: PathBuf, iterations: usize, co /// - the response. async fn sync_chain_mmr(state: &Arc, current_client_block_height: u32) -> SyncChainMmrRun { let start = Instant::now(); - let chain_tip = state.chain_tip(Finality::Committed); - state - .sync_chain_mmr(BlockNumber::from(current_client_block_height)..=chain_tip) + let view = state.view(); + let chain_tip = view.tip(); + view.sync_chain_mmr(BlockNumber::from(current_client_block_height)..=chain_tip) .await .unwrap(); let elapsed = start.elapsed(); diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index be7c69258..d85bded1f 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -306,6 +306,7 @@ impl BatchJob { .flat_map(AuthenticatedTransaction::unauthenticated_note_ids); self.state + .view() .get_batch_inputs( block_references.map(|(block_num, _)| block_num).collect(), unauthenticated_notes.collect(), diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index 5385d64c6..9b3501221 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -2,7 +2,7 @@ use std::ops::Deref; use std::sync::Arc; use anyhow::Context; -use miden_node_store::state::BlockWriter; +use miden_node_store::state::{BlockWriter, State}; use miden_node_utils::formatting::format_array; use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::spawn::spawn_blocking_in_current_span; @@ -31,6 +31,9 @@ pub struct BlockBuilder { /// The frequency at which blocks are produced. pub block_interval: Duration, + /// Read-only store state, used to fetch block inputs. + pub state: Arc, + /// The store's block-write capability, used for committing blocks. pub block_writer: BlockWriter, @@ -44,11 +47,17 @@ impl BlockBuilder { /// /// If the block prover URL is not set, the block builder will use the local block prover. pub fn new( + state: Arc, block_writer: BlockWriter, validator: BlockProducerValidatorClient, block_interval: Duration, ) -> Self { - Self { block_interval, block_writer, validator } + Self { + block_interval, + state, + block_writer, + validator, + } } /// Starts the [`BlockBuilder`], infinitely producing blocks at the configured interval. /// @@ -214,7 +223,8 @@ impl BlockBuilder { batch_iter.map(Deref::deref).flat_map(ProvenBatch::created_nullifiers); let inputs = self - .block_writer + .state + .view() .get_block_inputs( account_ids_iter.collect(), created_nullifiers_iter.collect(), diff --git a/crates/block-producer/src/proof_scheduler.rs b/crates/block-producer/src/proof_scheduler.rs index cbbc5bcf9..a611e9d19 100644 --- a/crates/block-producer/src/proof_scheduler.rs +++ b/crates/block-producer/src/proof_scheduler.rs @@ -18,7 +18,7 @@ use std::time::Duration; use anyhow::Context; use miden_node_proto::BlockProofRequest; -use miden_node_store::state::{Finality, ProofWriter, State}; +use miden_node_store::state::{ProofWriter, State}; use miden_node_utils::retry::{self, Retryable}; use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::tracing::miden_instrument; @@ -107,6 +107,7 @@ impl ProofTaskJoinSet { /// Transient errors are retried internally. pub(crate) async fn run( block_prover: Arc, + state: Arc, proof_writer: ProofWriter, mut chain_tip_rx: watch::Receiver, max_concurrent_proofs: NonZeroUsize, @@ -119,7 +120,7 @@ pub(crate) async fn run( // Next block number to schedule. Initialized from the proven tip's child so we skip // already-proven blocks on restart. - let mut next_to_prove = proof_writer.chain_tip(Finality::Proven).child(); + let mut next_to_prove = state.proven_tip().child(); // Completed proofs waiting to be committed in order. let mut pending: BTreeMap> = BTreeMap::new(); @@ -128,7 +129,7 @@ pub(crate) async fn run( // Schedule blocks up to chain_tip that haven't been scheduled yet. let chain_tip = *chain_tip_rx.borrow(); while proving_tasks.len() < max_concurrent_proofs.get() && next_to_prove <= chain_tip { - proving_tasks.spawn(proof_writer.state(), &block_prover, next_to_prove); + proving_tasks.spawn(&state, &block_prover, next_to_prove); next_to_prove = next_to_prove.child(); } @@ -145,7 +146,7 @@ pub(crate) async fn run( // Drain completed proofs in ascending order so the proven tip advances without // gaps. - let mut next = proof_writer.chain_tip(Finality::Proven).child(); + let mut next = state.proven_tip().child(); while let Some(proof_bytes) = pending.remove(&next) { proof_writer.apply_proof(next, proof_bytes).await?; next = next.child(); diff --git a/crates/block-producer/src/rpc_sync.rs b/crates/block-producer/src/rpc_sync.rs index e7cbe792f..9d6421bc2 100644 --- a/crates/block-producer/src/rpc_sync.rs +++ b/crates/block-producer/src/rpc_sync.rs @@ -5,7 +5,7 @@ use std::time::Duration; use anyhow::Context; use miden_node_proto::clients::RpcClient; use miden_node_proto::generated::rpc::{BlockSubscriptionRequest, ProofSubscriptionRequest}; -use miden_node_store::state::{BlockWriter, Finality, ProofWriter}; +use miden_node_store::state::{BlockWriter, ProofWriter, State}; use miden_node_utils::retry::{self, Retryable}; use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::tasks::Tasks; @@ -130,6 +130,8 @@ impl RpcReadiness { /// Synchronizes local state from an upstream RPC service. pub struct RpcSync { + /// Read-only store state, used by both loops to read tips and subscribe to commits. + pub state: Arc, /// The store's block-write capability, consumed by the block sync loop. pub block_writer: BlockWriter, /// The store's proof-write capability, consumed by the proof sync loop. @@ -143,11 +145,13 @@ impl RpcSync { pub async fn run(self, shutdown: CancellationToken) -> anyhow::Result<()> { let mut tasks = Tasks::new(); let block_sync = BlockSync { + state: Arc::clone(&self.state), writer: self.block_writer, source_rpc: self.source_rpc.clone(), readiness: self.readiness, }; let proof_sync = ProofSync { + state: self.state, writer: self.proof_writer, source_rpc: self.source_rpc, }; @@ -163,6 +167,7 @@ impl RpcSync { // ================================================================================================ struct BlockSync { + state: Arc, writer: BlockWriter, source_rpc: RpcClient, readiness: RpcReadiness, @@ -197,7 +202,7 @@ impl BlockSync { err, )] async fn sync(&self, shutdown: CancellationToken) -> anyhow::Result<()> { - let local_tip = self.writer.chain_tip(Finality::Committed); + let local_tip = self.state.committed_tip(); let mut client = self.source_rpc.clone(); let upstream_tip = BlockNumber::from(client.status(tonic::Request::new(())).await?.into_inner().chain_tip); @@ -225,13 +230,14 @@ impl BlockSync { .context("failed to deserialize block from upstream")?; self.writer.apply_block(block).await?; - let local_tip = self.writer.chain_tip(Finality::Committed); + let local_tip = self.state.committed_tip(); self.readiness.update(upstream_tip, local_tip).await; } } } struct ProofSync { + state: Arc, writer: ProofWriter, source_rpc: RpcClient, } @@ -267,7 +273,7 @@ impl ProofSync { async fn sync(&self, shutdown: CancellationToken) -> anyhow::Result<()> { // Subscribe from next proven tip. - let starting_block = self.writer.chain_tip(Finality::Proven).child(); + let starting_block = self.state.proven_tip().child(); info!( target: LOG_TARGET, block_from = %starting_block, @@ -280,7 +286,7 @@ impl ProofSync { .into_inner(); let mut expected = starting_block; - let mut committed_tip_rx = self.writer.subscribe_committed_tip(); + let mut committed_tip_rx = self.state.subscribe_committed_tip(); loop { let result = tokio::select! { () = shutdown.cancelled() => return Ok(()), diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index 599993c7e..fbcb3b478 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use std::time::Duration; use anyhow::Result; -use miden_node_store::state::{BlockWriter, Finality, ProofWriter, State}; +use miden_node_store::state::{BlockWriter, ProofWriter, State}; use miden_node_utils::formatting::{format_input_notes, format_output_notes}; use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::tasks::Tasks; @@ -111,11 +111,16 @@ impl Sequencer { let state = self.state; let validator = BlockProducerValidatorClient::new(self.validator_urls.clone(), self.validator_timeout)?; - let chain_tip = state.chain_tip(Finality::Committed); + let chain_tip = state.committed_tip(); tracing::info!(target: LOG_TARGET, "Sequencer initialized"); - let block_builder = BlockBuilder::new(self.block_writer, validator, self.block_interval); + let block_builder = BlockBuilder::new( + Arc::clone(&state), + self.block_writer, + validator, + self.block_interval, + ); let batch_intervals = BatchIntervals::derive_from(self.block_interval, self.batch_interval); let batch_builder = BatchBuilder::new( Arc::clone(&state), @@ -155,11 +160,13 @@ impl Sequencer { async { block_builder.run(mempool, shutdown).await } }); tasks.spawn("proof-scheduler", { + let state = Arc::clone(&api.state); let proof_writer = self.proof_writer; let shutdown = shutdown.clone(); async move { proof_scheduler::run( block_prover, + state, proof_writer, chain_tip_rx, self.max_concurrent_proofs, diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index d89d40f95..0d2d2843c 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -7,7 +7,7 @@ use miden_node_proto::decode; use miden_node_proto::decode::GrpcDecodeExt; use miden_node_proto::errors::ConversionError; use miden_node_proto::generated::sequencer; -use miden_node_store::state::{Finality, State, TransactionInputs as StoreTransactionInputs}; +use miden_node_store::state::{State, TransactionInputs as StoreTransactionInputs}; use miden_node_utils::formatting::format_opt; use miden_node_utils::tracing::miden_instrument; use miden_protocol::Word; @@ -185,7 +185,9 @@ pub async fn get_tx_inputs( let unauthenticated_note_commitments = proven_tx.unauthenticated_notes().map(|header| header.id().as_word()).collect(); - let store_inputs = state + // A single view scopes both the input query and the block height below to one snapshot. + let view = state.view(); + let store_inputs = view .get_transaction_inputs( proven_tx.account_id(), &nullifiers, @@ -202,7 +204,7 @@ pub async fn get_tx_inputs( return Err(StoreError::DuplicateAccountIdPrefix(proven_tx.account_id())); } - let current_block_height = state.chain_tip(Finality::Committed); + let current_block_height = view.tip(); let tx_inputs = TransactionInputs::from_store_inputs( proven_tx.account_id(), store_inputs, diff --git a/crates/rpc/src/server/api.rs b/crates/rpc/src/server/api.rs index 2b7844924..4995e7c79 100644 --- a/crates/rpc/src/server/api.rs +++ b/crates/rpc/src/server/api.rs @@ -1,5 +1,4 @@ use std::num::NonZeroUsize; -use std::ops::RangeInclusive; use std::sync::{Arc, LazyLock}; use std::time::Duration; @@ -10,7 +9,7 @@ use miden_node_proto::domain::block::InvalidBlockRange; use miden_node_proto::generated::rpc::MempoolStats as ProtoMempoolStats; use miden_node_proto::generated::rpc::api_server::Api; use miden_node_proto::generated::{self as proto}; -use miden_node_store::state::{Finality, State}; +use miden_node_store::state::State; use miden_node_store::{DatabaseError, GetBlockHeaderError}; use miden_node_utils::limiter::{ QueryParamAccountIdLimit, @@ -205,6 +204,7 @@ impl RpcService { let header = self .state + .view() .get_block_header(Some(block), false) .await .map_err(get_block_header_error_to_status)? @@ -234,25 +234,6 @@ impl RpcService { Ok(()) } - /// Fetches the committed chain tip and ensures the requested range does not extend beyond it. - /// - /// Returns the chain tip so callers can reuse it (e.g. in the response's pagination info) - /// without issuing a second query. - fn range_bounds_check( - &self, - range: &RangeInclusive, - ) -> Result { - let chain_tip = self.state.chain_tip(Finality::Committed); - if *range.end() > chain_tip { - return Err(Status::invalid_argument(format!( - "block_to ({}) is greater than chain tip ({chain_tip})", - range.end() - ))); - } - - Ok(chain_tip) - } - /// Errors if any of `candidate_ids` is classified as a network account by the store. Callers /// should pre-filter to post-deployment, public-account ids; `Ok(())` on empty. async fn reject_if_any_network_accounts( @@ -265,7 +246,7 @@ impl RpcService { } let network_accounts = - self.state.filter_network_accounts(&account_ids).await.map_err(|err| { + self.state.view().filter_network_accounts(&account_ids).await.map_err(|err| { Status::internal(format!("network-account classification failed: {err}")) })?; @@ -311,6 +292,7 @@ fn database_error_to_status(err: &DatabaseError) -> Status { | DatabaseError::AccountsNotFoundInDb(_) | DatabaseError::AccountNotPublic(_) => Status::not_found(message), DatabaseError::TransactionPageExceedsPayloadLimit { .. } => Status::out_of_range(message), + DatabaseError::RangeBeyondTip(_) => Status::invalid_argument(message), _ => Status::internal(message), } } diff --git a/crates/rpc/src/server/api/get_account.rs b/crates/rpc/src/server/api/get_account.rs index 24d01b95d..4e18383da 100644 --- a/crates/rpc/src/server/api/get_account.rs +++ b/crates/rpc/src/server/api/get_account.rs @@ -58,8 +58,12 @@ impl proto::server::rpc_api::GetAccount for RpcService { validate_storage_request(&details.storage_request)?; } - let account_data = - self.state.get_account(request).await.map_err(get_account_error_to_status)?; + let account_data = self + .state + .view() + .get_account(request) + .await + .map_err(get_account_error_to_status)?; Ok(account_data) } } diff --git a/crates/rpc/src/server/api/get_block_header_by_number.rs b/crates/rpc/src/server/api/get_block_header_by_number.rs index 27265587f..6a24df67f 100644 --- a/crates/rpc/src/server/api/get_block_header_by_number.rs +++ b/crates/rpc/src/server/api/get_block_header_by_number.rs @@ -39,6 +39,7 @@ impl proto::server::rpc_api::GetBlockHeaderByNumber for RpcService { let block_num = request.block_num.map(BlockNumber::from); let (block_header, mmr_proof) = self .state + .view() .get_block_header(block_num, request.include_mmr_proof.unwrap_or(false)) .await .map_err(super::get_block_header_error_to_status)?; diff --git a/crates/rpc/src/server/api/get_note_script_by_root.rs b/crates/rpc/src/server/api/get_note_script_by_root.rs index 437baf57e..ed4a413d3 100644 --- a/crates/rpc/src/server/api/get_note_script_by_root.rs +++ b/crates/rpc/src/server/api/get_note_script_by_root.rs @@ -44,6 +44,7 @@ impl proto::server::rpc_api::GetNoteScriptByRoot for RpcService { let script = self .state + .view() .get_note_script_by_root(root) .await .map_err(|err| database_error_to_status(&err))?; diff --git a/crates/rpc/src/server/api/get_notes_by_id.rs b/crates/rpc/src/server/api/get_notes_by_id.rs index 8f9ddd1f2..9926a7f3c 100644 --- a/crates/rpc/src/server/api/get_notes_by_id.rs +++ b/crates/rpc/src/server/api/get_notes_by_id.rs @@ -46,6 +46,7 @@ impl proto::server::rpc_api::GetNotesById for RpcService { let notes = self .state + .view() .get_notes_by_id(note_ids) .await .map_err(|err| database_error_to_status(&err))? diff --git a/crates/rpc/src/server/api/status.rs b/crates/rpc/src/server/api/status.rs index 2b937d670..f5003df89 100644 --- a/crates/rpc/src/server/api/status.rs +++ b/crates/rpc/src/server/api/status.rs @@ -3,7 +3,7 @@ use miden_node_proto::generated as proto; use miden_node_utils::tracing::miden_instrument; use tracing::debug; -use super::{Finality, ProtoMempoolStats, Request, RpcMode, RpcService}; +use super::{ProtoMempoolStats, Request, RpcMode, RpcService}; use crate::{COMPONENT, LOG_TARGET}; #[tonic::async_trait] @@ -48,7 +48,7 @@ impl proto::server::rpc_api::Status for RpcService { Ok(proto::rpc::RpcStatus { version: env!("CARGO_PKG_VERSION").to_string(), - chain_tip: self.state.chain_tip(Finality::Committed).as_u32(), + chain_tip: self.state.committed_tip().as_u32(), block_producer: block_producer_status.or(Some(proto::rpc::BlockProducerStatus { status: "unreachable".to_string(), version: "-".to_string(), diff --git a/crates/rpc/src/server/api/sync_account_storage_maps.rs b/crates/rpc/src/server/api/sync_account_storage_maps.rs index 8fcf75387..ebfc6129f 100644 --- a/crates/rpc/src/server/api/sync_account_storage_maps.rs +++ b/crates/rpc/src/server/api/sync_account_storage_maps.rs @@ -58,9 +58,9 @@ impl proto::server::rpc_api::SyncAccountStorageMaps for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range)?; - let storage_maps_page = self - .state + let view = self.state.view(); + let chain_tip = view.tip(); + let storage_maps_page = view .sync_account_storage_maps(account_id, block_range) .await .map_err(|err| database_error_to_status(&err))?; diff --git a/crates/rpc/src/server/api/sync_account_vault.rs b/crates/rpc/src/server/api/sync_account_vault.rs index 9e965a8c5..5fc2bd668 100644 --- a/crates/rpc/src/server/api/sync_account_vault.rs +++ b/crates/rpc/src/server/api/sync_account_vault.rs @@ -58,9 +58,9 @@ impl proto::server::rpc_api::SyncAccountVault for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range)?; - let (last_included_block, updates) = self - .state + let view = self.state.view(); + let chain_tip = view.tip(); + let (last_included_block, updates) = view .sync_account_vault(account_id, block_range) .await .map_err(|err| database_error_to_status(&err))?; diff --git a/crates/rpc/src/server/api/sync_chain_mmr.rs b/crates/rpc/src/server/api/sync_chain_mmr.rs index e75481596..55ef9425c 100644 --- a/crates/rpc/src/server/api/sync_chain_mmr.rs +++ b/crates/rpc/src/server/api/sync_chain_mmr.rs @@ -1,10 +1,11 @@ use miden_node_proto::generated as proto; +use miden_node_store::StateSyncError; use miden_node_utils::tracing::miden_instrument; use miden_protocol::block::BlockNumber; use tonic::Status; use tracing::debug; -use super::{Finality, RpcService}; +use super::RpcService; use crate::{COMPONENT, LOG_TARGET}; #[tonic::async_trait] @@ -39,11 +40,15 @@ impl proto::server::rpc_api::SyncChainMmr for RpcService { debug!(target: LOG_TARGET, "Syncing chain MMR"); let current_client_block_height = BlockNumber::from(request.current_client_block_height); + let view = self.state.view(); let sync_target = match request.finality_level() { proto::rpc::FinalityLevel::Committed | proto::rpc::FinalityLevel::Unspecified => { - self.state.chain_tip(Finality::Committed) + view.tip() }, - proto::rpc::FinalityLevel::Proven => self.state.chain_tip(Finality::Proven), + // The proven tip is read from a watch channel, not the view's snapshot, so clamp it to + // the view's tip: a block could be committed and proven between taking the view and + // reading the proven tip, and the view cannot serve blocks beyond its snapshot. + proto::rpc::FinalityLevel::Proven => self.state.proven_tip().min(view.tip()), }; if current_client_block_height > sync_target { @@ -53,11 +58,11 @@ impl proto::server::rpc_api::SyncChainMmr for RpcService { } let block_range = current_client_block_height..=sync_target; - let (mmr_delta, block_header, block_signatures) = self - .state - .sync_chain_mmr(block_range.clone()) - .await - .map_err(|err| Status::internal(err.to_string()))?; + let (mmr_delta, block_header, block_signatures) = + view.sync_chain_mmr(block_range.clone()).await.map_err(|err| match err { + StateSyncError::RangeBeyondTip(_) => Status::invalid_argument(err.to_string()), + _ => Status::internal(err.to_string()), + })?; Ok(proto::rpc::SyncChainMmrResponse { block_range: Some(proto::rpc::BlockRange { diff --git a/crates/rpc/src/server/api/sync_notes.rs b/crates/rpc/src/server/api/sync_notes.rs index adeb14b4b..961819f1d 100644 --- a/crates/rpc/src/server/api/sync_notes.rs +++ b/crates/rpc/src/server/api/sync_notes.rs @@ -47,10 +47,10 @@ impl proto::server::rpc_api::SyncNotes for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range)?; + let view = self.state.view(); + let chain_tip = view.tip(); - let (results, last_block_checked) = self - .state + let (results, last_block_checked) = view .sync_notes(request.note_tags, block_range) .await .map_err(note_sync_error_to_status)?; @@ -94,7 +94,7 @@ fn note_sync_error_to_status(err: NoteSyncError) -> Status { match err { NoteSyncError::DatabaseError(err) => super::database_error_to_status(&err), NoteSyncError::InvalidBlockRange(_) - | NoteSyncError::FutureBlock { .. } + | NoteSyncError::RangeBeyondTip(_) | NoteSyncError::DeserializationFailed(_) => Status::invalid_argument(message), NoteSyncError::UnderlyingDatabaseError(_) | NoteSyncError::EmptyBlockHeadersTable diff --git a/crates/rpc/src/server/api/sync_nullifiers.rs b/crates/rpc/src/server/api/sync_nullifiers.rs index a6ae3a053..63865ea25 100644 --- a/crates/rpc/src/server/api/sync_nullifiers.rs +++ b/crates/rpc/src/server/api/sync_nullifiers.rs @@ -58,10 +58,10 @@ impl proto::server::rpc_api::SyncNullifiers for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range)?; + let view = self.state.view(); + let chain_tip = view.tip(); - let (nullifiers, block_num) = self - .state + let (nullifiers, block_num) = view .sync_nullifiers(request.prefix_len, request.nullifiers, block_range) .await .map_err(|err| database_error_to_status(&err))?; diff --git a/crates/rpc/src/server/api/sync_transactions.rs b/crates/rpc/src/server/api/sync_transactions.rs index ef8441aba..32c0ab472 100644 --- a/crates/rpc/src/server/api/sync_transactions.rs +++ b/crates/rpc/src/server/api/sync_transactions.rs @@ -61,10 +61,10 @@ impl proto::server::rpc_api::SyncTransactions for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range)?; + let view = self.state.view(); + let chain_tip = view.tip(); let account_ids = read_account_ids::(request.account_ids)?; - let (last_block_included, transaction_records_db) = self - .state + let (last_block_included, transaction_records_db) = view .sync_transactions(account_ids, block_range) .await .map_err(|err| database_error_to_status(&err))?; diff --git a/crates/rpc/src/server/mod.rs b/crates/rpc/src/server/mod.rs index 3391aba7c..fafe0c57a 100644 --- a/crates/rpc/src/server/mod.rs +++ b/crates/rpc/src/server/mod.rs @@ -13,7 +13,7 @@ use miden_node_proto::clients::{ }; use miden_node_proto::server::{rpc_api, sequencer_api}; use miden_node_proto_build::rpc_api_descriptor; -use miden_node_store::state::{BlockWriter, Finality, ProofWriter, State}; +use miden_node_store::state::{BlockWriter, ProofWriter, State}; use miden_node_utils::clap::{GrpcOptionsExternal, GrpcOptionsInternal}; use miden_node_utils::cors::cors_for_grpc_web_layer; use miden_node_utils::grpc; @@ -221,7 +221,7 @@ impl Rpc { tonic_health::ServingStatus::Serving, ) .await; - let chain_tip = self.state.chain_tip(Finality::Committed); + let chain_tip = self.state.committed_tip(); log_node_ready(mode, endpoint, chain_tip); }, RpcMode::FullNode { source_rpc, readiness_threshold, .. } => { @@ -238,6 +238,7 @@ impl Rpc { tasks.spawn( "RPC sync", RpcSync { + state: Arc::clone(&self.state), block_writer, proof_writer, source_rpc: *source_rpc, diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index efce5c7bd..fb1f41843 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -81,6 +81,8 @@ pub enum DatabaseError { Diesel(#[from] diesel::result::Error), #[error(transparent)] QueryParamLimit(#[from] QueryLimitError), + #[error(transparent)] + RangeBeyondTip(#[from] RangeBeyondTip), // OTHER ERRORS // --------------------------------------------------------------------------------------------- @@ -251,6 +253,14 @@ pub enum ApplyBlockWithProvingInputsError { ApplyBlock(#[source] ApplyBlockError), } +/// A requested block range extends beyond the chain tip of the state view serving the request. +#[derive(Error, Debug)] +#[error("block_to ({block_to}) is greater than chain tip ({chain_tip})")] +pub struct RangeBeyondTip { + pub chain_tip: BlockNumber, + pub block_to: BlockNumber, +} + #[derive(Error, Debug)] pub enum GetBlockHeaderError { #[error("database error")] @@ -282,6 +292,8 @@ pub enum StateSyncError { EmptyBlockHeadersTable, #[error("failed to build MMR delta")] FailedToBuildMmrDelta(#[from] MmrError), + #[error(transparent)] + RangeBeyondTip(#[from] RangeBeyondTip), } impl From for StateSyncError { @@ -302,11 +314,8 @@ pub enum NoteSyncError { MmrError(#[from] MmrError), #[error("invalid block range")] InvalidBlockRange(#[from] InvalidBlockRange), - #[error("block_to ({block_to}) is greater than chain tip ({chain_tip})")] - FutureBlock { - chain_tip: BlockNumber, - block_to: BlockNumber, - }, + #[error(transparent)] + RangeBeyondTip(#[from] RangeBeyondTip), #[error("malformed note tags")] DeserializationFailed(#[from] ConversionError), } diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 6a11cb39f..7ace507dd 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -35,10 +35,11 @@ pub use errors::{ GetBlockHeaderError, GetBlockInputsError, NoteSyncError, + RangeBeyondTip, StateSyncError, }; pub use genesis::GenesisState; -pub use state::{BlockWriter, LoadedState, ProofWriter, State, WriterTask}; +pub use state::{BlockWriter, LoadedState, ProofWriter, State, StateView, WriterTask}; /// Returns the store crate version. pub fn version() -> &'static str { diff --git a/crates/store/src/state/inputs.rs b/crates/store/src/state/inputs.rs deleted file mode 100644 index cc03f4191..000000000 --- a/crates/store/src/state/inputs.rs +++ /dev/null @@ -1,369 +0,0 @@ -//! Block-producer input queries: batch, block, and transaction inputs. -//! -//! These read paths combine in-memory snapshot data (tree witnesses, partial MMRs) with database -//! lookups, scoping the latter by the snapshot's tip so both sources describe the same block -//! height. - -use std::collections::{BTreeMap, BTreeSet, HashSet}; -use std::ops::ControlFlow; - -use miden_node_proto::domain::batch::BatchInputs; -use miden_node_utils::formatting::format_array; -use miden_node_utils::tracing::miden_instrument; -use miden_protocol::Word; -use miden_protocol::account::AccountId; -use miden_protocol::block::account_tree::AccountWitness; -use miden_protocol::block::nullifier_tree::NullifierWitness; -use miden_protocol::block::{BlockInputs, BlockNumber}; -use miden_protocol::crypto::merkle::mmr::PartialMmr; -use miden_protocol::note::Nullifier; -use miden_protocol::transaction::PartialBlockchain; - -use crate::COMPONENT; -use crate::db::NullifierInfo; -use crate::errors::{DatabaseError, GetBatchInputsError, GetBlockInputsError}; -use crate::state::State; - -// STRUCTURES -// ================================================================================================ - -#[derive(Debug, Default)] -pub struct TransactionInputs { - pub account_commitment: Word, - pub nullifiers: Vec, - pub found_unauthenticated_notes: HashSet, - pub new_account_id_prefix_is_unique: Option, -} - -type BlockInputWitnesses = ( - BlockNumber, - BTreeMap, - BTreeMap, - PartialMmr, -); - -// INPUT QUERIES -// ================================================================================================ - -impl State { - /// Fetches the inputs for a transaction batch from the database. - /// - /// ## Inputs - /// - /// The function takes as input: - /// - The tx reference blocks are the set of blocks referenced by transactions in the batch. - /// - The unauthenticated note commitments are the set of commitments of unauthenticated notes - /// consumed by all transactions in the batch. For these notes, we attempt to find inclusion - /// proofs. Not all notes will exist in the DB necessarily, as some notes can be created and - /// consumed within the same batch. - /// - /// ## Outputs - /// - /// The function will return: - /// - A block inclusion proof for all tx reference blocks and for all blocks which are - /// referenced by a note inclusion proof. - /// - Note inclusion proofs for all notes that were found in the DB. - /// - The block header that the batch should reference, i.e. the latest known block. - pub async fn get_batch_inputs( - &self, - tx_reference_blocks: BTreeSet, - unauthenticated_note_commitments: BTreeSet, - ) -> Result { - if tx_reference_blocks.is_empty() { - return Err(GetBatchInputsError::TransactionBlockReferencesEmpty); - } - - // First we grab note inclusion proofs for the known notes. These proofs only prove that the - // note was included in a given block. We then also need to prove that each of those blocks - // is included in the chain. - let note_proofs = self - .db - .select_note_inclusion_proofs(unauthenticated_note_commitments) - .await - .map_err(GetBatchInputsError::SelectNoteInclusionProofError)?; - - // The set of blocks that the notes are included in. - let note_blocks = note_proofs.values().map(|proof| proof.location().block_num()); - - // Collect all blocks we need to query without duplicates, which is: - // - all blocks for which we need to prove note inclusion. - // - all blocks referenced by transactions in the batch. - let mut blocks: BTreeSet = tx_reference_blocks; - blocks.extend(note_blocks); - - let (batch_reference_block, partial_mmr) = { - let snapshot = self.snapshot(); - - let latest_block_num = snapshot.latest_block_num(); - - let highest_block_num = - *blocks.last().expect("we should have checked for empty block references"); - if highest_block_num > latest_block_num { - return Err(GetBatchInputsError::UnknownTransactionBlockReference { - highest_block_num, - latest_block_num, - }); - } - - // Remove the latest block from the to-be-tracked blocks as it will be the reference - // block for the batch itself and thus added to the MMR within the batch kernel, so - // there is no need to prove its inclusion. - blocks.remove(&latest_block_num); - - // SAFETY: - // - The latest block num was retrieved from the inner blockchain from which we will - // also retrieve the proofs, so it is guaranteed to exist in that chain. - // - We have checked that no block number in the blocks set is greater than latest block - // number *and* latest block num was removed from the set. Therefore only block - // numbers smaller than latest block num remain in the set. Therefore all the block - // numbers are guaranteed to exist in the chain state at latest block num. - let partial_mmr = snapshot - .blockchain - .partial_mmr_from_blocks(&blocks, latest_block_num) - .expect("latest block num should exist and all blocks in set should be < than latest block"); - - (latest_block_num, partial_mmr) - }; - - // Fetch the reference block of the batch as part of this query, so we can avoid looking it - // up in a separate DB access. - let mut headers = self - .db - .select_block_headers(blocks.into_iter().chain(std::iter::once(batch_reference_block))) - .await - .map_err(GetBatchInputsError::SelectBlockHeaderError)?; - - // Find and remove the batch reference block as we don't want to add it to the chain MMR. - let header_index = headers - .iter() - .enumerate() - .find_map(|(index, header)| { - (header.block_num() == batch_reference_block).then_some(index) - }) - .expect("DB should have returned the header of the batch reference block"); - - // The order doesn't matter for PartialBlockchain::new, so swap remove is fine. - let batch_reference_block_header = headers.swap_remove(header_index); - - // SAFETY: This should not error because: - // - we're passing exactly the block headers that we've added to the partial MMR, - // - so none of the block headers block numbers should exceed the chain length of the - // partial MMR, - // - and we've added blocks to a BTreeSet, so there can be no duplicates. - // - // We construct headers and partial MMR in concert, so they are consistent. This is why we - // can call the unchecked constructor. - let partial_block_chain = PartialBlockchain::new_unchecked(partial_mmr, headers) - .expect("partial mmr and block headers should be consistent"); - - Ok(BatchInputs { - batch_reference_block_header, - note_proofs, - partial_block_chain, - }) - } - - /// Returns data needed by the block producer to construct and prove the next block. - pub async fn get_block_inputs( - &self, - account_ids: Vec, - nullifiers: Vec, - unauthenticated_note_commitments: BTreeSet, - reference_blocks: BTreeSet, - ) -> Result { - // Get the note inclusion proofs from the DB. We do this first so we have to acquire the - // lock to the state just once. There we need the reference blocks of the note proofs to get - // their authentication paths in the chain MMR. - let unauthenticated_note_proofs = self - .db - .select_note_inclusion_proofs(unauthenticated_note_commitments) - .await - .map_err(GetBlockInputsError::SelectNoteInclusionProofError)?; - - // The set of blocks that the notes are included in. - let note_proof_reference_blocks = - unauthenticated_note_proofs.values().map(|proof| proof.location().block_num()); - - // Collect all blocks we need to prove inclusion for, without duplicates. - let mut blocks = reference_blocks; - blocks.extend(note_proof_reference_blocks); - - let (latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr) = - self.get_block_inputs_witnesses(&mut blocks, &account_ids, &nullifiers)?; - - // Fetch the block headers for all blocks in the partial MMR plus the latest one which will - // be used as the previous block header of the block being built. - let mut headers = self - .db - .select_block_headers(blocks.into_iter().chain(std::iter::once(latest_block_number))) - .await - .map_err(GetBlockInputsError::SelectBlockHeaderError)?; - - // Find and remove the latest block as we must not add it to the chain MMR, since it is not - // yet in the chain. - let latest_block_header_index = headers - .iter() - .enumerate() - .find_map(|(index, header)| { - (header.block_num() == latest_block_number).then_some(index) - }) - .expect("DB should have returned the header of the latest block header"); - - // The order doesn't matter for PartialBlockchain::new, so swap remove is fine. - let latest_block_header = headers.swap_remove(latest_block_header_index); - - // SAFETY: This should not error because: - // - we're passing exactly the block headers that we've added to the partial MMR, - // - so none of the block header's block numbers should exceed the chain length of the - // partial MMR, - // - and we've added blocks to a BTreeSet, so there can be no duplicates. - // - // We construct headers and partial MMR in concert, so they are consistent. This is why we - // can call the unchecked constructor. - let partial_block_chain = PartialBlockchain::new_unchecked(partial_mmr, headers) - .expect("partial mmr and block headers should be consistent"); - - Ok(BlockInputs::new( - latest_block_header, - partial_block_chain, - account_witnesses, - nullifier_witnesses, - unauthenticated_note_proofs, - )) - } - - /// Get account and nullifier witnesses for the requested account IDs and nullifier as well as - /// the [`PartialMmr`] for the given blocks. The MMR won't contain the latest block and its - /// number is removed from `blocks` and returned separately. - /// - /// This method acquires the lock to the inner state and does not access the DB so we release - /// the lock asap. - fn get_block_inputs_witnesses( - &self, - blocks: &mut BTreeSet, - account_ids: &[AccountId], - nullifiers: &[Nullifier], - ) -> Result { - self.with_inner_read_blocking(|inner| { - let latest_block_number = inner.latest_block_num(); - - // If `blocks` is empty, use the latest block number which will never trigger the error. - let highest_block_number = blocks.last().copied().unwrap_or(latest_block_number); - if highest_block_number > latest_block_number { - return Err(GetBlockInputsError::UnknownBatchBlockReference { - highest_block_number, - latest_block_number, - }); - } - - // The latest block is not yet in the chain MMR, so we can't (and don't need to) prove - // its inclusion in the chain. - blocks.remove(&latest_block_number); - - // Fetch the partial MMR at the state of the latest block with authentication paths for - // the provided set of blocks. - // - // SAFETY: - // - The latest block num was retrieved from the inner blockchain from which we will - // also retrieve the proofs, so it is guaranteed to exist in that chain. - // - We have checked that no block number in the blocks set is greater than latest block - // number *and* latest block num was removed from the set. Therefore only block - // numbers smaller than latest block num remain in the set. Therefore all the block - // numbers are guaranteed to exist in the chain state at latest block num. - let partial_mmr = - inner.blockchain.partial_mmr_from_blocks(blocks, latest_block_number).expect( - "latest block num should exist and all blocks in set should be < than latest block", - ); - - // Fetch witnesses for all accounts. - let account_witnesses = account_ids - .iter() - .copied() - .map(|account_id| (account_id, inner.account_tree.open_latest(account_id))) - .collect::>(); - - // Fetch witnesses for all nullifiers. We don't check whether the nullifiers are spent - // or not as this is done as part of proposing the block. - let nullifier_witnesses: BTreeMap = nullifiers - .iter() - .copied() - .map(|nullifier| (nullifier, inner.nullifier_tree.open(&nullifier))) - .collect(); - - Ok((latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr)) - }) - } - - /// Returns data needed by the block producer to verify transactions validity. - #[miden_instrument( - target = COMPONENT, - skip_all, - fields( - account.id=%account_id, - nullifiers = %format_array(nullifiers), - ), - )] - pub async fn get_transaction_inputs( - &self, - account_id: AccountId, - nullifiers: &[Nullifier], - unauthenticated_note_commitments: Vec, - ) -> Result { - let tree_inputs = self.with_inner_read_blocking(|inner| { - let account_commitment = inner.account_tree.get_latest_commitment(account_id); - - let new_account_id_prefix_is_unique = if account_commitment.is_empty() { - Some(!inner.account_tree.contains_account_id_prefix_in_latest(account_id.prefix())) - } else { - None - }; - - // Non-unique account Id prefixes for new accounts are not allowed, so the transaction - // cannot be valid and the response is already complete. - if let Some(false) = new_account_id_prefix_is_unique { - return ControlFlow::Break(TransactionInputs { - new_account_id_prefix_is_unique, - ..Default::default() - }); - } - - let nullifiers = nullifiers - .iter() - .map(|nullifier| NullifierInfo { - nullifier: *nullifier, - block_num: inner.nullifier_tree.get_block_num(nullifier).unwrap_or_default(), - }) - .collect(); - - ControlFlow::Continue(( - account_commitment, - nullifiers, - new_account_id_prefix_is_unique, - inner.latest_block_num(), - )) - }); - // `Break` carries a complete response (duplicate account ID prefix), so it is returned - // as-is without the note lookup below; `Continue` carries the tree reads needed to build - // the full response. - let (account_commitment, nullifiers, new_account_id_prefix_is_unique, latest_block_num) = - match tree_inputs { - ControlFlow::Continue(inputs) => inputs, - ControlFlow::Break(response) => return Ok(response), - }; - - // Scope the note lookup by the snapshot's tip so the result is consistent with the tree - // reads above: mid-apply, the DB may already contain notes from a block the snapshot does - // not include yet. - let found_unauthenticated_notes = self - .db - .select_existing_note_commitments(unauthenticated_note_commitments, latest_block_num) - .await?; - - Ok(TransactionInputs { - account_commitment, - nullifiers, - found_unauthenticated_notes, - new_account_id_prefix_is_unique, - }) - } -} diff --git a/crates/store/src/state/lifecycle.rs b/crates/store/src/state/lifecycle.rs index 837311d0d..405e1e618 100644 --- a/crates/store/src/state/lifecycle.rs +++ b/crates/store/src/state/lifecycle.rs @@ -34,7 +34,7 @@ use crate::state::loader::{ verify_tree_consistency, }; use crate::state::writer::{WriteRequest, WriteWorker}; -use crate::state::{BlockCache, InMemoryState, ProofCache, SnapshotGuard, State}; +use crate::state::{BlockCache, ProofCache, SnapshotGuard, State, StateSnapshot}; use crate::{COMPONENT, DataDirectory, DatabaseOptions}; /// Number of recent committed blocks held in the in-memory cache for replica subscriptions. @@ -65,7 +65,9 @@ impl LoadedState { /// [`Arc`] is the read-only view shared with every component that queries or /// subscribes; it exposes no mutating methods. [`BlockWriter`] and [`ProofWriter`] are the /// only handles able to mutate the store — hand them to the single task driving each write - /// path (block production or sync, and proof scheduling or sync respectively). + /// path (block production or sync, and proof scheduling or sync respectively). The + /// capabilities expose no read access: tasks that both read and write receive the + /// [`Arc`] alongside their capability. /// /// The writer exits once the shutdown token passed to [`State::load`] is cancelled or the /// [`BlockWriter`] (holding the only request sender) is dropped — an in-flight block write @@ -78,7 +80,7 @@ impl LoadedState { let writer_task = tokio::spawn(self.writer.run()); let state = Arc::new(self.state); let block_writer = BlockWriter { - state: Arc::clone(&state), + block_store: Arc::clone(&state.block_store), write_tx: self.write_tx, }; let proof_writer = ProofWriter { state: Arc::clone(&state) }; @@ -94,28 +96,19 @@ impl LoadedState { /// Only handle able to apply blocks; obtained exactly once from [`LoadedState::start`] and /// deliberately not cloneable, so granting it to a single task (the block builder in sequencer /// mode, the block sync loop in full-node mode) statically prevents every other component from -/// writing blocks. Dereferences to [`State`] for read access. +/// writing blocks. +/// +/// Exposes no read access: holders that also need to query the store receive the [`Arc`] +/// returned alongside this capability by [`LoadedState::start`]. pub struct BlockWriter { - state: Arc, + /// The block store, used to persist proving inputs alongside applied blocks. + pub(super) block_store: Arc, /// Sender for block-write requests to the [`WriteWorker`](crate::state::writer::WriteWorker) /// task. Never cloned out of this struct: the writer exits once it is dropped. pub(super) write_tx: mpsc::Sender, } -impl std::ops::Deref for BlockWriter { - type Target = State; - - fn deref(&self) -> &State { - &self.state - } -} - impl BlockWriter { - /// Returns the shared read-only state. - pub fn state(&self) -> &Arc { - &self.state - } - /// Stops the store, waiting until the write worker has released the tree storage it owns. /// /// Consumes the capability — closing the write channel the write worker listens on — and then @@ -144,24 +137,13 @@ impl BlockWriter { /// Only handle able to commit block proofs and advance the proven tip; obtained exactly once from /// [`LoadedState::start`] and deliberately not cloneable, so granting it to a single task (the /// proof scheduler in sequencer mode, the proof sync loop in full-node mode) statically prevents -/// every other component from writing proofs. Dereferences to [`State`] for read access. +/// every other component from writing proofs. +/// +/// Exposes no read access: the held state is only used internally to commit proofs and advance +/// the proven tip. Holders that also need to query the store receive the [`Arc`] returned +/// alongside this capability by [`LoadedState::start`]. pub struct ProofWriter { - state: Arc, -} - -impl std::ops::Deref for ProofWriter { - type Target = State; - - fn deref(&self) -> &State { - &self.state - } -} - -impl ProofWriter { - /// Returns the shared read-only state. - pub fn state(&self) -> &Arc { - &self.state - } + pub(super) state: Arc, } // WRITER TASK @@ -299,18 +281,18 @@ impl State { let snapshots_live = Arc::new(AtomicUsize::new(0)); // Create the initial snapshot from reader views of the just-loaded trees. - let initial_snapshot = Arc::new(InMemoryState { - nullifier_tree: nullifier_tree + let initial_snapshot = Arc::new(StateSnapshot::new( + nullifier_tree .reader() .map_err(|e| StateInitializationError::NullifierTreeIoError(e.as_report()))?, - account_tree: account_tree.reader(), - forest: forest + blockchain.clone(), + account_tree.reader(), + forest .reader() .map_err(|e| StateInitializationError::AccountStateForestIoError(e.as_report()))?, - blockchain: blockchain.clone(), - _guard: SnapshotGuard::new(Arc::clone(&snapshots_live), latest_block_num), - }); - let in_memory = Arc::new(ArcSwap::from(initial_snapshot)); + SnapshotGuard::new(Arc::clone(&snapshots_live), latest_block_num), + )); + let latest_snapshot = Arc::new(ArcSwap::from(initial_snapshot)); // Assemble the write worker. It owns the writable trees and processes write requests // serially, publishing a new snapshot after each committed block. The caller runs it; it @@ -320,7 +302,7 @@ impl State { let block_writer = WriteWorker { db: Arc::clone(&db), block_store: Arc::clone(&block_store), - in_memory: Arc::clone(&in_memory), + latest_snapshot: Arc::clone(&latest_snapshot), committed_tip_tx: Arc::clone(&committed_tip_tx), block_cache: block_cache.clone(), rx: write_rx, @@ -335,7 +317,7 @@ impl State { data_directory: data_path.to_path_buf(), db, block_store, - in_memory, + latest_snapshot, proven_tip, committed_tip_tx, block_cache, diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 7f3cd1f1c..c9fdc92fd 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -14,44 +14,24 @@ use crate::blocks::BlockStore; use crate::db::Db; use crate::proven_tip::ProvenTipWriter; -mod loader; - -mod replica; -pub use replica::{BlockCache, BlockNotification, ProofCache, ProofNotification}; - -mod account; - -mod apply_block; -mod apply_proof; mod block_lifecycle; mod bootstrap; mod disk_monitor; -mod sync_state; - -mod inputs; -pub use inputs::TransactionInputs; +mod loader; mod lifecycle; pub use lifecycle::{BlockWriter, LoadedState, ProofWriter, WriterTask}; -mod queries; - -mod snapshot; -pub(crate) use snapshot::{InMemoryState, SNAPSHOTS_LIVE_WARN_THRESHOLD, SnapshotGuard}; +mod replica; +pub use replica::{BlockCache, BlockNotification, ProofCache, ProofNotification}; -mod writer; +mod tip; -// FINALITY -// ================================================================================================ +mod view; +use view::{SnapshotGuard, StateSnapshot}; +pub use view::{StateView, TransactionInputs}; -/// The finality level for chain tip queries. -#[derive(Debug, Clone, Copy)] -pub enum Finality { - /// The latest committed (but not necessarily proven) block. - Committed, - /// The latest block that has been proven in an unbroken sequence from genesis. - Proven, -} +mod writer; // CHAIN STATE // ================================================================================================ @@ -60,6 +40,10 @@ pub enum Finality { /// /// Mutations go through the [`BlockWriter`] and [`ProofWriter`] capabilities returned by /// [`LoadedState::start`]; every holder of this type can only query and subscribe. +/// +/// All tree and database reads go through a request-scoped [`StateView`] obtained from +/// [`State::view`]; this type itself only exposes chain-tip queries, subscriptions, and +/// block-store access. pub struct State { /// Root directory containing the store's on-disk data. data_directory: PathBuf, @@ -76,7 +60,7 @@ pub struct State { /// Readers load the snapshot wait-free via [`ArcSwap::load_full`]; the /// [`WriteWorker`](writer::WriteWorker) task atomically replaces the pointer after each /// committed block. Readers holding an old snapshot are unaffected by the swap. - in_memory: Arc>, + latest_snapshot: Arc>, /// The latest proven-in-sequence block number, updated by the proof scheduler or `apply_proof`. proven_tip: ProvenTipWriter, diff --git a/crates/store/src/state/queries.rs b/crates/store/src/state/queries.rs deleted file mode 100644 index d12fc9cb9..000000000 --- a/crates/store/src/state/queries.rs +++ /dev/null @@ -1,96 +0,0 @@ -//! General read queries over the store state. - -use std::collections::HashSet; - -use miden_node_utils::tracing::miden_instrument; -use miden_protocol::Word; -use miden_protocol::account::AccountId; -use miden_protocol::block::{BlockHeader, BlockNumber}; -use miden_protocol::crypto::merkle::mmr::MmrProof; -use miden_protocol::note::{NoteId, NoteScript}; - -use crate::COMPONENT; -use crate::db::NoteRecord; -use crate::errors::{DatabaseError, GetBlockHeaderError}; -use crate::state::{Finality, State}; - -impl State { - /// Queries a [BlockHeader] from the database, and returns it alongside its inclusion proof. - /// - /// If [None] is given as the value of `block_num`, the data for the latest [BlockHeader] is - /// returned. - #[miden_instrument( - level = "debug", - target = COMPONENT, - skip_all, - err, - )] - pub async fn get_block_header( - &self, - block_num: Option, - include_mmr_proof: bool, - ) -> Result<(Option, Option), GetBlockHeaderError> { - // Resolve "latest" against the in-memory snapshot rather than the DB: mid-apply, the DB may - // already contain a block that the snapshot's blockchain cannot prove yet. Scoping the DB - // query by the snapshot's tip keeps the header and MMR proof consistent. - let snapshot = self.snapshot(); - let latest_block_num = snapshot.latest_block_num(); - let block_num = block_num.unwrap_or(latest_block_num); - if block_num > latest_block_num { - return Ok((None, None)); - } - - let block_header = self.db.select_block_header_by_block_num(Some(block_num)).await?; - if let Some(header) = block_header { - let mmr_proof = if include_mmr_proof { - let mmr_proof = snapshot.blockchain.open(header.block_num())?; - Some(mmr_proof) - } else { - None - }; - Ok((Some(header), mmr_proof)) - } else { - Ok((None, None)) - } - } - - /// Queries a list of notes from the database. - /// - /// If the provided list of [`NoteId`] given is empty or no note matches the provided - /// [`NoteId`] an empty list is returned. - pub async fn get_notes_by_id( - &self, - note_ids: Vec, - ) -> Result, DatabaseError> { - self.db.select_notes_by_id(note_ids).await - } - - /// Returns the script for a note by its root. - pub async fn get_note_script_by_root( - &self, - root: Word, - ) -> Result, DatabaseError> { - self.db.select_note_script_by_root(root).await - } - - /// Filters `account_ids` down to the subset classified as network accounts. - pub async fn filter_network_accounts( - &self, - account_ids: &[AccountId], - ) -> Result, DatabaseError> { - self.db.select_network_accounts_subset(account_ids.to_vec()).await - } - - /// Returns the effective chain tip for the given finality level. - /// - /// - [`Finality::Committed`]: returns the latest committed block number (from the in-memory - /// snapshot). - /// - [`Finality::Proven`]: returns the latest proven-in-sequence block number (cached via watch - /// channel, updated by the proof scheduler). - pub fn chain_tip(&self, finality: Finality) -> BlockNumber { - match finality { - Finality::Committed => self.snapshot().latest_block_num(), - Finality::Proven => self.proven_tip.read(), - } - } -} diff --git a/crates/store/src/state/replica.rs b/crates/store/src/state/replica.rs index 0399f7e21..e24d3acff 100644 --- a/crates/store/src/state/replica.rs +++ b/crates/store/src/state/replica.rs @@ -1,11 +1,23 @@ +//! Block and proof serving for replica subscriptions. +//! +//! Replica subscribers stream committed blocks and block proofs from this node. The writer pushes +//! each freshly committed block (and the proof path each proven block) into a FIFO cache here, so +//! subscribers keeping up with the tip are served from memory; a subscriber that has fallen +//! behind the cache window falls back to reading the block store. +//! +//! These reads serve raw block/proof bytes from the caches and the block store — they never touch +//! the database or tree state, which is why they live on [`State`] directly rather than on a +//! [`StateView`](super::StateView). The tip checks gating them are live availability checks +//! ("is this block committed/proven yet?"), deliberately race-tolerant: a `None` for a block that +//! commits an instant later is corrected by the next tip-watch wakeup. + use std::sync::Arc; use miden_node_utils::block_cache::BlockOrderedCache; use miden_protocol::block::BlockNumber; -use tokio::sync::watch; use crate::errors::DatabaseError; -use crate::state::{Finality, State}; +use crate::state::State; // BLOCK NOTIFICATION // ================================================================================================ @@ -74,23 +86,13 @@ pub type ProofCache = BlockOrderedCache; // ================================================================================================ impl State { - /// Returns a watch receiver that wakes every time a new block is committed. - pub fn subscribe_committed_tip(&self) -> watch::Receiver { - self.committed_tip_tx.subscribe() - } - - /// Returns a watch receiver that wakes every time the proven-in-sequence tip advances. - pub fn subscribe_proven_tip(&self) -> watch::Receiver { - self.proven_tip.subscribe() - } - /// Loads a block from the in-memory replica cache or block store. Return `Ok(None)` if the /// block is not found. pub async fn load_block( &self, block_num: BlockNumber, ) -> Result>, DatabaseError> { - if block_num > self.chain_tip(Finality::Committed) { + if block_num > self.committed_tip() { return Ok(None); } if let Some(block) = self.block_cache.get(block_num) { @@ -105,7 +107,7 @@ impl State { &self, block_num: BlockNumber, ) -> Result>, DatabaseError> { - if block_num > self.chain_tip(Finality::Proven) { + if block_num > self.proven_tip() { return Ok(None); } if let Some(proof) = self.proof_cache.get(block_num) { diff --git a/crates/store/src/state/tip.rs b/crates/store/src/state/tip.rs new file mode 100644 index 000000000..fee824047 --- /dev/null +++ b/crates/store/src/state/tip.rs @@ -0,0 +1,45 @@ +//! Live chain-tip queries and tip subscriptions. +//! +//! Both tips are published through watch channels by their single writers (the block writer for +//! the committed tip, the proof scheduler or proof sync for the proven tip). Everything here +//! reads or subscribes to those channels; none of it touches state snapshots, so these values +//! advance independently of any [`StateView`](super::StateView). + +use miden_protocol::block::BlockNumber; +use tokio::sync::watch; + +use super::State; + +// TIP QUERIES & SUBSCRIPTIONS +// ================================================================================================ + +impl State { + /// Returns the latest committed (but not necessarily proven) block number. + /// + /// This is a live value: it advances independently of any [`StateView`](super::StateView). + /// Reads that must be consistent with data must use a view's tip instead. + /// + /// The committed tip is published after the corresponding state snapshot, so it never reports + /// a block that a freshly created view cannot serve. + pub fn committed_tip(&self) -> BlockNumber { + *self.committed_tip_tx.borrow() + } + + /// Returns the latest block number proven in an unbroken sequence from genesis. + /// + /// This is a live value published by the proof scheduler (sequencer mode) or the proof sync + /// loop (full-node mode); it is always at or behind the committed tip. + pub fn proven_tip(&self) -> BlockNumber { + self.proven_tip.read() + } + + /// Returns a watch receiver that wakes every time a new block is committed. + pub fn subscribe_committed_tip(&self) -> watch::Receiver { + self.committed_tip_tx.subscribe() + } + + /// Returns a watch receiver that wakes every time the proven-in-sequence tip advances. + pub fn subscribe_proven_tip(&self) -> watch::Receiver { + self.proven_tip.subscribe() + } +} diff --git a/crates/store/src/state/view/account/mod.rs b/crates/store/src/state/view/account/mod.rs new file mode 100644 index 000000000..723cd58d6 --- /dev/null +++ b/crates/store/src/state/view/account/mod.rs @@ -0,0 +1,414 @@ +use std::collections::HashSet; + +use miden_node_proto::domain::account::{ + AccountDetailRequest, + AccountDetails, + AccountRequest, + AccountResponse, + AccountStorageDetails, + AccountStorageMapDetails, + AccountStorageRequest, + AccountVaultDetails, + SlotData, + StorageMapEntries, + StorageMapRequest, +}; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::account::{AccountId, AccountStorageHeader, StorageSlotName, StorageSlotType}; +use miden_protocol::block::BlockNumber; +use miden_protocol::block::account_tree::AccountWitness; + +use super::StateView; +use crate::COMPONENT; +use crate::account_state_forest::AccountStorageMapResult; +use crate::errors::{DatabaseError, GetAccountError}; + +impl StateView { + /// Returns an account witness and optionally account details at a specific block. + /// + /// The witness is a Merkle proof of inclusion in the account tree, proving the account's + /// state commitment. If `details` is requested, the method also returns the account's code, + /// vault assets, and storage data. Account details are only available for public accounts. + /// + /// If `block_num` is provided, returns the state at that historical block; otherwise, returns + /// the latest state. Note that historical states are only available for recent blocks close + /// to the chain tip. + #[miden_instrument( + target = COMPONENT, + skip_all, + )] + pub async fn get_account( + &self, + account_request: AccountRequest, + ) -> Result { + let AccountRequest { block_num, account_id, details } = account_request; + + if details.is_some() && !account_id.is_public() { + return Err(GetAccountError::AccountNotPublic(account_id)); + } + + let (block_num, witness) = self.get_account_witness(block_num, account_id).await?; + + let details = if let Some(request) = details { + Some( + self.fetch_public_account_details(account_id, block_num, &witness, request) + .await?, + ) + } else { + None + }; + + Ok(AccountResponse { block_num, witness, details }) + } + + /// Returns an account witness (Merkle proof of inclusion in the account tree). + /// + /// If `block_num` is provided, returns the witness at that historical block; + /// otherwise, returns the witness at the latest block. + #[miden_instrument( + target = COMPONENT, + skip_all, + )] + async fn get_account_witness( + &self, + block_num: Option, + account_id: AccountId, + ) -> Result<(BlockNumber, AccountWitness), GetAccountError> { + self.with_inner_read_blocking(|inner_state| { + // Determine which block to query + let (block_num, witness) = if let Some(requested_block) = block_num { + // Historical query: use the account tree with history + let witness = inner_state + .account_tree + .open_at(account_id, requested_block) + .ok_or_else(|| { + let latest_block = inner_state.account_tree.block_number_latest(); + if requested_block > latest_block { + GetAccountError::UnknownBlock(requested_block) + } else { + GetAccountError::BlockPruned(requested_block) + } + })?; + (requested_block, witness) + } else { + // Latest query: use the latest state + let block_num = inner_state.account_tree.block_number_latest(); + let witness = inner_state.account_tree.open_latest(account_id); + (block_num, witness) + }; + + Ok((block_num, witness)) + }) + } + + /// Returns storage map details from the forest for a specific account and storage slot. + /// + /// The forest can only be used if all hashed keys in the storage map are known in the + /// reverse-key LRU cache. If any hashed key is unknown, the method returns `Ok(None)` to signal + /// that the caller should fall back to reconstructing the storage map details from the + /// database. + #[miden_instrument( + target = COMPONENT, + skip_all, + )] + fn get_storage_map_details_from_forest( + &self, + account_id: AccountId, + slot_name: &StorageSlotName, + block_num: BlockNumber, + ) -> Result, DatabaseError> { + self.with_forest_read_blocking(|forest| { + match forest + .get_storage_map_details_for_all_entries(account_id, slot_name.clone(), block_num) + .map_err(DatabaseError::MerkleError)? + { + AccountStorageMapResult::NotFound => Err(DatabaseError::StorageRootNotFound { + account_id, + slot_name: slot_name.to_string(), + block_num, + }), + AccountStorageMapResult::Details(details) => Ok(Some(details)), + AccountStorageMapResult::CannotReconstructKeysFromCache => Ok(None), + } + }) + } + + /// Returns vault details by reconstructing the vault from the database. + async fn reconstruct_vault_details_from_db( + &self, + account_id: AccountId, + block_num: BlockNumber, + ) -> Result { + let assets = self.db().select_account_vault_at_block(account_id, block_num).await?; + + if assets.len() > AccountVaultDetails::MAX_RETURN_ENTRIES { + return Ok(AccountVaultDetails::LimitExceeded); + } + + let keys = assets.iter().map(miden_protocol::asset::Asset::id); + + // The reverse-key caches are shared between the writer and all snapshots, so caching via + // the current snapshot's forest is visible everywhere. + self.with_forest_read_blocking(|forest| { + forest + .vault_key_cache + .put_many(keys.into_iter().map(|raw_key| (raw_key.hash(), raw_key))); + }); + + Ok(AccountVaultDetails::from_assets(assets)) + } + + /// Returns storage map details by reconstructing the storage map from the database. + async fn reconstruct_storage_map_details_from_db( + &self, + account_id: AccountId, + slot_name: StorageSlotName, + block_num: BlockNumber, + ) -> Result { + let details = self + .db() + .reconstruct_storage_map_from_db( + account_id, + slot_name, + block_num, + Some(AccountStorageMapDetails::MAX_RETURN_ENTRIES), + ) + .await?; + + if let StorageMapEntries::AllEntries(entries) = &details.entries { + self.with_forest_read_blocking(|forest| { + forest.cache_storage_map_keys(entries.iter().map(|(raw_key, _)| *raw_key)); + }); + } + + Ok(details) + } + + /// Fetches the account details (code, vault, storage) for a public account at the specified + /// block. + /// + /// This method queries the database to fetch the account state and processes the detail + /// request to return only the requested information. + /// + /// For specific key queries (`SlotData::MapKeys`), the forest is used to provide SMT proofs. + /// Returns an error if the forest doesn't have data for the requested slot. + /// All-entries queries (`SlotData::All`) use the forest when all hashed keys are known in the + /// reverse-key LRU cache, otherwise they fall back to database reconstruction. + #[miden_instrument( + target = COMPONENT, + skip_all, + )] + async fn fetch_public_account_details( + &self, + account_id: AccountId, + block_num: BlockNumber, + witness: &AccountWitness, + detail_request: AccountDetailRequest, + ) -> Result { + let AccountDetailRequest { + code_commitment, + asset_vault_commitment, + storage_request, + } = detail_request; + + if !account_id.is_public() { + return Err(GetAccountError::AccountNotPublic(account_id)); + } + + // Validate block exists in the blockchain before querying the database. The view's tip is + // the same snapshot the witness was resolved against, so the witness and the DB reads below + // observe a single consistent block height. + if block_num > self.tip() { + return Err(GetAccountError::UnknownBlock(block_num)); + } + + // Query account header and storage header together in a single DB call + let (account_header, storage_header) = self + .db() + .select_account_header_with_storage_header_at_block(account_id, block_num) + .await? + .ok_or(GetAccountError::AccountNotFound(account_id, block_num))?; + + let should_apply_response_budget = + matches!(&storage_request, AccountStorageRequest::AllStorageMaps); + let storage_requests = expand_account_storage_request(storage_request, &storage_header); + + let account_code = match code_commitment { + Some(commitment) if commitment == account_header.code_commitment() => None, + Some(_) => { + self.db() + .select_account_code_by_commitment(account_header.code_commitment()) + .await? + }, + None => None, + }; + + // Query account state forest for vault details on commitment mismatch. + // + // The forest can only reconstruct the vault if all hashed vault keys are known in the + // reverse-key LRU cache. If any hashed key is unknown, the forest returns `None` and we + // fall back to reconstructing the vault details from the database. + let vault_details = match asset_vault_commitment { + Some(commitment) if commitment == account_header.vault_root() => { + AccountVaultDetails::empty() + }, + Some(_) => { + let forest_details = self.with_forest_read_blocking(|forest| { + forest.get_vault_details(account_id, block_num).map_err(|err| { + DatabaseError::DataCorrupted(format!( + "failed to reconstruct vault for account {account_id} at block {block_num}: {err}" + )) + }) + })?; + + match forest_details { + Some(details) => details, + None => self.reconstruct_vault_details_from_db(account_id, block_num).await?, + } + }, + None => AccountVaultDetails::empty(), + }; + + // Split storage map requests into two categories: + // - slots with explicit keys (including proofs) + // - slots with "all entries" + let mut storage_map_details = + Vec::::with_capacity(storage_requests.len()); + let mut map_keys_requests = Vec::new(); + let mut all_entries_requests = Vec::new(); + let mut storage_request_slots = Vec::with_capacity(storage_requests.len()); + + for (index, StorageMapRequest { slot_name, slot_data }) in + storage_requests.into_iter().enumerate() + { + storage_request_slots.push(slot_name.clone()); + match slot_data { + SlotData::MapKeys(keys) => { + map_keys_requests.push((index, slot_name, keys)); + }, + SlotData::All => { + all_entries_requests.push((index, slot_name)); + }, + } + } + + let mut storage_map_details_by_index = vec![None; storage_request_slots.len()]; + + // Handle slots with explicit key requests + if !map_keys_requests.is_empty() { + self.with_forest_read_blocking(|forest| { + for (index, slot_name, keys) in map_keys_requests { + let details = forest + .get_storage_map_details_for_keys( + account_id, + slot_name.clone(), + block_num, + &keys, + ) + .ok_or_else(|| DatabaseError::StorageRootNotFound { + account_id, + slot_name: slot_name.to_string(), + block_num, + })? + .map_err(DatabaseError::MerkleError)?; + storage_map_details_by_index[index] = Some(details); + } + Ok::<(), DatabaseError>(()) + })?; + } + + // Handle slots with "all entries" requests + for (index, slot_name) in all_entries_requests { + let details = match self + .get_storage_map_details_from_forest(account_id, &slot_name, block_num)? + { + Some(details) => details, + None => { + self.reconstruct_storage_map_details_from_db(account_id, slot_name, block_num) + .await? + }, + }; + storage_map_details_by_index[index] = Some(details); + } + + for (details, slot_name) in + storage_map_details_by_index.into_iter().zip(storage_request_slots.iter()) + { + let details = details.ok_or_else(|| DatabaseError::StorageRootNotFound { + account_id, + slot_name: slot_name.to_string(), + block_num, + })?; + storage_map_details.push(details); + } + + // In case of an "all storage maps" request we have to be careful: even with the per-slot + // limit of [`AccountStorageMapDetails::MAX_RETURN_ENTRIES`] we might go over the response + // size limit. Here we make sure that we're within that limit by potentially truncating the + // response. + if should_apply_response_budget { + return Ok(apply_all_storage_maps_response_budget( + block_num, + witness, + account_header, + account_code, + vault_details, + storage_header, + storage_map_details, + storage_request_slots, + MAX_ALL_STORAGE_MAPS_RESPONSE_PAYLOAD_WITH_BUDGET_RESERVED_FOR_LIMIT_EXCEEDED_SLOTS, + )); + } + + Ok(AccountDetails { + account_header, + account_code, + vault_details, + storage_details: AccountStorageDetails { + header: storage_header, + map_details: storage_map_details, + }, + }) + } +} + +// HELPERS +// ================================================================================================ + +/// Expand [`AccountStorageRequest`] to a vector of slot requests. +fn expand_account_storage_request( + storage_request: AccountStorageRequest, + storage_header: &AccountStorageHeader, +) -> Vec { + match storage_request { + AccountStorageRequest::None => Vec::new(), + AccountStorageRequest::Explicit(requests) => requests, + AccountStorageRequest::AllStorageMaps => storage_header + .slots() + .filter(|slot| slot.slot_type() == StorageSlotType::Map) + .map(|slot| StorageMapRequest { + slot_name: slot.name().clone(), + slot_data: SlotData::All, + }) + .collect(), + } +} + +mod response_budget; +use response_budget::{ + MAX_ALL_STORAGE_MAPS_RESPONSE_PAYLOAD_WITH_BUDGET_RESERVED_FOR_LIMIT_EXCEEDED_SLOTS, + apply_all_storage_maps_response_budget, +}; + +// NETWORK ACCOUNT CLASSIFICATION +// ================================================================================================ + +impl StateView { + /// Filters `account_ids` down to the subset classified as network accounts. + pub async fn filter_network_accounts( + &self, + account_ids: &[AccountId], + ) -> Result, DatabaseError> { + self.db().select_network_accounts_subset(account_ids.to_vec()).await + } +} diff --git a/crates/store/src/state/account.rs b/crates/store/src/state/view/account/response_budget.rs similarity index 50% rename from crates/store/src/state/account.rs rename to crates/store/src/state/view/account/response_budget.rs index 490dcd70b..b8ecaa699 100644 --- a/crates/store/src/state/account.rs +++ b/crates/store/src/state/view/account/response_budget.rs @@ -1,416 +1,33 @@ +//! Response-size budgeting for `all-storage-maps` account detail responses. +//! +//! Even with the per-slot entry limit, an `all-storage-maps` request can exceed the response +//! payload cap. The budget below keeps the encoded response within the cap by replacing map +//! contents with "limit exceeded" markers once the budget is spent, reserving space for those +//! markers up front. + use miden_node_proto::domain::account::{ - AccountDetailRequest, AccountDetails, - AccountRequest, AccountResponse, AccountStorageDetails, AccountStorageMapDetails, - AccountStorageRequest, AccountVaultDetails, - SlotData, StorageMapEntries, - StorageMapRequest, }; use miden_node_proto::generated as proto; use miden_node_proto::prost::Message as _; use miden_node_proto::prost::encoding::{encoded_len_varint, key_len}; use miden_node_utils::limiter::MAX_RESPONSE_PAYLOAD_BYTES; -use miden_node_utils::tracing::miden_instrument; -use miden_protocol::account::{ - AccountHeader, - AccountId, - AccountStorageHeader, - StorageSlotName, - StorageSlotType, -}; +use miden_protocol::account::{AccountHeader, AccountStorageHeader, StorageSlotName}; use miden_protocol::block::BlockNumber; use miden_protocol::block::account_tree::AccountWitness; -use super::State; -use crate::COMPONENT; -use crate::account_state_forest::AccountStorageMapResult; -use crate::errors::{DatabaseError, GetAccountError}; - -impl State { - /// Returns an account witness and optionally account details at a specific block. - /// - /// The witness is a Merkle proof of inclusion in the account tree, proving the account's - /// state commitment. If `details` is requested, the method also returns the account's code, - /// vault assets, and storage data. Account details are only available for public accounts. - /// - /// If `block_num` is provided, returns the state at that historical block; otherwise, returns - /// the latest state. Note that historical states are only available for recent blocks close - /// to the chain tip. - #[miden_instrument( - target = COMPONENT, - skip_all, - )] - pub async fn get_account( - &self, - account_request: AccountRequest, - ) -> Result { - let AccountRequest { block_num, account_id, details } = account_request; - - if details.is_some() && !account_id.is_public() { - return Err(GetAccountError::AccountNotPublic(account_id)); - } - - let (block_num, witness) = self.get_account_witness(block_num, account_id).await?; - - let details = if let Some(request) = details { - Some( - self.fetch_public_account_details(account_id, block_num, &witness, request) - .await?, - ) - } else { - None - }; - - Ok(AccountResponse { block_num, witness, details }) - } - - /// Returns an account witness (Merkle proof of inclusion in the account tree). - /// - /// If `block_num` is provided, returns the witness at that historical block; - /// otherwise, returns the witness at the latest block. - #[miden_instrument( - target = COMPONENT, - skip_all, - )] - async fn get_account_witness( - &self, - block_num: Option, - account_id: AccountId, - ) -> Result<(BlockNumber, AccountWitness), GetAccountError> { - self.with_inner_read_blocking(|inner_state| { - // Determine which block to query - let (block_num, witness) = if let Some(requested_block) = block_num { - // Historical query: use the account tree with history - let witness = inner_state - .account_tree - .open_at(account_id, requested_block) - .ok_or_else(|| { - let latest_block = inner_state.account_tree.block_number_latest(); - if requested_block > latest_block { - GetAccountError::UnknownBlock(requested_block) - } else { - GetAccountError::BlockPruned(requested_block) - } - })?; - (requested_block, witness) - } else { - // Latest query: use the latest state - let block_num = inner_state.account_tree.block_number_latest(); - let witness = inner_state.account_tree.open_latest(account_id); - (block_num, witness) - }; - - Ok((block_num, witness)) - }) - } - - /// Returns storage map details from the forest for a specific account and storage slot. - /// - /// The forest can only be used if all hashed keys in the storage map are known in the - /// reverse-key LRU cache. If any hashed key is unknown, the method returns `Ok(None)` to signal - /// that the caller should fall back to reconstructing the storage map details from the - /// database. - #[miden_instrument( - target = COMPONENT, - skip_all, - )] - fn get_storage_map_details_from_forest( - &self, - account_id: AccountId, - slot_name: &StorageSlotName, - block_num: BlockNumber, - ) -> Result, DatabaseError> { - self.with_forest_read_blocking(|forest| { - match forest - .get_storage_map_details_for_all_entries(account_id, slot_name.clone(), block_num) - .map_err(DatabaseError::MerkleError)? - { - AccountStorageMapResult::NotFound => Err(DatabaseError::StorageRootNotFound { - account_id, - slot_name: slot_name.to_string(), - block_num, - }), - AccountStorageMapResult::Details(details) => Ok(Some(details)), - AccountStorageMapResult::CannotReconstructKeysFromCache => Ok(None), - } - }) - } - - /// Returns vault details by reconstructing the vault from the database. - async fn reconstruct_vault_details_from_db( - &self, - account_id: AccountId, - block_num: BlockNumber, - ) -> Result { - let assets = self.db.select_account_vault_at_block(account_id, block_num).await?; - - if assets.len() > AccountVaultDetails::MAX_RETURN_ENTRIES { - return Ok(AccountVaultDetails::LimitExceeded); - } - - let keys = assets.iter().map(miden_protocol::asset::Asset::id); - - // The reverse-key caches are shared between the writer and all snapshots, so caching via - // the current snapshot's forest is visible everywhere. - self.with_forest_read_blocking(|forest| { - forest - .vault_key_cache - .put_many(keys.into_iter().map(|raw_key| (raw_key.hash(), raw_key))); - }); - - Ok(AccountVaultDetails::from_assets(assets)) - } - - /// Returns storage map details by reconstructing the storage map from the database. - async fn reconstruct_storage_map_details_from_db( - &self, - account_id: AccountId, - slot_name: StorageSlotName, - block_num: BlockNumber, - ) -> Result { - let details = self - .db - .reconstruct_storage_map_from_db( - account_id, - slot_name, - block_num, - Some(AccountStorageMapDetails::MAX_RETURN_ENTRIES), - ) - .await?; - - if let StorageMapEntries::AllEntries(entries) = &details.entries { - self.with_forest_read_blocking(|forest| { - forest.cache_storage_map_keys(entries.iter().map(|(raw_key, _)| *raw_key)); - }); - } - - Ok(details) - } - - /// Fetches the account details (code, vault, storage) for a public account at the specified - /// block. - /// - /// This method queries the database to fetch the account state and processes the detail - /// request to return only the requested information. - /// - /// For specific key queries (`SlotData::MapKeys`), the forest is used to provide SMT proofs. - /// Returns an error if the forest doesn't have data for the requested slot. - /// All-entries queries (`SlotData::All`) use the forest when all hashed keys are known in the - /// reverse-key LRU cache, otherwise they fall back to database reconstruction. - #[miden_instrument( - target = COMPONENT, - skip_all, - )] - async fn fetch_public_account_details( - &self, - account_id: AccountId, - block_num: BlockNumber, - witness: &AccountWitness, - detail_request: AccountDetailRequest, - ) -> Result { - let AccountDetailRequest { - code_commitment, - asset_vault_commitment, - storage_request, - } = detail_request; - - if !account_id.is_public() { - return Err(GetAccountError::AccountNotPublic(account_id)); - } - - // Validate block exists in the blockchain before querying the database - { - let latest_block_num = self.snapshot().latest_block_num(); - - if block_num > latest_block_num { - return Err(GetAccountError::UnknownBlock(block_num)); - } - } - - // Query account header and storage header together in a single DB call - let (account_header, storage_header) = self - .db - .select_account_header_with_storage_header_at_block(account_id, block_num) - .await? - .ok_or(GetAccountError::AccountNotFound(account_id, block_num))?; - - let should_apply_response_budget = - matches!(&storage_request, AccountStorageRequest::AllStorageMaps); - let storage_requests = expand_account_storage_request(storage_request, &storage_header); - - let account_code = match code_commitment { - Some(commitment) if commitment == account_header.code_commitment() => None, - Some(_) => { - self.db - .select_account_code_by_commitment(account_header.code_commitment()) - .await? - }, - None => None, - }; - - // Query account state forest for vault details on commitment mismatch. - // - // The forest can only reconstruct the vault if all hashed vault keys are known in the - // reverse-key LRU cache. If any hashed key is unknown, the forest returns `None` and we - // fall back to reconstructing the vault details from the database. - let vault_details = match asset_vault_commitment { - Some(commitment) if commitment == account_header.vault_root() => { - AccountVaultDetails::empty() - }, - Some(_) => { - let forest_details = self.with_forest_read_blocking(|forest| { - forest.get_vault_details(account_id, block_num).map_err(|err| { - DatabaseError::DataCorrupted(format!( - "failed to reconstruct vault for account {account_id} at block {block_num}: {err}" - )) - }) - })?; - - match forest_details { - Some(details) => details, - None => self.reconstruct_vault_details_from_db(account_id, block_num).await?, - } - }, - None => AccountVaultDetails::empty(), - }; - - // Split storage map requests into two categories: - // - slots with explicit keys (including proofs) - // - slots with "all entries" - let mut storage_map_details = - Vec::::with_capacity(storage_requests.len()); - let mut map_keys_requests = Vec::new(); - let mut all_entries_requests = Vec::new(); - let mut storage_request_slots = Vec::with_capacity(storage_requests.len()); - - for (index, StorageMapRequest { slot_name, slot_data }) in - storage_requests.into_iter().enumerate() - { - storage_request_slots.push(slot_name.clone()); - match slot_data { - SlotData::MapKeys(keys) => { - map_keys_requests.push((index, slot_name, keys)); - }, - SlotData::All => { - all_entries_requests.push((index, slot_name)); - }, - } - } - - let mut storage_map_details_by_index = vec![None; storage_request_slots.len()]; - - // Handle slots with explicit key requests - if !map_keys_requests.is_empty() { - self.with_forest_read_blocking(|forest| { - for (index, slot_name, keys) in map_keys_requests { - let details = forest - .get_storage_map_details_for_keys( - account_id, - slot_name.clone(), - block_num, - &keys, - ) - .ok_or_else(|| DatabaseError::StorageRootNotFound { - account_id, - slot_name: slot_name.to_string(), - block_num, - })? - .map_err(DatabaseError::MerkleError)?; - storage_map_details_by_index[index] = Some(details); - } - Ok::<(), DatabaseError>(()) - })?; - } - - // Handle slots with "all entries" requests - for (index, slot_name) in all_entries_requests { - let details = match self - .get_storage_map_details_from_forest(account_id, &slot_name, block_num)? - { - Some(details) => details, - None => { - self.reconstruct_storage_map_details_from_db(account_id, slot_name, block_num) - .await? - }, - }; - storage_map_details_by_index[index] = Some(details); - } - - for (details, slot_name) in - storage_map_details_by_index.into_iter().zip(storage_request_slots.iter()) - { - let details = details.ok_or_else(|| DatabaseError::StorageRootNotFound { - account_id, - slot_name: slot_name.to_string(), - block_num, - })?; - storage_map_details.push(details); - } - - // In case of an "all storage maps" request we have to be careful: even with the per-slot - // limit of [`AccountStorageMapDetails::MAX_RETURN_ENTRIES`] we might go over the response - // size limit. Here we make sure that we're within that limit by potentially truncating the - // response. - if should_apply_response_budget { - return Ok(apply_all_storage_maps_response_budget( - block_num, - witness, - account_header, - account_code, - vault_details, - storage_header, - storage_map_details, - storage_request_slots, - MAX_ALL_STORAGE_MAPS_RESPONSE_PAYLOAD_WITH_BUDGET_RESERVED_FOR_LIMIT_EXCEEDED_SLOTS, - )); - } - - Ok(AccountDetails { - account_header, - account_code, - vault_details, - storage_details: AccountStorageDetails { - header: storage_header, - map_details: storage_map_details, - }, - }) - } -} - -// HELPERS -// ================================================================================================ - -/// Expand [`AccountStorageRequest`] to a vector of slot requests. -fn expand_account_storage_request( - storage_request: AccountStorageRequest, - storage_header: &AccountStorageHeader, -) -> Vec { - match storage_request { - AccountStorageRequest::None => Vec::new(), - AccountStorageRequest::Explicit(requests) => requests, - AccountStorageRequest::AllStorageMaps => storage_header - .slots() - .filter(|slot| slot.slot_type() == StorageSlotType::Map) - .map(|slot| StorageMapRequest { - slot_name: slot.name().clone(), - slot_data: SlotData::All, - }) - .collect(), - } -} - // This is intentionally conservative. Storage slot names can be up to u8::MAX bytes, and a // `limit_exceeded` map detail stores only the slot name plus the `too_many_entries` flag. const STORAGE_MAP_LIMIT_EXCEEDED_FIELD_MAX_LEN: usize = 263; // A conservative limit that makes sure that limit exceeded messages can be appended for all slots // in the response. -const MAX_ALL_STORAGE_MAPS_RESPONSE_PAYLOAD_WITH_BUDGET_RESERVED_FOR_LIMIT_EXCEEDED_SLOTS: usize = +pub(super) const MAX_ALL_STORAGE_MAPS_RESPONSE_PAYLOAD_WITH_BUDGET_RESERVED_FOR_LIMIT_EXCEEDED_SLOTS: usize = MAX_RESPONSE_PAYLOAD_BYTES - 256 * STORAGE_MAP_LIMIT_EXCEEDED_FIELD_MAX_LEN - 8192; // Conservative max length for storage map entries: key-value pairs, each one is four `fixed64` @@ -450,7 +67,7 @@ fn estimate_storage_map_details_field_len(details: &AccountStorageMapDetails) -> /// We reserve space for the "limit exceeded" responses in advance so we're safe to start appending /// "limit exceeded" at any point during iteration. #[expect(clippy::too_many_arguments)] -fn apply_all_storage_maps_response_budget( +pub(super) fn apply_all_storage_maps_response_budget( block_num: BlockNumber, witness: &AccountWitness, account_header: AccountHeader, @@ -507,7 +124,6 @@ fn apply_all_storage_maps_response_budget( }, } } - // TESTS // ================================================================================================ @@ -539,7 +155,8 @@ mod tests { use miden_protocol::testing::account_id::AccountIdBuilder; use miden_protocol::{EMPTY_WORD, Felt, Word}; - use super::{apply_all_storage_maps_response_budget, expand_account_storage_request}; + use super::super::expand_account_storage_request; + use super::apply_all_storage_maps_response_budget; fn storage_header() -> AccountStorageHeader { AccountStorageHeader::new(vec![ diff --git a/crates/store/src/state/view/batch_inputs.rs b/crates/store/src/state/view/batch_inputs.rs new file mode 100644 index 000000000..c1e6a5c40 --- /dev/null +++ b/crates/store/src/state/view/batch_inputs.rs @@ -0,0 +1,129 @@ +//! Batch input query for the block producer. +//! +//! Combines in-memory snapshot data (partial MMR) with database lookups, scoping the latter by +//! the view's tip so both sources describe the same block height. + +use std::collections::BTreeSet; + +use miden_node_proto::domain::batch::BatchInputs; +use miden_protocol::Word; +use miden_protocol::block::BlockNumber; +use miden_protocol::transaction::PartialBlockchain; + +use super::StateView; +use crate::errors::GetBatchInputsError; + +impl StateView { + /// Fetches the inputs for a transaction batch from the database. + /// + /// ## Inputs + /// + /// The function takes as input: + /// - The tx reference blocks are the set of blocks referenced by transactions in the batch. + /// - The unauthenticated note commitments are the set of commitments of unauthenticated notes + /// consumed by all transactions in the batch. For these notes, we attempt to find inclusion + /// proofs. Not all notes will exist in the DB necessarily, as some notes can be created and + /// consumed within the same batch. + /// + /// ## Outputs + /// + /// The function will return: + /// - A block inclusion proof for all tx reference blocks and for all blocks which are + /// referenced by a note inclusion proof. + /// - Note inclusion proofs for all notes that were found in the DB. + /// - The block header that the batch should reference, i.e. the latest known block. + pub async fn get_batch_inputs( + &self, + tx_reference_blocks: BTreeSet, + unauthenticated_note_commitments: BTreeSet, + ) -> Result { + if tx_reference_blocks.is_empty() { + return Err(GetBatchInputsError::TransactionBlockReferencesEmpty); + } + + // First we grab note inclusion proofs for the known notes. These proofs only prove that the + // note was included in a given block. We then also need to prove that each of those blocks + // is included in the chain. + let note_proofs = self + .db() + .select_note_inclusion_proofs(unauthenticated_note_commitments) + .await + .map_err(GetBatchInputsError::SelectNoteInclusionProofError)?; + + // The set of blocks that the notes are included in. + let note_blocks = note_proofs.values().map(|proof| proof.location().block_num()); + + // Collect all blocks we need to query without duplicates, which is: + // - all blocks for which we need to prove note inclusion. + // - all blocks referenced by transactions in the batch. + let mut blocks: BTreeSet = tx_reference_blocks; + blocks.extend(note_blocks); + + let latest_block_num = self.tip(); + + let highest_block_num = + *blocks.last().expect("we should have checked for empty block references"); + if highest_block_num > latest_block_num { + return Err(GetBatchInputsError::UnknownTransactionBlockReference { + highest_block_num, + latest_block_num, + }); + } + + // Remove the latest block from the to-be-tracked blocks as it will be the reference block + // for the batch itself and thus added to the MMR within the batch kernel, so there is no + // need to prove its inclusion. + blocks.remove(&latest_block_num); + + // SAFETY: + // - The latest block num was retrieved from the view's blockchain from which we will + // also retrieve the proofs, so it is guaranteed to exist in that chain. + // - We have checked that no block number in the blocks set is greater than latest block + // number *and* latest block num was removed from the set. Therefore only block + // numbers smaller than latest block num remain in the set. Therefore all the block + // numbers are guaranteed to exist in the chain state at latest block num. + let partial_mmr = + self.blockchain().partial_mmr_from_blocks(&blocks, latest_block_num).expect( + "latest block num should exist and all blocks in set should be < than latest block", + ); + + let batch_reference_block = latest_block_num; + + // Fetch the reference block of the batch as part of this query, so we can avoid looking it + // up in a separate DB access. + let mut headers = self + .db() + .select_block_headers(blocks.into_iter().chain(std::iter::once(batch_reference_block))) + .await + .map_err(GetBatchInputsError::SelectBlockHeaderError)?; + + // Find and remove the batch reference block as we don't want to add it to the chain MMR. + let header_index = headers + .iter() + .enumerate() + .find_map(|(index, header)| { + (header.block_num() == batch_reference_block).then_some(index) + }) + .expect("DB should have returned the header of the batch reference block"); + + // The order doesn't matter for PartialBlockchain::new, so swap remove is fine. + let batch_reference_block_header = headers.swap_remove(header_index); + + // SAFETY: This should not error because: + // - we're passing exactly the block headers that we've added to the partial MMR, + // - so none of the block headers block numbers should exceed the chain length of the + // partial MMR, + // - and we've added blocks to a BTreeSet, so there can be no duplicates. + // + // We construct headers and partial MMR in concert, so they are consistent. This is why we + // can call the unchecked constructor. + let partial_block_chain = PartialBlockchain::new_unchecked(partial_mmr, headers) + .expect("partial mmr and block headers should be consistent"); + + Ok(BatchInputs { + batch_reference_block_header, + note_proofs, + partial_block_chain, + }) + } +} diff --git a/crates/store/src/state/view/block.rs b/crates/store/src/state/view/block.rs new file mode 100644 index 000000000..a2ae77b8f --- /dev/null +++ b/crates/store/src/state/view/block.rs @@ -0,0 +1,49 @@ +//! Block header reads. + +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::block::{BlockHeader, BlockNumber}; +use miden_protocol::crypto::merkle::mmr::MmrProof; + +use super::StateView; +use crate::COMPONENT; +use crate::errors::GetBlockHeaderError; + +impl StateView { + /// Queries a [BlockHeader] from the database, and returns it alongside its inclusion proof. + /// + /// If [None] is given as the value of `block_num`, the data for the latest [BlockHeader] is + /// returned. + #[miden_instrument( + level = "debug", + target = COMPONENT, + skip_all, + err, + )] + pub async fn get_block_header( + &self, + block_num: Option, + include_mmr_proof: bool, + ) -> Result<(Option, Option), GetBlockHeaderError> { + // Resolve "latest" against the view's snapshot rather than the DB: mid-apply, the DB may + // already contain a block that the snapshot's blockchain cannot prove yet. Scoping the DB + // query by the view's tip keeps the header and MMR proof consistent. + let latest_block_num = self.tip(); + let block_num = block_num.unwrap_or(latest_block_num); + if block_num > latest_block_num { + return Ok((None, None)); + } + + let block_header = self.db().select_block_header_by_block_num(Some(block_num)).await?; + if let Some(header) = block_header { + let mmr_proof = if include_mmr_proof { + let mmr_proof = self.blockchain().open(header.block_num())?; + Some(mmr_proof) + } else { + None + }; + Ok((Some(header), mmr_proof)) + } else { + Ok((None, None)) + } + } +} diff --git a/crates/store/src/state/view/block_inputs.rs b/crates/store/src/state/view/block_inputs.rs new file mode 100644 index 000000000..e4175f075 --- /dev/null +++ b/crates/store/src/state/view/block_inputs.rs @@ -0,0 +1,155 @@ +//! Block input query for the block producer. +//! +//! Combines in-memory snapshot data (tree witnesses, partial MMR) with database lookups, scoping +//! the latter by the view's tip so both sources describe the same block height. + +use std::collections::{BTreeMap, BTreeSet}; + +use miden_protocol::Word; +use miden_protocol::account::AccountId; +use miden_protocol::block::account_tree::AccountWitness; +use miden_protocol::block::nullifier_tree::NullifierWitness; +use miden_protocol::block::{BlockInputs, BlockNumber}; +use miden_protocol::crypto::merkle::mmr::PartialMmr; +use miden_protocol::note::Nullifier; +use miden_protocol::transaction::PartialBlockchain; + +use super::StateView; +use crate::errors::GetBlockInputsError; + +type BlockInputWitnesses = ( + BlockNumber, + BTreeMap, + BTreeMap, + PartialMmr, +); + +impl StateView { + /// Returns data needed by the block producer to construct and prove the next block. + pub async fn get_block_inputs( + &self, + account_ids: Vec, + nullifiers: Vec, + unauthenticated_note_commitments: BTreeSet, + reference_blocks: BTreeSet, + ) -> Result { + // Get the note inclusion proofs from the DB. We do this first so we have to acquire the + // lock to the state just once. There we need the reference blocks of the note proofs to get + // their authentication paths in the chain MMR. + let unauthenticated_note_proofs = self + .db() + .select_note_inclusion_proofs(unauthenticated_note_commitments) + .await + .map_err(GetBlockInputsError::SelectNoteInclusionProofError)?; + + // The set of blocks that the notes are included in. + let note_proof_reference_blocks = + unauthenticated_note_proofs.values().map(|proof| proof.location().block_num()); + + // Collect all blocks we need to prove inclusion for, without duplicates. + let mut blocks = reference_blocks; + blocks.extend(note_proof_reference_blocks); + + let (latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr) = + self.get_block_inputs_witnesses(&mut blocks, &account_ids, &nullifiers)?; + + // Fetch the block headers for all blocks in the partial MMR plus the latest one which will + // be used as the previous block header of the block being built. + let mut headers = self + .db() + .select_block_headers(blocks.into_iter().chain(std::iter::once(latest_block_number))) + .await + .map_err(GetBlockInputsError::SelectBlockHeaderError)?; + + // Find and remove the latest block as we must not add it to the chain MMR, since it is not + // yet in the chain. + let latest_block_header_index = headers + .iter() + .enumerate() + .find_map(|(index, header)| { + (header.block_num() == latest_block_number).then_some(index) + }) + .expect("DB should have returned the header of the latest block header"); + + // The order doesn't matter for PartialBlockchain::new, so swap remove is fine. + let latest_block_header = headers.swap_remove(latest_block_header_index); + + // SAFETY: This should not error because: + // - we're passing exactly the block headers that we've added to the partial MMR, + // - so none of the block header's block numbers should exceed the chain length of the + // partial MMR, + // - and we've added blocks to a BTreeSet, so there can be no duplicates. + // + // We construct headers and partial MMR in concert, so they are consistent. This is why we + // can call the unchecked constructor. + let partial_block_chain = PartialBlockchain::new_unchecked(partial_mmr, headers) + .expect("partial mmr and block headers should be consistent"); + + Ok(BlockInputs::new( + latest_block_header, + partial_block_chain, + account_witnesses, + nullifier_witnesses, + unauthenticated_note_proofs, + )) + } + + /// Get account and nullifier witnesses for the requested account IDs and nullifier as well as + /// the [`PartialMmr`] for the given blocks. The MMR won't contain the latest block and its + /// number is removed from `blocks` and returned separately. + fn get_block_inputs_witnesses( + &self, + blocks: &mut BTreeSet, + account_ids: &[AccountId], + nullifiers: &[Nullifier], + ) -> Result { + self.with_inner_read_blocking(|inner| { + let latest_block_number = inner.latest_block_num(); + + // If `blocks` is empty, use the latest block number which will never trigger the error. + let highest_block_number = blocks.last().copied().unwrap_or(latest_block_number); + if highest_block_number > latest_block_number { + return Err(GetBlockInputsError::UnknownBatchBlockReference { + highest_block_number, + latest_block_number, + }); + } + + // The latest block is not yet in the chain MMR, so we can't (and don't need to) prove + // its inclusion in the chain. + blocks.remove(&latest_block_number); + + // Fetch the partial MMR at the state of the latest block with authentication paths for + // the provided set of blocks. + // + // SAFETY: + // - The latest block num was retrieved from the inner blockchain from which we will + // also retrieve the proofs, so it is guaranteed to exist in that chain. + // - We have checked that no block number in the blocks set is greater than latest block + // number *and* latest block num was removed from the set. Therefore only block + // numbers smaller than latest block num remain in the set. Therefore all the block + // numbers are guaranteed to exist in the chain state at latest block num. + let partial_mmr = + inner.blockchain.partial_mmr_from_blocks(blocks, latest_block_number).expect( + "latest block num should exist and all blocks in set should be < than latest block", + ); + + // Fetch witnesses for all accounts. + let account_witnesses = account_ids + .iter() + .copied() + .map(|account_id| (account_id, inner.account_tree.open_latest(account_id))) + .collect::>(); + + // Fetch witnesses for all nullifiers. We don't check whether the nullifiers are spent + // or not as this is done as part of proposing the block. + let nullifier_witnesses: BTreeMap = nullifiers + .iter() + .copied() + .map(|nullifier| (nullifier, inner.nullifier_tree.open(&nullifier))) + .collect(); + + Ok((latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr)) + }) + } +} diff --git a/crates/store/src/state/view/mod.rs b/crates/store/src/state/view/mod.rs new file mode 100644 index 000000000..94c8dd821 --- /dev/null +++ b/crates/store/src/state/view/mod.rs @@ -0,0 +1,123 @@ +//! Request-scoped, consistent read view of the store. +//! +//! All store reads go through [`StateView`]: it pins one state snapshot for its whole lifetime, +//! and every database query it exposes is scoped by that snapshot's block height. This makes it +//! impossible to implement a read whose tree and database halves observe different chain tips — +//! mid-apply, the database may already contain rows for a block the snapshot cannot prove yet. +//! +//! The submodules hold the read endpoints, all `impl StateView`; the snapshot internals +//! ([`StateSnapshot`]) are only visible within this module tree, so no other part of the store +//! can reach the trees directly. + +use std::ops::RangeInclusive; +use std::sync::Arc; + +use miden_protocol::block::{BlockNumber, Blockchain}; +use tracing::Span; + +use crate::account_state_forest::{AccountStateForest, AccountStateForestBackendReader}; +use crate::db::Db; +use crate::errors::RangeBeyondTip; +use crate::state::State; + +mod snapshot; +pub(in crate::state) use snapshot::{SNAPSHOTS_LIVE_WARN_THRESHOLD, SnapshotGuard, StateSnapshot}; + +mod account; +mod batch_inputs; +mod block; +mod block_inputs; +mod note; +mod sync; + +mod transaction_inputs; +pub use transaction_inputs::TransactionInputs; + +// STATE VIEW +// ================================================================================================ + +/// A consistent read view of the store, pinned at its snapshot's block height. +/// +/// Obtained from [`State::view`]; create one per request and drop it when the request completes. +/// Holding a view pins a snapshot generation (and thereby the `RocksDB` snapshots backing the +/// trees), so it must not be stored in long-lived structs; leaked or slow readers are reported by +/// the store's snapshot-lifetime warnings. +/// +/// Reads that are technically not block-scoped (e.g. content-addressed note scripts) also live +/// here so that every read path flows through a single, consistently-scoped type. +pub struct StateView { + snapshot: Arc, + db: Arc, +} + +impl State { + /// Returns a read view pinned at the current chain tip (wait-free, no lock required). + /// + /// The view is frozen: it is unaffected if the writer publishes a new snapshot while it is + /// held. + pub fn view(&self) -> StateView { + StateView { + snapshot: self.latest_snapshot.load_full(), + db: Arc::clone(&self.db), + } + } +} + +impl StateView { + /// The chain tip this view is pinned at. + pub fn tip(&self) -> BlockNumber { + self.snapshot.latest_block_num() + } + + /// Returns the pinned snapshot's blockchain MMR. + /// + /// The MMR is the only part of the snapshot that is purely in-memory and therefore safe to + /// access directly on an async worker thread. The account and nullifier trees may be backed + /// by `RocksDB` and are deliberately not reachable here — they must be accessed through + /// [`Self::with_inner_read_blocking`]. + fn blockchain(&self) -> &Blockchain { + &self.snapshot.blockchain + } + + /// Returns the database handle. + /// + /// Queries whose results depend on the chain tip must be scoped by [`Self::tip`] (or a block + /// number validated against it), never by a tip obtained elsewhere. + fn db(&self) -> &Db { + &self.db + } + + /// Ensures the given block range does not extend beyond this view's chain tip. + /// + /// Every range-scoped read on this type calls this before touching the database, so callers + /// never need to pre-validate ranges themselves. + fn check_range(&self, range: &RangeInclusive) -> Result<(), RangeBeyondTip> { + let tip = self.tip(); + if *range.end() > tip { + return Err(RangeBeyondTip { chain_tip: tip, block_to: *range.end() }); + } + Ok(()) + } + + /// Runs a synchronous read-only operation over the pinned state snapshot on Tokio's blocking + /// path. + /// + /// The account and nullifier trees may be backed by `RocksDB`, so tree access must not run on + /// an async worker thread directly. This helper preserves the current tracing span while + /// moving the closure body into `block_in_place`. + fn with_inner_read_blocking(&self, f: impl FnOnce(&StateSnapshot) -> R) -> R { + let span = Span::current(); + tokio::task::block_in_place(|| span.in_scope(|| f(&self.snapshot))) + } + + /// Runs a synchronous read-only operation over the account state forest snapshot on Tokio's + /// blocking path. + /// + /// See [`Self::with_inner_read_blocking`] for why this uses `block_in_place`. + fn with_forest_read_blocking( + &self, + f: impl FnOnce(&AccountStateForest) -> R, + ) -> R { + self.with_inner_read_blocking(|snapshot| f(&snapshot.forest)) + } +} diff --git a/crates/store/src/state/view/note.rs b/crates/store/src/state/view/note.rs new file mode 100644 index 000000000..82958b1a3 --- /dev/null +++ b/crates/store/src/state/view/note.rs @@ -0,0 +1,32 @@ +//! Note reads. +//! +//! These are content-addressed lookups and technically not block-scoped, but they live on +//! [`StateView`] so that every read path flows through the same type. + +use miden_protocol::Word; +use miden_protocol::note::{NoteId, NoteScript}; + +use super::StateView; +use crate::db::NoteRecord; +use crate::errors::DatabaseError; + +impl StateView { + /// Queries a list of notes from the database. + /// + /// If the provided list of [`NoteId`] given is empty or no note matches the provided + /// [`NoteId`] an empty list is returned. + pub async fn get_notes_by_id( + &self, + note_ids: Vec, + ) -> Result, DatabaseError> { + self.db().select_notes_by_id(note_ids).await + } + + /// Returns the script for a note by its root. + pub async fn get_note_script_by_root( + &self, + root: Word, + ) -> Result, DatabaseError> { + self.db().select_note_script_by_root(root).await + } +} diff --git a/crates/store/src/state/snapshot.rs b/crates/store/src/state/view/snapshot.rs similarity index 61% rename from crates/store/src/state/snapshot.rs rename to crates/store/src/state/view/snapshot.rs index 364acb052..b0d34f1ac 100644 --- a/crates/store/src/state/snapshot.rs +++ b/crates/store/src/state/view/snapshot.rs @@ -1,6 +1,6 @@ //! In-memory snapshot machinery for lock-free reads. //! -//! Readers access the store's tree state through immutable [`InMemoryState`] snapshots published +//! Readers access the store's tree state through immutable [`StateSnapshot`] snapshots published //! by the block writer after each committed block. [`SnapshotGuard`] tracks how many snapshot //! generations are pinned by readers, since each generation pins a `RocksDB` snapshot. @@ -11,12 +11,10 @@ use std::time::{Duration, Instant}; use miden_protocol::block::nullifier_tree::NullifierTree; use miden_protocol::block::{BlockNumber, Blockchain}; use miden_protocol::crypto::merkle::smt::LargeSmt; -use tracing::Span; use crate::COMPONENT; use crate::account_state_forest::{AccountStateForest, AccountStateForestBackendReader}; use crate::accounts::AccountTreeWithHistory; -use crate::state::State; use crate::state::loader::TreeStorageReader; /// Snapshot lifetime above which [`SnapshotGuard`] logs a warning on release. @@ -31,14 +29,14 @@ const SNAPSHOT_LIFETIME_WARN_THRESHOLD: Duration = Duration::from_secs(10); /// Steady state is 1-2 generations: the just-published snapshot plus predecessors briefly pinned /// by in-flight requests. A sustained higher count means slow or leaked readers are holding old /// generations alive (see [`SnapshotGuard`]). -pub(crate) const SNAPSHOTS_LIVE_WARN_THRESHOLD: u64 = 4; +pub(in crate::state) const SNAPSHOTS_LIVE_WARN_THRESHOLD: u64 = 4; // SNAPSHOT GUARD // ================================================================================================ -/// RAII member of [`InMemoryState`] that tracks the number of live snapshot generations. +/// RAII member of [`StateSnapshot`] that tracks the number of live snapshot generations. /// -/// [`InMemoryState`] is dropped exactly when the last [`Arc`] reference to it is released, so the +/// [`StateSnapshot`] is dropped exactly when the last [`Arc`] reference to it is released, so the /// shared counter reports how many distinct snapshot generations are currently pinned by readers. /// A sustained count above 1-2 means slow readers are holding old generations alive. Each /// generation pins a `RocksDB` snapshot, which delays garbage collection of superseded key @@ -47,14 +45,14 @@ pub(crate) const SNAPSHOTS_LIVE_WARN_THRESHOLD: u64 = 4; /// /// Readers are expected to be request-scoped, so snapshot lifetimes should be well under a block /// interval. A lifetime exceeding [`SNAPSHOT_LIFETIME_WARN_THRESHOLD`] is logged at warn level. -pub(crate) struct SnapshotGuard { +pub(in crate::state) struct SnapshotGuard { live: Arc, created_at: Instant, block_num: BlockNumber, } impl SnapshotGuard { - pub(crate) fn new(live: Arc, block_num: BlockNumber) -> Self { + pub(in crate::state) fn new(live: Arc, block_num: BlockNumber) -> Self { live.fetch_add(1, Ordering::Relaxed); Self { live, @@ -90,68 +88,49 @@ impl Drop for SnapshotGuard { } } -// IN-MEMORY STATE +// STATE SNAPSHOT // ================================================================================================ -/// Immutable snapshot of the in-memory tree state published after each committed block. +/// Immutable snapshot of the store's tree state published after each committed block. /// /// The trees are backed by read-only snapshot storage ([`TreeStorageReader`] / /// [`AccountStateForestBackendReader`]), so any number of readers can access the data concurrently /// without holding a lock and without blocking the writer. -pub(crate) struct InMemoryState { - pub(crate) nullifier_tree: NullifierTree>, - pub(crate) blockchain: Blockchain, - pub(crate) account_tree: AccountTreeWithHistory, - pub(crate) forest: AccountStateForest, +/// +/// The writer and lifecycle can *construct* snapshots via [`Self::new`], but the fields are only +/// readable within the view module tree: every read outside it must go through +/// [`StateView`](super::StateView). +pub(in crate::state) struct StateSnapshot { + pub(super) nullifier_tree: NullifierTree>, + pub(super) blockchain: Blockchain, + pub(super) account_tree: AccountTreeWithHistory, + pub(super) forest: AccountStateForest, /// Keeps the live-snapshot count accurate; see [`SnapshotGuard`]. - pub(crate) _guard: SnapshotGuard, + _guard: SnapshotGuard, } -impl InMemoryState { +impl StateSnapshot { + /// Assembles a snapshot from reader views of the trees and the guard tracking its generation. + pub(in crate::state) fn new( + nullifier_tree: NullifierTree>, + blockchain: Blockchain, + account_tree: AccountTreeWithHistory, + forest: AccountStateForest, + guard: SnapshotGuard, + ) -> Self { + Self { + nullifier_tree, + blockchain, + account_tree, + forest, + _guard: guard, + } + } + /// Returns the latest block number. - pub(crate) fn latest_block_num(&self) -> BlockNumber { + pub(super) fn latest_block_num(&self) -> BlockNumber { self.blockchain .chain_tip() .expect("chain should always have at least the genesis block") } } - -// SNAPSHOT HELPERS -// ================================================================================================ - -impl State { - /// Returns the current in-memory state snapshot (wait-free, no lock required). - /// - /// The returned snapshot is a frozen view: it is unaffected if the writer publishes a new - /// snapshot while it is held. - pub(super) fn snapshot(&self) -> Arc { - self.in_memory.load_full() - } - - /// Runs a synchronous read-only operation over the current in-memory state snapshot on Tokio's - /// blocking path. - /// - /// The account and nullifier trees may be backed by `RocksDB`, so tree access must not run on - /// an async worker thread directly. This helper preserves the current tracing span while - /// moving the closure body into `block_in_place`. - pub(super) fn with_inner_read_blocking(&self, f: impl FnOnce(&InMemoryState) -> R) -> R { - let span = Span::current(); - tokio::task::block_in_place(|| { - span.in_scope(|| { - let snapshot = self.snapshot(); - f(&snapshot) - }) - }) - } - - /// Runs a synchronous read-only operation over the account state forest snapshot on Tokio's - /// blocking path. - /// - /// See [`Self::with_inner_read_blocking`] for why this uses `block_in_place`. - pub(super) fn with_forest_read_blocking( - &self, - f: impl FnOnce(&AccountStateForest) -> R, - ) -> R { - self.with_inner_read_blocking(|snapshot| f(&snapshot.forest)) - } -} diff --git a/crates/store/src/state/sync_state.rs b/crates/store/src/state/view/sync.rs similarity index 72% rename from crates/store/src/state/sync_state.rs rename to crates/store/src/state/view/sync.rs index 54b67d1b9..1511846ff 100644 --- a/crates/store/src/state/sync_state.rs +++ b/crates/store/src/state/view/sync.rs @@ -5,7 +5,7 @@ use miden_protocol::account::AccountId; use miden_protocol::block::{BlockHeader, BlockNumber, BlockSignatures}; use miden_protocol::crypto::merkle::mmr::{Forest, MmrDelta, MmrProof}; -use super::State; +use super::StateView; use crate::COMPONENT; use crate::db::models::queries::StorageMapValuesPage; use crate::db::{AccountVaultValue, NoteSyncUpdate, NullifierInfo}; @@ -14,18 +14,25 @@ use crate::errors::{DatabaseError, NoteSyncError, StateSyncError}; // STATE SYNCHRONIZATION ENDPOINTS // ================================================================================================ -impl State { +impl StateView { /// Returns the complete transaction records for the specified accounts within the specified /// block range, including state commitments and note IDs. + /// + /// Returns [`RangeBeyondTip`](crate::errors::RangeBeyondTip) if the range extends beyond this + /// view's chain tip. pub async fn sync_transactions( &self, account_ids: Vec, block_range: RangeInclusive, ) -> Result<(BlockNumber, Vec), DatabaseError> { - self.db.select_transactions_records(account_ids, block_range).await + self.check_range(&block_range)?; + self.db().select_transactions_records(account_ids, block_range).await } /// Returns the chain MMR delta and the `block_to` block header for the specified block range. + /// + /// Returns [`RangeBeyondTip`](crate::errors::RangeBeyondTip) if the range extends beyond this + /// view's chain tip. #[miden_instrument( level = "debug", target = COMPONENT, @@ -36,13 +43,15 @@ impl State { &self, block_range: RangeInclusive, ) -> Result<(MmrDelta, BlockHeader, BlockSignatures), StateSyncError> { + self.check_range(&block_range)?; + let block_from = *block_range.start(); let block_to = *block_range.end(); - // SAFETY: block_to has been validated to be <= the effective tip (chain tip or latest - // proven block) by the caller, so it must exist in the database. + // SAFETY: block_to <= this view's tip (checked above), so it is committed and must exist in + // the database. let (block_header, signatures) = self - .db + .db() .select_block_header_and_signatures_by_block_num(block_to) .await? .expect("block_to should exist in the database"); @@ -71,8 +80,7 @@ impl State { let to_forest = block_to.as_usize(); let mmr_delta = self - .snapshot() - .blockchain + .blockchain() .as_mmr() .get_delta( Forest::new(from_forest).expect("from_forest fits in u32"), @@ -91,6 +99,9 @@ impl State { /// /// Also returns the last block number checked. If this equals `block_range.end()`, the /// sync is complete. + /// + /// Returns [`RangeBeyondTip`](crate::errors::RangeBeyondTip) if the range extends beyond this + /// view's chain tip. #[miden_instrument( level = "debug", target = COMPONENT, @@ -102,25 +113,22 @@ impl State { note_tags: Vec, block_range: RangeInclusive, ) -> Result<(Vec<(NoteSyncUpdate, MmrProof)>, BlockNumber), NoteSyncError> { + self.check_range(&block_range)?; + let block_end = *block_range.end(); // The MMR at forest N contains proofs for blocks 0..N-1, so we use block_end + 1 to include - // the proof for block_end. SAFETY: it is ensured that block_end <= chain_tip, and the - // blockchain MMR always has at least chain_tip + 1 leaves. + // the proof for block_end. SAFETY: block_end <= this view's tip (checked above), and the + // view's blockchain MMR always has at least tip + 1 leaves. let mmr_checkpoint = block_end + 1; - let note_syncs = self.db.get_note_sync_multi(block_range, note_tags.into()).await?; + let note_syncs = self.db().get_note_sync_multi(block_range, note_tags.into()).await?; let mut results = Vec::new(); - { - let snapshot = self.snapshot(); - - for note_sync in note_syncs { - let mmr_proof = snapshot - .blockchain - .open_at(note_sync.block_header.block_num(), mmr_checkpoint)?; - results.push((note_sync, mmr_proof)); - } + for note_sync in note_syncs { + let mmr_proof = + self.blockchain().open_at(note_sync.block_header.block_num(), mmr_checkpoint)?; + results.push((note_sync, mmr_proof)); } // if results is empty, return `block_end` since the sync is complete. @@ -130,13 +138,18 @@ impl State { Ok((results, last_block_checked)) } + /// Returns nullifiers matching the given prefixes that were created within a block range. + /// + /// Returns [`RangeBeyondTip`](crate::errors::RangeBeyondTip) if the range extends beyond this + /// view's chain tip. pub async fn sync_nullifiers( &self, prefix_len: u32, nullifier_prefixes: Vec, block_range: RangeInclusive, ) -> Result<(Vec, BlockNumber), DatabaseError> { - self.db + self.check_range(&block_range)?; + self.db() .select_nullifiers_by_prefix(prefix_len, nullifier_prefixes, block_range) .await } @@ -145,20 +158,28 @@ impl State { // -------------------------------------------------------------------------------------------- /// Returns account vault updates for specified account within a block range. + /// + /// Returns [`RangeBeyondTip`](crate::errors::RangeBeyondTip) if the range extends beyond this + /// view's chain tip. pub async fn sync_account_vault( &self, account_id: AccountId, block_range: RangeInclusive, ) -> Result<(BlockNumber, Vec), DatabaseError> { - self.db.get_account_vault_sync(account_id, block_range).await + self.check_range(&block_range)?; + self.db().get_account_vault_sync(account_id, block_range).await } /// Returns storage map values for syncing within a block range. + /// + /// Returns [`RangeBeyondTip`](crate::errors::RangeBeyondTip) if the range extends beyond this + /// view's chain tip. pub async fn sync_account_storage_maps( &self, account_id: AccountId, block_range: RangeInclusive, ) -> Result { - self.db.select_storage_map_sync_values(account_id, block_range, None).await + self.check_range(&block_range)?; + self.db().select_storage_map_sync_values(account_id, block_range, None).await } } diff --git a/crates/store/src/state/view/transaction_inputs.rs b/crates/store/src/state/view/transaction_inputs.rs new file mode 100644 index 000000000..a38fae6a1 --- /dev/null +++ b/crates/store/src/state/view/transaction_inputs.rs @@ -0,0 +1,93 @@ +//! Transaction input query for the block producer's transaction validation. + +use std::collections::HashSet; +use std::ops::ControlFlow; + +use miden_node_utils::formatting::format_array; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::Word; +use miden_protocol::account::AccountId; +use miden_protocol::note::Nullifier; + +use super::StateView; +use crate::COMPONENT; +use crate::db::NullifierInfo; +use crate::errors::DatabaseError; + +/// Store-level inputs for validating a proven transaction. +#[derive(Debug, Default)] +pub struct TransactionInputs { + pub account_commitment: Word, + pub nullifiers: Vec, + pub found_unauthenticated_notes: HashSet, + pub new_account_id_prefix_is_unique: Option, +} + +impl StateView { + /// Returns data needed by the block producer to verify transactions validity. + #[miden_instrument( + target = COMPONENT, + skip_all, + fields( + account.id=%account_id, + nullifiers = %format_array(nullifiers), + ), + )] + pub async fn get_transaction_inputs( + &self, + account_id: AccountId, + nullifiers: &[Nullifier], + unauthenticated_note_commitments: Vec, + ) -> Result { + let tree_inputs = self.with_inner_read_blocking(|inner| { + let account_commitment = inner.account_tree.get_latest_commitment(account_id); + + let new_account_id_prefix_is_unique = if account_commitment.is_empty() { + Some(!inner.account_tree.contains_account_id_prefix_in_latest(account_id.prefix())) + } else { + None + }; + + // Non-unique account Id prefixes for new accounts are not allowed, so the transaction + // cannot be valid and the response is already complete. + if let Some(false) = new_account_id_prefix_is_unique { + return ControlFlow::Break(TransactionInputs { + new_account_id_prefix_is_unique, + ..Default::default() + }); + } + + let nullifiers = nullifiers + .iter() + .map(|nullifier| NullifierInfo { + nullifier: *nullifier, + block_num: inner.nullifier_tree.get_block_num(nullifier).unwrap_or_default(), + }) + .collect(); + + ControlFlow::Continue((account_commitment, nullifiers, new_account_id_prefix_is_unique)) + }); + // `Break` carries a complete response (duplicate account ID prefix), so it is returned + // as-is without the note lookup below; `Continue` carries the tree reads needed to build + // the full response. + let (account_commitment, nullifiers, new_account_id_prefix_is_unique) = match tree_inputs { + ControlFlow::Continue(inputs) => inputs, + ControlFlow::Break(response) => return Ok(response), + }; + + // Scope the note lookup by the view's tip so the result is consistent with the tree reads + // above: mid-apply, the DB may already contain notes from a block the snapshot does not + // include yet. + let found_unauthenticated_notes = self + .db() + .select_existing_note_commitments(unauthenticated_note_commitments, self.tip()) + .await?; + + Ok(TransactionInputs { + account_commitment, + nullifiers, + found_unauthenticated_notes, + new_account_id_prefix_is_unique, + }) + } +} diff --git a/crates/store/src/state/apply_block.rs b/crates/store/src/state/writer/apply_block.rs similarity index 100% rename from crates/store/src/state/apply_block.rs rename to crates/store/src/state/writer/apply_block.rs diff --git a/crates/store/src/state/apply_proof.rs b/crates/store/src/state/writer/apply_proof.rs similarity index 83% rename from crates/store/src/state/apply_proof.rs rename to crates/store/src/state/writer/apply_proof.rs index cd7b87423..ecd4e5af5 100644 --- a/crates/store/src/state/apply_proof.rs +++ b/crates/store/src/state/writer/apply_proof.rs @@ -4,7 +4,7 @@ use miden_protocol::block::{BlockNumber, BlockProof}; use miden_protocol::utils::serde::Deserializable; use crate::COMPONENT; -use crate::state::{Finality, ProofNotification, ProofWriter}; +use crate::state::{ProofNotification, ProofWriter}; impl ProofWriter { /// Saves a block proof, advances the proven-in-sequence tip, and notifies replica subscribers. @@ -26,13 +26,13 @@ impl ProofWriter { block_num: BlockNumber, proof_bytes: Vec, ) -> anyhow::Result<()> { - let expected = self.proven_tip.read().child(); + let expected = self.state.proven_tip().child(); ensure!( block_num == expected, "out-of-sequence proof: expected block {expected}, got {block_num}", ); - let committed_tip = self.chain_tip(Finality::Committed); + let committed_tip = self.state.committed_tip(); ensure!( block_num <= committed_tip, "proof for uncommitted block {block_num} exceeds committed tip {committed_tip}", @@ -40,11 +40,12 @@ impl ProofWriter { verify_block_proof(block_num, &proof_bytes)?; - self.block_store.commit_proof(block_num, &proof_bytes).await?; - self.proof_cache + self.state.block_store.commit_proof(block_num, &proof_bytes).await?; + self.state + .proof_cache .push(block_num, ProofNotification::new(block_num, proof_bytes)) .expect("proof cache receives sequential block numbers"); - self.proven_tip.advance(block_num); + self.state.proven_tip.advance(block_num); Ok(()) } } diff --git a/crates/store/src/state/writer.rs b/crates/store/src/state/writer/mod.rs similarity index 92% rename from crates/store/src/state/writer.rs rename to crates/store/src/state/writer/mod.rs index 168d9a619..2dae7f205 100644 --- a/crates/store/src/state/writer.rs +++ b/crates/store/src/state/writer/mod.rs @@ -2,8 +2,14 @@ //! //! A single [`WriteWorker`] task owns the mutable trees and processes incoming [`WriteRequest`]s //! one at a time via an mpsc channel. After each successful commit it publishes a new -//! [`InMemoryState`] snapshot via an [`ArcSwap`], making the updated trees immediately visible to +//! [`StateSnapshot`] snapshot via an [`ArcSwap`], making the updated trees immediately visible to //! wait-free readers. +//! +//! The submodules hold the [`BlockWriter`] and [`ProofWriter`](crate::state::ProofWriter) +//! capability entry points that feed this worker. + +mod apply_block; +mod apply_proof; use std::fmt::Display; use std::sync::Arc; @@ -35,14 +41,8 @@ use crate::db::{Db, NoteRecord}; use crate::errors::{ApplyBlockError, InvalidBlockError}; use crate::state::block_lifecycle::{BlockLifecycle, lifecycle_events_enabled}; use crate::state::loader::TreeStorage; -use crate::state::{ - BlockCache, - BlockNotification, - BlockWriter, - InMemoryState, - SNAPSHOTS_LIVE_WARN_THRESHOLD, - SnapshotGuard, -}; +use crate::state::view::{SNAPSHOTS_LIVE_WARN_THRESHOLD, SnapshotGuard, StateSnapshot}; +use crate::state::{BlockCache, BlockNotification, BlockWriter}; use crate::{COMPONENT, HistoricalError, LOG_TARGET}; // WRITE REQUEST @@ -57,7 +57,8 @@ pub(super) struct WriteRequest { impl BlockWriter { /// Apply changes of a new block to the DB and in-memory data structures. /// - /// Blocks are forwarded to the [`WriteWorker`] task, which processes them one at a time. + /// Blocks are forwarded to the store's write worker task, which processes them one at a + /// time. /// Readers are unaffected while a block is being applied: they keep reading from the previous /// in-memory snapshot until the writer atomically publishes the new one. #[miden_instrument( @@ -82,27 +83,27 @@ impl BlockWriter { /// /// The writer owns the writable trees directly, so no locks are held at any point: validation and /// mutation-computation read the owned trees, the DB commit runs without touching them, and the -/// new [`InMemoryState`] snapshot is published atomically at the end. +/// new [`StateSnapshot`] snapshot is published atomically at the end. pub(super) struct WriteWorker { - pub db: Arc, - pub block_store: Arc, + pub(in crate::state) db: Arc, + pub(in crate::state) block_store: Arc, /// Atomically swappable pointer through which new snapshots are published. - pub in_memory: Arc>, - pub committed_tip_tx: Arc>, - pub block_cache: BlockCache, - pub rx: mpsc::Receiver, + pub(in crate::state) latest_snapshot: Arc>, + pub(in crate::state) committed_tip_tx: Arc>, + pub(in crate::state) block_cache: BlockCache, + pub(in crate::state) rx: mpsc::Receiver, /// Token signalling node shutdown; the writer stops accepting new requests once cancelled. - pub shutdown: CancellationToken, + pub(in crate::state) shutdown: CancellationToken, /// The mutable nullifier tree owned by this writer. - pub nullifier_tree: NullifierTree>, + pub(in crate::state) nullifier_tree: NullifierTree>, /// The mutable account tree owned by this writer. - pub account_tree: AccountTreeWithHistory, + pub(in crate::state) account_tree: AccountTreeWithHistory, /// The blockchain MMR owned by this writer. - pub blockchain: Blockchain, + pub(in crate::state) blockchain: Blockchain, /// The mutable account state forest owned by this writer. - pub forest: AccountStateForest, + pub(in crate::state) forest: AccountStateForest, /// Shared counter of live snapshot generations, for observability. - pub snapshots_live: Arc, + pub(in crate::state) snapshots_live: Arc, } /// Note records and state mutations computed from a validated block, before any modifications. @@ -220,7 +221,7 @@ impl WriteWorker { // Atomically publish the new state. Readers that call `snapshot()` after this point will // see the updated state. Readers holding the old snapshot continue unaffected. - self.in_memory.store(snapshot); + self.latest_snapshot.store(snapshot); let snapshots_live = self.check_live_snapshots(block_num); miden_span_record!(snapshots.live = snapshots_live); @@ -306,7 +307,7 @@ impl WriteWorker { nullifier_tree_update: NullifierMutationSet, account_tree_update: AccountMutationSet, account_forest_update: PreparedAccountStateForestBlockUpdate, - ) -> Arc { + ) -> Arc { self.nullifier_tree .apply_mutations(nullifier_tree_update) .unwrap_or_else(|error| { @@ -325,16 +326,15 @@ impl WriteWorker { Self::abort_after_post_commit_failure("account-state forest", &error) }); - Arc::new(InMemoryState { - nullifier_tree: self - .nullifier_tree + Arc::new(StateSnapshot::new( + self.nullifier_tree .reader() .expect("nullifier tree snapshot creation should not fail"), - account_tree: self.account_tree.reader(), - blockchain: self.blockchain.clone(), - forest: self.forest.reader().expect("forest snapshot creation should not fail"), - _guard: SnapshotGuard::new(Arc::clone(&self.snapshots_live), block_num), - }) + self.blockchain.clone(), + self.account_tree.reader(), + self.forest.reader().expect("forest snapshot creation should not fail"), + SnapshotGuard::new(Arc::clone(&self.snapshots_live), block_num), + )) } /// Terminates after a persistent state failure that occurred after the canonical DB commit. From d9b0f04f90b92e61a34688a8858fe06e338bbb5e Mon Sep 17 00:00:00 2001 From: sergerad Date: Fri, 31 Jul 2026 13:48:17 +1200 Subject: [PATCH 30/35] Add ScopedBlockNum --- crates/store/src/db/mod.rs | 42 ++++++++++++++----- crates/store/src/db/tests.rs | 14 ++++++- crates/store/src/lib.rs | 11 ++++- crates/store/src/state/mod.rs | 2 +- crates/store/src/state/view/account/mod.rs | 25 ++++++----- crates/store/src/state/view/mod.rs | 24 +++++++++-- crates/store/src/state/view/sync.rs | 18 ++++---- .../src/state/view/transaction_inputs.rs | 2 +- 8 files changed, 101 insertions(+), 37 deletions(-) diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 9b1abcbf8..900b70f84 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -54,6 +54,7 @@ use crate::db::models::queries::{ }; use crate::errors::{DatabaseError, NoteSyncError}; use crate::genesis::GenesisBlock; +use crate::state::{ScopedBlockNum, ScopedBlockRange}; use crate::{COMPONENT, LOG_TARGET}; const STORAGE_MAP_VALUE_PER_ROW_BYTES: usize = @@ -311,8 +312,9 @@ impl Db { &self, prefix_len: u32, nullifier_prefixes: Vec, - block_range: RangeInclusive, + block_range: ScopedBlockRange, ) -> Result<(Vec, BlockNumber)> { + let block_range = block_range.into_inner(); assert_eq!(prefix_len, 16, "Only 16-bit prefixes are supported"); self.transact("nullifieres by prefix", move |conn| { @@ -511,8 +513,9 @@ impl Db { pub async fn select_account_header_with_storage_header_at_block( &self, account_id: AccountId, - block_num: BlockNumber, + block_num: ScopedBlockNum, ) -> Result> { + let block_num = block_num.get(); self.transact("Get account header with storage header at block", move |conn| { queries::select_account_header_with_storage_header_at_block(conn, account_id, block_num) }) @@ -527,9 +530,10 @@ impl Db { )] pub async fn get_note_sync_multi( &self, - block_range: RangeInclusive, + block_range: ScopedBlockRange, note_tags: Arc<[u32]>, ) -> Result, NoteSyncError> { + let block_range = block_range.into_inner(); self.transact("notes sync task", move |conn| { queries::get_note_sync_multi(conn, ¬e_tags, block_range, MAX_RESPONSE_PAYLOAD_BYTES) }) @@ -562,8 +566,9 @@ impl Db { pub async fn select_existing_note_commitments( &self, note_commitments: Vec, - up_to_block: BlockNumber, + up_to_block: ScopedBlockNum, ) -> Result> { + let up_to_block = up_to_block.get(); self.transact("note by commitment", move |conn| { queries::select_existing_note_commitments( conn, @@ -643,6 +648,18 @@ impl Db { /// The returned values are the latest known values up to `block_range.end()`, and no values /// earlier than `block_range.start()` are returned. pub(crate) async fn select_storage_map_sync_values( + &self, + account_id: AccountId, + block_range: ScopedBlockRange, + entries_limit: Option, + ) -> Result { + self.select_storage_map_sync_values_raw(account_id, block_range.into_inner(), entries_limit) + .await + } + + /// Raw variant of [`Self::select_storage_map_sync_values`] for internal pagination, where + /// sub-ranges are derived from a bound that was already validated by the caller. + async fn select_storage_map_sync_values_raw( &self, account_id: AccountId, block_range: RangeInclusive, @@ -677,12 +694,14 @@ impl Db { &self, account_id: AccountId, slot_name: miden_protocol::account::StorageSlotName, - block_num: BlockNumber, + block_num: ScopedBlockNum, entries_limit: Option, ) -> Result { use miden_node_proto::domain::account::{AccountStorageMapDetails, StorageMapEntries}; use miden_protocol::EMPTY_WORD; + let block_num = block_num.get(); + // TODO this remains expensive with a large history until we implement pruning for DB // columns let mut values = Vec::new(); @@ -690,7 +709,7 @@ impl Db { let entries_limit = entries_limit.unwrap_or_else(default_storage_map_entries_limit); let mut page = self - .select_storage_map_sync_values( + .select_storage_map_sync_values_raw( account_id, block_range_start..=block_num, Some(entries_limit), @@ -714,7 +733,7 @@ impl Db { block_range_start = page.last_block_included.child(); page = self - .select_storage_map_sync_values( + .select_storage_map_sync_values_raw( account_id, block_range_start..=block_num, Some(entries_limit), @@ -767,8 +786,9 @@ impl Db { pub async fn select_account_vault_at_block( &self, account_id: AccountId, - block_num: BlockNumber, + block_num: ScopedBlockNum, ) -> Result, DatabaseError> { + let block_num = block_num.get(); self.transact("select account vault at block", move |conn| { queries::select_account_vault_at_block(conn, account_id, block_num) }) @@ -778,8 +798,9 @@ impl Db { pub async fn get_account_vault_sync( &self, account_id: AccountId, - block_range: RangeInclusive, + block_range: ScopedBlockRange, ) -> Result<(BlockNumber, Vec)> { + let block_range = block_range.into_inner(); self.transact("account vault sync", move |conn| { queries::select_account_vault_assets(conn, account_id, block_range) }) @@ -803,8 +824,9 @@ impl Db { pub async fn select_transactions_records( &self, account_ids: Vec, - block_range: RangeInclusive, + block_range: ScopedBlockRange, ) -> Result<(BlockNumber, Vec)> { + let block_range = block_range.into_inner(); self.transact("full transactions records", move |conn| { queries::select_transactions_records(conn, &account_ids, block_range) }) diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 614867462..626a20e59 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -1614,7 +1614,12 @@ async fn reconstruct_storage_map_from_db_pages_until_latest() { .unwrap(); let details = db - .reconstruct_storage_map_from_db(account_id, slot_name.clone(), block3, Some(1)) + .reconstruct_storage_map_from_db( + account_id, + slot_name.clone(), + crate::state::ScopedBlockNum::new_unchecked(block3), + Some(1), + ) .await .unwrap(); @@ -1672,7 +1677,12 @@ async fn reconstruct_storage_map_from_db_returns_limit_exceeded_for_single_block // Use limit=1 so that 3 entries in a single block exceed the limit. block_range_start is block5 // (the first block with data), and the target is also block5. let details = db - .reconstruct_storage_map_from_db(account_id, slot_name.clone(), block5, Some(1)) + .reconstruct_storage_map_from_db( + account_id, + slot_name.clone(), + crate::state::ScopedBlockNum::new_unchecked(block5), + Some(1), + ) .await .unwrap(); diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 7ace507dd..ec40d3fe3 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -39,7 +39,16 @@ pub use errors::{ StateSyncError, }; pub use genesis::GenesisState; -pub use state::{BlockWriter, LoadedState, ProofWriter, State, StateView, WriterTask}; +pub use state::{ + BlockWriter, + LoadedState, + ProofWriter, + ScopedBlockNum, + ScopedBlockRange, + State, + StateView, + WriterTask, +}; /// Returns the store crate version. pub fn version() -> &'static str { diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index c9fdc92fd..8c32fc9d7 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -28,8 +28,8 @@ pub use replica::{BlockCache, BlockNotification, ProofCache, ProofNotification}; mod tip; mod view; +pub use view::{ScopedBlockNum, ScopedBlockRange, StateView, TransactionInputs}; use view::{SnapshotGuard, StateSnapshot}; -pub use view::{StateView, TransactionInputs}; mod writer; diff --git a/crates/store/src/state/view/account/mod.rs b/crates/store/src/state/view/account/mod.rs index 723cd58d6..f03238fa1 100644 --- a/crates/store/src/state/view/account/mod.rs +++ b/crates/store/src/state/view/account/mod.rs @@ -18,7 +18,7 @@ use miden_protocol::account::{AccountId, AccountStorageHeader, StorageSlotName, use miden_protocol::block::BlockNumber; use miden_protocol::block::account_tree::AccountWitness; -use super::StateView; +use super::{ScopedBlockNum, StateView}; use crate::COMPONENT; use crate::account_state_forest::AccountStorageMapResult; use crate::errors::{DatabaseError, GetAccountError}; @@ -137,7 +137,7 @@ impl StateView { async fn reconstruct_vault_details_from_db( &self, account_id: AccountId, - block_num: BlockNumber, + block_num: ScopedBlockNum, ) -> Result { let assets = self.db().select_account_vault_at_block(account_id, block_num).await?; @@ -163,7 +163,7 @@ impl StateView { &self, account_id: AccountId, slot_name: StorageSlotName, - block_num: BlockNumber, + block_num: ScopedBlockNum, ) -> Result { let details = self .db() @@ -218,14 +218,13 @@ impl StateView { // Validate block exists in the blockchain before querying the database. The view's tip is // the same snapshot the witness was resolved against, so the witness and the DB reads below // observe a single consistent block height. - if block_num > self.tip() { - return Err(GetAccountError::UnknownBlock(block_num)); - } + let scoped_block = + self.scope_block(block_num).ok_or(GetAccountError::UnknownBlock(block_num))?; // Query account header and storage header together in a single DB call let (account_header, storage_header) = self .db() - .select_account_header_with_storage_header_at_block(account_id, block_num) + .select_account_header_with_storage_header_at_block(account_id, scoped_block) .await? .ok_or(GetAccountError::AccountNotFound(account_id, block_num))?; @@ -263,7 +262,9 @@ impl StateView { match forest_details { Some(details) => details, - None => self.reconstruct_vault_details_from_db(account_id, block_num).await?, + None => { + self.reconstruct_vault_details_from_db(account_id, scoped_block).await? + }, } }, None => AccountVaultDetails::empty(), @@ -324,8 +325,12 @@ impl StateView { { Some(details) => details, None => { - self.reconstruct_storage_map_details_from_db(account_id, slot_name, block_num) - .await? + self.reconstruct_storage_map_details_from_db( + account_id, + slot_name, + scoped_block, + ) + .await? }, }; storage_map_details_by_index[index] = Some(details); diff --git a/crates/store/src/state/view/mod.rs b/crates/store/src/state/view/mod.rs index 94c8dd821..5dcbe1b14 100644 --- a/crates/store/src/state/view/mod.rs +++ b/crates/store/src/state/view/mod.rs @@ -20,6 +20,9 @@ use crate::db::Db; use crate::errors::RangeBeyondTip; use crate::state::State; +mod scoped; +pub use scoped::{ScopedBlockNum, ScopedBlockRange}; + mod snapshot; pub(in crate::state) use snapshot::{SNAPSHOTS_LIVE_WARN_THRESHOLD, SnapshotGuard, StateSnapshot}; @@ -87,16 +90,31 @@ impl StateView { &self.db } - /// Ensures the given block range does not extend beyond this view's chain tip. + /// Returns this view's tip as a scoped block number for tip-bounded database queries. + fn scoped_tip(&self) -> ScopedBlockNum { + ScopedBlockNum::new(self.tip()) + } + + /// Validates that `block_num` does not exceed this view's chain tip, returning the scoped block + /// number required by block-bounded database queries. + fn scope_block(&self, block_num: BlockNumber) -> Option { + (block_num <= self.tip()).then(|| ScopedBlockNum::new(block_num)) + } + + /// Validates that `range` does not extend beyond this view's chain tip, returning the scoped + /// range required by range-bounded database queries. /// /// Every range-scoped read on this type calls this before touching the database, so callers /// never need to pre-validate ranges themselves. - fn check_range(&self, range: &RangeInclusive) -> Result<(), RangeBeyondTip> { + fn scope_range( + &self, + range: RangeInclusive, + ) -> Result { let tip = self.tip(); if *range.end() > tip { return Err(RangeBeyondTip { chain_tip: tip, block_to: *range.end() }); } - Ok(()) + Ok(ScopedBlockRange::new(range)) } /// Runs a synchronous read-only operation over the pinned state snapshot on Tokio's blocking diff --git a/crates/store/src/state/view/sync.rs b/crates/store/src/state/view/sync.rs index 1511846ff..89fbb9bff 100644 --- a/crates/store/src/state/view/sync.rs +++ b/crates/store/src/state/view/sync.rs @@ -25,7 +25,7 @@ impl StateView { account_ids: Vec, block_range: RangeInclusive, ) -> Result<(BlockNumber, Vec), DatabaseError> { - self.check_range(&block_range)?; + let block_range = self.scope_range(block_range)?; self.db().select_transactions_records(account_ids, block_range).await } @@ -43,10 +43,10 @@ impl StateView { &self, block_range: RangeInclusive, ) -> Result<(MmrDelta, BlockHeader, BlockSignatures), StateSyncError> { - self.check_range(&block_range)?; + let block_range = self.scope_range(block_range)?; - let block_from = *block_range.start(); - let block_to = *block_range.end(); + let block_from = block_range.start(); + let block_to = block_range.end(); // SAFETY: block_to <= this view's tip (checked above), so it is committed and must exist in // the database. @@ -113,9 +113,9 @@ impl StateView { note_tags: Vec, block_range: RangeInclusive, ) -> Result<(Vec<(NoteSyncUpdate, MmrProof)>, BlockNumber), NoteSyncError> { - self.check_range(&block_range)?; + let block_range = self.scope_range(block_range)?; - let block_end = *block_range.end(); + let block_end = block_range.end(); // The MMR at forest N contains proofs for blocks 0..N-1, so we use block_end + 1 to include // the proof for block_end. SAFETY: block_end <= this view's tip (checked above), and the // view's blockchain MMR always has at least tip + 1 leaves. @@ -148,7 +148,7 @@ impl StateView { nullifier_prefixes: Vec, block_range: RangeInclusive, ) -> Result<(Vec, BlockNumber), DatabaseError> { - self.check_range(&block_range)?; + let block_range = self.scope_range(block_range)?; self.db() .select_nullifiers_by_prefix(prefix_len, nullifier_prefixes, block_range) .await @@ -166,7 +166,7 @@ impl StateView { account_id: AccountId, block_range: RangeInclusive, ) -> Result<(BlockNumber, Vec), DatabaseError> { - self.check_range(&block_range)?; + let block_range = self.scope_range(block_range)?; self.db().get_account_vault_sync(account_id, block_range).await } @@ -179,7 +179,7 @@ impl StateView { account_id: AccountId, block_range: RangeInclusive, ) -> Result { - self.check_range(&block_range)?; + let block_range = self.scope_range(block_range)?; self.db().select_storage_map_sync_values(account_id, block_range, None).await } } diff --git a/crates/store/src/state/view/transaction_inputs.rs b/crates/store/src/state/view/transaction_inputs.rs index a38fae6a1..01aaf7ad2 100644 --- a/crates/store/src/state/view/transaction_inputs.rs +++ b/crates/store/src/state/view/transaction_inputs.rs @@ -80,7 +80,7 @@ impl StateView { // include yet. let found_unauthenticated_notes = self .db() - .select_existing_note_commitments(unauthenticated_note_commitments, self.tip()) + .select_existing_note_commitments(unauthenticated_note_commitments, self.scoped_tip()) .await?; Ok(TransactionInputs { From db5fe1e27e2feaf5d83d39fbcc599009359cd069 Mon Sep 17 00:00:00 2001 From: sergerad Date: Fri, 31 Jul 2026 13:51:18 +1200 Subject: [PATCH 31/35] Lint --- crates/store/src/state/view/scoped.rs | 62 +++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 crates/store/src/state/view/scoped.rs diff --git a/crates/store/src/state/view/scoped.rs b/crates/store/src/state/view/scoped.rs new file mode 100644 index 000000000..85e49a923 --- /dev/null +++ b/crates/store/src/state/view/scoped.rs @@ -0,0 +1,62 @@ +//! Proof-of-validation block-number types issued by [`StateView`](super::StateView). +//! +//! Tip-scoped database queries take these types instead of raw block numbers, so a query whose +//! bound was not validated against a state view's tip is not expressible: the only constructors +//! live on [`StateView`](super::StateView), which checks the bound against its pinned snapshot. +//! +//! Chain tips are monotonic, so a value issued by an older view remains valid for any later +//! state — holding one across view lifetimes is sound, if pointless. + +use std::ops::RangeInclusive; + +use miden_protocol::block::BlockNumber; + +/// A block number proven to be at or below the issuing view's chain tip. +#[derive(Debug, Clone, Copy)] +pub struct ScopedBlockNum(BlockNumber); + +impl ScopedBlockNum { + /// Issued by [`StateView`](super::StateView) after validating the bound against its tip. + pub(super) fn new(block_num: BlockNumber) -> Self { + Self(block_num) + } + + /// Constructs a scoped block number without validation. + /// + /// Test-only: lets database tests exercise scoped queries without a running state. + #[cfg(test)] + pub(crate) fn new_unchecked(block_num: BlockNumber) -> Self { + Self(block_num) + } + + /// Returns the validated block number. + pub(crate) fn get(self) -> BlockNumber { + self.0 + } +} + +/// A block range whose upper bound is proven to be at or below the issuing view's chain tip. +#[derive(Debug, Clone)] +pub struct ScopedBlockRange(RangeInclusive); + +impl ScopedBlockRange { + /// Issued by [`StateView`](super::StateView) after validating the range against its tip. + pub(super) fn new(range: RangeInclusive) -> Self { + Self(range) + } + + /// Returns the start of the validated range. + pub(crate) fn start(&self) -> BlockNumber { + *self.0.start() + } + + /// Returns the end of the validated range. + pub(crate) fn end(&self) -> BlockNumber { + *self.0.end() + } + + /// Returns the validated range. + pub(crate) fn into_inner(self) -> RangeInclusive { + self.0 + } +} From f12d4cea902e07000c724088243ec818cdfc74d3 Mon Sep 17 00:00:00 2001 From: sergerad Date: Sat, 1 Aug 2026 08:24:17 +1200 Subject: [PATCH 32/35] Fix worker module position --- crates/store/src/state/lifecycle.rs | 112 +---- crates/store/src/state/mod.rs | 3 +- crates/store/src/state/writer/mod.rs | 564 ++++-------------------- crates/store/src/state/writer/worker.rs | 513 +++++++++++++++++++++ 4 files changed, 626 insertions(+), 566 deletions(-) create mode 100644 crates/store/src/state/writer/worker.rs diff --git a/crates/store/src/state/lifecycle.rs b/crates/store/src/state/lifecycle.rs index 405e1e618..4afe7afe7 100644 --- a/crates/store/src/state/lifecycle.rs +++ b/crates/store/src/state/lifecycle.rs @@ -1,12 +1,9 @@ //! Store lifecycle: loading the state, starting its write worker, and stopping the store. -use std::future::Future; use std::num::NonZeroUsize; use std::path::Path; -use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::AtomicUsize; -use std::task::{Context, Poll}; use arc_swap::ArcSwap; use miden_node_utils::ErrorReport; @@ -33,8 +30,16 @@ use crate::state::loader::{ verify_account_state_forest_consistency, verify_tree_consistency, }; -use crate::state::writer::{WriteRequest, WriteWorker}; -use crate::state::{BlockCache, ProofCache, SnapshotGuard, State, StateSnapshot}; +use crate::state::writer::{WriteRequest, WriteWorker, WriterTask}; +use crate::state::{ + BlockCache, + BlockWriter, + ProofCache, + ProofWriter, + SnapshotGuard, + State, + StateSnapshot, +}; use crate::{COMPONENT, DataDirectory, DatabaseOptions}; /// Number of recent committed blocks held in the in-memory cache for replica subscriptions. @@ -88,86 +93,7 @@ impl LoadedState { } } -// WRITE CAPABILITIES -// ================================================================================================ - -/// The store's block-write capability. -/// -/// Only handle able to apply blocks; obtained exactly once from [`LoadedState::start`] and -/// deliberately not cloneable, so granting it to a single task (the block builder in sequencer -/// mode, the block sync loop in full-node mode) statically prevents every other component from -/// writing blocks. -/// -/// Exposes no read access: holders that also need to query the store receive the [`Arc`] -/// returned alongside this capability by [`LoadedState::start`]. -pub struct BlockWriter { - /// The block store, used to persist proving inputs alongside applied blocks. - pub(super) block_store: Arc, - /// Sender for block-write requests to the [`WriteWorker`](crate::state::writer::WriteWorker) - /// task. Never cloned out of this struct: the writer exits once it is dropped. - pub(super) write_tx: mpsc::Sender, -} - -impl BlockWriter { - /// Stops the store, waiting until the write worker has released the tree storage it owns. - /// - /// Consumes the capability — closing the write channel the write worker listens on — and then - /// joins the writer task returned by [`LoadedState::start`]. The drop must precede the join - /// or the write worker never observes the closed channel; doing both here keeps that ordering - /// out of caller hands. Read-only [`State`] references may outlive the stop. - /// - /// Callers that need the storage released deterministically must use this method instead of - /// dropping: the node's `recover` command stops the store before the process exits, and the - /// stress-test's store seeding stops it so the same data directory can be re-loaded (or its - /// temporary directory deleted) immediately afterwards. The running node does not use this - /// method — its writer exits via the shutdown token passed to [`State::load`] and is joined - /// through the node's task set. - /// - /// # Panics - /// - /// Panics if the writer task panicked. - pub async fn stop(self, writer_task: WriterTask) { - drop(self); - writer_task.await.expect("write worker task should not panic"); - } -} - -/// The store's proof-write capability. -/// -/// Only handle able to commit block proofs and advance the proven tip; obtained exactly once from -/// [`LoadedState::start`] and deliberately not cloneable, so granting it to a single task (the -/// proof scheduler in sequencer mode, the proof sync loop in full-node mode) statically prevents -/// every other component from writing proofs. -/// -/// Exposes no read access: the held state is only used internally to commit proofs and advance -/// the proven tip. Holders that also need to query the store receive the [`Arc`] returned -/// alongside this capability by [`LoadedState::start`]. -pub struct ProofWriter { - pub(super) state: Arc, -} - -// WRITER TASK -// ================================================================================================ - -/// Handle of the store's write worker task, returned by [`LoadedState::start`]. -/// -/// Awaiting it resolves once the writer has exited and released the tree storage it owns; a join -/// error carries a writer panic. The newtype ensures [`BlockWriter::stop`] can only be given the -/// store's own writer task, and deliberately does not expose [`tokio::task::JoinHandle::abort`]: -/// aborting the writer mid-write could leave the trees lagging the committed database state, -/// voiding the guarantee that an in-flight block write always completes. -#[must_use = "await the writer task to observe its exit, or pass it to `BlockWriter::stop`"] -pub struct WriterTask(tokio::task::JoinHandle<()>); - -impl Future for WriterTask { - type Output = Result<(), tokio::task::JoinError>; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - Pin::new(&mut self.0).poll(cx) - } -} - -// LOAD & STOP +// LOAD // ================================================================================================ impl State { @@ -299,20 +225,20 @@ impl State { // exits when the shutdown token is cancelled or the `BlockWriter` (holding the only request // sender) is dropped. let (write_tx, write_rx) = mpsc::channel(1); - let block_writer = WriteWorker { - db: Arc::clone(&db), - block_store: Arc::clone(&block_store), - latest_snapshot: Arc::clone(&latest_snapshot), - committed_tip_tx: Arc::clone(&committed_tip_tx), - block_cache: block_cache.clone(), - rx: write_rx, + let block_writer = WriteWorker::new( + Arc::clone(&db), + Arc::clone(&block_store), + Arc::clone(&latest_snapshot), + Arc::clone(&committed_tip_tx), + block_cache.clone(), + write_rx, shutdown, nullifier_tree, account_tree, blockchain, forest, snapshots_live, - }; + ); let state = Self { data_directory: data_path.to_path_buf(), db, diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 8c32fc9d7..37c556409 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -20,7 +20,7 @@ mod disk_monitor; mod loader; mod lifecycle; -pub use lifecycle::{BlockWriter, LoadedState, ProofWriter, WriterTask}; +pub use lifecycle::LoadedState; mod replica; pub use replica::{BlockCache, BlockNotification, ProofCache, ProofNotification}; @@ -32,6 +32,7 @@ pub use view::{ScopedBlockNum, ScopedBlockRange, StateView, TransactionInputs}; use view::{SnapshotGuard, StateSnapshot}; mod writer; +pub use writer::{BlockWriter, ProofWriter, WriterTask}; // CHAIN STATE // ================================================================================================ diff --git a/crates/store/src/state/writer/mod.rs b/crates/store/src/state/writer/mod.rs index 2dae7f205..0ae34d5a7 100644 --- a/crates/store/src/state/writer/mod.rs +++ b/crates/store/src/state/writer/mod.rs @@ -5,45 +5,84 @@ //! [`StateSnapshot`] snapshot via an [`ArcSwap`], making the updated trees immediately visible to //! wait-free readers. //! -//! The submodules hold the [`BlockWriter`] and [`ProofWriter`](crate::state::ProofWriter) -//! capability entry points that feed this worker. +//! The [`BlockWriter`] and [`ProofWriter`] capabilities defined here are the only handles able +//! to feed this worker and to commit proofs; the submodules hold their entry points. mod apply_block; mod apply_proof; -use std::fmt::Display; +mod worker; +use std::future::Future; +use std::pin::Pin; use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::task::{Context, Poll}; -use arc_swap::ArcSwap; use miden_node_utils::ErrorReport; -use miden_node_utils::shutdown::CancellationToken; -use miden_node_utils::tracing::{miden_instrument, miden_span_record}; -use miden_protocol::Word; -use miden_protocol::account::AccountUpdateDetails; -use miden_protocol::block::account_tree::AccountMutationSet; -use miden_protocol::block::nullifier_tree::{NullifierMutationSet, NullifierTree}; -use miden_protocol::block::{BlockBody, BlockHeader, BlockNumber, Blockchain, SignedBlock}; -use miden_protocol::crypto::merkle::smt::LargeSmt; -use miden_protocol::note::{NoteDetails, Nullifier}; -use miden_protocol::transaction::OutputNote; -use miden_protocol::utils::serde::Serializable; -use tokio::sync::{mpsc, oneshot, watch}; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::block::SignedBlock; +use tokio::sync::{mpsc, oneshot}; +pub(in crate::state) use worker::WriteWorker; -use crate::account_state_forest::{ - AccountStateForest, - AccountStateForestBackend, - PreparedAccountStateForestBlockUpdate, -}; -use crate::accounts::AccountTreeWithHistory; +use crate::COMPONENT; use crate::blocks::BlockStore; -use crate::db::{Db, NoteRecord}; -use crate::errors::{ApplyBlockError, InvalidBlockError}; -use crate::state::block_lifecycle::{BlockLifecycle, lifecycle_events_enabled}; -use crate::state::loader::TreeStorage; -use crate::state::view::{SNAPSHOTS_LIVE_WARN_THRESHOLD, SnapshotGuard, StateSnapshot}; -use crate::state::{BlockCache, BlockNotification, BlockWriter}; -use crate::{COMPONENT, HistoricalError, LOG_TARGET}; +use crate::errors::ApplyBlockError; + +// WRITE CAPABILITIES +// ================================================================================================ + +/// The store's block-write capability. +/// +/// Only handle able to apply blocks; obtained exactly once from [`LoadedState::start`](crate::state::LoadedState::start) and +/// deliberately not cloneable, so granting it to a single task (the block builder in sequencer +/// mode, the block sync loop in full-node mode) statically prevents every other component from +/// writing blocks. +/// +/// Exposes no read access: holders that also need to query the store receive the +/// [`Arc`](crate::state::State) returned alongside this capability by +/// [`LoadedState::start`](crate::state::LoadedState::start). +pub struct BlockWriter { + /// The block store, used to persist proving inputs alongside applied blocks. + pub(super) block_store: Arc, + /// Sender for block-write requests to the [`WriteWorker`] task. Never cloned out of this + /// struct: the writer exits once it is dropped. + pub(super) write_tx: mpsc::Sender, +} + +/// The store's proof-write capability. +/// +/// Only handle able to commit block proofs and advance the proven tip; obtained exactly once from +/// [`LoadedState::start`](crate::state::LoadedState::start) and deliberately not cloneable, so granting it to a single task (the +/// proof scheduler in sequencer mode, the proof sync loop in full-node mode) statically prevents +/// every other component from writing proofs. +/// +/// Exposes no read access: the held state is only used internally to commit proofs and advance +/// the proven tip. Holders that also need to query the store receive the +/// [`Arc`](crate::state::State) returned alongside this capability by +/// [`LoadedState::start`](crate::state::LoadedState::start). +pub struct ProofWriter { + pub(super) state: Arc, +} + +// WRITER TASK +// ================================================================================================ + +/// Handle of the store's write worker task, returned by [`LoadedState::start`](crate::state::LoadedState::start). +/// +/// Awaiting it resolves once the writer has exited and released the tree storage it owns; a join +/// error carries a writer panic. The newtype ensures [`BlockWriter::stop`] can only be given the +/// store's own writer task, and deliberately does not expose [`tokio::task::JoinHandle::abort`]: +/// aborting the writer mid-write could leave the trees lagging the committed database state, +/// voiding the guarantee that an in-flight block write always completes. +#[must_use = "await the writer task to observe its exit, or pass it to `BlockWriter::stop`"] +pub struct WriterTask(pub(super) tokio::task::JoinHandle<()>); + +impl Future for WriterTask { + type Output = Result<(), tokio::task::JoinError>; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + Pin::new(&mut self.0).poll(cx) + } +} // WRITE REQUEST // ================================================================================================ @@ -55,6 +94,29 @@ pub(super) struct WriteRequest { } impl BlockWriter { + /// Stops the store, waiting until the write worker has released the tree storage it owns. + /// + /// Consumes the capability — closing the write channel the write worker listens on — and then + /// joins the writer task returned by [`LoadedState::start`](crate::state::LoadedState::start). The drop must precede the join + /// or the write worker never observes the closed channel; doing both here keeps that ordering + /// out of caller hands. Read-only [`State`](crate::state::State) references may outlive the + /// stop. + /// + /// Callers that need the storage released deterministically must use this method instead of + /// dropping: the node's `recover` command stops the store before the process exits, and the + /// stress-test's store seeding stops it so the same data directory can be re-loaded (or its + /// temporary directory deleted) immediately afterwards. The running node does not use this + /// method — its writer exits via the shutdown token passed to + /// [`State::load`](crate::state::State::load) and is joined through the node's task set. + /// + /// # Panics + /// + /// Panics if the writer task panicked. + pub async fn stop(self, writer_task: WriterTask) { + drop(self); + writer_task.await.expect("write worker task should not panic"); + } + /// Apply changes of a new block to the DB and in-memory data structures. /// /// Blocks are forwarded to the store's write worker task, which processes them one at a @@ -75,445 +137,3 @@ impl BlockWriter { result_rx.await? } } - -// WRITE WORKER -// ================================================================================================ - -/// Single-task owner of the mutable trees. Processes [`WriteRequest`]s serially. -/// -/// The writer owns the writable trees directly, so no locks are held at any point: validation and -/// mutation-computation read the owned trees, the DB commit runs without touching them, and the -/// new [`StateSnapshot`] snapshot is published atomically at the end. -pub(super) struct WriteWorker { - pub(in crate::state) db: Arc, - pub(in crate::state) block_store: Arc, - /// Atomically swappable pointer through which new snapshots are published. - pub(in crate::state) latest_snapshot: Arc>, - pub(in crate::state) committed_tip_tx: Arc>, - pub(in crate::state) block_cache: BlockCache, - pub(in crate::state) rx: mpsc::Receiver, - /// Token signalling node shutdown; the writer stops accepting new requests once cancelled. - pub(in crate::state) shutdown: CancellationToken, - /// The mutable nullifier tree owned by this writer. - pub(in crate::state) nullifier_tree: NullifierTree>, - /// The mutable account tree owned by this writer. - pub(in crate::state) account_tree: AccountTreeWithHistory, - /// The blockchain MMR owned by this writer. - pub(in crate::state) blockchain: Blockchain, - /// The mutable account state forest owned by this writer. - pub(in crate::state) forest: AccountStateForest, - /// Shared counter of live snapshot generations, for observability. - pub(in crate::state) snapshots_live: Arc, -} - -/// Note records and state mutations computed from a validated block, before any modifications. -struct PreparedBlockUpdate { - notes: Vec<(NoteRecord, Option)>, - nullifier_tree_update: NullifierMutationSet, - account_tree_update: AccountMutationSet, - account_forest_update: PreparedAccountStateForestBlockUpdate, -} - -impl WriteWorker { - /// Runs the writer loop, processing requests until shutdown is signalled or the - /// [`BlockWriter`] (holding the only request sender) is dropped. - /// - /// Cancellation is only observed between requests: an in-flight block write always runs to - /// completion, so shutdown never leaves the trees lagging the committed database state. - /// Requests still queued when cancellation fires are dropped, failing their senders. - pub async fn run(mut self) { - loop { - let req = tokio::select! { - biased; - () = self.shutdown.cancelled() => break, - req = self.rx.recv() => match req { - Some(req) => req, - None => break, - }, - }; - let result = self.write_block(req.signed_block).await; - let _ = req.result_tx.send(result); - } - } - - /// Validates and commits a signed block to all persistent and in-memory stores. - /// - /// ## Note on state consistency - /// - /// Readers access the in-memory state through frozen snapshots, so consistency is maintained - /// by ordering the commit steps rather than by locking: - /// - /// - the block is validated against the writer-owned trees and the DB prior to starting any - /// modifications. - /// - the block is saved to the block store. Such blocks are considered candidates and are not - /// yet available for reading because the latest block pointer is not updated yet. - /// - the DB transaction is committed. Concurrent readers still see the previous in-memory - /// snapshot; queries that combine DB and in-memory data are scoped by block number. - /// - the in-memory structures owned by the writer are updated. On a crash in between, the trees - /// lag the DB by one block, which is detected by the consistency checks at startup (the same - /// crash semantics as the previous lock-based implementation). - /// - the new snapshot is published atomically, making the block visible to readers. - #[miden_instrument( - target = COMPONENT, - skip_all, - err, - )] - async fn write_block(&mut self, signed_block: SignedBlock) -> Result<(), ApplyBlockError> { - let header = signed_block.header(); - let body = signed_block.body(); - - let block_num = header.block_num(); - let block_commitment = header.commitment(); - let num_transactions = body.transactions().as_slice().len(); - - miden_span_record!( - block.number = %block_num, - block.commitment = %block_commitment, - block.transactions.count = num_transactions, - ); - - self.validate_block_header(header, body).await?; - - let block_lifecycle = - lifecycle_events_enabled().then(|| BlockLifecycle::from_block_body(block_num, body)); - let unresolved_note_nullifiers = block_lifecycle - .as_ref() - .map_or_else(Vec::new, BlockLifecycle::unresolved_note_nullifiers); - - // Compute the tree and forest mutations and note records upfront, before any modifications. - // The writer is the sole forest mutator, so the precomputed forest update stays valid until - // it is applied after the DB commit below. - let PreparedBlockUpdate { - notes, - nullifier_tree_update, - account_tree_update, - account_forest_update, - } = tokio::task::block_in_place(|| self.prepare_block_update(header, body))?; - let precomputed_public_states = account_forest_update.account_states.clone(); - - // Save the block to the block store. In a case of a failed DB transaction, the in-memory - // state will be unchanged, but the file might still be written. Such blocks should be - // considered candidates, not finalized blocks. - let signed_block_bytes = signed_block.to_bytes(); - // Clone before moving into the block-save call so we can cache for replicas at commit. - let cache_bytes = signed_block_bytes.clone(); - self.block_store.save_block(block_num, &signed_block_bytes).await?; - - // Commit to the DB. Readers continue to see the previous in-memory snapshot while the DB - // commits; queries that combine DB and in-memory data are scoped by block number. - let resolved_note_ids = self - .db - .apply_block(signed_block, notes, precomputed_public_states, unresolved_note_nullifiers) - .await - .map_err(|err| ApplyBlockError::DbUpdateTaskFailed(err.as_report()))?; - - // The DB is committed at this point, so the prepared mutations must be applied and any - // failure to do so aborts the process. - let snapshot = tokio::task::block_in_place(|| { - self.apply_prepared_mutations( - block_num, - block_commitment, - nullifier_tree_update, - account_tree_update, - account_forest_update, - ) - }); - - // Atomically publish the new state. Readers that call `snapshot()` after this point will - // see the updated state. Readers holding the old snapshot continue unaffected. - self.latest_snapshot.store(snapshot); - - let snapshots_live = self.check_live_snapshots(block_num); - miden_span_record!(snapshots.live = snapshots_live); - - // Push to cache and notify replica subscribers. - self.block_cache - .push(block_num, BlockNotification::new(block_num, cache_bytes)) - .expect("block cache receives sequential block numbers"); - let _ = self.committed_tip_tx.send(block_num); - - if let Some(block_lifecycle) = block_lifecycle { - block_lifecycle.emit(&resolved_note_ids); - } - tracing::debug!(target: LOG_TARGET, "Block applied"); - - Ok(()) - } - - /// Returns the number of live snapshot generations, warning when slow readers are pinning too - /// many old generations in memory. - /// - /// The count is returned rather than recorded here because `miden_span_record!` must be used - /// within a `#[miden_instrument]` function. - fn check_live_snapshots(&self, block_num: BlockNumber) -> u64 { - let snapshots_live = self.snapshots_live.load(Ordering::Relaxed) as u64; - if snapshots_live > SNAPSHOTS_LIVE_WARN_THRESHOLD { - tracing::warn!( - target: COMPONENT, - block_num = block_num.as_u32(), - snapshots.live = snapshots_live, - "too many live state snapshots; slow readers are pinning old generations", - ); - } - snapshots_live - } - - /// Computes the note records and all tree and forest mutations for a block, without mutating - /// any state. - /// - /// May block on backend I/O, so it must run on Tokio's blocking path. The returned forest - /// update is bound to the forest state observed here; it remains valid until applied because - /// the writer is the sole forest mutator. - fn prepare_block_update( - &self, - header: &BlockHeader, - body: &BlockBody, - ) -> Result { - let notes = Self::build_note_records(header, body)?; - let (nullifier_tree_update, account_tree_update) = - self.compute_tree_mutations(header, body)?; - - // Public account updates carry patches; private accounts are filtered out since they don't - // expose their state changes. - let account_patches = - body.updated_accounts().iter().filter_map(|update| match update.details() { - AccountUpdateDetails::Public(patch) => Some(patch.clone()), - AccountUpdateDetails::Private => None, - }); - let account_forest_update = self - .forest - .compute_block_update_mutations(header.block_num(), account_patches) - .map_err(ApplyBlockError::AccountStateForestPreparation)?; - - Ok(PreparedBlockUpdate { - notes, - nullifier_tree_update, - account_tree_update, - account_forest_update, - }) - } - - /// Applies the prepared mutations to the writer-owned trees and builds the new snapshot from - /// reader views of them. The reader views are point-in-time storage snapshots, so no tree data - /// is copied. - /// - /// Must only be called after the corresponding DB commit: at that point the mutations are part - /// of canonical state, so a failure to apply them leaves the trees divergent and aborts the - /// process. May block on backend I/O, so it must run on Tokio's blocking path. - fn apply_prepared_mutations( - &mut self, - block_num: BlockNumber, - block_commitment: Word, - nullifier_tree_update: NullifierMutationSet, - account_tree_update: AccountMutationSet, - account_forest_update: PreparedAccountStateForestBlockUpdate, - ) -> Arc { - self.nullifier_tree - .apply_mutations(nullifier_tree_update) - .unwrap_or_else(|error| { - Self::abort_after_post_commit_failure("nullifier tree", &error) - }); - - self.account_tree - .apply_mutations(account_tree_update) - .unwrap_or_else(|error| Self::abort_after_post_commit_failure("account tree", &error)); - - self.blockchain.push(block_commitment); - - self.forest - .apply_precomputed_block_update(block_num, account_forest_update) - .unwrap_or_else(|error| { - Self::abort_after_post_commit_failure("account-state forest", &error) - }); - - Arc::new(StateSnapshot::new( - self.nullifier_tree - .reader() - .expect("nullifier tree snapshot creation should not fail"), - self.blockchain.clone(), - self.account_tree.reader(), - self.forest.reader().expect("forest snapshot creation should not fail"), - SnapshotGuard::new(Arc::clone(&self.snapshots_live), block_num), - )) - } - - /// Terminates after a persistent state failure that occurred after the canonical DB commit. - /// - /// Returning would expose components at different block heights. Tests panic so the fatal path - /// can be asserted without terminating the test process; production aborts immediately. - fn abort_after_post_commit_failure(component: &str, error: &impl Display) -> ! { - tracing::error!( - target: LOG_TARGET, - component, - error = %error, - "persistent state update failed after database commit; aborting" - ); - - #[cfg(test)] - panic!("{component} update failed after database commit: {error}"); - - #[cfg(not(test))] - std::process::abort(); - } - - /// Validates that the block header is consistent with the block body and the current state. - #[miden_instrument( - target = COMPONENT, - skip_all, - err, - )] - async fn validate_block_header( - &self, - header: &BlockHeader, - body: &BlockBody, - ) -> Result<(), ApplyBlockError> { - // Validate that header and body match. - let tx_commitment = body.transactions().commitment(); - if header.tx_commitment() != tx_commitment { - return Err(InvalidBlockError::InvalidBlockTxCommitment { - expected: tx_commitment, - actual: header.tx_commitment(), - } - .into()); - } - - let block_num = header.block_num(); - - // Validate that the applied block is the next block in sequence. - let prev_block = self - .db - .select_block_header_by_block_num(None) - .await? - .ok_or(ApplyBlockError::DbBlockHeaderEmpty)?; - let expected_block_num = prev_block.block_num().child(); - if block_num != expected_block_num { - return Err(InvalidBlockError::NewBlockInvalidBlockNum { - expected: expected_block_num, - submitted: block_num, - } - .into()); - } - if header.prev_block_commitment() != prev_block.commitment() { - return Err(InvalidBlockError::NewBlockInvalidPrevCommitment.into()); - } - - Ok(()) - } - - /// Computes nullifier and account tree mutations, validating roots against the block header. - #[miden_instrument( - target = COMPONENT, - skip_all, - err, - )] - fn compute_tree_mutations( - &self, - header: &BlockHeader, - body: &BlockBody, - ) -> Result<(NullifierMutationSet, AccountMutationSet), ApplyBlockError> { - let block_num = header.block_num(); - - // A nullifier can only ever be created once, so the block is invalid if any of its - // nullifiers are already recorded in the tree. - let duplicate_nullifiers: Vec<_> = body - .created_nullifiers() - .iter() - .filter(|&nullifier| self.nullifier_tree.get_block_num(nullifier).is_some()) - .copied() - .collect(); - if !duplicate_nullifiers.is_empty() { - return Err(InvalidBlockError::DuplicatedNullifiers(duplicate_nullifiers).into()); - } - - // The header's chain commitment must equal the chain MMR root prior to this block. - let peaks = self.blockchain.peaks(); - if peaks.hash_peaks() != header.chain_commitment() { - return Err(InvalidBlockError::NewBlockInvalidChainCommitment.into()); - } - - // Compute the nullifier tree mutations and verify that they produce the nullifier root - // claimed in the header. - let nullifier_tree_update = self - .nullifier_tree - .compute_mutations( - body.created_nullifiers().iter().map(|nullifier| (*nullifier, block_num)), - ) - .map_err(InvalidBlockError::NewBlockNullifierAlreadySpent)?; - - if nullifier_tree_update.as_mutation_set().root() != header.nullifier_root() { - return Err(InvalidBlockError::NewBlockInvalidNullifierRoot.into()); - } - - // Compute the account tree mutations and verify that they produce the account root claimed - // in the header. - let account_tree_update = self - .account_tree - .compute_mutations( - body.updated_accounts() - .iter() - .map(|update| (update.account_id(), update.final_state_commitment())), - ) - .map_err(|e| match e { - HistoricalError::AccountTreeError(err) => { - InvalidBlockError::NewBlockDuplicateAccountIdPrefix(err) - }, - HistoricalError::MerkleError(_) => { - panic!("Unexpected MerkleError during account tree mutation computation") - }, - })?; - - if account_tree_update.as_mutation_set().root() != header.account_root() { - return Err(InvalidBlockError::NewBlockInvalidAccountRoot.into()); - } - - Ok((nullifier_tree_update, account_tree_update)) - } - - /// Builds note records with inclusion proofs from the block body. - #[miden_instrument( - target = COMPONENT, - skip_all, - err, - )] - fn build_note_records( - header: &BlockHeader, - body: &BlockBody, - ) -> Result)>, ApplyBlockError> { - let block_num = header.block_num(); - - let note_tree = body.compute_block_note_tree(); - if note_tree.root() != header.note_root() { - return Err(InvalidBlockError::NewBlockInvalidNoteRoot.into()); - } - - let notes = body - .output_notes() - .map(|(note_index, note)| { - let (details, attachments, nullifier) = match note { - OutputNote::Public(public) => ( - Some(NoteDetails::from(public.as_note())), - public.as_note().attachments().clone(), - Some(public.as_note().nullifier()), - ), - OutputNote::Private(private) => (None, private.attachments().clone(), None), - }; - - let inclusion_path = note_tree.open(note_index); - - let note_record = NoteRecord { - block_num, - note_index, - note_id: note.id().as_word(), - metadata: *note.metadata(), - details, - attachments, - inclusion_path, - }; - - Ok((note_record, nullifier)) - }) - .collect::, InvalidBlockError>>()?; - - Ok(notes) - } -} diff --git a/crates/store/src/state/writer/worker.rs b/crates/store/src/state/writer/worker.rs new file mode 100644 index 000000000..26639612c --- /dev/null +++ b/crates/store/src/state/writer/worker.rs @@ -0,0 +1,513 @@ +//! The write worker: single-task owner of the store's mutable trees. + +use std::fmt::Display; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use arc_swap::ArcSwap; +use miden_node_utils::ErrorReport; +use miden_node_utils::shutdown::CancellationToken; +use miden_node_utils::tracing::{miden_instrument, miden_span_record}; +use miden_protocol::Word; +use miden_protocol::account::AccountUpdateDetails; +use miden_protocol::block::account_tree::AccountMutationSet; +use miden_protocol::block::nullifier_tree::{NullifierMutationSet, NullifierTree}; +use miden_protocol::block::{BlockBody, BlockHeader, BlockNumber, Blockchain, SignedBlock}; +use miden_protocol::crypto::merkle::smt::LargeSmt; +use miden_protocol::note::{NoteDetails, Nullifier}; +use miden_protocol::transaction::OutputNote; +use miden_protocol::utils::serde::Serializable; +use tokio::sync::{mpsc, watch}; + +use super::WriteRequest; +use crate::account_state_forest::{ + AccountStateForest, + AccountStateForestBackend, + PreparedAccountStateForestBlockUpdate, +}; +use crate::accounts::AccountTreeWithHistory; +use crate::blocks::BlockStore; +use crate::db::{Db, NoteRecord}; +use crate::errors::{ApplyBlockError, InvalidBlockError}; +use crate::state::block_lifecycle::{BlockLifecycle, lifecycle_events_enabled}; +use crate::state::loader::TreeStorage; +use crate::state::view::{SNAPSHOTS_LIVE_WARN_THRESHOLD, SnapshotGuard, StateSnapshot}; +use crate::state::{BlockCache, BlockNotification}; +use crate::{COMPONENT, HistoricalError, LOG_TARGET}; + +// WRITE WORKER +// ================================================================================================ + +/// Single-task owner of the mutable trees. Processes [`WriteRequest`]s serially. +/// +/// The writer owns the writable trees directly, so no locks are held at any point: validation and +/// mutation-computation read the owned trees, the DB commit runs without touching them, and the +/// new [`StateSnapshot`] snapshot is published atomically at the end. +pub(in crate::state) struct WriteWorker { + db: Arc, + block_store: Arc, + /// Atomically swappable pointer through which new snapshots are published. + latest_snapshot: Arc>, + committed_tip_tx: Arc>, + block_cache: BlockCache, + rx: mpsc::Receiver, + /// Token signalling node shutdown; the writer stops accepting new requests once cancelled. + shutdown: CancellationToken, + /// The mutable nullifier tree owned by this writer. + nullifier_tree: NullifierTree>, + /// The mutable account tree owned by this writer. + account_tree: AccountTreeWithHistory, + /// The blockchain MMR owned by this writer. + blockchain: Blockchain, + /// The mutable account state forest owned by this writer. + forest: AccountStateForest, + /// Shared counter of live snapshot generations, for observability. + snapshots_live: Arc, +} + +/// Note records and state mutations computed from a validated block, before any modifications. +struct PreparedBlockUpdate { + notes: Vec<(NoteRecord, Option)>, + nullifier_tree_update: NullifierMutationSet, + account_tree_update: AccountMutationSet, + account_forest_update: PreparedAccountStateForestBlockUpdate, +} + +impl WriteWorker { + /// Assembles the write worker from the loaded trees and the store's shared infrastructure. + /// + /// Only construction is exposed outside this module: once assembled, the worker's internals + /// are reachable solely through [`Self::run`]. + #[expect(clippy::too_many_arguments)] + pub(in crate::state) fn new( + db: Arc, + block_store: Arc, + latest_snapshot: Arc>, + committed_tip_tx: Arc>, + block_cache: BlockCache, + rx: mpsc::Receiver, + shutdown: CancellationToken, + nullifier_tree: NullifierTree>, + account_tree: AccountTreeWithHistory, + blockchain: Blockchain, + forest: AccountStateForest, + snapshots_live: Arc, + ) -> Self { + Self { + db, + block_store, + latest_snapshot, + committed_tip_tx, + block_cache, + rx, + shutdown, + nullifier_tree, + account_tree, + blockchain, + forest, + snapshots_live, + } + } + + /// Runs the writer loop, processing requests until shutdown is signalled or the + /// [`BlockWriter`] (holding the only request sender) is dropped. + /// + /// Cancellation is only observed between requests: an in-flight block write always runs to + /// completion, so shutdown never leaves the trees lagging the committed database state. + /// Requests still queued when cancellation fires are dropped, failing their senders. + pub async fn run(mut self) { + loop { + let req = tokio::select! { + biased; + () = self.shutdown.cancelled() => break, + req = self.rx.recv() => match req { + Some(req) => req, + None => break, + }, + }; + let result = self.write_block(req.signed_block).await; + let _ = req.result_tx.send(result); + } + } + + /// Validates and commits a signed block to all persistent and in-memory stores. + /// + /// ## Note on state consistency + /// + /// Readers access the in-memory state through frozen snapshots, so consistency is maintained + /// by ordering the commit steps rather than by locking: + /// + /// - the block is validated against the writer-owned trees and the DB prior to starting any + /// modifications. + /// - the block is saved to the block store. Such blocks are considered candidates and are not + /// yet available for reading because the latest block pointer is not updated yet. + /// - the DB transaction is committed. Concurrent readers still see the previous in-memory + /// snapshot; queries that combine DB and in-memory data are scoped by block number. + /// - the in-memory structures owned by the writer are updated. On a crash in between, the trees + /// lag the DB by one block, which is detected by the consistency checks at startup (the same + /// crash semantics as the previous lock-based implementation). + /// - the new snapshot is published atomically, making the block visible to readers. + #[miden_instrument( + target = COMPONENT, + skip_all, + err, + )] + async fn write_block(&mut self, signed_block: SignedBlock) -> Result<(), ApplyBlockError> { + let header = signed_block.header(); + let body = signed_block.body(); + + let block_num = header.block_num(); + let block_commitment = header.commitment(); + let num_transactions = body.transactions().as_slice().len(); + + miden_span_record!( + block.number = %block_num, + block.commitment = %block_commitment, + block.transactions.count = num_transactions, + ); + + self.validate_block_header(header, body).await?; + + let block_lifecycle = + lifecycle_events_enabled().then(|| BlockLifecycle::from_block_body(block_num, body)); + let unresolved_note_nullifiers = block_lifecycle + .as_ref() + .map_or_else(Vec::new, BlockLifecycle::unresolved_note_nullifiers); + + // Compute the tree and forest mutations and note records upfront, before any modifications. + // The writer is the sole forest mutator, so the precomputed forest update stays valid until + // it is applied after the DB commit below. + let PreparedBlockUpdate { + notes, + nullifier_tree_update, + account_tree_update, + account_forest_update, + } = tokio::task::block_in_place(|| self.prepare_block_update(header, body))?; + let precomputed_public_states = account_forest_update.account_states.clone(); + + // Save the block to the block store. In a case of a failed DB transaction, the in-memory + // state will be unchanged, but the file might still be written. Such blocks should be + // considered candidates, not finalized blocks. + let signed_block_bytes = signed_block.to_bytes(); + // Clone before moving into the block-save call so we can cache for replicas at commit. + let cache_bytes = signed_block_bytes.clone(); + self.block_store.save_block(block_num, &signed_block_bytes).await?; + + // Commit to the DB. Readers continue to see the previous in-memory snapshot while the DB + // commits; queries that combine DB and in-memory data are scoped by block number. + let resolved_note_ids = self + .db + .apply_block(signed_block, notes, precomputed_public_states, unresolved_note_nullifiers) + .await + .map_err(|err| ApplyBlockError::DbUpdateTaskFailed(err.as_report()))?; + + // The DB is committed at this point, so the prepared mutations must be applied and any + // failure to do so aborts the process. + let snapshot = tokio::task::block_in_place(|| { + self.apply_prepared_mutations( + block_num, + block_commitment, + nullifier_tree_update, + account_tree_update, + account_forest_update, + ) + }); + + // Atomically publish the new state. Readers that call `snapshot()` after this point will + // see the updated state. Readers holding the old snapshot continue unaffected. + self.latest_snapshot.store(snapshot); + + let snapshots_live = self.check_live_snapshots(block_num); + miden_span_record!(snapshots.live = snapshots_live); + + // Push to cache and notify replica subscribers. + self.block_cache + .push(block_num, BlockNotification::new(block_num, cache_bytes)) + .expect("block cache receives sequential block numbers"); + let _ = self.committed_tip_tx.send(block_num); + + if let Some(block_lifecycle) = block_lifecycle { + block_lifecycle.emit(&resolved_note_ids); + } + tracing::debug!(target: LOG_TARGET, "Block applied"); + + Ok(()) + } + + /// Returns the number of live snapshot generations, warning when slow readers are pinning too + /// many old generations in memory. + /// + /// The count is returned rather than recorded here because `miden_span_record!` must be used + /// within a `#[miden_instrument]` function. + fn check_live_snapshots(&self, block_num: BlockNumber) -> u64 { + let snapshots_live = self.snapshots_live.load(Ordering::Relaxed) as u64; + if snapshots_live > SNAPSHOTS_LIVE_WARN_THRESHOLD { + tracing::warn!( + target: COMPONENT, + block_num = block_num.as_u32(), + snapshots.live = snapshots_live, + "too many live state snapshots; slow readers are pinning old generations", + ); + } + snapshots_live + } + + /// Computes the note records and all tree and forest mutations for a block, without mutating + /// any state. + /// + /// May block on backend I/O, so it must run on Tokio's blocking path. The returned forest + /// update is bound to the forest state observed here; it remains valid until applied because + /// the writer is the sole forest mutator. + fn prepare_block_update( + &self, + header: &BlockHeader, + body: &BlockBody, + ) -> Result { + let notes = Self::build_note_records(header, body)?; + let (nullifier_tree_update, account_tree_update) = + self.compute_tree_mutations(header, body)?; + + // Public account updates carry patches; private accounts are filtered out since they don't + // expose their state changes. + let account_patches = + body.updated_accounts().iter().filter_map(|update| match update.details() { + AccountUpdateDetails::Public(patch) => Some(patch.clone()), + AccountUpdateDetails::Private => None, + }); + let account_forest_update = self + .forest + .compute_block_update_mutations(header.block_num(), account_patches) + .map_err(ApplyBlockError::AccountStateForestPreparation)?; + + Ok(PreparedBlockUpdate { + notes, + nullifier_tree_update, + account_tree_update, + account_forest_update, + }) + } + + /// Applies the prepared mutations to the writer-owned trees and builds the new snapshot from + /// reader views of them. The reader views are point-in-time storage snapshots, so no tree data + /// is copied. + /// + /// Must only be called after the corresponding DB commit: at that point the mutations are part + /// of canonical state, so a failure to apply them leaves the trees divergent and aborts the + /// process. May block on backend I/O, so it must run on Tokio's blocking path. + fn apply_prepared_mutations( + &mut self, + block_num: BlockNumber, + block_commitment: Word, + nullifier_tree_update: NullifierMutationSet, + account_tree_update: AccountMutationSet, + account_forest_update: PreparedAccountStateForestBlockUpdate, + ) -> Arc { + self.nullifier_tree + .apply_mutations(nullifier_tree_update) + .unwrap_or_else(|error| { + Self::abort_after_post_commit_failure("nullifier tree", &error) + }); + + self.account_tree + .apply_mutations(account_tree_update) + .unwrap_or_else(|error| Self::abort_after_post_commit_failure("account tree", &error)); + + self.blockchain.push(block_commitment); + + self.forest + .apply_precomputed_block_update(block_num, account_forest_update) + .unwrap_or_else(|error| { + Self::abort_after_post_commit_failure("account-state forest", &error) + }); + + Arc::new(StateSnapshot::new( + self.nullifier_tree + .reader() + .expect("nullifier tree snapshot creation should not fail"), + self.blockchain.clone(), + self.account_tree.reader(), + self.forest.reader().expect("forest snapshot creation should not fail"), + SnapshotGuard::new(Arc::clone(&self.snapshots_live), block_num), + )) + } + + /// Terminates after a persistent state failure that occurred after the canonical DB commit. + /// + /// Returning would expose components at different block heights. Tests panic so the fatal path + /// can be asserted without terminating the test process; production aborts immediately. + fn abort_after_post_commit_failure(component: &str, error: &impl Display) -> ! { + tracing::error!( + target: LOG_TARGET, + component, + error = %error, + "persistent state update failed after database commit; aborting" + ); + + #[cfg(test)] + panic!("{component} update failed after database commit: {error}"); + + #[cfg(not(test))] + std::process::abort(); + } + + /// Validates that the block header is consistent with the block body and the current state. + #[miden_instrument( + target = COMPONENT, + skip_all, + err, + )] + async fn validate_block_header( + &self, + header: &BlockHeader, + body: &BlockBody, + ) -> Result<(), ApplyBlockError> { + // Validate that header and body match. + let tx_commitment = body.transactions().commitment(); + if header.tx_commitment() != tx_commitment { + return Err(InvalidBlockError::InvalidBlockTxCommitment { + expected: tx_commitment, + actual: header.tx_commitment(), + } + .into()); + } + + let block_num = header.block_num(); + + // Validate that the applied block is the next block in sequence. + let prev_block = self + .db + .select_block_header_by_block_num(None) + .await? + .ok_or(ApplyBlockError::DbBlockHeaderEmpty)?; + let expected_block_num = prev_block.block_num().child(); + if block_num != expected_block_num { + return Err(InvalidBlockError::NewBlockInvalidBlockNum { + expected: expected_block_num, + submitted: block_num, + } + .into()); + } + if header.prev_block_commitment() != prev_block.commitment() { + return Err(InvalidBlockError::NewBlockInvalidPrevCommitment.into()); + } + + Ok(()) + } + + /// Computes nullifier and account tree mutations, validating roots against the block header. + #[miden_instrument( + target = COMPONENT, + skip_all, + err, + )] + fn compute_tree_mutations( + &self, + header: &BlockHeader, + body: &BlockBody, + ) -> Result<(NullifierMutationSet, AccountMutationSet), ApplyBlockError> { + let block_num = header.block_num(); + + // A nullifier can only ever be created once, so the block is invalid if any of its + // nullifiers are already recorded in the tree. + let duplicate_nullifiers: Vec<_> = body + .created_nullifiers() + .iter() + .filter(|&nullifier| self.nullifier_tree.get_block_num(nullifier).is_some()) + .copied() + .collect(); + if !duplicate_nullifiers.is_empty() { + return Err(InvalidBlockError::DuplicatedNullifiers(duplicate_nullifiers).into()); + } + + // The header's chain commitment must equal the chain MMR root prior to this block. + let peaks = self.blockchain.peaks(); + if peaks.hash_peaks() != header.chain_commitment() { + return Err(InvalidBlockError::NewBlockInvalidChainCommitment.into()); + } + + // Compute the nullifier tree mutations and verify that they produce the nullifier root + // claimed in the header. + let nullifier_tree_update = self + .nullifier_tree + .compute_mutations( + body.created_nullifiers().iter().map(|nullifier| (*nullifier, block_num)), + ) + .map_err(InvalidBlockError::NewBlockNullifierAlreadySpent)?; + + if nullifier_tree_update.as_mutation_set().root() != header.nullifier_root() { + return Err(InvalidBlockError::NewBlockInvalidNullifierRoot.into()); + } + + // Compute the account tree mutations and verify that they produce the account root claimed + // in the header. + let account_tree_update = self + .account_tree + .compute_mutations( + body.updated_accounts() + .iter() + .map(|update| (update.account_id(), update.final_state_commitment())), + ) + .map_err(|e| match e { + HistoricalError::AccountTreeError(err) => { + InvalidBlockError::NewBlockDuplicateAccountIdPrefix(err) + }, + HistoricalError::MerkleError(_) => { + panic!("Unexpected MerkleError during account tree mutation computation") + }, + })?; + + if account_tree_update.as_mutation_set().root() != header.account_root() { + return Err(InvalidBlockError::NewBlockInvalidAccountRoot.into()); + } + + Ok((nullifier_tree_update, account_tree_update)) + } + + /// Builds note records with inclusion proofs from the block body. + #[miden_instrument( + target = COMPONENT, + skip_all, + err, + )] + fn build_note_records( + header: &BlockHeader, + body: &BlockBody, + ) -> Result)>, ApplyBlockError> { + let block_num = header.block_num(); + + let note_tree = body.compute_block_note_tree(); + if note_tree.root() != header.note_root() { + return Err(InvalidBlockError::NewBlockInvalidNoteRoot.into()); + } + + let notes = body + .output_notes() + .map(|(note_index, note)| { + let (details, attachments, nullifier) = match note { + OutputNote::Public(public) => ( + Some(NoteDetails::from(public.as_note())), + public.as_note().attachments().clone(), + Some(public.as_note().nullifier()), + ), + OutputNote::Private(private) => (None, private.attachments().clone(), None), + }; + + let inclusion_path = note_tree.open(note_index); + + let note_record = NoteRecord { + block_num, + note_index, + note_id: note.id().as_word(), + metadata: *note.metadata(), + details, + attachments, + inclusion_path, + }; + + Ok((note_record, nullifier)) + }) + .collect::, InvalidBlockError>>()?; + + Ok(notes) + } +} From da541fbd219d59d8d5d33944cc3b365d2994131f Mon Sep 17 00:00:00 2001 From: sergerad Date: Sat, 1 Aug 2026 15:14:38 +1200 Subject: [PATCH 33/35] with_view --- bin/stress-test/src/store/mod.rs | 30 +++++++---- crates/block-producer/src/store/mod.rs | 26 ++++++---- .../server/api/sync_account_storage_maps.rs | 13 ++--- .../rpc/src/server/api/sync_account_vault.rs | 14 +++--- crates/rpc/src/server/api/sync_chain_mmr.rs | 50 +++++++++++-------- crates/rpc/src/server/api/sync_notes.rs | 14 +++--- crates/rpc/src/server/api/sync_nullifiers.rs | 16 +++--- .../rpc/src/server/api/sync_transactions.rs | 14 +++--- crates/store/src/state/view/mod.rs | 19 +++++++ 9 files changed, 122 insertions(+), 74 deletions(-) diff --git a/bin/stress-test/src/store/mod.rs b/bin/stress-test/src/store/mod.rs index d21af7436..9163febd5 100644 --- a/bin/stress-test/src/store/mod.rs +++ b/bin/stress-test/src/store/mod.rs @@ -518,12 +518,18 @@ pub async fn sync_transactions( block_to: u32, ) -> (Duration, proto::rpc::SyncTransactionsResponse) { let start = Instant::now(); - let view = state.view(); - let (last_block_included, records) = view - .sync_transactions(account_ids, BlockNumber::from(block_from)..=BlockNumber::from(block_to)) - .await - .unwrap(); - let chain_tip = view.tip(); + let (chain_tip, sync_result) = state + .with_view(async |view| { + let result = view + .sync_transactions( + account_ids, + BlockNumber::from(block_from)..=BlockNumber::from(block_to), + ) + .await; + (view.tip(), result) + }) + .await; + let (last_block_included, records) = sync_result.unwrap(); let response = proto::rpc::SyncTransactionsResponse { pagination_info: Some(proto::rpc::PaginationInfo { chain_tip: chain_tip.as_u32(), @@ -635,11 +641,13 @@ pub async fn bench_sync_chain_mmr(data_directory: PathBuf, iterations: usize, co /// - the response. async fn sync_chain_mmr(state: &Arc, current_client_block_height: u32) -> SyncChainMmrRun { let start = Instant::now(); - let view = state.view(); - let chain_tip = view.tip(); - view.sync_chain_mmr(BlockNumber::from(current_client_block_height)..=chain_tip) - .await - .unwrap(); + state + .with_view(async |view| { + view.sync_chain_mmr(BlockNumber::from(current_client_block_height)..=view.tip()) + .await + .unwrap(); + }) + .await; let elapsed = start.elapsed(); SyncChainMmrRun { duration: elapsed } } diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index 0d2d2843c..8be96a175 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -185,16 +185,21 @@ pub async fn get_tx_inputs( let unauthenticated_note_commitments = proven_tx.unauthenticated_notes().map(|header| header.id().as_word()).collect(); - // A single view scopes both the input query and the block height below to one snapshot. - let view = state.view(); - let store_inputs = view - .get_transaction_inputs( - proven_tx.account_id(), - &nullifiers, - unauthenticated_note_commitments, - ) - .await - .map_err(StoreError::GetTransactionInputsFailed)?; + // A single view scopes both the input query and the block height to one snapshot; the view is + // released as soon as the query completes. + let (current_block_height, store_inputs) = state + .with_view(async |view| { + let inputs = view + .get_transaction_inputs( + proven_tx.account_id(), + &nullifiers, + unauthenticated_note_commitments, + ) + .await; + (view.tip(), inputs) + }) + .await; + let store_inputs = store_inputs.map_err(StoreError::GetTransactionInputsFailed)?; if !store_inputs.new_account_id_prefix_is_unique.unwrap_or(true) { debug_assert!( @@ -204,7 +209,6 @@ pub async fn get_tx_inputs( return Err(StoreError::DuplicateAccountIdPrefix(proven_tx.account_id())); } - let current_block_height = view.tip(); let tx_inputs = TransactionInputs::from_store_inputs( proven_tx.account_id(), store_inputs, diff --git a/crates/rpc/src/server/api/sync_account_storage_maps.rs b/crates/rpc/src/server/api/sync_account_storage_maps.rs index ebfc6129f..7845c841d 100644 --- a/crates/rpc/src/server/api/sync_account_storage_maps.rs +++ b/crates/rpc/src/server/api/sync_account_storage_maps.rs @@ -58,12 +58,13 @@ impl proto::server::rpc_api::SyncAccountStorageMaps for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let view = self.state.view(); - let chain_tip = view.tip(); - let storage_maps_page = view - .sync_account_storage_maps(account_id, block_range) - .await - .map_err(|err| database_error_to_status(&err))?; + let (chain_tip, sync_result) = self + .state + .with_view(async |view| { + (view.tip(), view.sync_account_storage_maps(account_id, block_range).await) + }) + .await; + let storage_maps_page = sync_result.map_err(|err| database_error_to_status(&err))?; let updates = storage_maps_page .values .into_iter() diff --git a/crates/rpc/src/server/api/sync_account_vault.rs b/crates/rpc/src/server/api/sync_account_vault.rs index 5fc2bd668..6a68b41bc 100644 --- a/crates/rpc/src/server/api/sync_account_vault.rs +++ b/crates/rpc/src/server/api/sync_account_vault.rs @@ -58,12 +58,14 @@ impl proto::server::rpc_api::SyncAccountVault for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let view = self.state.view(); - let chain_tip = view.tip(); - let (last_included_block, updates) = view - .sync_account_vault(account_id, block_range) - .await - .map_err(|err| database_error_to_status(&err))?; + let (chain_tip, sync_result) = self + .state + .with_view(async |view| { + (view.tip(), view.sync_account_vault(account_id, block_range).await) + }) + .await; + let (last_included_block, updates) = + sync_result.map_err(|err| database_error_to_status(&err))?; let updates = updates .into_iter() .map(|update| { diff --git a/crates/rpc/src/server/api/sync_chain_mmr.rs b/crates/rpc/src/server/api/sync_chain_mmr.rs index 55ef9425c..66c803bac 100644 --- a/crates/rpc/src/server/api/sync_chain_mmr.rs +++ b/crates/rpc/src/server/api/sync_chain_mmr.rs @@ -40,29 +40,39 @@ impl proto::server::rpc_api::SyncChainMmr for RpcService { debug!(target: LOG_TARGET, "Syncing chain MMR"); let current_client_block_height = BlockNumber::from(request.current_client_block_height); - let view = self.state.view(); - let sync_target = match request.finality_level() { - proto::rpc::FinalityLevel::Committed | proto::rpc::FinalityLevel::Unspecified => { - view.tip() - }, - // The proven tip is read from a watch channel, not the view's snapshot, so clamp it to - // the view's tip: a block could be committed and proven between taking the view and - // reading the proven tip, and the view cannot serve blocks beyond its snapshot. - proto::rpc::FinalityLevel::Proven => self.state.proven_tip().min(view.tip()), - }; + let finality_level = request.finality_level(); + let (block_range, sync_result) = self + .state + .with_view(async |view| { + let sync_target = match finality_level { + proto::rpc::FinalityLevel::Committed + | proto::rpc::FinalityLevel::Unspecified => view.tip(), + // The proven tip is read from a watch channel, not the view's snapshot, so + // clamp it to the view's tip: a block could be committed and proven between + // taking the view and reading the proven tip, and the view cannot serve blocks + // beyond its snapshot. + proto::rpc::FinalityLevel::Proven => self.state.proven_tip().min(view.tip()), + }; + let block_range = current_client_block_height..=sync_target; + let result = if current_client_block_height > sync_target { + None + } else { + Some(view.sync_chain_mmr(block_range.clone()).await) + }; + (block_range, result) + }) + .await; - if current_client_block_height > sync_target { + let Some(sync_result) = sync_result else { return Err(Status::invalid_argument(format!( - "start block is not known: current client block height {current_client_block_height} is greater than chain tip {sync_target}" + "start block is not known: current client block height {current_client_block_height} is greater than chain tip {}", + block_range.end(), ))); - } - - let block_range = current_client_block_height..=sync_target; - let (mmr_delta, block_header, block_signatures) = - view.sync_chain_mmr(block_range.clone()).await.map_err(|err| match err { - StateSyncError::RangeBeyondTip(_) => Status::invalid_argument(err.to_string()), - _ => Status::internal(err.to_string()), - })?; + }; + let (mmr_delta, block_header, block_signatures) = sync_result.map_err(|err| match err { + StateSyncError::RangeBeyondTip(_) => Status::invalid_argument(err.to_string()), + _ => Status::internal(err.to_string()), + })?; Ok(proto::rpc::SyncChainMmrResponse { block_range: Some(proto::rpc::BlockRange { diff --git a/crates/rpc/src/server/api/sync_notes.rs b/crates/rpc/src/server/api/sync_notes.rs index 961819f1d..cc93c530f 100644 --- a/crates/rpc/src/server/api/sync_notes.rs +++ b/crates/rpc/src/server/api/sync_notes.rs @@ -47,13 +47,13 @@ impl proto::server::rpc_api::SyncNotes for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let view = self.state.view(); - let chain_tip = view.tip(); - - let (results, last_block_checked) = view - .sync_notes(request.note_tags, block_range) - .await - .map_err(note_sync_error_to_status)?; + let (chain_tip, sync_result) = self + .state + .with_view(async |view| { + (view.tip(), view.sync_notes(request.note_tags, block_range).await) + }) + .await; + let (results, last_block_checked) = sync_result.map_err(note_sync_error_to_status)?; let blocks = results .into_iter() .map(|(state, mmr_proof)| proto::rpc::sync_notes_response::NoteSyncBlock { diff --git a/crates/rpc/src/server/api/sync_nullifiers.rs b/crates/rpc/src/server/api/sync_nullifiers.rs index 63865ea25..5e7e7d46f 100644 --- a/crates/rpc/src/server/api/sync_nullifiers.rs +++ b/crates/rpc/src/server/api/sync_nullifiers.rs @@ -58,13 +58,15 @@ impl proto::server::rpc_api::SyncNullifiers for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let view = self.state.view(); - let chain_tip = view.tip(); - - let (nullifiers, block_num) = view - .sync_nullifiers(request.prefix_len, request.nullifiers, block_range) - .await - .map_err(|err| database_error_to_status(&err))?; + let (chain_tip, sync_result) = self + .state + .with_view(async |view| { + let result = + view.sync_nullifiers(request.prefix_len, request.nullifiers, block_range).await; + (view.tip(), result) + }) + .await; + let (nullifiers, block_num) = sync_result.map_err(|err| database_error_to_status(&err))?; let nullifiers = nullifiers .into_iter() .map(|nullifier_info| proto::rpc::sync_nullifiers_response::NullifierUpdate { diff --git a/crates/rpc/src/server/api/sync_transactions.rs b/crates/rpc/src/server/api/sync_transactions.rs index 32c0ab472..030bbcab7 100644 --- a/crates/rpc/src/server/api/sync_transactions.rs +++ b/crates/rpc/src/server/api/sync_transactions.rs @@ -61,13 +61,15 @@ impl proto::server::rpc_api::SyncTransactions for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let view = self.state.view(); - let chain_tip = view.tip(); let account_ids = read_account_ids::(request.account_ids)?; - let (last_block_included, transaction_records_db) = view - .sync_transactions(account_ids, block_range) - .await - .map_err(|err| database_error_to_status(&err))?; + let (chain_tip, sync_result) = self + .state + .with_view(async |view| { + (view.tip(), view.sync_transactions(account_ids, block_range).await) + }) + .await; + let (last_block_included, transaction_records_db) = + sync_result.map_err(|err| database_error_to_status(&err))?; let transactions = transaction_records_db.into_iter().map(transaction_record_to_proto).collect(); diff --git a/crates/store/src/state/view/mod.rs b/crates/store/src/state/view/mod.rs index 5dcbe1b14..0091fffea 100644 --- a/crates/store/src/state/view/mod.rs +++ b/crates/store/src/state/view/mod.rs @@ -58,12 +58,31 @@ impl State { /// /// The view is frozen: it is unaffected if the writer publishes a new snapshot while it is /// held. + /// + /// Use this only for single-expression reads (`state.view().get_account(..)`), where the + /// temporary view drops at the end of the statement. Binding the view to a variable keeps its + /// snapshot generation pinned until the end of the scope; for reads spanning multiple + /// statements use [`Self::with_view`] instead, which releases the view as soon as the read + /// completes. pub fn view(&self) -> StateView { StateView { snapshot: self.latest_snapshot.load_full(), db: Arc::clone(&self.db), } } + + /// Runs a read operation over a view pinned at the current chain tip, dropping the view — and + /// releasing its snapshot generation — as soon as the operation completes. + /// + /// The closure receives a shared reference, so the view cannot escape the call. Prefer this + /// over [`Self::view`] whenever a read spans multiple statements: it keeps the snapshot from + /// being pinned through unrelated work that follows (e.g. response encoding). Single-expression + /// reads (`state.view().get_account(..)`) don't need it — the temporary view already drops at + /// the end of the statement. + pub async fn with_view(&self, f: impl AsyncFnOnce(&StateView) -> R) -> R { + let view = self.view(); + f(&view).await + } } impl StateView { From e1d3e186794d4c177160a06189076797b72d363d Mon Sep 17 00:00:00 2001 From: sergerad Date: Sat, 1 Aug 2026 17:03:24 +1200 Subject: [PATCH 34/35] Fix sync_chain_mmr view usage with tip and add comments --- crates/rpc/src/server/api/sync_chain_mmr.rs | 52 +++++++++------------ crates/store/src/state/tip.rs | 12 ++++- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/crates/rpc/src/server/api/sync_chain_mmr.rs b/crates/rpc/src/server/api/sync_chain_mmr.rs index 66c803bac..8a91f155f 100644 --- a/crates/rpc/src/server/api/sync_chain_mmr.rs +++ b/crates/rpc/src/server/api/sync_chain_mmr.rs @@ -40,39 +40,31 @@ impl proto::server::rpc_api::SyncChainMmr for RpcService { debug!(target: LOG_TARGET, "Syncing chain MMR"); let current_client_block_height = BlockNumber::from(request.current_client_block_height); - let finality_level = request.finality_level(); - let (block_range, sync_result) = self - .state - .with_view(async |view| { - let sync_target = match finality_level { - proto::rpc::FinalityLevel::Committed - | proto::rpc::FinalityLevel::Unspecified => view.tip(), - // The proven tip is read from a watch channel, not the view's snapshot, so - // clamp it to the view's tip: a block could be committed and proven between - // taking the view and reading the proven tip, and the view cannot serve blocks - // beyond its snapshot. - proto::rpc::FinalityLevel::Proven => self.state.proven_tip().min(view.tip()), - }; - let block_range = current_client_block_height..=sync_target; - let result = if current_client_block_height > sync_target { - None - } else { - Some(view.sync_chain_mmr(block_range.clone()).await) - }; - (block_range, result) - }) - .await; - let Some(sync_result) = sync_result else { + // Read the target tip before creating the view: tips are monotonic and each committed tip + // is published after its snapshot, so the view below is guaranteed to be able to serve + // `sync_target`. + let sync_target = match request.finality_level() { + proto::rpc::FinalityLevel::Committed | proto::rpc::FinalityLevel::Unspecified => { + self.state.committed_tip() + }, + proto::rpc::FinalityLevel::Proven => self.state.proven_tip(), + }; + + if current_client_block_height > sync_target { return Err(Status::invalid_argument(format!( - "start block is not known: current client block height {current_client_block_height} is greater than chain tip {}", - block_range.end(), + "start block is not known: current client block height {current_client_block_height} is greater than chain tip {sync_target}" ))); - }; - let (mmr_delta, block_header, block_signatures) = sync_result.map_err(|err| match err { - StateSyncError::RangeBeyondTip(_) => Status::invalid_argument(err.to_string()), - _ => Status::internal(err.to_string()), - })?; + } + + let block_range = current_client_block_height..=sync_target; + let (mmr_delta, block_header, block_signatures) = + self.state.view().sync_chain_mmr(block_range.clone()).await.map_err( + |err| match err { + StateSyncError::RangeBeyondTip(_) => Status::invalid_argument(err.to_string()), + _ => Status::internal(err.to_string()), + }, + )?; Ok(proto::rpc::SyncChainMmrResponse { block_range: Some(proto::rpc::BlockRange { diff --git a/crates/store/src/state/tip.rs b/crates/store/src/state/tip.rs index fee824047..ea9ecf1af 100644 --- a/crates/store/src/state/tip.rs +++ b/crates/store/src/state/tip.rs @@ -19,8 +19,10 @@ impl State { /// This is a live value: it advances independently of any [`StateView`](super::StateView). /// Reads that must be consistent with data must use a view's tip instead. /// - /// The committed tip is published after the corresponding state snapshot, so it never reports - /// a block that a freshly created view cannot serve. + /// Ordering matters when combining this with a view: read the tip *before* creating the view. + /// Tips are monotonic and the committed tip is published after its snapshot, so a view + /// created afterwards can always serve a tip read earlier. Reading a live tip *after* + /// creating a view is racy — the tip may have advanced past the view's pinned snapshot. pub fn committed_tip(&self) -> BlockNumber { *self.committed_tip_tx.borrow() } @@ -29,6 +31,12 @@ impl State { /// /// This is a live value published by the proof scheduler (sequencer mode) or the proof sync /// loop (full-node mode); it is always at or behind the committed tip. + /// + /// Ordering matters when combining this with a view: read the tip *before* creating the view. + /// Proofs are only applied to committed blocks and each committed tip is published after its + /// snapshot, so a view created afterwards can always serve a proven tip read earlier. Reading + /// it *after* creating a view is racy — a block may commit and prove concurrently, putting + /// the proven tip past the view's pinned snapshot. pub fn proven_tip(&self) -> BlockNumber { self.proven_tip.read() } From d5ea358dd692a414df5c23c57bb8fdbae74fc020 Mon Sep 17 00:00:00 2001 From: sergerad Date: Sat, 1 Aug 2026 18:21:15 +1200 Subject: [PATCH 35/35] map results and move map_err into closure --- bin/stress-test/src/store/mod.rs | 17 ++++++++--------- crates/block-producer/src/store/mod.rs | 19 +++++++++---------- .../server/api/sync_account_storage_maps.rs | 10 ++++++---- .../rpc/src/server/api/sync_account_vault.rs | 11 ++++++----- crates/rpc/src/server/api/sync_notes.rs | 10 ++++++---- crates/rpc/src/server/api/sync_nullifiers.rs | 12 ++++++------ .../rpc/src/server/api/sync_transactions.rs | 11 ++++++----- 7 files changed, 47 insertions(+), 43 deletions(-) diff --git a/bin/stress-test/src/store/mod.rs b/bin/stress-test/src/store/mod.rs index 9163febd5..f9d0db615 100644 --- a/bin/stress-test/src/store/mod.rs +++ b/bin/stress-test/src/store/mod.rs @@ -518,18 +518,17 @@ pub async fn sync_transactions( block_to: u32, ) -> (Duration, proto::rpc::SyncTransactionsResponse) { let start = Instant::now(); - let (chain_tip, sync_result) = state + let (chain_tip, (last_block_included, records)) = state .with_view(async |view| { - let result = view - .sync_transactions( - account_ids, - BlockNumber::from(block_from)..=BlockNumber::from(block_to), - ) - .await; - (view.tip(), result) + view.sync_transactions( + account_ids, + BlockNumber::from(block_from)..=BlockNumber::from(block_to), + ) + .await + .map(|records| (view.tip(), records)) + .unwrap() }) .await; - let (last_block_included, records) = sync_result.unwrap(); let response = proto::rpc::SyncTransactionsResponse { pagination_info: Some(proto::rpc::PaginationInfo { chain_tip: chain_tip.as_u32(), diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index 8be96a175..c3f98ab0c 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -189,17 +189,16 @@ pub async fn get_tx_inputs( // released as soon as the query completes. let (current_block_height, store_inputs) = state .with_view(async |view| { - let inputs = view - .get_transaction_inputs( - proven_tx.account_id(), - &nullifiers, - unauthenticated_note_commitments, - ) - .await; - (view.tip(), inputs) + view.get_transaction_inputs( + proven_tx.account_id(), + &nullifiers, + unauthenticated_note_commitments, + ) + .await + .map(|inputs| (view.tip(), inputs)) + .map_err(StoreError::GetTransactionInputsFailed) }) - .await; - let store_inputs = store_inputs.map_err(StoreError::GetTransactionInputsFailed)?; + .await?; if !store_inputs.new_account_id_prefix_is_unique.unwrap_or(true) { debug_assert!( diff --git a/crates/rpc/src/server/api/sync_account_storage_maps.rs b/crates/rpc/src/server/api/sync_account_storage_maps.rs index 7845c841d..9904b5e72 100644 --- a/crates/rpc/src/server/api/sync_account_storage_maps.rs +++ b/crates/rpc/src/server/api/sync_account_storage_maps.rs @@ -58,13 +58,15 @@ impl proto::server::rpc_api::SyncAccountStorageMaps for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let (chain_tip, sync_result) = self + let (chain_tip, storage_maps_page) = self .state .with_view(async |view| { - (view.tip(), view.sync_account_storage_maps(account_id, block_range).await) + view.sync_account_storage_maps(account_id, block_range) + .await + .map(|page| (view.tip(), page)) + .map_err(|err| database_error_to_status(&err)) }) - .await; - let storage_maps_page = sync_result.map_err(|err| database_error_to_status(&err))?; + .await?; let updates = storage_maps_page .values .into_iter() diff --git a/crates/rpc/src/server/api/sync_account_vault.rs b/crates/rpc/src/server/api/sync_account_vault.rs index 6a68b41bc..4fad5a220 100644 --- a/crates/rpc/src/server/api/sync_account_vault.rs +++ b/crates/rpc/src/server/api/sync_account_vault.rs @@ -58,14 +58,15 @@ impl proto::server::rpc_api::SyncAccountVault for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let (chain_tip, sync_result) = self + let (chain_tip, (last_included_block, updates)) = self .state .with_view(async |view| { - (view.tip(), view.sync_account_vault(account_id, block_range).await) + view.sync_account_vault(account_id, block_range) + .await + .map(|updates| (view.tip(), updates)) + .map_err(|err| database_error_to_status(&err)) }) - .await; - let (last_included_block, updates) = - sync_result.map_err(|err| database_error_to_status(&err))?; + .await?; let updates = updates .into_iter() .map(|update| { diff --git a/crates/rpc/src/server/api/sync_notes.rs b/crates/rpc/src/server/api/sync_notes.rs index cc93c530f..a2c540f1d 100644 --- a/crates/rpc/src/server/api/sync_notes.rs +++ b/crates/rpc/src/server/api/sync_notes.rs @@ -47,13 +47,15 @@ impl proto::server::rpc_api::SyncNotes for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let (chain_tip, sync_result) = self + let (chain_tip, (results, last_block_checked)) = self .state .with_view(async |view| { - (view.tip(), view.sync_notes(request.note_tags, block_range).await) + view.sync_notes(request.note_tags, block_range) + .await + .map(|notes| (view.tip(), notes)) + .map_err(note_sync_error_to_status) }) - .await; - let (results, last_block_checked) = sync_result.map_err(note_sync_error_to_status)?; + .await?; let blocks = results .into_iter() .map(|(state, mmr_proof)| proto::rpc::sync_notes_response::NoteSyncBlock { diff --git a/crates/rpc/src/server/api/sync_nullifiers.rs b/crates/rpc/src/server/api/sync_nullifiers.rs index 5e7e7d46f..9816a1c12 100644 --- a/crates/rpc/src/server/api/sync_nullifiers.rs +++ b/crates/rpc/src/server/api/sync_nullifiers.rs @@ -58,15 +58,15 @@ impl proto::server::rpc_api::SyncNullifiers for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let (chain_tip, sync_result) = self + let (chain_tip, (nullifiers, block_num)) = self .state .with_view(async |view| { - let result = - view.sync_nullifiers(request.prefix_len, request.nullifiers, block_range).await; - (view.tip(), result) + view.sync_nullifiers(request.prefix_len, request.nullifiers, block_range) + .await + .map(|nullifiers| (view.tip(), nullifiers)) + .map_err(|err| database_error_to_status(&err)) }) - .await; - let (nullifiers, block_num) = sync_result.map_err(|err| database_error_to_status(&err))?; + .await?; let nullifiers = nullifiers .into_iter() .map(|nullifier_info| proto::rpc::sync_nullifiers_response::NullifierUpdate { diff --git a/crates/rpc/src/server/api/sync_transactions.rs b/crates/rpc/src/server/api/sync_transactions.rs index 030bbcab7..84ff96992 100644 --- a/crates/rpc/src/server/api/sync_transactions.rs +++ b/crates/rpc/src/server/api/sync_transactions.rs @@ -62,14 +62,15 @@ impl proto::server::rpc_api::SyncTransactions for RpcService { .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; let account_ids = read_account_ids::(request.account_ids)?; - let (chain_tip, sync_result) = self + let (chain_tip, (last_block_included, transaction_records_db)) = self .state .with_view(async |view| { - (view.tip(), view.sync_transactions(account_ids, block_range).await) + view.sync_transactions(account_ids, block_range) + .await + .map(|records| (view.tip(), records)) + .map_err(|err| database_error_to_status(&err)) }) - .await; - let (last_block_included, transaction_records_db) = - sync_result.map_err(|err| database_error_to_status(&err))?; + .await?; let transactions = transaction_records_db.into_iter().map(transaction_record_to_proto).collect();