diff --git a/Cargo.lock b/Cargo.lock index bf0af484b..e56671c1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3631,6 +3631,7 @@ name = "miden-node-store" version = "0.16.0-alpha.2" dependencies = [ "anyhow", + "arc-swap", "assert_matches", "build-rs", "criterion", diff --git a/Cargo.toml b/Cargo.toml index b2b084c82..0258b8b86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,7 @@ miden-crypto = { version = "0.28" } # External dependencies anyhow = { version = "1.0" } +arc-swap = { version = "1.7" } assert_matches = { version = "1.5" } axum = { version = "0.8" } backon = { version = "1.6" } diff --git a/bin/node/src/commands/modes.rs b/bin/node/src/commands/modes.rs index 74e0966a5..5082792b4 100644 --- a/bin/node/src/commands/modes.rs +++ b/bin/node/src/commands/modes.rs @@ -12,7 +12,7 @@ use miden_node_proto::clients::{ ValidatorClient, }; use miden_node_rpc::{PreAuthSubmission, Rpc, RpcMode, SequencerInternal, ValidatorClients}; -use miden_node_store::State; +use miden_node_store::{BlockWriter, ProofWriter, State, WriterTask}; use miden_node_utils::clap::{GrpcOptionsInternal, duration_to_human_readable_string}; use miden_node_utils::formatting::format_endpoint; use miden_node_utils::shutdown::CancellationToken; @@ -57,11 +57,14 @@ impl SequencerCommand { let runtime = self.runtime.runtime_config(&self.store); self.block_producer.validate()?; let network_tx_auth = self.runtime.rpc.network_tx_auth()?; - let state = load_state(&runtime).await?; + let (state, block_writer, proof_writer, writer_task) = + load_state(&runtime, shutdown.clone()).await?; let _disk_monitor = state.spawn_disk_monitor(shutdown.clone()); let sequencer = Sequencer { - store: Arc::clone(&state), + state: Arc::clone(&state), + block_writer, + proof_writer, validator_urls: self.external_services.validator_urls.clone(), validator_timeout: self.external_services.validator_timeout, batch_prover_url: self.block_producer.batch.prover_url, @@ -75,17 +78,17 @@ impl SequencerCommand { batch_workers: self.block_producer.batch.workers, } .spawn(shutdown.clone()) - .await .context("failed to spawn sequencer")?; let block_producer = sequencer.api(); let rpc = Rpc { listener: bind_rpc(runtime.rpc_listen).await?, - store: state, + state, mode: RpcMode::sequencer( block_producer.clone(), self.external_services.validator_clients()?, ), + sync_writers: None, ntx_builder: Some(self.external_services.ntx_builder_client()?), grpc_options: runtime.external_grpc_options, network_tx_auth, @@ -93,6 +96,7 @@ impl SequencerCommand { let mut tasks = Tasks::new(); tasks.spawn("sequencer", sequencer.wait()); tasks.spawn("RPC server", rpc.serve(shutdown.clone())); + tasks.spawn("store block writer", join_store_writer(writer_task)); if let Some(internal_listen) = self.internal { let sequencer_internal = SequencerInternal { listener: bind_rpc(internal_listen).await?, @@ -240,19 +244,22 @@ impl FullNodeCommand { let source_rpc = self.sync.source_rpc_client()?; let pre_auth = self.pre_auth_submission()?; let network_tx_auth = self.runtime.rpc.network_tx_auth()?; - let state = load_state(&runtime).await?; + let (state, block_writer, proof_writer, writer_task) = + load_state(&runtime, shutdown.clone()).await?; let _disk_monitor = state.spawn_disk_monitor(shutdown.clone()); let rpc = Rpc { listener: bind_rpc(runtime.rpc_listen).await?, - store: state, + state, mode: RpcMode::full_node(source_rpc, self.sync.readiness_threshold, pre_auth), + sync_writers: Some((block_writer, proof_writer)), ntx_builder: None, grpc_options: runtime.external_grpc_options, network_tx_auth, }; let mut tasks = Tasks::new(); tasks.spawn("RPC server", rpc.serve(shutdown.clone())); + tasks.spawn("store block writer", join_store_writer(writer_task)); tasks.join_next_or_cancelled(shutdown).await } @@ -326,16 +333,29 @@ impl SyncOptions { } } -async fn load_state(runtime: &RuntimeConfig) -> anyhow::Result> { - let state = State::load_with_database_options( +async fn load_state( + runtime: &RuntimeConfig, + shutdown: CancellationToken, +) -> anyhow::Result<(Arc, BlockWriter, ProofWriter, WriterTask)> { + let loaded = State::load_with_database_options( &runtime.data_directory, runtime.storage_options.clone(), runtime.database_options, + shutdown, ) .await .context("failed to load state")?; - Ok(Arc::new(state)) + Ok(loaded.start()) +} + +/// Supervises the store's write worker task. +/// +/// On shutdown the task-drain loop waits for the writer to finish any in-flight block write and +/// close its storage; an early exit or panic surfaces through the task set like any other task +/// failure. +async fn join_store_writer(writer_task: WriterTask) -> anyhow::Result<()> { + writer_task.await.map_err(anyhow::Error::from) } async fn bind_rpc(listen: SocketAddr) -> anyhow::Result { diff --git a/bin/node/src/commands/recover.rs b/bin/node/src/commands/recover.rs index 343850f99..94cd00baf 100644 --- a/bin/node/src/commands/recover.rs +++ b/bin/node/src/commands/recover.rs @@ -5,8 +5,8 @@ use std::time::Duration; use anyhow::Context; use miden_node_proto::clients::{Builder, ValidatorClient}; use miden_node_proto::generated::validator::BlockSubscriptionRequest; -use miden_node_store::State; -use miden_node_store::state::Finality; +use miden_node_store::{BlockWriter, State, WriterTask}; +use miden_node_utils::shutdown::CancellationToken; use miden_protocol::block::{BlockNumber, SignedBlock}; use miden_protocol::utils::serde::Deserializable; use tokio_stream::StreamExt; @@ -36,21 +36,28 @@ pub struct RecoverCommand { impl RecoverCommand { pub async fn handle(self) -> anyhow::Result<()> { - let state = self.load_state().await?; + let (state, block_writer, writer_task) = self.load_state().await?; let validator = self.validator_client()?; - recover_from_validator(&state, validator).await + let result = recover_from_validator(&state, &block_writer, validator).await; + // Wait for the writer to drain and release the backing storage before the process exits. + block_writer.stop(writer_task).await; + result } - async fn load_state(&self) -> anyhow::Result> { - let state = State::load_with_database_options( + async fn load_state(&self) -> anyhow::Result<(Arc, BlockWriter, WriterTask)> { + // Recovery is not wired into the node's shutdown token; the writer exits once the + // `BlockWriter` (holding the only write handle) is dropped after recovery completes. + let loaded = State::load_with_database_options( &self.data_directory, self.store.storage.clone().into(), self.store.sqlite.database_options(), + CancellationToken::new(), ) .await .context("failed to load state")?; - Ok(Arc::new(state)) + let (state, block_writer, _proof_writer, writer_task) = loaded.start(); + Ok((state, block_writer, writer_task)) } fn validator_client(&self) -> anyhow::Result { @@ -66,7 +73,8 @@ impl RecoverCommand { /// Streams blocks from the validator into the local store until the chain tip is reached. async fn recover_from_validator( - state: &Arc, + state: &State, + block_writer: &BlockWriter, mut validator: ValidatorClient, ) -> anyhow::Result<()> { // Capture the validator's chain tip as the recovery target. The validator's block stream @@ -81,7 +89,7 @@ async fn recover_from_validator( .chain_tip, ); - let local_tip = state.chain_tip(Finality::Committed).await; + let local_tip = state.committed_tip(); if local_tip >= validator_tip { info!( target: LOG_TARGET, @@ -111,7 +119,10 @@ async fn recover_from_validator( let block = SignedBlock::read_from_bytes(&event.block) .context("failed to deserialize block from validator")?; let block_num = block.header().block_num(); - state.apply_block(block).await.context("failed to apply recovered block")?; + block_writer + .apply_block(block) + .await + .context("failed to apply recovered block")?; info!(target: LOG_TARGET, block_number = %block_num.as_u32(), "Applied recovered block"); // Stop once we reach the tip captured at the start of recovery. @@ -121,7 +132,7 @@ async fn recover_from_validator( } // The stream can end before reaching the tip if the validator restarts or drops the connection. - let final_tip = state.chain_tip(Finality::Committed).await; + let final_tip = state.committed_tip(); anyhow::ensure!( final_tip >= validator_tip, "validator block stream ended at block {} before reaching the chain tip {}", diff --git a/bin/stress-test/src/seeding/mod.rs b/bin/stress-test/src/seeding/mod.rs index eeb21fae5..e64bb1d82 100644 --- a/bin/stress-test/src/seeding/mod.rs +++ b/bin/stress-test/src/seeding/mod.rs @@ -6,8 +6,9 @@ use std::time::Instant; use metrics::SeedingMetrics; use miden_node_proto::domain::batch::BatchInputs; -use miden_node_store::{DataDirectory, GenesisState, State}; +use miden_node_store::{BlockWriter, DataDirectory, GenesisState, State, WriterTask}; use miden_node_utils::clap::StorageOptions; +use miden_node_utils::shutdown::CancellationToken; use miden_protocol::account::auth::AuthScheme; use miden_protocol::account::{ Account, @@ -235,7 +236,7 @@ pub async fn seed_store_with_readers( let genesis_header = genesis_block.inner().header().clone(); State::bootstrap(genesis_block, &data_directory).expect("store should bootstrap"); - let store_state = load_state(data_directory.clone()).await; + let (state, block_writer, writer_task) = load_state(data_directory.clone()).await; // Recreate the deterministic genesis benchmark accounts after bootstrapping instead of keeping // another copy of their potentially very large maps alive while the genesis block is built. @@ -259,7 +260,7 @@ pub async fn seed_store_with_readers( let stop_readers = Arc::new(AtomicBool::new(false)); let reader_tasks: Vec<_> = (0..readers) .map(|_| { - let state = Arc::clone(&store_state); + let state = Arc::clone(&state); let stop = Arc::clone(&stop_readers); tokio::spawn(async move { read_latest_header_until(&state, &stop).await }) }) @@ -270,6 +271,7 @@ pub async fn seed_store_with_readers( let data_directory = miden_node_store::DataDirectory::load(data_directory).expect("data directory should exist"); let metrics = Box::pin(generate_blocks( + &state, account_batches, initial_accounts, if seed_public_accounts_at_genesis { @@ -279,7 +281,7 @@ pub async fn seed_store_with_readers( }, faucet, genesis_header, - &store_state, + &block_writer, data_directory, accounts_filepath, &signer, @@ -290,8 +292,8 @@ pub async fn seed_store_with_readers( )) .await; - // Stop the readers and report their latencies; they hold state references and the read phase - // should not include post-write quiescence. + // Stop the readers and report their latencies before stopping the store: they hold state + // references, and the read phase should not include post-write quiescence. let write_load_elapsed = start.elapsed(); stop_readers.store(true, Ordering::Relaxed); let mut read_latencies = Vec::new(); @@ -302,6 +304,10 @@ pub async fn seed_store_with_readers( report_read_latencies(&read_latencies, readers, write_load_elapsed); } + // Wait for the store to release its backing storage so callers can immediately re-load the + // state from the same data directory. + block_writer.stop(writer_task).await; + println!("Total time: {:.3} seconds", start.elapsed().as_secs_f64()); println!("{metrics}"); } @@ -309,9 +315,9 @@ pub async fn seed_store_with_readers( /// Loops `get_block_header` (latest header with MMR proof) against the state until `stop` is set, /// returning the latency of every request. /// -/// The request combines an in-memory read (opening the latest block's MMR proof, which contends -/// with the writer's lock during block application) with a database header lookup, so it exercises -/// the read path most exposed to concurrent block application. +/// The request combines an in-memory read (opening the latest block's MMR proof against the +/// snapshot's blockchain) with a database header lookup scoped to the snapshot's tip, so it +/// exercises the read path most exposed to concurrent block application. async fn read_latest_header_until( state: &Arc, stop: &AtomicBool, @@ -320,6 +326,7 @@ async fn read_latest_header_until( while !stop.load(Ordering::Relaxed) { let start = Instant::now(); let (header, proof) = state + .view() .get_block_header(None, true) .await .expect("get_block_header should succeed during seeding"); @@ -353,12 +360,13 @@ fn report_read_latencies( #[expect(clippy::too_many_arguments)] #[expect(clippy::too_many_lines)] async fn generate_blocks( + state: &State, account_batches: Vec, initial_accounts: Vec, first_account_index: u64, mut faucet: Account, genesis_header: BlockHeader, - store_state: &Arc, + block_writer: &BlockWriter, data_directory: DataDirectory, accounts_filepath: PathBuf, signer: &EcdsaSecretKey, @@ -439,7 +447,7 @@ async fn generate_blocks( .collect(); // create the block and send it to the store - let block_inputs = get_block_inputs(store_state, &batches, &mut metrics).await; + let block_inputs = get_block_inputs(state, &batches, &mut metrics).await; // update blocks let block_kind = if has_pending_account_creations { @@ -450,7 +458,7 @@ async fn generate_blocks( prev_block_header = apply_block( batches, block_inputs, - store_state, + block_writer, &mut metrics, signer, block_kind, @@ -461,8 +469,7 @@ async fn generate_blocks( .extend(pending_consumed_accounts.into_iter().map(|account| (account.id(), account))); // create the consume notes txs to be used in the next block - let batch_inputs = - get_batch_inputs(store_state, &prev_block_header, ¬es, &mut metrics).await; + let batch_inputs = get_batch_inputs(state, &prev_block_header, ¬es, &mut metrics).await; (pending_consumed_accounts, consume_notes_txs) = create_consume_note_txs( &prev_block_header, accounts, @@ -486,11 +493,11 @@ async fn generate_blocks( .par_chunks(TRANSACTIONS_PER_BATCH) .map(|txs| create_batch(txs, &prev_block_header)) .collect(); - let block_inputs = get_block_inputs(store_state, &batches, &mut metrics).await; + let block_inputs = get_block_inputs(state, &batches, &mut metrics).await; prev_block_header = apply_block( batches, block_inputs, - store_state, + block_writer, &mut metrics, signer, metrics::BlockKind::AccountCreation, @@ -526,11 +533,11 @@ async fn generate_blocks( let emit_note_tx = create_emit_note_tx(&prev_block_header, &mut faucet, notes.clone()); let batches = vec![create_batch(std::slice::from_ref(&emit_note_tx), &prev_block_header)]; - let block_inputs = get_block_inputs(store_state, &batches, &mut metrics).await; + let block_inputs = get_block_inputs(state, &batches, &mut metrics).await; prev_block_header = apply_block( batches, block_inputs, - store_state, + block_writer, &mut metrics, signer, metrics::BlockKind::UpdateNoteEmission, @@ -538,8 +545,7 @@ async fn generate_blocks( ) .await; - let batch_inputs = - get_batch_inputs(store_state, &prev_block_header, ¬es, &mut metrics).await; + let batch_inputs = get_batch_inputs(state, &prev_block_header, ¬es, &mut metrics).await; let accounts = selected_account_ids .iter() .map(|account_id| { @@ -562,11 +568,11 @@ async fn generate_blocks( .par_chunks(TRANSACTIONS_PER_BATCH) .map(|txs| create_batch(txs, &prev_block_header)) .collect(); - let block_inputs = get_block_inputs(store_state, &batches, &mut metrics).await; + let block_inputs = get_block_inputs(state, &batches, &mut metrics).await; prev_block_header = apply_block( batches, block_inputs, - store_state, + block_writer, &mut metrics, signer, metrics::BlockKind::AccountUpdate, @@ -595,7 +601,7 @@ async fn generate_blocks( async fn apply_block( batches: Vec, block_inputs: BlockInputs, - store_state: &Arc, + block_writer: &BlockWriter, metrics: &mut SeedingMetrics, signer: &EcdsaSecretKey, block_kind: metrics::BlockKind, @@ -613,7 +619,7 @@ async fn apply_block( let ordered_batches = proposed_block.batches().clone(); let start = Instant::now(); - store_state + block_writer .apply_block_with_proving_inputs(ordered_batches, block_inputs, signed_block) .await .unwrap(); @@ -1063,7 +1069,7 @@ fn create_emit_note_tx( /// Gets the batch inputs from the store and tracks the query time on the metrics. async fn get_batch_inputs( - store_state: &Arc, + state: &State, block_ref: &BlockHeader, notes: &[Note], metrics: &mut SeedingMetrics, @@ -1071,7 +1077,8 @@ async fn get_batch_inputs( let start = Instant::now(); // Mark every note as unauthenticated, so that the store returns the inclusion proofs for all of // them - let batch_inputs = store_state + let batch_inputs = state + .view() .get_batch_inputs( [block_ref.block_num()].into_iter().collect(), notes.iter().map(|note| note.id().as_word()).collect(), @@ -1084,12 +1091,13 @@ async fn get_batch_inputs( /// Gets the block inputs from the store and tracks the query time on the metrics. async fn get_block_inputs( - store_state: &Arc, + state: &State, batches: &[ProvenBatch], metrics: &mut SeedingMetrics, ) -> BlockInputs { let start = Instant::now(); - let inputs = store_state + let inputs = state + .view() .get_block_inputs( batches.iter().flat_map(ProvenBatch::updated_accounts).collect(), batches.iter().flat_map(ProvenBatch::created_nullifiers).collect(), @@ -1111,14 +1119,28 @@ async fn get_block_inputs( inputs } -/// Loads the store state from the given data directory. +/// Loads the store state from the given data directory, detaching the block writer task. +/// +/// Intended for benches that run until process exit and never need the storage released +/// deterministically; use [`load_state`] when the writer must be joined. The write capability is +/// leaked to keep the block writer alive for the process lifetime, as before the read/write +/// split. pub async fn start_store(data_directory: PathBuf) -> Arc { - load_state(data_directory).await + let (state, block_writer, _writer_task) = load_state(data_directory).await; + std::mem::forget(block_writer); + state } -async fn load_state(data_directory: PathBuf) -> Arc { - let state = State::load(&data_directory, StorageOptions::bench()) - .await - .expect("store state should load"); - Arc::new(state) +/// Loads the store state and spawns its block writer, returning the write capability and the +/// writer's join handle. +/// +/// The writer exits once the returned [`BlockWriter`] is dropped; awaiting the handle after that +/// guarantees the backing storage has been released. +async fn load_state(data_directory: PathBuf) -> (Arc, BlockWriter, WriterTask) { + let (state, block_writer, _proof_writer, writer_task) = + State::load(&data_directory, StorageOptions::bench(), CancellationToken::new()) + .await + .expect("store state should load") + .start(); + (state, block_writer, writer_task) } diff --git a/bin/stress-test/src/seeding/tests.rs b/bin/stress-test/src/seeding/tests.rs index e0be9a7b2..420b96de3 100644 --- a/bin/stress-test/src/seeding/tests.rs +++ b/bin/stress-test/src/seeding/tests.rs @@ -170,8 +170,9 @@ async fn seed_store_persists_one_public_account_and_applies_one_map_update() { assert_eq!(account_ids.len(), 1); let account_id = AccountId::from_hex(account_ids[0]).unwrap(); - let state = load_state(data_directory).await; + let (state, block_writer, writer_task) = load_state(data_directory).await; let response = state + .view() .get_account(AccountRequest { account_id, block_num: None, @@ -198,6 +199,10 @@ async fn seed_store_persists_one_public_account_and_applies_one_map_update() { .map(|(_, value)| *value), Some(benchmark_storage_map_update_value(0, 0, 1)) ); + + // Release the backing storage before the temporary directory is deleted. + drop(state); + block_writer.stop(writer_task).await; } #[tokio::test(flavor = "multi_thread")] @@ -213,8 +218,9 @@ async fn seed_store_handles_map_larger_than_transaction_account_update_limit() { assert_eq!(account_ids.len(), 1); let account_id = AccountId::from_hex(account_ids[0]).unwrap(); - let state = load_state(data_directory).await; + let (state, block_writer, writer_task) = load_state(data_directory).await; let response = state + .view() .get_account(AccountRequest { account_id, block_num: None, @@ -223,4 +229,8 @@ async fn seed_store_handles_map_larger_than_transaction_account_update_limit() { .await .unwrap(); assert_ne!(response.witness.state_commitment(), Word::empty()); + + // Release the backing storage before the temporary directory is deleted. + drop(state); + block_writer.stop(writer_task).await; } diff --git a/bin/stress-test/src/store/mod.rs b/bin/stress-test/src/store/mod.rs index 6f9440148..d21af7436 100644 --- a/bin/stress-test/src/store/mod.rs +++ b/bin/stress-test/src/store/mod.rs @@ -5,8 +5,9 @@ use std::time::{Duration, Instant}; use futures::{StreamExt, stream}; use miden_node_proto::domain::account::AccountRequest; use miden_node_proto::generated::{self as proto}; -use miden_node_store::state::{Finality, State}; +use miden_node_store::state::State; use miden_node_utils::clap::StorageOptions; +use miden_node_utils::shutdown::CancellationToken; use miden_protocol::Word; use miden_protocol::account::AccountId; use miden_protocol::block::BlockNumber; @@ -127,7 +128,8 @@ async fn get_account( let start = Instant::now(); let request = AccountRequest::try_from(request).expect("request should be valid"); - let response: proto::rpc::AccountResponse = state.get_account(request).await.unwrap().into(); + let response: proto::rpc::AccountResponse = + state.view().get_account(request).await.unwrap().into(); let duration = start.elapsed(); let details = response.details; @@ -232,7 +234,7 @@ pub async fn bench_sync_notes(data_directory: PathBuf, iterations: usize, concur let store_state = start_store(data_directory).await; // Get the latest block number to determine the range - let chain_tip = store_state.chain_tip(Finality::Committed).await.as_u32(); + let chain_tip = store_state.committed_tip().as_u32(); // each request will have `ACCOUNTS_PER_SYNC_NOTES` note tags and will be sent with block number // 0. @@ -268,6 +270,7 @@ pub async fn sync_notes( .collect::>(); let start = Instant::now(); state + .view() .sync_notes(note_tags, BlockNumber::from(0)..=BlockNumber::from(chain_tip)) .await .unwrap(); @@ -304,7 +307,7 @@ pub async fn bench_sync_nullifiers( .collect(); // Get the latest block number to determine the range - let chain_tip = store_state.chain_tip(Finality::Committed).await.as_u32(); + let chain_tip = store_state.committed_tip().as_u32(); // Get all nullifier prefixes from the store using sync_notes let mut nullifier_prefixes: Vec = vec![]; @@ -316,6 +319,7 @@ pub async fn bench_sync_nullifiers( .map(|id| u32::from(NoteTag::with_account_target(*id))) .collect(); let (blocks, last_block_checked) = store_state + .view() .sync_notes( note_tags, BlockNumber::from(current_block_num)..=BlockNumber::from(chain_tip), @@ -338,6 +342,7 @@ pub async fn bench_sync_nullifiers( note_ids.iter().take(NOTE_IDS_PER_NULLIFIERS_CHECK).copied().collect(); if !note_ids_to_fetch.is_empty() { let notes = store_state + .view() .get_notes_by_id(note_ids_to_fetch) .await .unwrap() @@ -393,6 +398,7 @@ async fn sync_nullifiers( ) -> (Duration, usize) { let start = Instant::now(); let (nullifiers, _) = state + .view() .sync_nullifiers( 16, nullifiers_prefixes, @@ -438,7 +444,7 @@ pub async fn bench_sync_transactions( let store_state = start_store(data_directory).await; // Get the latest block number to determine the range - let chain_tip = store_state.chain_tip(Finality::Committed).await.as_u32(); + let chain_tip = store_state.committed_tip().as_u32(); // each request will have `accounts_per_request` account ids and will query a range of blocks let request = |_| { @@ -512,11 +518,12 @@ pub async fn sync_transactions( block_to: u32, ) -> (Duration, proto::rpc::SyncTransactionsResponse) { let start = Instant::now(); - let (last_block_included, records) = state + let view = state.view(); + let (last_block_included, records) = view .sync_transactions(account_ids, BlockNumber::from(block_from)..=BlockNumber::from(block_to)) .await .unwrap(); - let chain_tip = state.chain_tip(Finality::Committed).await; + let chain_tip = view.tip(); let response = proto::rpc::SyncTransactionsResponse { pagination_info: Some(proto::rpc::PaginationInfo { chain_tip: chain_tip.as_u32(), @@ -599,7 +606,7 @@ async fn sync_transactions_paginated( pub async fn bench_sync_chain_mmr(data_directory: PathBuf, iterations: usize, concurrency: usize) { let store_state = start_store(data_directory).await; - let chain_tip = store_state.chain_tip(Finality::Committed).await.as_u32(); + let chain_tip = store_state.committed_tip().as_u32(); let request = |_| { let state = Arc::clone(&store_state); @@ -628,9 +635,9 @@ pub async fn bench_sync_chain_mmr(data_directory: PathBuf, iterations: usize, co /// - the response. async fn sync_chain_mmr(state: &Arc, current_client_block_height: u32) -> SyncChainMmrRun { let start = Instant::now(); - let chain_tip = state.chain_tip(Finality::Committed).await; - state - .sync_chain_mmr(BlockNumber::from(current_client_block_height)..=chain_tip) + let view = state.view(); + let chain_tip = view.tip(); + view.sync_chain_mmr(BlockNumber::from(current_client_block_height)..=chain_tip) .await .unwrap(); let elapsed = start.elapsed(); @@ -685,7 +692,11 @@ fn transaction_record_to_proto( pub async fn load_state(data_directory: &Path) { let start = Instant::now(); - let _state = State::load(data_directory, StorageOptions::default()).await.unwrap(); + // The writer is never started: this bench only measures load time, and dropping the un-started + // state releases the tree storage the writer owns. + let _loaded = State::load(data_directory, StorageOptions::default(), CancellationToken::new()) + .await + .unwrap(); let elapsed = start.elapsed(); // Get database path and run SQL commands to count records diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index 5cc4c3693..d85bded1f 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -45,7 +45,7 @@ pub struct BatchBuilder { /// /// If not provided, a local batch prover is used. batch_prover: BatchProver, - store: Arc, + state: Arc, } pub(crate) struct BatchIntervals { @@ -79,7 +79,7 @@ impl BatchBuilder { /// /// If no batch prover URL is provided, a local batch prover is used instead. pub fn new( - store: Arc, + state: Arc, num_workers: NonZeroUsize, batch_prover_url: Option, intervals: BatchIntervals, @@ -92,7 +92,7 @@ impl BatchBuilder { num_workers, intervals, batch_prover, - store, + state, }) } @@ -160,7 +160,7 @@ impl BatchBuilder { transactions.unauthenticated_notes.count = telemetry.unauthenticated_notes_count, ); let job = BatchJob { - store: self.store.clone(), + state: self.state.clone(), mempool, batch_prover: self.batch_prover.clone(), }; @@ -240,7 +240,7 @@ impl BatchBuilder { /// /// Recoverable errors are handled internally. Mempool poison is propagated as a fatal error. struct BatchJob { - store: Arc, + state: Arc, batch_prover: BatchProver, mempool: SharedMempool, } @@ -305,7 +305,8 @@ impl BatchJob { .map(Deref::deref) .flat_map(AuthenticatedTransaction::unauthenticated_note_ids); - self.store + self.state + .view() .get_batch_inputs( block_references.map(|(block_num, _)| block_num).collect(), unauthenticated_notes.collect(), diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index 4d0b28bd3..9b3501221 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -2,7 +2,7 @@ use std::ops::Deref; use std::sync::Arc; use anyhow::Context; -use miden_node_store::state::State; +use miden_node_store::state::{BlockWriter, State}; use miden_node_utils::formatting::format_array; use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::spawn::spawn_blocking_in_current_span; @@ -31,23 +31,33 @@ pub struct BlockBuilder { /// The frequency at which blocks are produced. pub block_interval: Duration, - /// The store state for committing blocks. - pub store: Arc, + /// Read-only store state, used to fetch block inputs. + pub state: Arc, + + /// The store's block-write capability, used for committing blocks. + pub block_writer: BlockWriter, /// The validator RPC client for validating blocks. pub validator: BlockProducerValidatorClient, } impl BlockBuilder { - /// Creates a new [`BlockBuilder`] with the given store state and optional block prover URL. + /// Creates a new [`BlockBuilder`] with the given block-write capability and optional block + /// prover URL. /// /// If the block prover URL is not set, the block builder will use the local block prover. pub fn new( - store: Arc, + state: Arc, + block_writer: BlockWriter, validator: BlockProducerValidatorClient, block_interval: Duration, ) -> Self { - Self { block_interval, store, validator } + Self { + block_interval, + state, + block_writer, + validator, + } } /// Starts the [`BlockBuilder`], infinitely producing blocks at the configured interval. /// @@ -213,7 +223,8 @@ impl BlockBuilder { batch_iter.map(Deref::deref).flat_map(ProvenBatch::created_nullifiers); let inputs = self - .store + .state + .view() .get_block_inputs( account_ids_iter.collect(), created_nullifiers_iter.collect(), @@ -371,7 +382,7 @@ impl BlockBuilder { tracing::debug!(target: LOG_TARGET, transactions = %format_array(transaction_ids), "Included transactions"); } - self.store + self.block_writer .apply_block_with_proving_inputs(ordered_batches, block_inputs, signed_block) .await .map_err(StoreError::ApplyBlockFailed) diff --git a/crates/block-producer/src/mempool/tests.rs b/crates/block-producer/src/mempool/tests.rs index 890b2e38e..e056f90ae 100644 --- a/crates/block-producer/src/mempool/tests.rs +++ b/crates/block-producer/src/mempool/tests.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::time::Duration; use miden_protocol::Word; use miden_protocol::block::{BlockHeader, BlockNumber}; @@ -143,18 +144,30 @@ async fn add_transaction_traces_are_correct() { let (mut uut, _) = Mempool::for_tests(); let txs = MockProvenTxBuilder::sequential(); + let tx_id = txs[0].id().to_string(); uut.add_transaction(txs[0].clone()).unwrap(); - let span_data = rx_export.recv().await.unwrap(); - assert_eq!(span_data.name, "mempool.add_transaction"); + // The exporter is global, so skip spans leaked by tests running concurrently on other threads + // by matching on this test's transaction ID. + let span_data = tokio::time::timeout(Duration::from_secs(30), async { + loop { + let span_data = rx_export.recv().await.unwrap(); + if span_data.name == "mempool.add_transaction" + && span_data + .attributes + .iter() + .any(|kv| kv.key == "transaction.id".into() && kv.value.to_string() == tx_id) + { + break span_data; + } + } + }) + .await + .expect("span for the added transaction should be exported"); + assert!(span_data.attributes.iter().any(|kv| kv.key == "code.module.name".into() && kv.value == "miden_node_block_producer::mempool".into())); - assert!( - span_data - .attributes - .iter() - .any(|kv| kv.key == "transaction.id".into() && kv.value.to_string().starts_with("0x")) - ); + assert!(tx_id.starts_with("0x")); } // BATCH FAILED TESTS diff --git a/crates/block-producer/src/proof_scheduler.rs b/crates/block-producer/src/proof_scheduler.rs index c050dc0a3..a611e9d19 100644 --- a/crates/block-producer/src/proof_scheduler.rs +++ b/crates/block-producer/src/proof_scheduler.rs @@ -18,7 +18,7 @@ use std::time::Duration; use anyhow::Context; use miden_node_proto::BlockProofRequest; -use miden_node_store::state::{Finality, State}; +use miden_node_store::state::{ProofWriter, State}; use miden_node_utils::retry::{self, Retryable}; use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::tracing::miden_instrument; @@ -108,6 +108,7 @@ impl ProofTaskJoinSet { pub(crate) async fn run( block_prover: Arc, state: Arc, + proof_writer: ProofWriter, mut chain_tip_rx: watch::Receiver, max_concurrent_proofs: NonZeroUsize, shutdown: CancellationToken, @@ -119,7 +120,7 @@ pub(crate) async fn run( // Next block number to schedule. Initialized from the proven tip's child so we skip // already-proven blocks on restart. - let mut next_to_prove = state.chain_tip(Finality::Proven).await.child(); + let mut next_to_prove = state.proven_tip().child(); // Completed proofs waiting to be committed in order. let mut pending: BTreeMap> = BTreeMap::new(); @@ -145,9 +146,9 @@ pub(crate) async fn run( // Drain completed proofs in ascending order so the proven tip advances without // gaps. - let mut next = state.chain_tip(Finality::Proven).await.child(); + let mut next = state.proven_tip().child(); while let Some(proof_bytes) = pending.remove(&next) { - state.apply_proof(next, proof_bytes).await?; + proof_writer.apply_proof(next, proof_bytes).await?; next = next.child(); } }, diff --git a/crates/block-producer/src/rpc_sync.rs b/crates/block-producer/src/rpc_sync.rs index be684dac0..9d6421bc2 100644 --- a/crates/block-producer/src/rpc_sync.rs +++ b/crates/block-producer/src/rpc_sync.rs @@ -5,7 +5,7 @@ use std::time::Duration; use anyhow::Context; use miden_node_proto::clients::RpcClient; use miden_node_proto::generated::rpc::{BlockSubscriptionRequest, ProofSubscriptionRequest}; -use miden_node_store::state::{Finality, State}; +use miden_node_store::state::{BlockWriter, ProofWriter, State}; use miden_node_utils::retry::{self, Retryable}; use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::tasks::Tasks; @@ -130,7 +130,12 @@ impl RpcReadiness { /// Synchronizes local state from an upstream RPC service. pub struct RpcSync { + /// Read-only store state, used by both loops to read tips and subscribe to commits. pub state: Arc, + /// The store's block-write capability, consumed by the block sync loop. + pub block_writer: BlockWriter, + /// The store's proof-write capability, consumed by the proof sync loop. + pub proof_writer: ProofWriter, pub source_rpc: RpcClient, pub readiness: RpcReadiness, } @@ -141,11 +146,13 @@ impl RpcSync { let mut tasks = Tasks::new(); let block_sync = BlockSync { state: Arc::clone(&self.state), + writer: self.block_writer, source_rpc: self.source_rpc.clone(), readiness: self.readiness, }; let proof_sync = ProofSync { state: self.state, + writer: self.proof_writer, source_rpc: self.source_rpc, }; @@ -161,6 +168,7 @@ impl RpcSync { struct BlockSync { state: Arc, + writer: BlockWriter, source_rpc: RpcClient, readiness: RpcReadiness, } @@ -194,7 +202,7 @@ impl BlockSync { err, )] async fn sync(&self, shutdown: CancellationToken) -> anyhow::Result<()> { - let local_tip = self.state.chain_tip(Finality::Committed).await; + let local_tip = self.state.committed_tip(); let mut client = self.source_rpc.clone(); let upstream_tip = BlockNumber::from(client.status(tonic::Request::new(())).await?.into_inner().chain_tip); @@ -220,9 +228,9 @@ impl BlockSync { let upstream_tip = BlockNumber::from(event.committed_chain_tip); let block = SignedBlock::read_from_bytes(&event.block) .context("failed to deserialize block from upstream")?; - self.state.apply_block(block).await?; + self.writer.apply_block(block).await?; - let local_tip = self.state.chain_tip(Finality::Committed).await; + let local_tip = self.state.committed_tip(); self.readiness.update(upstream_tip, local_tip).await; } } @@ -230,6 +238,7 @@ impl BlockSync { struct ProofSync { state: Arc, + writer: ProofWriter, source_rpc: RpcClient, } @@ -264,7 +273,7 @@ impl ProofSync { async fn sync(&self, shutdown: CancellationToken) -> anyhow::Result<()> { // Subscribe from next proven tip. - let starting_block = self.state.chain_tip(Finality::Proven).await.child(); + let starting_block = self.state.proven_tip().child(); info!( target: LOG_TARGET, block_from = %starting_block, @@ -303,7 +312,7 @@ impl ProofSync { }, } - self.state.apply_proof(block_num, event.proof).await?; + self.writer.apply_proof(block_num, event.proof).await?; expected = expected.child(); } diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index 25e9e81ba..fbcb3b478 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use std::time::Duration; use anyhow::Result; -use miden_node_store::state::{Finality, State}; +use miden_node_store::state::{BlockWriter, ProofWriter, State}; use miden_node_utils::formatting::{format_input_notes, format_output_notes}; use miden_node_utils::shutdown::CancellationToken; use miden_node_utils::tasks::Tasks; @@ -69,8 +69,12 @@ impl BlockProducerApiConfig { /// /// Specifies how to connect to the batch prover and block prover components. pub struct Sequencer { - /// The store state shared with the block producer. - pub store: Arc, + /// The read-only store state shared with the block producer. + pub state: Arc, + /// The store's block-write capability, handed to the block builder. + pub block_writer: BlockWriter, + /// The store's proof-write capability, handed to the proof scheduler. + pub proof_writer: ProofWriter, /// The addresses of the validator components. pub validator_urls: Vec, /// The request timeout for calls to the validator components. @@ -102,19 +106,24 @@ pub struct Sequencer { impl Sequencer { /// Spawns the sequencer tasks and returns its in-process API. - pub async fn spawn(self, shutdown: CancellationToken) -> Result { + pub fn spawn(self, shutdown: CancellationToken) -> Result { tracing::info!(target: LOG_TARGET, "Initializing sequencer"); - let store = self.store; + let state = self.state; let validator = BlockProducerValidatorClient::new(self.validator_urls.clone(), self.validator_timeout)?; - let chain_tip = store.chain_tip(Finality::Committed).await; + let chain_tip = state.committed_tip(); tracing::info!(target: LOG_TARGET, "Sequencer initialized"); - let block_builder = BlockBuilder::new(Arc::clone(&store), validator, self.block_interval); + let block_builder = BlockBuilder::new( + Arc::clone(&state), + self.block_writer, + validator, + self.block_interval, + ); let batch_intervals = BatchIntervals::derive_from(self.block_interval, self.batch_interval); let batch_builder = BatchBuilder::new( - Arc::clone(&store), + Arc::clone(&state), self.batch_workers, self.batch_prover_url, batch_intervals, @@ -125,13 +134,13 @@ impl Sequencer { mempool_tx_capacity: self.mempool_tx_capacity, }; let mempool = Mempool::shared(chain_tip, api_config.mempool_config()); - let api = BlockProducerApi::from_shared_mempool(mempool.clone(), store, shutdown.clone()); + let api = BlockProducerApi::from_shared_mempool(mempool.clone(), state, shutdown.clone()); let block_prover = if let Some(url) = self.block_prover_url { Arc::new(BlockProver::remote(url)?) } else { Arc::new(BlockProver::local()) }; - let chain_tip_rx = api.store.subscribe_committed_tip(); + let chain_tip_rx = api.state.subscribe_committed_tip(); // Spawn batch builder, block builder, and proof scheduler. The builders communicate // indirectly via a shared mempool. @@ -151,12 +160,14 @@ impl Sequencer { async { block_builder.run(mempool, shutdown).await } }); tasks.spawn("proof-scheduler", { - let store = Arc::clone(&api.store); + let state = Arc::clone(&api.state); + let proof_writer = self.proof_writer; let shutdown = shutdown.clone(); async move { proof_scheduler::run( block_prover, - store, + state, + proof_writer, chain_tip_rx, self.max_concurrent_proofs, shutdown, @@ -168,14 +179,6 @@ impl Sequencer { Ok(SequencerHandle { api, task }) } - - /// Serves the sequencer tasks. - /// - /// Executes in place (i.e. not spawned) and will run indefinitely until a fatal error is - /// encountered. - pub async fn serve(self) -> anyhow::Result<()> { - self.spawn(CancellationToken::new()).await?.wait().await - } } /// Running sequencer tasks plus the API used to submit work to them. @@ -210,7 +213,7 @@ pub struct BlockProducerApi { /// the block-producers usage of the mempool. mempool: Arc>, - store: Arc, + state: Arc, /// Cached mempool statistics that are updated periodically to avoid locking the mempool for /// each status request. @@ -238,17 +241,17 @@ pub struct BlockProducerStatus { impl BlockProducerApi { /// Creates an API backed by a fresh mempool. - pub fn new(store: Arc, chain_tip: BlockNumber, config: BlockProducerApiConfig) -> Self { + pub fn new(state: Arc, chain_tip: BlockNumber, config: BlockProducerApiConfig) -> Self { Self::from_shared_mempool( Mempool::shared(chain_tip, config.mempool_config()), - store, + state, CancellationToken::new(), ) } fn from_shared_mempool( mempool: SharedMempool, - store: Arc, + state: Arc, shutdown: CancellationToken, ) -> Self { let cached_mempool_stats = mempool @@ -257,7 +260,7 @@ impl BlockProducerApi { .unwrap_or_default(); let api = Self { mempool: Arc::new(Mutex::new(mempool)), - store, + state, cached_mempool_stats: Arc::new(RwLock::new(cached_mempool_stats)), }; api.spawn_mempool_stats_updater(shutdown); @@ -329,7 +332,7 @@ impl BlockProducerApi { tracing::debug!(target: COMPONENT, proof = ?tx.proof()); // Authenticate against the local store, then add to the mempool. - let inputs = get_tx_inputs(&self.store, &tx) + let inputs = get_tx_inputs(&self.state, &tx) .await .map_err(MempoolSubmissionError::StoreStateReadFailed)?; // SAFETY: we assume that the rpc component has verified the transaction proof already. @@ -376,7 +379,7 @@ impl BlockProducerApi { let mut inputs = Vec::with_capacity(batch.transactions().len()); for tx in batch.transactions() { inputs.push( - get_tx_inputs(&self.store, tx) + get_tx_inputs(&self.state, tx) .await .map_err(MempoolSubmissionError::StoreStateReadFailed)?, ); diff --git a/crates/block-producer/src/server/tests.rs b/crates/block-producer/src/server/tests.rs index 5ee80cd96..ed3764dc1 100644 --- a/crates/block-producer/src/server/tests.rs +++ b/crates/block-producer/src/server/tests.rs @@ -1,10 +1,8 @@ use std::num::NonZeroUsize; -use std::sync::Arc; use std::time::Duration; use miden_node_store::GenesisState; use miden_node_store::state::State; -use miden_node_utils::clap::StorageOptions; use miden_node_utils::fee::test_fee_params; use miden_protocol::block::{BlockNumber, ValidatorKeys}; use miden_protocol::testing::random_secret_key::random_secret_key; @@ -23,10 +21,12 @@ use crate::{ async fn block_producer_starts_with_store_state() { let data_directory = tempfile::tempdir().expect("tempdir should be created"); bootstrap_store(data_directory.path()); - let store = load_state(data_directory.path()).await; + let (state, block_writer, proof_writer) = State::for_tests(data_directory.path()).await; let block_producer = Sequencer { - store, + state, + block_writer, + proof_writer, validator_urls: vec![Url::parse("http://127.0.0.1:0").unwrap()], validator_timeout: DEFAULT_VALIDATOR_TIMEOUT, batch_prover_url: None, @@ -40,7 +40,6 @@ async fn block_producer_starts_with_store_state() { batch_workers: DEFAULT_BATCH_WORKERS, } .spawn(miden_node_utils::shutdown::CancellationToken::new()) - .await .unwrap(); let status = block_producer.api().status().await; @@ -61,8 +60,3 @@ fn bootstrap_store(path: &std::path::Path) { State::bootstrap(genesis_block, path).expect("store should bootstrap"); } - -async fn load_state(path: &std::path::Path) -> Arc { - let state = State::load(path, StorageOptions::default()).await.expect("state should load"); - Arc::new(state) -} diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index 670c284a9..0d2d2843c 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -7,7 +7,7 @@ use miden_node_proto::decode; use miden_node_proto::decode::GrpcDecodeExt; use miden_node_proto::errors::ConversionError; use miden_node_proto::generated::sequencer; -use miden_node_store::state::{Finality, State, TransactionInputs as StoreTransactionInputs}; +use miden_node_store::state::{State, TransactionInputs as StoreTransactionInputs}; use miden_node_utils::formatting::format_opt; use miden_node_utils::tracing::miden_instrument; use miden_protocol::Word; @@ -185,7 +185,9 @@ pub async fn get_tx_inputs( let unauthenticated_note_commitments = proven_tx.unauthenticated_notes().map(|header| header.id().as_word()).collect(); - let store_inputs = state + // A single view scopes both the input query and the block height below to one snapshot. + let view = state.view(); + let store_inputs = view .get_transaction_inputs( proven_tx.account_id(), &nullifiers, @@ -202,7 +204,7 @@ pub async fn get_tx_inputs( return Err(StoreError::DuplicateAccountIdPrefix(proven_tx.account_id())); } - let current_block_height = state.chain_tip(Finality::Committed).await; + let current_block_height = view.tip(); let tx_inputs = TransactionInputs::from_store_inputs( proven_tx.account_id(), store_inputs, diff --git a/crates/rpc/src/server/api.rs b/crates/rpc/src/server/api.rs index 688801590..4995e7c79 100644 --- a/crates/rpc/src/server/api.rs +++ b/crates/rpc/src/server/api.rs @@ -1,5 +1,4 @@ use std::num::NonZeroUsize; -use std::ops::RangeInclusive; use std::sync::{Arc, LazyLock}; use std::time::Duration; @@ -10,7 +9,7 @@ use miden_node_proto::domain::block::InvalidBlockRange; use miden_node_proto::generated::rpc::MempoolStats as ProtoMempoolStats; use miden_node_proto::generated::rpc::api_server::Api; use miden_node_proto::generated::{self as proto}; -use miden_node_store::state::{Finality, State}; +use miden_node_store::state::State; use miden_node_store::{DatabaseError, GetBlockHeaderError}; use miden_node_utils::limiter::{ QueryParamAccountIdLimit, @@ -112,7 +111,7 @@ impl From for RpcInvalidBlockRange { // ================================================================================================ pub struct RpcService { - store: Arc, + state: Arc, mode: RpcMode, ntx_builder: Option, network_tx_auth: Option, @@ -125,14 +124,14 @@ pub struct RpcService { impl RpcService { pub(crate) fn new( - store: Arc, + state: Arc, mode: RpcMode, ntx_builder: Option, commitment_cache_capacity: NonZeroUsize, network_tx_auth: Option, ) -> Self { Self { - store, + state, mode, ntx_builder, network_tx_auth, @@ -204,7 +203,8 @@ impl RpcService { } let header = self - .store + .state + .view() .get_block_header(Some(block), false) .await .map_err(get_block_header_error_to_status)? @@ -234,25 +234,6 @@ impl RpcService { Ok(()) } - /// Fetches the committed chain tip and ensures the requested range does not extend beyond it. - /// - /// Returns the chain tip so callers can reuse it (e.g. in the response's pagination info) - /// without issuing a second query. - async fn range_bounds_check( - &self, - range: &RangeInclusive, - ) -> Result { - let chain_tip = self.store.chain_tip(Finality::Committed).await; - if *range.end() > chain_tip { - return Err(Status::invalid_argument(format!( - "block_to ({}) is greater than chain tip ({chain_tip})", - range.end() - ))); - } - - Ok(chain_tip) - } - /// Errors if any of `candidate_ids` is classified as a network account by the store. Callers /// should pre-filter to post-deployment, public-account ids; `Ok(())` on empty. async fn reject_if_any_network_accounts( @@ -265,7 +246,7 @@ impl RpcService { } let network_accounts = - self.store.filter_network_accounts(&account_ids).await.map_err(|err| { + self.state.view().filter_network_accounts(&account_ids).await.map_err(|err| { Status::internal(format!("network-account classification failed: {err}")) })?; @@ -311,6 +292,7 @@ fn database_error_to_status(err: &DatabaseError) -> Status { | DatabaseError::AccountsNotFoundInDb(_) | DatabaseError::AccountNotPublic(_) => Status::not_found(message), DatabaseError::TransactionPageExceedsPayloadLimit { .. } => Status::out_of_range(message), + DatabaseError::RangeBeyondTip(_) => Status::invalid_argument(message), _ => Status::internal(message), } } diff --git a/crates/rpc/src/server/api/get_account.rs b/crates/rpc/src/server/api/get_account.rs index 18e5ea0b5..4e18383da 100644 --- a/crates/rpc/src/server/api/get_account.rs +++ b/crates/rpc/src/server/api/get_account.rs @@ -58,8 +58,12 @@ impl proto::server::rpc_api::GetAccount for RpcService { validate_storage_request(&details.storage_request)?; } - let account_data = - self.store.get_account(request).await.map_err(get_account_error_to_status)?; + let account_data = self + .state + .view() + .get_account(request) + .await + .map_err(get_account_error_to_status)?; Ok(account_data) } } diff --git a/crates/rpc/src/server/api/get_block_by_number.rs b/crates/rpc/src/server/api/get_block_by_number.rs index d59e989a3..b69141854 100644 --- a/crates/rpc/src/server/api/get_block_by_number.rs +++ b/crates/rpc/src/server/api/get_block_by_number.rs @@ -38,12 +38,12 @@ impl proto::server::rpc_api::GetBlockByNumber for RpcService { let block_num = BlockNumber::from(request.block_num); let block = self - .store + .state .load_block(block_num) .await .map_err(|err| database_error_to_status(&err))?; let proof = if request.include_proof.unwrap_or_default() { - self.store + self.state .load_proof(block_num) .await .map_err(|err| database_error_to_status(&err))? diff --git a/crates/rpc/src/server/api/get_block_header_by_number.rs b/crates/rpc/src/server/api/get_block_header_by_number.rs index e39002cb4..6a24df67f 100644 --- a/crates/rpc/src/server/api/get_block_header_by_number.rs +++ b/crates/rpc/src/server/api/get_block_header_by_number.rs @@ -38,7 +38,8 @@ impl proto::server::rpc_api::GetBlockHeaderByNumber for RpcService { let block_num = request.block_num.map(BlockNumber::from); let (block_header, mmr_proof) = self - .store + .state + .view() .get_block_header(block_num, request.include_mmr_proof.unwrap_or(false)) .await .map_err(super::get_block_header_error_to_status)?; diff --git a/crates/rpc/src/server/api/get_note_script_by_root.rs b/crates/rpc/src/server/api/get_note_script_by_root.rs index 4cc123b6c..ed4a413d3 100644 --- a/crates/rpc/src/server/api/get_note_script_by_root.rs +++ b/crates/rpc/src/server/api/get_note_script_by_root.rs @@ -43,7 +43,8 @@ impl proto::server::rpc_api::GetNoteScriptByRoot for RpcService { debug!(target: LOG_TARGET, "Getting note script by root"); let script = self - .store + .state + .view() .get_note_script_by_root(root) .await .map_err(|err| database_error_to_status(&err))?; diff --git a/crates/rpc/src/server/api/get_notes_by_id.rs b/crates/rpc/src/server/api/get_notes_by_id.rs index f27c32d84..9926a7f3c 100644 --- a/crates/rpc/src/server/api/get_notes_by_id.rs +++ b/crates/rpc/src/server/api/get_notes_by_id.rs @@ -45,7 +45,8 @@ impl proto::server::rpc_api::GetNotesById for RpcService { let note_ids: Vec = note_ids.into_iter().map(NoteId::from_raw).collect(); let notes = self - .store + .state + .view() .get_notes_by_id(note_ids) .await .map_err(|err| database_error_to_status(&err))? diff --git a/crates/rpc/src/server/api/status.rs b/crates/rpc/src/server/api/status.rs index 026e9c622..f5003df89 100644 --- a/crates/rpc/src/server/api/status.rs +++ b/crates/rpc/src/server/api/status.rs @@ -3,7 +3,7 @@ use miden_node_proto::generated as proto; use miden_node_utils::tracing::miden_instrument; use tracing::debug; -use super::{Finality, ProtoMempoolStats, Request, RpcMode, RpcService}; +use super::{ProtoMempoolStats, Request, RpcMode, RpcService}; use crate::{COMPONENT, LOG_TARGET}; #[tonic::async_trait] @@ -48,7 +48,7 @@ impl proto::server::rpc_api::Status for RpcService { Ok(proto::rpc::RpcStatus { version: env!("CARGO_PKG_VERSION").to_string(), - chain_tip: self.store.chain_tip(Finality::Committed).await.as_u32(), + chain_tip: self.state.committed_tip().as_u32(), block_producer: block_producer_status.or(Some(proto::rpc::BlockProducerStatus { status: "unreachable".to_string(), version: "-".to_string(), diff --git a/crates/rpc/src/server/api/submit_proven_tx.rs b/crates/rpc/src/server/api/submit_proven_tx.rs index 610641502..a4e7cc4bc 100644 --- a/crates/rpc/src/server/api/submit_proven_tx.rs +++ b/crates/rpc/src/server/api/submit_proven_tx.rs @@ -169,7 +169,7 @@ impl RpcService { request: proto::transaction::ProvenTransaction, rebuilt_tx: ProvenTransaction, ) -> tonic::Result { - let tx_inputs = get_tx_inputs(&self.store, &rebuilt_tx).await.map_err(|err| { + let tx_inputs = get_tx_inputs(&self.state, &rebuilt_tx).await.map_err(|err| { Status::internal(err.as_report_context("failed to get transaction inputs")) })?; diff --git a/crates/rpc/src/server/api/submit_proven_tx_batch.rs b/crates/rpc/src/server/api/submit_proven_tx_batch.rs index 04f48835a..f04f9b701 100644 --- a/crates/rpc/src/server/api/submit_proven_tx_batch.rs +++ b/crates/rpc/src/server/api/submit_proven_tx_batch.rs @@ -167,7 +167,7 @@ impl RpcService { let mut auth_inputs = Vec::with_capacity(proposed_batch.transactions().len()); for tx in proposed_batch.transactions() { - let inputs = get_tx_inputs(&self.store, tx).await.map_err(|err| { + let inputs = get_tx_inputs(&self.state, tx).await.map_err(|err| { Status::internal(err.as_report_context("failed to authenticate transaction")) })?; auth_inputs.push(inputs.into()); diff --git a/crates/rpc/src/server/api/subscription/stream/mod.rs b/crates/rpc/src/server/api/subscription/stream/mod.rs index 0645a48dc..e1fda85dc 100644 --- a/crates/rpc/src/server/api/subscription/stream/mod.rs +++ b/crates/rpc/src/server/api/subscription/stream/mod.rs @@ -61,14 +61,14 @@ impl SubscriptionStream { from: BlockNumber, client_ip: Option, ) -> tonic::Result { - let store = Arc::clone(&rpc.store); + let store = Arc::clone(&rpc.state); Self::create( from, client_ip, Arc::clone(&rpc.subscription_ban), Arc::clone(&rpc.block_subscription_semaphore), - rpc.store.subscribe_committed_tip(), + rpc.state.subscribe_committed_tip(), move |block| { let store = Arc::clone(&store); async move { store.load_block(block).await } @@ -82,14 +82,14 @@ impl SubscriptionStream { from: BlockNumber, client_ip: Option, ) -> tonic::Result { - let store = Arc::clone(&rpc.store); + let store = Arc::clone(&rpc.state); Self::create( from, client_ip, Arc::clone(&rpc.subscription_ban), Arc::clone(&rpc.proof_subscription_semaphore), - rpc.store.subscribe_proven_tip(), + rpc.state.subscribe_proven_tip(), move |block| { let store = Arc::clone(&store); async move { store.load_proof(block).await } diff --git a/crates/rpc/src/server/api/sync_account_storage_maps.rs b/crates/rpc/src/server/api/sync_account_storage_maps.rs index 623fa7ff5..ebfc6129f 100644 --- a/crates/rpc/src/server/api/sync_account_storage_maps.rs +++ b/crates/rpc/src/server/api/sync_account_storage_maps.rs @@ -58,9 +58,9 @@ impl proto::server::rpc_api::SyncAccountStorageMaps for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range).await?; - let storage_maps_page = self - .store + let view = self.state.view(); + let chain_tip = view.tip(); + let storage_maps_page = view .sync_account_storage_maps(account_id, block_range) .await .map_err(|err| database_error_to_status(&err))?; diff --git a/crates/rpc/src/server/api/sync_account_vault.rs b/crates/rpc/src/server/api/sync_account_vault.rs index 389f3a47e..5fc2bd668 100644 --- a/crates/rpc/src/server/api/sync_account_vault.rs +++ b/crates/rpc/src/server/api/sync_account_vault.rs @@ -58,9 +58,9 @@ impl proto::server::rpc_api::SyncAccountVault for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range).await?; - let (last_included_block, updates) = self - .store + let view = self.state.view(); + let chain_tip = view.tip(); + let (last_included_block, updates) = view .sync_account_vault(account_id, block_range) .await .map_err(|err| database_error_to_status(&err))?; diff --git a/crates/rpc/src/server/api/sync_chain_mmr.rs b/crates/rpc/src/server/api/sync_chain_mmr.rs index e55c04513..55ef9425c 100644 --- a/crates/rpc/src/server/api/sync_chain_mmr.rs +++ b/crates/rpc/src/server/api/sync_chain_mmr.rs @@ -1,10 +1,11 @@ use miden_node_proto::generated as proto; +use miden_node_store::StateSyncError; use miden_node_utils::tracing::miden_instrument; use miden_protocol::block::BlockNumber; use tonic::Status; use tracing::debug; -use super::{Finality, RpcService}; +use super::RpcService; use crate::{COMPONENT, LOG_TARGET}; #[tonic::async_trait] @@ -39,11 +40,15 @@ impl proto::server::rpc_api::SyncChainMmr for RpcService { debug!(target: LOG_TARGET, "Syncing chain MMR"); let current_client_block_height = BlockNumber::from(request.current_client_block_height); + let view = self.state.view(); let sync_target = match request.finality_level() { proto::rpc::FinalityLevel::Committed | proto::rpc::FinalityLevel::Unspecified => { - self.store.chain_tip(Finality::Committed).await + view.tip() }, - proto::rpc::FinalityLevel::Proven => self.store.chain_tip(Finality::Proven).await, + // The proven tip is read from a watch channel, not the view's snapshot, so clamp it to + // the view's tip: a block could be committed and proven between taking the view and + // reading the proven tip, and the view cannot serve blocks beyond its snapshot. + proto::rpc::FinalityLevel::Proven => self.state.proven_tip().min(view.tip()), }; if current_client_block_height > sync_target { @@ -53,11 +58,11 @@ impl proto::server::rpc_api::SyncChainMmr for RpcService { } let block_range = current_client_block_height..=sync_target; - let (mmr_delta, block_header, block_signatures) = self - .store - .sync_chain_mmr(block_range.clone()) - .await - .map_err(|err| Status::internal(err.to_string()))?; + let (mmr_delta, block_header, block_signatures) = + view.sync_chain_mmr(block_range.clone()).await.map_err(|err| match err { + StateSyncError::RangeBeyondTip(_) => Status::invalid_argument(err.to_string()), + _ => Status::internal(err.to_string()), + })?; Ok(proto::rpc::SyncChainMmrResponse { block_range: Some(proto::rpc::BlockRange { diff --git a/crates/rpc/src/server/api/sync_notes.rs b/crates/rpc/src/server/api/sync_notes.rs index 411d653b7..961819f1d 100644 --- a/crates/rpc/src/server/api/sync_notes.rs +++ b/crates/rpc/src/server/api/sync_notes.rs @@ -47,10 +47,10 @@ impl proto::server::rpc_api::SyncNotes for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range).await?; + let view = self.state.view(); + let chain_tip = view.tip(); - let (results, last_block_checked) = self - .store + let (results, last_block_checked) = view .sync_notes(request.note_tags, block_range) .await .map_err(note_sync_error_to_status)?; @@ -94,7 +94,7 @@ fn note_sync_error_to_status(err: NoteSyncError) -> Status { match err { NoteSyncError::DatabaseError(err) => super::database_error_to_status(&err), NoteSyncError::InvalidBlockRange(_) - | NoteSyncError::FutureBlock { .. } + | NoteSyncError::RangeBeyondTip(_) | NoteSyncError::DeserializationFailed(_) => Status::invalid_argument(message), NoteSyncError::UnderlyingDatabaseError(_) | NoteSyncError::EmptyBlockHeadersTable diff --git a/crates/rpc/src/server/api/sync_nullifiers.rs b/crates/rpc/src/server/api/sync_nullifiers.rs index bcd05eaab..63865ea25 100644 --- a/crates/rpc/src/server/api/sync_nullifiers.rs +++ b/crates/rpc/src/server/api/sync_nullifiers.rs @@ -58,10 +58,10 @@ impl proto::server::rpc_api::SyncNullifiers for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range).await?; + let view = self.state.view(); + let chain_tip = view.tip(); - let (nullifiers, block_num) = self - .store + let (nullifiers, block_num) = view .sync_nullifiers(request.prefix_len, request.nullifiers, block_range) .await .map_err(|err| database_error_to_status(&err))?; diff --git a/crates/rpc/src/server/api/sync_transactions.rs b/crates/rpc/src/server/api/sync_transactions.rs index 489e2a483..32c0ab472 100644 --- a/crates/rpc/src/server/api/sync_transactions.rs +++ b/crates/rpc/src/server/api/sync_transactions.rs @@ -61,10 +61,10 @@ impl proto::server::rpc_api::SyncTransactions for RpcService { let block_range = range .into_inclusive_range::() .map_err(invalid_block_range_to_status)?; - let chain_tip = self.range_bounds_check(&block_range).await?; + let view = self.state.view(); + let chain_tip = view.tip(); let account_ids = read_account_ids::(request.account_ids)?; - let (last_block_included, transaction_records_db) = self - .store + let (last_block_included, transaction_records_db) = view .sync_transactions(account_ids, block_range) .await .map_err(|err| database_error_to_status(&err))?; diff --git a/crates/rpc/src/server/mod.rs b/crates/rpc/src/server/mod.rs index 22beebe61..fafe0c57a 100644 --- a/crates/rpc/src/server/mod.rs +++ b/crates/rpc/src/server/mod.rs @@ -13,7 +13,7 @@ use miden_node_proto::clients::{ }; use miden_node_proto::server::{rpc_api, sequencer_api}; use miden_node_proto_build::rpc_api_descriptor; -use miden_node_store::state::{Finality, State}; +use miden_node_store::state::{BlockWriter, ProofWriter, State}; use miden_node_utils::clap::{GrpcOptionsExternal, GrpcOptionsInternal}; use miden_node_utils::cors::cors_for_grpc_web_layer; use miden_node_utils::grpc; @@ -45,8 +45,13 @@ mod health; /// It uses the supplied store state and mode-specific submission handling. pub struct Rpc { pub listener: TcpListener, - pub store: Arc, + pub state: Arc, pub mode: RpcMode, + /// Store write capabilities consumed by the full-node sync loop. + /// + /// Must be provided in full-node mode and omitted in sequencer mode. The RPC service itself + /// only ever reads through `store`; these are passed through untouched to the sync tasks. + pub sync_writers: Option<(BlockWriter, ProofWriter)>, pub ntx_builder: Option, pub grpc_options: GrpcOptionsExternal, pub network_tx_auth: Option, @@ -188,7 +193,7 @@ impl Rpc { let endpoint = self.listener.local_addr().context("failed to read RPC listen address")?; let mode = self.mode.as_str(); let mut api = api::RpcService::new( - self.store.clone(), + self.state.clone(), self.mode.clone(), self.ntx_builder.clone(), NonZeroUsize::new(1_000_000).unwrap(), @@ -216,7 +221,7 @@ impl Rpc { tonic_health::ServingStatus::Serving, ) .await; - let chain_tip = self.store.chain_tip(Finality::Committed).await; + let chain_tip = self.state.committed_tip(); log_node_ready(mode, endpoint, chain_tip); }, RpcMode::FullNode { source_rpc, readiness_threshold, .. } => { @@ -227,10 +232,15 @@ impl Rpc { ) .await; let readiness = RpcReadiness::new(health_reporter, readiness_threshold); + let (block_writer, proof_writer) = self.sync_writers.context( + "full-node RPC requires the store write capabilities for its sync loop", + )?; tasks.spawn( "RPC sync", RpcSync { - state: Arc::clone(&self.store), + state: Arc::clone(&self.state), + block_writer, + proof_writer, source_rpc: *source_rpc, readiness, } diff --git a/crates/rpc/src/tests.rs b/crates/rpc/src/tests.rs index fa5156596..4c86cfd29 100644 --- a/crates/rpc/src/tests.rs +++ b/crates/rpc/src/tests.rs @@ -22,7 +22,7 @@ use miden_node_proto::generated::{self as proto}; use miden_node_proto::server::{ntx_builder_api, rpc_api, validator_api}; use miden_node_store::genesis::config::GenesisConfig; use miden_node_store::state::State; -use miden_node_utils::clap::{GrpcOptionsExternal, StorageOptions}; +use miden_node_utils::clap::GrpcOptionsExternal; use miden_node_utils::limiter::{ QueryParamAccountIdLimit, QueryParamLimiter, @@ -94,7 +94,7 @@ impl TestStore { async fn start() -> Self { let data_directory = new_tempdir(); let genesis_commitment = Self::bootstrap(&data_directory); - let state = load_state(&data_directory).await; + let (state, ..) = State::for_tests(&data_directory).await; Self { state, genesis_commitment, @@ -115,11 +115,6 @@ impl TestStore { } } -async fn load_state(path: &std::path::Path) -> Arc { - let state = State::load(path, StorageOptions::default()).await.expect("state should load"); - Arc::new(state) -} - /// Byte offset of the account delta commitment in serialized `ProvenTransaction`. Layout: /// `AccountId` (15) + `initial_commitment` (32) + `final_commitment` (32) = 79 const DELTA_COMMITMENT_BYTE_OFFSET: usize = 15 + 32 + 32; @@ -631,8 +626,8 @@ async fn start_source_rpc( let store = TestStore::start().await; let block_producer_dir = new_tempdir(); TestStore::bootstrap(&block_producer_dir); - let block_producer_state = load_state(&block_producer_dir).await; - let store_state = Arc::clone(&store.state); + let (block_producer_state, ..) = State::for_tests(&block_producer_dir).await; + let state = Arc::clone(&store.state); let listener = TcpListener::bind("127.0.0.1:0").await.expect("Failed to bind source RPC"); let addr = listener.local_addr().expect("Failed to get source RPC address"); @@ -644,7 +639,7 @@ async fn start_source_rpc( BlockProducerApiConfig::default(), ); let source_rpc = RpcService::new( - store_state, + state, RpcMode::sequencer(block_producer, ValidatorClients::new(vec![validator]).unwrap()), Some(ntx_builder), NonZeroUsize::new(1_000_000).unwrap(), @@ -1108,8 +1103,8 @@ async fn start_rpc_with_options( let store = TestStore::start().await; let block_producer_dir = new_tempdir(); TestStore::bootstrap(&block_producer_dir); - let block_producer_state = load_state(&block_producer_dir).await; - let store_state = Arc::clone(&store.state); + let (block_producer_state, ..) = State::for_tests(&block_producer_dir).await; + let state = Arc::clone(&store.state); // Start the rpc component. let rpc_listener = TcpListener::bind("127.0.0.1:0").await.expect("Failed to bind rpc"); @@ -1131,11 +1126,12 @@ async fn start_rpc_with_options( .connect_lazy::(); Rpc { listener: rpc_listener, - store: store_state, + state, mode: RpcMode::sequencer( block_producer, ValidatorClients::new(vec![validator]).unwrap(), ), + sync_writers: None, ntx_builder: None, grpc_options, network_tx_auth: None, diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index bd20adc7c..35bdd7fe0 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -19,6 +19,7 @@ doctest = false [dependencies] anyhow = { workspace = true } +arc-swap = { workspace = true } deadpool = { features = ["managed", "rt_tokio_1"], workspace = true } deadpool-diesel = { features = ["sqlite"], workspace = true } diesel = { features = ["numeric", "sqlite"], workspace = true } diff --git a/crates/store/src/account_state_forest/mod.rs b/crates/store/src/account_state_forest/mod.rs index 159ecf734..ca0347ddb 100644 --- a/crates/store/src/account_state_forest/mod.rs +++ b/crates/store/src/account_state_forest/mod.rs @@ -4,7 +4,7 @@ use std::num::NonZeroUsize; use miden_crypto::hash::rpo::Rpo256; #[cfg(feature = "rocksdb")] use miden_crypto::merkle::smt::ForestPersistentBackend; -use miden_crypto::merkle::smt::{Backend, ForestInMemoryBackend}; +use miden_crypto::merkle::smt::{Backend, BackendReader, ForestInMemoryBackend}; use miden_node_proto::domain::account::{ AccountStorageMapDetails, AccountVaultDetails, @@ -73,6 +73,9 @@ pub(crate) type AccountStateForestBackend = ForestPersistentBackend; #[cfg(not(feature = "rocksdb"))] pub(crate) type AccountStateForestBackend = ForestInMemoryBackend; +/// The read-only snapshot backend for the forest, used by in-memory state snapshots. +pub(crate) type AccountStateForestBackendReader = ::Reader; + const fn empty_smt_root() -> Word { *EmptySubtreeRoots::entry(SMT_DEPTH, 0) } @@ -89,7 +92,7 @@ pub enum AccountStorageMapResult { } /// Container for forest-related state that needs to be updated atomically. -pub(crate) struct AccountStateForest { +pub(crate) struct AccountStateForest { /// `LargeSmtForest` for efficient account storage reconstruction. Populated during block import /// with storage and vault SMTs. forest: LargeSmtForest, @@ -153,7 +156,7 @@ impl AccountStateForest { } } -impl AccountStateForest { +impl AccountStateForest { pub(crate) fn from_backend(backend: B) -> Result { Ok(Self { forest: LargeSmtForest::new(backend)?, @@ -738,6 +741,37 @@ impl AccountStateForest { ))) } + /// Retrieves the most recent vault SMT root for an account. If no vault root is found for the + /// account, returns an empty SMT root. + pub(crate) fn get_latest_vault_root(&self, account_id: AccountId) -> Word { + let lineage = Self::vault_lineage_id(account_id); + self.forest.latest_root(lineage).unwrap_or_else(empty_smt_root) + } + + /// Retrieves the most recent storage map SMT root for an account slot. + pub(crate) fn get_latest_storage_map_root( + &self, + account_id: AccountId, + slot_name: &StorageSlotName, + ) -> Word { + let lineage = Self::storage_lineage_id(account_id, slot_name); + self.forest.latest_root(lineage).unwrap_or_else(empty_smt_root) + } +} + +impl AccountStateForest { + /// Returns a read-only snapshot of this forest backed by a reader view of the backend. + /// + /// The reverse-key caches are shallow clones sharing the same underlying storage, so cache + /// entries added by the writer are visible to snapshot readers and vice versa. + pub(crate) fn reader(&self) -> Result, LargeSmtForestError> { + Ok(AccountStateForest { + forest: self.forest.reader()?, + storage_map_key_cache: self.storage_map_key_cache.clone(), + vault_key_cache: self.vault_key_cache.clone(), + }) + } + // PUBLIC INTERFACE // -------------------------------------------------------------------------------------------- @@ -860,23 +894,6 @@ impl AccountStateForest { self.apply_account_updates_without_pruning(block_num, account_updates) } - /// Retrieves the most recent vault SMT root for an account. If no vault root is found for the - /// account, returns an empty SMT root. - pub(crate) fn get_latest_vault_root(&self, account_id: AccountId) -> Word { - let lineage = Self::vault_lineage_id(account_id); - self.forest.latest_root(lineage).unwrap_or_else(empty_smt_root) - } - - /// Retrieves the most recent storage map SMT root for an account slot. - pub(crate) fn get_latest_storage_map_root( - &self, - account_id: AccountId, - slot_name: &StorageSlotName, - ) -> Word { - let lineage = Self::storage_lineage_id(account_id, slot_name); - self.forest.latest_root(lineage).unwrap_or_else(empty_smt_root) - } - // PRUNING // -------------------------------------------------------------------------------------------- diff --git a/crates/store/src/accounts/mod.rs b/crates/store/src/accounts/mod.rs index 560e208bc..9b4e10dbd 100644 --- a/crates/store/src/accounts/mod.rs +++ b/crates/store/src/accounts/mod.rs @@ -16,6 +16,7 @@ use miden_protocol::crypto::merkle::smt::{ SMT_DEPTH, SmtLeaf, SmtStorage, + SmtStorageReader, }; use miden_protocol::crypto::merkle::{ EmptySubtreeRoots, @@ -118,7 +119,7 @@ impl HistoricalOverlay { /// reversion data (mutations that undo changes). Historical witnesses are reconstructed /// by starting from the latest state and applying reversion overlays backwards in time. #[derive(Debug)] -pub struct AccountTreeWithHistory { +pub struct AccountTreeWithHistory { /// The current block number (latest state). block_number: BlockNumber, /// The latest account tree state. @@ -127,7 +128,7 @@ pub struct AccountTreeWithHistory { overlays: BTreeMap, } -impl AccountTreeWithHistory { +impl AccountTreeWithHistory { /// Maximum number of historical blocks to maintain. pub const MAX_HISTORY: usize = 50; @@ -355,7 +356,9 @@ impl AccountTreeWithHistory { let path = SparseMerklePath::try_from(path).ok()?; Some((path, leaf)) } +} +impl AccountTreeWithHistory { // PUBLIC MUTATORS // -------------------------------------------------------------------------------------------- @@ -409,4 +412,14 @@ impl AccountTreeWithHistory { Ok(()) } + + /// Returns a read-only snapshot of this tree backed by a reader view of the storage. + pub fn reader(&self) -> AccountTreeWithHistory { + let latest = self.latest.reader().expect("snapshot creation should not fail"); + AccountTreeWithHistory { + block_number: self.block_number, + latest, + overlays: self.overlays.clone(), + } + } } diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 238b80b80..900b70f84 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -36,7 +36,6 @@ use miden_protocol::note::{ }; use miden_protocol::transaction::TransactionHeader; use miden_protocol::utils::serde::Deserializable; -use tokio::sync::oneshot; use tracing::info; use crate::db::migrations::{migrate_database, verify_latest_schema}; @@ -55,6 +54,7 @@ use crate::db::models::queries::{ }; use crate::errors::{DatabaseError, NoteSyncError}; use crate::genesis::GenesisBlock; +use crate::state::{ScopedBlockNum, ScopedBlockRange}; use crate::{COMPONENT, LOG_TARGET}; const STORAGE_MAP_VALUE_PER_ROW_BYTES: usize = @@ -312,8 +312,9 @@ impl Db { &self, prefix_len: u32, nullifier_prefixes: Vec, - block_range: RangeInclusive, + block_range: ScopedBlockRange, ) -> Result<(Vec, BlockNumber)> { + let block_range = block_range.into_inner(); assert_eq!(prefix_len, 16, "Only 16-bit prefixes are supported"); self.transact("nullifieres by prefix", move |conn| { @@ -512,8 +513,9 @@ impl Db { pub async fn select_account_header_with_storage_header_at_block( &self, account_id: AccountId, - block_num: BlockNumber, + block_num: ScopedBlockNum, ) -> Result> { + let block_num = block_num.get(); self.transact("Get account header with storage header at block", move |conn| { queries::select_account_header_with_storage_header_at_block(conn, account_id, block_num) }) @@ -528,9 +530,10 @@ impl Db { )] pub async fn get_note_sync_multi( &self, - block_range: RangeInclusive, + block_range: ScopedBlockRange, note_tags: Arc<[u32]>, ) -> Result, NoteSyncError> { + let block_range = block_range.into_inner(); self.transact("notes sync task", move |conn| { queries::get_note_sync_multi(conn, ¬e_tags, block_range, MAX_RESPONSE_PAYLOAD_BYTES) }) @@ -552,7 +555,8 @@ impl Db { .await } - /// Returns all note commitments from the DB that match the provided ones. + /// Returns all note commitments from the DB that match the provided ones and were committed at + /// or before `up_to_block`. #[miden_instrument( level = "debug", target = COMPONENT, @@ -562,9 +566,15 @@ impl Db { pub async fn select_existing_note_commitments( &self, note_commitments: Vec, + up_to_block: ScopedBlockNum, ) -> Result> { + let up_to_block = up_to_block.get(); self.transact("note by commitment", move |conn| { - queries::select_existing_note_commitments(conn, note_commitments.as_slice()) + queries::select_existing_note_commitments( + conn, + note_commitments.as_slice(), + up_to_block, + ) }) .await } @@ -588,8 +598,8 @@ impl Db { /// Inserts the data of a new block into the DB. /// - /// `allow_acquire` and `acquire_done` are used to synchronize writes to the DB with writes to - /// the in-memory trees. Further details available on [`super::state::State::apply_block`]. + /// The transaction is committed when this method returns. Synchronization with the in-memory + /// trees is handled by the block writer task; see [`super::state::State::apply_block`]. /// /// Consumed note IDs omitted from transaction headers are resolved from /// `unresolved_note_nullifiers` on a best-effort basis. The returned mapping is used only for @@ -603,8 +613,6 @@ impl Db { )] pub(crate) async fn apply_block( &self, - allow_acquire: oneshot::Sender<()>, - acquire_done: oneshot::Receiver<()>, signed_block: SignedBlock, notes: Vec<(NoteRecord, Option)>, precomputed_public_states: PrecomputedPublicAccountStates, @@ -612,22 +620,10 @@ impl Db { ) -> Result> { self.transact("apply block", move |conn| { models::queries::apply_block(conn, &signed_block, ¬es, &precomputed_public_states)?; - - // XXX FIXME TODO free floating mutex MUST NOT exist it doesn't bind it properly to the - // data locked! - { - let _span = tracing::info_span!(target: COMPONENT, "acquire_write_lock").entered(); - if allow_acquire.send(()).is_err() { - tracing::warn!(target: COMPONENT, "failed to send notification for successful block application, potential deadlock"); - } - } - models::queries::prune_history(conn, signed_block.header().block_num())?; let mut resolved_note_ids = BTreeMap::new(); - for chunk in - unresolved_note_nullifiers.chunks(QueryParamNoteCommitmentLimit::LIMIT) - { + for chunk in unresolved_note_nullifiers.chunks(QueryParamNoteCommitmentLimit::LIMIT) { match queries::select_note_ids_by_nullifier(conn, chunk) { Ok(note_ids) => resolved_note_ids.extend(note_ids), Err(err) => { @@ -642,10 +638,6 @@ impl Db { } } - let _span = - tracing::info_span!(target: COMPONENT, "acquire_done_lock").entered(); - acquire_done.blocking_recv()?; - Ok(resolved_note_ids) }) .await @@ -656,6 +648,18 @@ impl Db { /// The returned values are the latest known values up to `block_range.end()`, and no values /// earlier than `block_range.start()` are returned. pub(crate) async fn select_storage_map_sync_values( + &self, + account_id: AccountId, + block_range: ScopedBlockRange, + entries_limit: Option, + ) -> Result { + self.select_storage_map_sync_values_raw(account_id, block_range.into_inner(), entries_limit) + .await + } + + /// Raw variant of [`Self::select_storage_map_sync_values`] for internal pagination, where + /// sub-ranges are derived from a bound that was already validated by the caller. + async fn select_storage_map_sync_values_raw( &self, account_id: AccountId, block_range: RangeInclusive, @@ -690,12 +694,14 @@ impl Db { &self, account_id: AccountId, slot_name: miden_protocol::account::StorageSlotName, - block_num: BlockNumber, + block_num: ScopedBlockNum, entries_limit: Option, ) -> Result { use miden_node_proto::domain::account::{AccountStorageMapDetails, StorageMapEntries}; use miden_protocol::EMPTY_WORD; + let block_num = block_num.get(); + // TODO this remains expensive with a large history until we implement pruning for DB // columns let mut values = Vec::new(); @@ -703,7 +709,7 @@ impl Db { let entries_limit = entries_limit.unwrap_or_else(default_storage_map_entries_limit); let mut page = self - .select_storage_map_sync_values( + .select_storage_map_sync_values_raw( account_id, block_range_start..=block_num, Some(entries_limit), @@ -727,7 +733,7 @@ impl Db { block_range_start = page.last_block_included.child(); page = self - .select_storage_map_sync_values( + .select_storage_map_sync_values_raw( account_id, block_range_start..=block_num, Some(entries_limit), @@ -780,8 +786,9 @@ impl Db { pub async fn select_account_vault_at_block( &self, account_id: AccountId, - block_num: BlockNumber, + block_num: ScopedBlockNum, ) -> Result, DatabaseError> { + let block_num = block_num.get(); self.transact("select account vault at block", move |conn| { queries::select_account_vault_at_block(conn, account_id, block_num) }) @@ -791,8 +798,9 @@ impl Db { pub async fn get_account_vault_sync( &self, account_id: AccountId, - block_range: RangeInclusive, + block_range: ScopedBlockRange, ) -> Result<(BlockNumber, Vec)> { + let block_range = block_range.into_inner(); self.transact("account vault sync", move |conn| { queries::select_account_vault_assets(conn, account_id, block_range) }) @@ -816,8 +824,9 @@ impl Db { pub async fn select_transactions_records( &self, account_ids: Vec, - block_range: RangeInclusive, + block_range: ScopedBlockRange, ) -> Result<(BlockNumber, Vec)> { + let block_range = block_range.into_inner(); self.transact("full transactions records", move |conn| { queries::select_transactions_records(conn, &account_ids, block_range) }) diff --git a/crates/store/src/db/models/queries/notes.rs b/crates/store/src/db/models/queries/notes.rs index da9b78508..e0ee31720 100644 --- a/crates/store/src/db/models/queries/notes.rs +++ b/crates/store/src/db/models/queries/notes.rs @@ -221,7 +221,8 @@ pub(crate) fn select_notes_by_id( Ok(records) } -/// Select the subset of note commitments that already exist in the notes table +/// Select the subset of note commitments that already exist in the notes table and were +/// committed at or before `up_to_block`. /// /// # Raw SQL /// @@ -229,11 +230,12 @@ pub(crate) fn select_notes_by_id( /// SELECT /// notes.note_commitment /// FROM notes -/// WHERE note_commitment IN (?1) +/// WHERE note_commitment IN (?1) AND committed_at <= ?2 /// ``` pub(crate) fn select_existing_note_commitments( conn: &mut SqliteConnection, note_commitments: &[Word], + up_to_block: BlockNumber, ) -> Result, DatabaseError> { QueryParamNoteCommitmentLimit::check(note_commitments.len())?; @@ -241,6 +243,7 @@ pub(crate) fn select_existing_note_commitments( let raw_commitments = SelectDsl::select(schema::notes::table, schema::notes::note_id) .filter(schema::notes::note_id.eq_any(¬e_commitments)) + .filter(schema::notes::committed_at.le(up_to_block.to_raw_sql())) .load::>(conn)?; let commitments = raw_commitments diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 614867462..626a20e59 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -1614,7 +1614,12 @@ async fn reconstruct_storage_map_from_db_pages_until_latest() { .unwrap(); let details = db - .reconstruct_storage_map_from_db(account_id, slot_name.clone(), block3, Some(1)) + .reconstruct_storage_map_from_db( + account_id, + slot_name.clone(), + crate::state::ScopedBlockNum::new_unchecked(block3), + Some(1), + ) .await .unwrap(); @@ -1672,7 +1677,12 @@ async fn reconstruct_storage_map_from_db_returns_limit_exceeded_for_single_block // Use limit=1 so that 3 entries in a single block exceed the limit. block_range_start is block5 // (the first block with data), and the target is also block5. let details = db - .reconstruct_storage_map_from_db(account_id, slot_name.clone(), block5, Some(1)) + .reconstruct_storage_map_from_db( + account_id, + slot_name.clone(), + crate::state::ScopedBlockNum::new_unchecked(block5), + Some(1), + ) .await .unwrap(); diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index 0969a58ad..fb1f41843 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -81,6 +81,8 @@ pub enum DatabaseError { Diesel(#[from] diesel::result::Error), #[error(transparent)] QueryParamLimit(#[from] QueryLimitError), + #[error(transparent)] + RangeBeyondTip(#[from] RangeBeyondTip), // OTHER ERRORS // --------------------------------------------------------------------------------------------- @@ -231,16 +233,16 @@ pub enum ApplyBlockError { // OTHER ERRORS // --------------------------------------------------------------------------------------------- - #[error("block applying was cancelled because of closed channel on database side")] + #[error("block applying was cancelled because the writer task dropped the result channel")] ClosedChannel(#[from] RecvError), - #[error("concurrent write detected")] - ConcurrentWrite, #[error("account state forest update preparation failed")] AccountStateForestPreparation(#[source] AccountStateForestUpdateError), #[error("database doesn't have any block header data")] DbBlockHeaderEmpty, #[error("database update failed: {0}")] DbUpdateTaskFailed(String), + #[error("failed to send block to the writer task: {0}")] + WriterTaskSendFailed(String), } #[derive(Error, Debug)] @@ -251,6 +253,14 @@ pub enum ApplyBlockWithProvingInputsError { ApplyBlock(#[source] ApplyBlockError), } +/// A requested block range extends beyond the chain tip of the state view serving the request. +#[derive(Error, Debug)] +#[error("block_to ({block_to}) is greater than chain tip ({chain_tip})")] +pub struct RangeBeyondTip { + pub chain_tip: BlockNumber, + pub block_to: BlockNumber, +} + #[derive(Error, Debug)] pub enum GetBlockHeaderError { #[error("database error")] @@ -282,6 +292,8 @@ pub enum StateSyncError { EmptyBlockHeadersTable, #[error("failed to build MMR delta")] FailedToBuildMmrDelta(#[from] MmrError), + #[error(transparent)] + RangeBeyondTip(#[from] RangeBeyondTip), } impl From for StateSyncError { @@ -302,11 +314,8 @@ pub enum NoteSyncError { MmrError(#[from] MmrError), #[error("invalid block range")] InvalidBlockRange(#[from] InvalidBlockRange), - #[error("block_to ({block_to}) is greater than chain tip ({chain_tip})")] - FutureBlock { - chain_tip: BlockNumber, - block_to: BlockNumber, - }, + #[error(transparent)] + RangeBeyondTip(#[from] RangeBeyondTip), #[error("malformed note tags")] DeserializationFailed(#[from] ConversionError), } diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 416e08736..ec40d3fe3 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -35,10 +35,20 @@ pub use errors::{ GetBlockHeaderError, GetBlockInputsError, NoteSyncError, + RangeBeyondTip, StateSyncError, }; pub use genesis::GenesisState; -pub use state::State; +pub use state::{ + BlockWriter, + LoadedState, + ProofWriter, + ScopedBlockNum, + ScopedBlockRange, + State, + StateView, + WriterTask, +}; /// Returns the store crate version. pub fn version() -> &'static str { diff --git a/crates/store/src/state/apply_block.rs b/crates/store/src/state/apply_block.rs deleted file mode 100644 index c81c72e32..000000000 --- a/crates/store/src/state/apply_block.rs +++ /dev/null @@ -1,434 +0,0 @@ -use std::fmt::Display; -use std::sync::Arc; - -use miden_node_proto::domain::proof_request::BlockProofRequest; -use miden_node_utils::ErrorReport; -use miden_node_utils::tracing::{miden_instrument, miden_span_record}; -use miden_protocol::Word; -use miden_protocol::account::AccountUpdateDetails; -use miden_protocol::batch::OrderedBatches; -use miden_protocol::block::account_tree::AccountMutationSet; -use miden_protocol::block::nullifier_tree::NullifierMutationSet; -use miden_protocol::block::{BlockBody, BlockHeader, BlockInputs, BlockNumber, SignedBlock}; -use miden_protocol::note::{NoteDetails, Nullifier}; -use miden_protocol::transaction::OutputNote; -use miden_protocol::utils::serde::Serializable; -use tokio::sync::oneshot; -use tracing::{Instrument, info_span}; - -use crate::db::NoteRecord; -use crate::errors::{ApplyBlockError, ApplyBlockWithProvingInputsError, InvalidBlockError}; -use crate::state::block_lifecycle::{BlockLifecycle, lifecycle_events_enabled}; -use crate::state::{BlockNotification, InnerState, State}; -use crate::{COMPONENT, HistoricalError, LOG_TARGET}; - -impl State { - /// Saves proving inputs for a signed block and applies it to the state. - /// - /// Used by the in-process block producer after it has built and signed a block. - #[miden_instrument( - target = COMPONENT, - skip_all, - err, - )] - pub async fn apply_block_with_proving_inputs( - &self, - ordered_batches: OrderedBatches, - block_inputs: BlockInputs, - signed_block: SignedBlock, - ) -> Result<(), ApplyBlockWithProvingInputsError> { - let block_header = signed_block.header().clone(); - let block_num = block_header.block_num(); - - let proving_inputs = BlockProofRequest { - tx_batches: ordered_batches, - block_header, - block_inputs, - }; - - self.save_proving_inputs(block_num, &proving_inputs) - .await - .map_err(ApplyBlockWithProvingInputsError::SaveProvingInputs)?; - - self.apply_block(signed_block) - .await - .map_err(ApplyBlockWithProvingInputsError::ApplyBlock) - } - - /// Apply changes of a new block to the DB and in-memory data structures. - /// - /// ## Note on state consistency - /// - /// The server contains in-memory representations of the existing trees, the in-memory - /// representation must be kept consistent with the committed data, this is necessary so to - /// provide consistent results for all endpoints. In order to achieve consistency, the - /// following steps are used: - /// - /// - the request data is validated, prior to starting any modifications. - /// - block is being saved into the store in parallel with updating the DB, but before - /// committing. This block is considered as candidate and not yet available for reading - /// because the latest block pointer is not updated yet. - /// - a transaction is open in the DB and the writes are started. - /// - while the transaction is not committed, concurrent reads are allowed, both the DB and the - /// in-memory representations, which are consistent at this stage. - /// - prior to committing the changes to the DB, exclusive locks to the canonical in-memory - /// state and account-state forest are acquired, preventing readers from observing them at - /// different block heights. - /// - the DB transaction is committed, and requests that read only from the DB can proceed to - /// use the fresh data. - /// - the account-state forest and canonical in-memory structures are updated, including the - /// latest block pointer, and both locks are released. - /// - if any persistent tree update fails after the DB commit, the process aborts. The durable - /// database remains authoritative, and divergent tree storage must be rebuilt before the node - /// resumes normal processing. - // TODO: This span is logged in a root span, we should connect it to the parent span. - #[miden_instrument( - target = COMPONENT, - skip_all, - err, - )] - pub async fn apply_block(&self, signed_block: SignedBlock) -> Result<(), ApplyBlockError> { - let _lock = self.writer.try_lock().map_err(|_| ApplyBlockError::ConcurrentWrite)?; - - let header = signed_block.header(); - let body = signed_block.body(); - - let block_num = header.block_num(); - let block_commitment = header.commitment(); - let num_transactions = body.transactions().as_slice().len(); - - miden_span_record!( - block.number = %block_num, - block.commitment = %block_commitment, - block.transactions.count = num_transactions, - ); - - self.validate_block_header(header, body).await?; - - let block_lifecycle = - lifecycle_events_enabled().then(|| BlockLifecycle::from_block_body(block_num, body)); - let unresolved_note_nullifiers = block_lifecycle - .as_ref() - .map_or_else(Vec::new, BlockLifecycle::unresolved_note_nullifiers); - - // Save the block to the block store. In a case of a rolled-back DB transaction, the - // in-memory state will be unchanged, but the file might still be written. Such blocks - // should be considered candidates, not finalized blocks. - let signed_block_bytes = signed_block.to_bytes(); - // Clone before moving into the block-save task so we can cache for replicas at commit. - let cache_bytes = signed_block_bytes.clone(); - let store = Arc::clone(&self.block_store); - let block_save_task = tokio::spawn( - async move { store.save_block(block_num, &signed_block_bytes).await }.in_current_span(), - ); - - let ( - nullifier_tree_old_root, - nullifier_tree_update, - account_tree_old_root, - account_tree_update, - ) = self.compute_tree_mutations(header, body).await?; - - let notes = Self::build_note_records(header, body)?; - - // Signals the transaction is ready to be committed, and the write lock can be acquired. - let (allow_acquire, acquired_allowed) = oneshot::channel::<()>(); - // Signals the write lock has been acquired, and the transaction can be committed. - let (inform_acquire_done, acquire_done) = oneshot::channel::<()>(); - - // Extract public account updates with patches before block is moved into async task. - // Private accounts are filtered out since they don't expose their state changes. - let account_patches = - Vec::from_iter(body.updated_accounts().iter().filter_map( - |update| match update.details() { - AccountUpdateDetails::Public(patch) => Some(patch.clone()), - AccountUpdateDetails::Private => None, - }, - )); - let account_forest_update = self.with_forest_read_blocking(|forest| { - forest - .compute_block_update_mutations(block_num, account_patches) - .map_err(ApplyBlockError::AccountStateForestPreparation) - })?; - let precomputed_public_states = account_forest_update.account_states.clone(); - - // The DB and in-memory state updates need to be synchronized and are partially overlapping. - // Namely, the DB transaction only proceeds after this task acquires the in-memory write - // lock. This requires the DB update to run concurrently, so a new task is spawned. - let db = Arc::clone(&self.db); - let db_update_task = tokio::spawn( - async move { - db.apply_block( - allow_acquire, - acquire_done, - signed_block, - notes, - precomputed_public_states, - unresolved_note_nullifiers, - ) - .await - } - .in_current_span(), - ); - - // Wait for the message from the DB update task, that we ready to commit the DB transaction. - acquired_allowed - .instrument(info_span!(target: COMPONENT, "await_db_readiness")) - .await - .map_err(ApplyBlockError::ClosedChannel)?; - - // Awaiting the block saving task to complete without errors. - block_save_task.await??; - - let resolved_note_ids = self.with_inner_and_forest_write_blocking(|inner, forest| { - // We need to check that neither the nullifier tree nor the account tree have changed - // while we were waiting for the DB preparation task to complete. If either of them did - // change, we do not proceed with in-memory and database updates, since it may lead to - // an inconsistent state. - if inner.nullifier_tree.root() != nullifier_tree_old_root - || inner.account_tree.root_latest() != account_tree_old_root - { - return Err(ApplyBlockError::ConcurrentWrite); - } - - // Notify the DB update task that the write lock has been acquired, so it can commit the - // DB transaction. - inform_acquire_done - .send(()) - .map_err(|_| ApplyBlockError::DbUpdateTaskFailed("Receiver was dropped".into()))?; - - // TODO: shutdown #91 Await for successful commit of the DB transaction. If the commit - // fails, we mustn't change in-memory state, so we return a block applying error and - // don't proceed with in-memory updates. - let resolved_note_ids = tokio::runtime::Handle::current() - .block_on(db_update_task)? - .map_err(|err| ApplyBlockError::DbUpdateTaskFailed(err.as_report()))?; - - // The DB is now committed. Keep both write locks held while advancing the forest and - // canonical in-memory state so readers cannot observe different block heights. - let InnerState { nullifier_tree, blockchain, account_tree } = inner; - forest - .apply_precomputed_block_update(block_num, account_forest_update) - .unwrap_or_else(|error| { - Self::abort_after_post_commit_failure("account-state forest", &error) - }); - nullifier_tree.apply_mutations(nullifier_tree_update).unwrap_or_else(|error| { - Self::abort_after_post_commit_failure("nullifier tree", &error) - }); - account_tree.apply_mutations(account_tree_update).unwrap_or_else(|error| { - Self::abort_after_post_commit_failure("account tree", &error) - }); - blockchain.push(block_commitment); - - Ok(resolved_note_ids) - })?; - - // Push to cache and notify replica subscribers. - self.block_cache - .push(block_num, BlockNotification::new(block_num, cache_bytes)) - .expect("block cache receives sequential block numbers"); - let _ = self.committed_tip_tx.send(block_num); - - if let Some(block_lifecycle) = block_lifecycle { - block_lifecycle.emit(&resolved_note_ids); - } - tracing::debug!(target: LOG_TARGET, "Block applied"); - - Ok(()) - } - - /// Terminates after a persistent state failure that occurred after the canonical DB commit. - /// - /// Returning would expose components at different block heights. Tests panic so the fatal path - /// can be asserted without terminating the test process; production aborts immediately. - fn abort_after_post_commit_failure(component: &str, error: &impl Display) -> ! { - tracing::error!( - target: LOG_TARGET, - component, - error = %error, - "persistent state update failed after database commit; aborting" - ); - - #[cfg(test)] - panic!("{component} update failed after database commit: {error}"); - - #[cfg(not(test))] - std::process::abort(); - } - - /// Saves the proving inputs for the given block to the block store. - pub async fn save_proving_inputs( - &self, - block_num: BlockNumber, - proving_inputs: &BlockProofRequest, - ) -> std::io::Result<()> { - self.block_store - .save_proving_inputs(block_num, &proving_inputs.to_bytes()) - .await - } - - /// Validates that the block header is consistent with the block body and the current state. - #[miden_instrument( - target = COMPONENT, - skip_all, - err, - )] - async fn validate_block_header( - &self, - header: &BlockHeader, - body: &BlockBody, - ) -> Result<(), ApplyBlockError> { - // Validate that header and body match. - let tx_commitment = body.transactions().commitment(); - if header.tx_commitment() != tx_commitment { - return Err(InvalidBlockError::InvalidBlockTxCommitment { - expected: tx_commitment, - actual: header.tx_commitment(), - } - .into()); - } - - let block_num = header.block_num(); - - // Validate that the applied block is the next block in sequence. - let prev_block = self - .db - .select_block_header_by_block_num(None) - .await? - .ok_or(ApplyBlockError::DbBlockHeaderEmpty)?; - let expected_block_num = prev_block.block_num().child(); - if block_num != expected_block_num { - return Err(InvalidBlockError::NewBlockInvalidBlockNum { - expected: expected_block_num, - submitted: block_num, - } - .into()); - } - if header.prev_block_commitment() != prev_block.commitment() { - return Err(InvalidBlockError::NewBlockInvalidPrevCommitment.into()); - } - - Ok(()) - } - - /// Computes nullifier and account tree mutations, validating roots against the block header. - #[miden_instrument( - target = COMPONENT, - skip_all, - err, - )] - async fn compute_tree_mutations( - &self, - header: &BlockHeader, - body: &BlockBody, - ) -> Result<(Word, NullifierMutationSet, Word, AccountMutationSet), ApplyBlockError> { - self.with_inner_read_blocking(|inner| { - let block_num = header.block_num(); - - // nullifiers can be produced only once - let duplicate_nullifiers: Vec<_> = body - .created_nullifiers() - .iter() - .filter(|&nullifier| inner.nullifier_tree.get_block_num(nullifier).is_some()) - .copied() - .collect(); - if !duplicate_nullifiers.is_empty() { - return Err(InvalidBlockError::DuplicatedNullifiers(duplicate_nullifiers).into()); - } - - // new_block.chain_root must be equal to the chain MMR root prior to the update - let peaks = inner.blockchain.peaks(); - if peaks.hash_peaks() != header.chain_commitment() { - return Err(InvalidBlockError::NewBlockInvalidChainCommitment.into()); - } - - // compute update for nullifier tree - let nullifier_tree_update = inner - .nullifier_tree - .compute_mutations( - body.created_nullifiers().iter().map(|nullifier| (*nullifier, block_num)), - ) - .map_err(InvalidBlockError::NewBlockNullifierAlreadySpent)?; - - if nullifier_tree_update.as_mutation_set().root() != header.nullifier_root() { - return Err(InvalidBlockError::NewBlockInvalidNullifierRoot.into()); - } - - // compute update for account tree - let account_tree_update = inner - .account_tree - .compute_mutations( - body.updated_accounts() - .iter() - .map(|update| (update.account_id(), update.final_state_commitment())), - ) - .map_err(|e| match e { - HistoricalError::AccountTreeError(err) => { - InvalidBlockError::NewBlockDuplicateAccountIdPrefix(err) - }, - HistoricalError::MerkleError(_) => { - panic!("Unexpected MerkleError during account tree mutation computation") - }, - })?; - - if account_tree_update.as_mutation_set().root() != header.account_root() { - return Err(InvalidBlockError::NewBlockInvalidAccountRoot.into()); - } - - Ok(( - inner.nullifier_tree.root(), - nullifier_tree_update, - inner.account_tree.root_latest(), - account_tree_update, - )) - }) - } - - /// Builds note records with inclusion proofs from the block body. - #[miden_instrument( - target = COMPONENT, - skip_all, - err, - )] - fn build_note_records( - header: &BlockHeader, - body: &BlockBody, - ) -> Result)>, ApplyBlockError> { - let block_num = header.block_num(); - - let note_tree = body.compute_block_note_tree(); - if note_tree.root() != header.note_root() { - return Err(InvalidBlockError::NewBlockInvalidNoteRoot.into()); - } - - let notes = body - .output_notes() - .map(|(note_index, note)| { - let (details, attachments, nullifier) = match note { - OutputNote::Public(public) => ( - Some(NoteDetails::from(public.as_note())), - public.as_note().attachments().clone(), - Some(public.as_note().nullifier()), - ), - OutputNote::Private(private) => (None, private.attachments().clone(), None), - }; - - let inclusion_path = note_tree.open(note_index); - - let note_record = NoteRecord { - block_num, - note_index, - note_id: note.id().as_word(), - metadata: *note.metadata(), - details, - attachments, - inclusion_path, - }; - - Ok((note_record, nullifier)) - }) - .collect::, InvalidBlockError>>()?; - - Ok(notes) - } -} diff --git a/crates/store/src/state/lifecycle.rs b/crates/store/src/state/lifecycle.rs new file mode 100644 index 000000000..4afe7afe7 --- /dev/null +++ b/crates/store/src/state/lifecycle.rs @@ -0,0 +1,275 @@ +//! Store lifecycle: loading the state, starting its write worker, and stopping the store. + +use std::num::NonZeroUsize; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::AtomicUsize; + +use arc_swap::ArcSwap; +use miden_node_utils::ErrorReport; +use miden_node_utils::clap::StorageOptions; +use miden_node_utils::shutdown::CancellationToken; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::block::BlockNumber; +use tokio::sync::{mpsc, watch}; + +use crate::account_state_forest::AccountStateForestBackend; +use crate::accounts::AccountTreeWithHistory; +use crate::blocks::BlockStore; +use crate::db::Db; +use crate::errors::StateInitializationError; +use crate::proven_tip::ProvenTipWriter; +use crate::state::loader::{ + ACCOUNT_STATE_FOREST_STORAGE_DIR, + ACCOUNT_TREE_STORAGE_DIR, + AccountForestLoader, + NULLIFIER_TREE_STORAGE_DIR, + TreeStorage, + TreeStorageLoader, + load_mmr, + verify_account_state_forest_consistency, + verify_tree_consistency, +}; +use crate::state::writer::{WriteRequest, WriteWorker, WriterTask}; +use crate::state::{ + BlockCache, + BlockWriter, + ProofCache, + ProofWriter, + SnapshotGuard, + State, + StateSnapshot, +}; +use crate::{COMPONENT, DataDirectory, DatabaseOptions}; + +/// Number of recent committed blocks held in the in-memory cache for replica subscriptions. +const BLOCK_CACHE_CAPACITY: NonZeroUsize = NonZeroUsize::new(512).unwrap(); + +/// Number of recent block proofs held in the in-memory cache for replica subscriptions. +const PROOF_CACHE_CAPACITY: NonZeroUsize = NonZeroUsize::new(512).unwrap(); + +// LOADED STATE +// ================================================================================================ + +/// A loaded store state whose write worker has not been started yet. +/// +/// Returned by [`State::load`]. [`Self::start`] spawns the writer and yields the read-only +/// [`State`] together with the write capabilities; since this is the only way to obtain a +/// [`BlockWriter`], [`BlockWriter::apply_block`] can always make progress. +#[must_use = "call `start` to spawn the write worker and obtain the state"] +pub struct LoadedState { + state: State, + writer: WriteWorker, + write_tx: mpsc::Sender, +} + +impl LoadedState { + /// Spawns the write worker onto the current runtime and returns the read-only state together + /// with the write capabilities and the writer task's handle. + /// + /// [`Arc`] is the read-only view shared with every component that queries or + /// subscribes; it exposes no mutating methods. [`BlockWriter`] and [`ProofWriter`] are the + /// only handles able to mutate the store — hand them to the single task driving each write + /// path (block production or sync, and proof scheduling or sync respectively). The + /// capabilities expose no read access: tasks that both read and write receive the + /// [`Arc`] alongside their capability. + /// + /// The writer exits once the shutdown token passed to [`State::load`] is cancelled or the + /// [`BlockWriter`] (holding the only request sender) is dropped — an in-flight block write + /// always completes first. Awaiting the returned handle after either event guarantees the + /// writer has released the tree storage it owns; a join error carries a writer panic. + /// + /// Callers without a token to cancel should stop the store via [`BlockWriter::stop`] rather + /// than dropping and joining by hand. + pub fn start(self) -> (Arc, BlockWriter, ProofWriter, WriterTask) { + let writer_task = tokio::spawn(self.writer.run()); + let state = Arc::new(self.state); + let block_writer = BlockWriter { + block_store: Arc::clone(&state.block_store), + write_tx: self.write_tx, + }; + let proof_writer = ProofWriter { state: Arc::clone(&state) }; + (state, block_writer, proof_writer, WriterTask(writer_task)) + } +} + +// LOAD +// ================================================================================================ + +impl State { + /// Loads the state from the data directory. + /// + /// The loaded state owns all store data structures and exposes subscription methods for + /// sequencer and replica tasks. Call [`LoadedState::start`] on the result to spawn the block + /// writer and obtain the usable [`State`]. + #[miden_instrument( + target = COMPONENT, + skip_all, + )] + pub async fn load( + data_path: &Path, + storage_options: StorageOptions, + shutdown: CancellationToken, + ) -> Result { + Self::load_with_database_options( + data_path, + storage_options, + DatabaseOptions::default(), + shutdown, + ) + .await + } + + /// Loads the state from the data directory using explicit database options. + /// + /// The loaded state owns all store data structures and exposes subscription methods for + /// sequencer and replica tasks. Call [`LoadedState::start`] on the result to spawn the block + /// writer and obtain the usable [`State`]. + #[miden_instrument( + target = COMPONENT, + skip_all, + )] + pub async fn load_with_database_options( + data_path: &Path, + storage_options: StorageOptions, + database_options: DatabaseOptions, + shutdown: CancellationToken, + ) -> Result { + let data_directory = DataDirectory::load(data_path.to_path_buf()) + .map_err(StateInitializationError::DataDirectoryLoadError)?; + + let block_store = Arc::new( + BlockStore::load(data_directory.block_store_dir()) + .map_err(StateInitializationError::BlockStoreLoadError)?, + ); + + let database_filepath = data_directory.database_path(); + let mut db = Db::load_with_pool_size( + database_filepath.clone(), + database_options.connection_pool_size, + ) + .await + .map_err(StateInitializationError::DatabaseLoadError)?; + + let blockchain = load_mmr(&mut db).await?; + let latest_block_num = blockchain.chain_tip().unwrap_or(BlockNumber::GENESIS); + + #[cfg(feature = "rocksdb")] + let (account_storage_config, nullifier_storage_config, forest_storage_config) = ( + storage_options.account_tree.into(), + storage_options.nullifier_tree.into(), + storage_options.account_state_forest.into(), + ); + #[cfg(not(feature = "rocksdb"))] + let (account_storage_config, nullifier_storage_config, forest_storage_config) = { + let _ = &storage_options; + ((), (), ()) + }; + let account_storage = + TreeStorage::create(data_path, &account_storage_config, ACCOUNT_TREE_STORAGE_DIR)?; + let account_tree = account_storage.load_account_tree(&mut db).await?; + + let nullifier_storage = + TreeStorage::create(data_path, &nullifier_storage_config, NULLIFIER_TREE_STORAGE_DIR)?; + let nullifier_tree = nullifier_storage.load_nullifier_tree(&mut db).await?; + + // Verify that tree roots match the expected roots from the database. This catches any + // divergence between persistent storage and the database caused by corruption or incomplete + // shutdown. + verify_tree_consistency(account_tree.root(), nullifier_tree.root(), &mut db).await?; + + let account_tree = AccountTreeWithHistory::new(account_tree, latest_block_num); + + let forest_backend = AccountStateForestBackend::create( + data_path, + &forest_storage_config, + ACCOUNT_STATE_FOREST_STORAGE_DIR, + )?; + let forest = forest_backend.load_account_state_forest(&mut db, latest_block_num).await?; + verify_account_state_forest_consistency(&forest, &mut db).await?; + + let db = Arc::new(db); + + // Initialize the proven tip from the block store. + let proven_tip_init = block_store + .load_proven_tip() + .map_err(StateInitializationError::ProvenTipLoadError)?; + let (proven_tip, _rx) = ProvenTipWriter::new(proven_tip_init); + + // Committed-tip watch: fires after each successful apply_block. + let (committed_tip_tx, _rx) = watch::channel(latest_block_num); + let committed_tip_tx = Arc::new(committed_tip_tx); + + let block_cache = BlockCache::new(BLOCK_CACHE_CAPACITY); + let proof_cache = ProofCache::new(PROOF_CACHE_CAPACITY); + + // Shared counter of live snapshot generations, for observability. + let snapshots_live = Arc::new(AtomicUsize::new(0)); + + // Create the initial snapshot from reader views of the just-loaded trees. + let initial_snapshot = Arc::new(StateSnapshot::new( + nullifier_tree + .reader() + .map_err(|e| StateInitializationError::NullifierTreeIoError(e.as_report()))?, + blockchain.clone(), + account_tree.reader(), + forest + .reader() + .map_err(|e| StateInitializationError::AccountStateForestIoError(e.as_report()))?, + SnapshotGuard::new(Arc::clone(&snapshots_live), latest_block_num), + )); + let latest_snapshot = Arc::new(ArcSwap::from(initial_snapshot)); + + // Assemble the write worker. It owns the writable trees and processes write requests + // serially, publishing a new snapshot after each committed block. The caller runs it; it + // exits when the shutdown token is cancelled or the `BlockWriter` (holding the only request + // sender) is dropped. + let (write_tx, write_rx) = mpsc::channel(1); + let block_writer = WriteWorker::new( + Arc::clone(&db), + Arc::clone(&block_store), + Arc::clone(&latest_snapshot), + Arc::clone(&committed_tip_tx), + block_cache.clone(), + write_rx, + shutdown, + nullifier_tree, + account_tree, + blockchain, + forest, + snapshots_live, + ); + let state = Self { + data_directory: data_path.to_path_buf(), + db, + block_store, + latest_snapshot, + proven_tip, + committed_tip_tx, + block_cache, + proof_cache, + }; + + Ok(LoadedState { state, writer: block_writer, write_tx }) + } + + /// Loads the state with default options and starts its write worker, detaching the worker + /// task. + /// + /// Test-only helper for tests in sibling crates that don't manage the writer's lifecycle: + /// the detached writer exits once the returned [`BlockWriter`] is dropped. Hidden from public + /// docs and not part of the stable API. + /// + /// # Panics + /// + /// Panics if the state fails to load. + #[doc(hidden)] + pub async fn for_tests(data_path: &Path) -> (Arc, BlockWriter, ProofWriter) { + let (state, block_writer, proof_writer, _writer_task) = + Self::load(data_path, StorageOptions::default(), CancellationToken::new()) + .await + .expect("state should load") + .start(); + (state, block_writer, proof_writer) + } +} diff --git a/crates/store/src/state/loader.rs b/crates/store/src/state/loader.rs index 177617988..82d663025 100644 --- a/crates/store/src/state/loader.rs +++ b/crates/store/src/state/loader.rs @@ -76,6 +76,9 @@ pub type TreeStorage = RocksDbStorage; #[cfg(not(feature = "rocksdb"))] pub type TreeStorage = MemoryStorage; +/// The read-only snapshot storage backend for trees, used by in-memory state snapshots. +pub type TreeStorageReader = ::Reader; + // ERROR CONVERSION // ================================================================================================ diff --git a/crates/store/src/state/mod.rs b/crates/store/src/state/mod.rs index 608f6268d..37c556409 100644 --- a/crates/store/src/state/mod.rs +++ b/crates/store/src/state/mod.rs @@ -3,125 +3,48 @@ //! The [State] provides data access and modifications methods, its main purpose is to ensure that //! data is atomically written, and that reads are consistent. -use std::collections::{BTreeMap, BTreeSet, HashSet}; -use std::num::NonZeroUsize; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::sync::Arc; -use miden_node_proto::domain::batch::BatchInputs; -use miden_node_utils::clap::StorageOptions; -use miden_node_utils::formatting::format_array; -use miden_node_utils::tracing::miden_instrument; -use miden_protocol::Word; -use miden_protocol::account::AccountId; -use miden_protocol::block::account_tree::AccountWitness; -use miden_protocol::block::nullifier_tree::{NullifierTree, NullifierWitness}; -use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, Blockchain}; -use miden_protocol::crypto::merkle::mmr::{MmrProof, PartialMmr}; -use miden_protocol::crypto::merkle::smt::{LargeSmt, SmtStorage}; -use miden_protocol::note::{NoteId, NoteScript, Nullifier}; -use miden_protocol::transaction::PartialBlockchain; -use tokio::sync::{Mutex, RwLock, watch}; -use tracing::{Instrument, Span}; +use arc_swap::ArcSwap; +use miden_protocol::block::BlockNumber; +use tokio::sync::watch; -use crate::account_state_forest::{AccountStateForest, AccountStateForestBackend}; -use crate::accounts::AccountTreeWithHistory; use crate::blocks::BlockStore; -use crate::db::{Db, NoteRecord, NullifierInfo}; -use crate::errors::{ - DatabaseError, - GetBatchInputsError, - GetBlockHeaderError, - GetBlockInputsError, - StateInitializationError, -}; +use crate::db::Db; use crate::proven_tip::ProvenTipWriter; -use crate::{COMPONENT, DataDirectory, DatabaseOptions}; -/// Number of recent committed blocks held in the in-memory cache for replica subscriptions. -const BLOCK_CACHE_CAPACITY: NonZeroUsize = NonZeroUsize::new(512).unwrap(); - -/// Number of recent block proofs held in the in-memory cache for replica subscriptions. -const PROOF_CACHE_CAPACITY: NonZeroUsize = NonZeroUsize::new(512).unwrap(); - -mod loader; -use loader::{ - ACCOUNT_STATE_FOREST_STORAGE_DIR, - ACCOUNT_TREE_STORAGE_DIR, - AccountForestLoader, - NULLIFIER_TREE_STORAGE_DIR, - TreeStorage, - TreeStorageLoader, - load_mmr, - verify_account_state_forest_consistency, - verify_tree_consistency, -}; - -mod replica; -pub use replica::{BlockCache, BlockNotification, ProofCache, ProofNotification}; - -mod account; - -mod apply_block; -mod apply_proof; mod block_lifecycle; mod bootstrap; mod disk_monitor; -mod sync_state; - -// FINALITY -// ================================================================================================ - -/// The finality level for chain tip queries. -#[derive(Debug, Clone, Copy)] -pub enum Finality { - /// The latest committed (but not necessarily proven) block. - Committed, - /// The latest block that has been proven in an unbroken sequence from genesis. - Proven, -} +mod loader; -// STRUCTURES -// ================================================================================================ +mod lifecycle; +pub use lifecycle::LoadedState; -#[derive(Debug, Default)] -pub struct TransactionInputs { - pub account_commitment: Word, - pub nullifiers: Vec, - pub found_unauthenticated_notes: HashSet, - pub new_account_id_prefix_is_unique: Option, -} +mod replica; +pub use replica::{BlockCache, BlockNotification, ProofCache, ProofNotification}; -type BlockInputWitnesses = ( - BlockNumber, - BTreeMap, - BTreeMap, - PartialMmr, -); +mod tip; -/// Container for state that needs to be updated atomically. -struct InnerState -where - S: SmtStorage, -{ - nullifier_tree: NullifierTree>, - blockchain: Blockchain, - account_tree: AccountTreeWithHistory, -} +mod view; +pub use view::{ScopedBlockNum, ScopedBlockRange, StateView, TransactionInputs}; +use view::{SnapshotGuard, StateSnapshot}; -impl InnerState { - /// Returns the latest block number. - fn latest_block_num(&self) -> BlockNumber { - self.blockchain - .chain_tip() - .expect("chain should always have at least the genesis block") - } -} +mod writer; +pub use writer::{BlockWriter, ProofWriter, WriterTask}; // CHAIN STATE // ================================================================================================ -/// The rollup state. +/// The rollup state, read-only. +/// +/// Mutations go through the [`BlockWriter`] and [`ProofWriter`] capabilities returned by +/// [`LoadedState::start`]; every holder of this type can only query and subscribe. +/// +/// All tree and database reads go through a request-scoped [`StateView`] obtained from +/// [`State::view`]; this type itself only exposes chain-tip queries, subscriptions, and +/// block-store access. pub struct State { /// Root directory containing the store's on-disk data. data_directory: PathBuf, @@ -133,24 +56,19 @@ pub struct State { /// The block store which stores full block contents for all blocks. block_store: Arc, - /// Read-write lock used to prevent writing to a structure while it is being used. + /// Atomically swappable pointer to the latest in-memory state snapshot. /// - /// The lock is writer-preferring, meaning the writer won't be starved. - inner: RwLock>, - - /// Forest-related state `(SmtForest, storage_map_roots, vault_roots)` with its own lock. - forest: RwLock>, - - /// To allow readers to access the tree data while an update in being performed, and prevent - /// TOCTOU issues, there must be no concurrent writers. This locks to serialize the writers. - writer: Mutex<()>, + /// Readers load the snapshot wait-free via [`ArcSwap::load_full`]; the + /// [`WriteWorker`](writer::WriteWorker) task atomically replaces the pointer after each + /// committed block. Readers holding an old snapshot are unaffected by the swap. + latest_snapshot: Arc>, /// The latest proven-in-sequence block number, updated by the proof scheduler or `apply_proof`. proven_tip: ProvenTipWriter, /// Watch sender fired after each block is committed. Replicas subscribe via /// `subscribe_committed_tip()` to be woken when new blocks arrive. - committed_tip_tx: watch::Sender, + committed_tip_tx: Arc>, /// FIFO cache of recent committed blocks for replica subscriptions. When a subscriber needs a /// block that has been evicted, it falls back to loading from the block store. @@ -160,612 +78,3 @@ pub struct State { /// that has been evicted, it falls back to loading from the block store. pub(crate) proof_cache: ProofCache, } - -impl State { - // CONSTRUCTOR - // -------------------------------------------------------------------------------------------- - - /// Loads the state from the data directory. - /// - /// The loaded state owns all store data structures and exposes subscription methods for - /// sequencer and replica tasks. - #[miden_instrument( - target = COMPONENT, - skip_all, - )] - pub async fn load( - data_path: &Path, - storage_options: StorageOptions, - ) -> Result { - Self::load_with_database_options(data_path, storage_options, DatabaseOptions::default()) - .await - } - - /// Loads the state from the data directory using explicit database options. - /// - /// The loaded state owns all store data structures and exposes subscription methods for - /// sequencer and replica tasks. - #[miden_instrument( - target = COMPONENT, - skip_all, - )] - pub async fn load_with_database_options( - data_path: &Path, - storage_options: StorageOptions, - database_options: DatabaseOptions, - ) -> Result { - let data_directory = DataDirectory::load(data_path.to_path_buf()) - .map_err(StateInitializationError::DataDirectoryLoadError)?; - - let block_store = Arc::new( - BlockStore::load(data_directory.block_store_dir()) - .map_err(StateInitializationError::BlockStoreLoadError)?, - ); - - let database_filepath = data_directory.database_path(); - let mut db = Db::load_with_pool_size( - database_filepath.clone(), - database_options.connection_pool_size, - ) - .await - .map_err(StateInitializationError::DatabaseLoadError)?; - - let blockchain = load_mmr(&mut db).await?; - let latest_block_num = blockchain.chain_tip().unwrap_or(BlockNumber::GENESIS); - - #[cfg(feature = "rocksdb")] - let (account_storage_config, nullifier_storage_config, forest_storage_config) = ( - storage_options.account_tree.into(), - storage_options.nullifier_tree.into(), - storage_options.account_state_forest.into(), - ); - #[cfg(not(feature = "rocksdb"))] - let (account_storage_config, nullifier_storage_config, forest_storage_config) = { - let _ = &storage_options; - ((), (), ()) - }; - let account_storage = - TreeStorage::create(data_path, &account_storage_config, ACCOUNT_TREE_STORAGE_DIR)?; - let account_tree = account_storage.load_account_tree(&mut db).await?; - - let nullifier_storage = - TreeStorage::create(data_path, &nullifier_storage_config, NULLIFIER_TREE_STORAGE_DIR)?; - let nullifier_tree = nullifier_storage.load_nullifier_tree(&mut db).await?; - - // Verify that tree roots match the expected roots from the database. This catches any - // divergence between persistent storage and the database caused by corruption or incomplete - // shutdown. - verify_tree_consistency(account_tree.root(), nullifier_tree.root(), &mut db).await?; - - let account_tree = AccountTreeWithHistory::new(account_tree, latest_block_num); - - let forest_backend = AccountStateForestBackend::create( - data_path, - &forest_storage_config, - ACCOUNT_STATE_FOREST_STORAGE_DIR, - )?; - let forest = forest_backend.load_account_state_forest(&mut db, latest_block_num).await?; - verify_account_state_forest_consistency(&forest, &mut db).await?; - - let inner = RwLock::new(InnerState { nullifier_tree, blockchain, account_tree }); - - let forest = RwLock::new(forest); - let writer = Mutex::new(()); - let db = Arc::new(db); - - // Initialize the proven tip from the block store. - let proven_tip_init = block_store - .load_proven_tip() - .map_err(StateInitializationError::ProvenTipLoadError)?; - let (proven_tip, _rx) = ProvenTipWriter::new(proven_tip_init); - - // Committed-tip watch: fires after each successful apply_block. - let (committed_tip_tx, _rx) = watch::channel(latest_block_num); - - Ok(Self { - data_directory: data_path.to_path_buf(), - db, - block_store, - inner, - forest, - writer, - proven_tip, - committed_tip_tx, - block_cache: BlockCache::new(BLOCK_CACHE_CAPACITY), - proof_cache: ProofCache::new(PROOF_CACHE_CAPACITY), - }) - } - - /// Returns a watch receiver that wakes every time a new block is committed. - pub fn subscribe_committed_tip(&self) -> watch::Receiver { - self.committed_tip_tx.subscribe() - } - - /// Loads serialized block proving inputs from the block store. - pub async fn load_proving_inputs( - &self, - block_num: BlockNumber, - ) -> std::io::Result>> { - self.block_store.load_proving_inputs(block_num).await - } - - /// Returns a watch receiver that wakes every time the proven-in-sequence tip advances. - pub fn subscribe_proven_tip(&self) -> watch::Receiver { - self.proven_tip.subscribe() - } - - // HELPER FUNCTIONS TO AVOID BLOCKING CALLS IN ASYNC CONTEXT - // -------------------------------------------------------------------------------------------- - - /// Runs a synchronous read-only operation over the inner state on Tokio's blocking path. - /// - /// The account and nullifier trees may be backed by `RocksDB`, so tree access must not run on - /// an async worker thread directly. This helper preserves the current tracing span while - /// moving the blocking lock acquisition and closure body into `block_in_place`. - fn with_inner_read_blocking(&self, f: impl FnOnce(&InnerState) -> R) -> R { - let span = Span::current(); - tokio::task::block_in_place(|| { - span.in_scope(|| { - let inner = self.inner.blocking_read(); - f(&inner) - }) - }) - } - - /// Runs a synchronous mutable operation while holding both in-memory state write locks. - /// - /// Locks are always acquired in inner-state then forest order. Holding both across the database - /// commit window prevents readers from observing canonical tree state from one block and - /// account-state forest data from another. - fn with_inner_and_forest_write_blocking( - &self, - f: impl FnOnce( - &mut InnerState, - &mut AccountStateForest, - ) -> R, - ) -> R { - let span = Span::current(); - tokio::task::block_in_place(|| { - span.in_scope(|| { - let mut inner = self.inner.blocking_write(); - let mut forest = self.forest.blocking_write(); - f(&mut inner, &mut forest) - }) - }) - } - - /// Runs a synchronous read-only operation over the account state forest on Tokio's blocking - /// path. - /// - /// The forest may be backed by `RocksDB`, so accesses to the underlying `LargeSmtForest` must - /// not run directly on an async worker thread. - fn with_forest_read_blocking( - &self, - f: impl FnOnce(&AccountStateForest) -> R, - ) -> R { - let span = Span::current(); - tokio::task::block_in_place(|| { - span.in_scope(|| { - let forest = self.forest.blocking_read(); - f(&forest) - }) - }) - } - - // STATE ACCESSORS - // -------------------------------------------------------------------------------------------- - - /// Queries a [BlockHeader] from the database, and returns it alongside its inclusion proof. - /// - /// If [None] is given as the value of `block_num`, the data for the latest [BlockHeader] is - /// returned. - #[miden_instrument( - level = "debug", - target = COMPONENT, - skip_all, - err, - )] - pub async fn get_block_header( - &self, - block_num: Option, - include_mmr_proof: bool, - ) -> Result<(Option, Option), GetBlockHeaderError> { - let block_header = self.db.select_block_header_by_block_num(block_num).await?; - if let Some(header) = block_header { - let mmr_proof = if include_mmr_proof { - let inner = self.inner.read().await; - let mmr_proof = inner.blockchain.open(header.block_num())?; - Some(mmr_proof) - } else { - None - }; - Ok((Some(header), mmr_proof)) - } else { - Ok((None, None)) - } - } - - /// Queries a list of notes from the database. - /// - /// If the provided list of [`NoteId`] given is empty or no note matches the provided - /// [`NoteId`] an empty list is returned. - pub async fn get_notes_by_id( - &self, - note_ids: Vec, - ) -> Result, DatabaseError> { - self.db.select_notes_by_id(note_ids).await - } - - /// Fetches the inputs for a transaction batch from the database. - /// - /// ## Inputs - /// - /// The function takes as input: - /// - The tx reference blocks are the set of blocks referenced by transactions in the batch. - /// - The unauthenticated note commitments are the set of commitments of unauthenticated notes - /// consumed by all transactions in the batch. For these notes, we attempt to find inclusion - /// proofs. Not all notes will exist in the DB necessarily, as some notes can be created and - /// consumed within the same batch. - /// - /// ## Outputs - /// - /// The function will return: - /// - A block inclusion proof for all tx reference blocks and for all blocks which are - /// referenced by a note inclusion proof. - /// - Note inclusion proofs for all notes that were found in the DB. - /// - The block header that the batch should reference, i.e. the latest known block. - pub async fn get_batch_inputs( - &self, - tx_reference_blocks: BTreeSet, - unauthenticated_note_commitments: BTreeSet, - ) -> Result { - if tx_reference_blocks.is_empty() { - return Err(GetBatchInputsError::TransactionBlockReferencesEmpty); - } - - // First we grab note inclusion proofs for the known notes. These proofs only prove that the - // note was included in a given block. We then also need to prove that each of those blocks - // is included in the chain. - let note_proofs = self - .db - .select_note_inclusion_proofs(unauthenticated_note_commitments) - .await - .map_err(GetBatchInputsError::SelectNoteInclusionProofError)?; - - // The set of blocks that the notes are included in. - let note_blocks = note_proofs.values().map(|proof| proof.location().block_num()); - - // Collect all blocks we need to query without duplicates, which is: - // - all blocks for which we need to prove note inclusion. - // - all blocks referenced by transactions in the batch. - let mut blocks: BTreeSet = tx_reference_blocks; - blocks.extend(note_blocks); - - // Scoped block to automatically drop the read lock guard as soon as we're done. We also - // avoid accessing the db in the block as this would delay dropping the guard. - let (batch_reference_block, partial_mmr) = { - let inner_state = self.inner.read().await; - - let latest_block_num = inner_state.latest_block_num(); - - let highest_block_num = - *blocks.last().expect("we should have checked for empty block references"); - if highest_block_num > latest_block_num { - return Err(GetBatchInputsError::UnknownTransactionBlockReference { - highest_block_num, - latest_block_num, - }); - } - - // Remove the latest block from the to-be-tracked blocks as it will be the reference - // block for the batch itself and thus added to the MMR within the batch kernel, so - // there is no need to prove its inclusion. - blocks.remove(&latest_block_num); - - // SAFETY: - // - The latest block num was retrieved from the inner blockchain from which we will - // also retrieve the proofs, so it is guaranteed to exist in that chain. - // - We have checked that no block number in the blocks set is greater than latest block - // number *and* latest block num was removed from the set. Therefore only block - // numbers smaller than latest block num remain in the set. Therefore all the block - // numbers are guaranteed to exist in the chain state at latest block num. - let partial_mmr = inner_state - .blockchain - .partial_mmr_from_blocks(&blocks, latest_block_num) - .expect("latest block num should exist and all blocks in set should be < than latest block"); - - (latest_block_num, partial_mmr) - }; - - // Fetch the reference block of the batch as part of this query, so we can avoid looking it - // up in a separate DB access. - let mut headers = self - .db - .select_block_headers(blocks.into_iter().chain(std::iter::once(batch_reference_block))) - .await - .map_err(GetBatchInputsError::SelectBlockHeaderError)?; - - // Find and remove the batch reference block as we don't want to add it to the chain MMR. - let header_index = headers - .iter() - .enumerate() - .find_map(|(index, header)| { - (header.block_num() == batch_reference_block).then_some(index) - }) - .expect("DB should have returned the header of the batch reference block"); - - // The order doesn't matter for PartialBlockchain::new, so swap remove is fine. - let batch_reference_block_header = headers.swap_remove(header_index); - - // SAFETY: This should not error because: - // - we're passing exactly the block headers that we've added to the partial MMR, - // - so none of the block headers block numbers should exceed the chain length of the - // partial MMR, - // - and we've added blocks to a BTreeSet, so there can be no duplicates. - // - // We construct headers and partial MMR in concert, so they are consistent. This is why we - // can call the unchecked constructor. - let partial_block_chain = PartialBlockchain::new_unchecked(partial_mmr, headers) - .expect("partial mmr and block headers should be consistent"); - - Ok(BatchInputs { - batch_reference_block_header, - note_proofs, - partial_block_chain, - }) - } - - /// Returns data needed by the block producer to construct and prove the next block. - pub async fn get_block_inputs( - &self, - account_ids: Vec, - nullifiers: Vec, - unauthenticated_note_commitments: BTreeSet, - reference_blocks: BTreeSet, - ) -> Result { - // Get the note inclusion proofs from the DB. We do this first so we have to acquire the - // lock to the state just once. There we need the reference blocks of the note proofs to get - // their authentication paths in the chain MMR. - let unauthenticated_note_proofs = self - .db - .select_note_inclusion_proofs(unauthenticated_note_commitments) - .await - .map_err(GetBlockInputsError::SelectNoteInclusionProofError)?; - - // The set of blocks that the notes are included in. - let note_proof_reference_blocks = - unauthenticated_note_proofs.values().map(|proof| proof.location().block_num()); - - // Collect all blocks we need to prove inclusion for, without duplicates. - let mut blocks = reference_blocks; - blocks.extend(note_proof_reference_blocks); - - let (latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr) = - self.get_block_inputs_witnesses(&mut blocks, &account_ids, &nullifiers)?; - - // Fetch the block headers for all blocks in the partial MMR plus the latest one which will - // be used as the previous block header of the block being built. - let mut headers = self - .db - .select_block_headers(blocks.into_iter().chain(std::iter::once(latest_block_number))) - .await - .map_err(GetBlockInputsError::SelectBlockHeaderError)?; - - // Find and remove the latest block as we must not add it to the chain MMR, since it is not - // yet in the chain. - let latest_block_header_index = headers - .iter() - .enumerate() - .find_map(|(index, header)| { - (header.block_num() == latest_block_number).then_some(index) - }) - .expect("DB should have returned the header of the latest block header"); - - // The order doesn't matter for PartialBlockchain::new, so swap remove is fine. - let latest_block_header = headers.swap_remove(latest_block_header_index); - - // SAFETY: This should not error because: - // - we're passing exactly the block headers that we've added to the partial MMR, - // - so none of the block header's block numbers should exceed the chain length of the - // partial MMR, - // - and we've added blocks to a BTreeSet, so there can be no duplicates. - // - // We construct headers and partial MMR in concert, so they are consistent. This is why we - // can call the unchecked constructor. - let partial_block_chain = PartialBlockchain::new_unchecked(partial_mmr, headers) - .expect("partial mmr and block headers should be consistent"); - - Ok(BlockInputs::new( - latest_block_header, - partial_block_chain, - account_witnesses, - nullifier_witnesses, - unauthenticated_note_proofs, - )) - } - - /// Get account and nullifier witnesses for the requested account IDs and nullifier as well as - /// the [`PartialMmr`] for the given blocks. The MMR won't contain the latest block and its - /// number is removed from `blocks` and returned separately. - /// - /// This method acquires the lock to the inner state and does not access the DB so we release - /// the lock asap. - fn get_block_inputs_witnesses( - &self, - blocks: &mut BTreeSet, - account_ids: &[AccountId], - nullifiers: &[Nullifier], - ) -> Result { - self.with_inner_read_blocking(|inner| { - let latest_block_number = inner.latest_block_num(); - - // If `blocks` is empty, use the latest block number which will never trigger the error. - let highest_block_number = blocks.last().copied().unwrap_or(latest_block_number); - if highest_block_number > latest_block_number { - return Err(GetBlockInputsError::UnknownBatchBlockReference { - highest_block_number, - latest_block_number, - }); - } - - // The latest block is not yet in the chain MMR, so we can't (and don't need to) prove - // its inclusion in the chain. - blocks.remove(&latest_block_number); - - // Fetch the partial MMR at the state of the latest block with authentication paths for - // the provided set of blocks. - // - // SAFETY: - // - The latest block num was retrieved from the inner blockchain from which we will - // also retrieve the proofs, so it is guaranteed to exist in that chain. - // - We have checked that no block number in the blocks set is greater than latest block - // number *and* latest block num was removed from the set. Therefore only block - // numbers smaller than latest block num remain in the set. Therefore all the block - // numbers are guaranteed to exist in the chain state at latest block num. - let partial_mmr = - inner.blockchain.partial_mmr_from_blocks(blocks, latest_block_number).expect( - "latest block num should exist and all blocks in set should be < than latest block", - ); - - // Fetch witnesses for all accounts. - let account_witnesses = account_ids - .iter() - .copied() - .map(|account_id| (account_id, inner.account_tree.open_latest(account_id))) - .collect::>(); - - // Fetch witnesses for all nullifiers. We don't check whether the nullifiers are spent - // or not as this is done as part of proposing the block. - let nullifier_witnesses: BTreeMap = nullifiers - .iter() - .copied() - .map(|nullifier| (nullifier, inner.nullifier_tree.open(&nullifier))) - .collect(); - - Ok((latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr)) - }) - } - - /// Returns data needed by the block producer to verify transactions validity. - #[miden_instrument( - target = COMPONENT, - skip_all, - fields( - account.id=%account_id, - nullifiers = %format_array(nullifiers), - ), - )] - pub async fn get_transaction_inputs( - &self, - account_id: AccountId, - nullifiers: &[Nullifier], - unauthenticated_note_commitments: Vec, - ) -> Result { - let tree_inputs = self.with_inner_read_blocking(|inner| { - let account_commitment = inner.account_tree.get_latest_commitment(account_id); - - let new_account_id_prefix_is_unique = if account_commitment.is_empty() { - Some(!inner.account_tree.contains_account_id_prefix_in_latest(account_id.prefix())) - } else { - None - }; - - // Non-unique account Id prefixes for new accounts are not allowed. - if let Some(false) = new_account_id_prefix_is_unique { - return Err(TransactionInputs { - new_account_id_prefix_is_unique, - ..Default::default() - }); - } - - let nullifiers = nullifiers - .iter() - .map(|nullifier| NullifierInfo { - nullifier: *nullifier, - block_num: inner.nullifier_tree.get_block_num(nullifier).unwrap_or_default(), - }) - .collect(); - - Ok((account_commitment, nullifiers, new_account_id_prefix_is_unique)) - }); - let (account_commitment, nullifiers, new_account_id_prefix_is_unique) = match tree_inputs { - Ok(inputs) => inputs, - Err(inputs) => return Ok(inputs), - }; - - let found_unauthenticated_notes = self - .db - .select_existing_note_commitments(unauthenticated_note_commitments) - .await?; - - Ok(TransactionInputs { - account_commitment, - nullifiers, - found_unauthenticated_notes, - new_account_id_prefix_is_unique, - }) - } - - /// Filters `account_ids` down to the subset classified as network accounts. - pub async fn filter_network_accounts( - &self, - account_ids: &[AccountId], - ) -> Result, DatabaseError> { - self.db.select_network_accounts_subset(account_ids.to_vec()).await - } - - /// Returns the effective chain tip for the given finality level. - /// - /// - [`Finality::Committed`]: returns the latest committed block number (from in-memory MMR). - /// - [`Finality::Proven`]: returns the latest proven-in-sequence block number (cached via watch - /// channel, updated by the proof scheduler). - pub async fn chain_tip(&self, finality: Finality) -> BlockNumber { - match finality { - Finality::Committed => self - .inner - .read() - .instrument(tracing::info_span!("acquire_inner")) - .await - .latest_block_num(), - Finality::Proven => self.proven_tip.read(), - } - } - - /// Loads a block from the in-memory replica cache or block store. Return `Ok(None)` if the - /// block is not found. - pub async fn load_block( - &self, - block_num: BlockNumber, - ) -> Result>, DatabaseError> { - if block_num > self.chain_tip(Finality::Committed).await { - return Ok(None); - } - if let Some(block) = self.block_cache.get(block_num) { - return Ok(Some(block.block_bytes().to_vec())); - } - self.block_store.load_block(block_num).await.map_err(Into::into) - } - - /// Loads a block proof from the in-memory replica cache or block store. Returns `Ok(None)` if - /// the proof is not found. - pub async fn load_proof( - &self, - block_num: BlockNumber, - ) -> Result>, DatabaseError> { - if block_num > self.chain_tip(Finality::Proven).await { - return Ok(None); - } - if let Some(proof) = self.proof_cache.get(block_num) { - return Ok(Some(proof.proof_bytes().to_vec())); - } - self.block_store.load_proof(block_num).await.map_err(Into::into) - } - - /// Returns the script for a note by its root. - pub async fn get_note_script_by_root( - &self, - root: Word, - ) -> Result, DatabaseError> { - self.db.select_note_script_by_root(root).await - } -} diff --git a/crates/store/src/state/replica.rs b/crates/store/src/state/replica.rs index 61533de8b..e24d3acff 100644 --- a/crates/store/src/state/replica.rs +++ b/crates/store/src/state/replica.rs @@ -1,8 +1,24 @@ +//! Block and proof serving for replica subscriptions. +//! +//! Replica subscribers stream committed blocks and block proofs from this node. The writer pushes +//! each freshly committed block (and the proof path each proven block) into a FIFO cache here, so +//! subscribers keeping up with the tip are served from memory; a subscriber that has fallen +//! behind the cache window falls back to reading the block store. +//! +//! These reads serve raw block/proof bytes from the caches and the block store — they never touch +//! the database or tree state, which is why they live on [`State`] directly rather than on a +//! [`StateView`](super::StateView). The tip checks gating them are live availability checks +//! ("is this block committed/proven yet?"), deliberately race-tolerant: a `None` for a block that +//! commits an instant later is corrected by the next tip-watch wakeup. + use std::sync::Arc; use miden_node_utils::block_cache::BlockOrderedCache; use miden_protocol::block::BlockNumber; +use crate::errors::DatabaseError; +use crate::state::State; + // BLOCK NOTIFICATION // ================================================================================================ @@ -65,3 +81,46 @@ pub type BlockCache = BlockOrderedCache; /// FIFO cache of recent block proofs for replica subscriptions. pub type ProofCache = BlockOrderedCache; + +// REPLICA STATE ACCESS +// ================================================================================================ + +impl State { + /// Loads a block from the in-memory replica cache or block store. Return `Ok(None)` if the + /// block is not found. + pub async fn load_block( + &self, + block_num: BlockNumber, + ) -> Result>, DatabaseError> { + if block_num > self.committed_tip() { + return Ok(None); + } + if let Some(block) = self.block_cache.get(block_num) { + return Ok(Some(block.block_bytes().to_vec())); + } + self.block_store.load_block(block_num).await.map_err(Into::into) + } + + /// Loads a block proof from the in-memory replica cache or block store. Returns `Ok(None)` if + /// the proof is not found. + pub async fn load_proof( + &self, + block_num: BlockNumber, + ) -> Result>, DatabaseError> { + if block_num > self.proven_tip() { + return Ok(None); + } + if let Some(proof) = self.proof_cache.get(block_num) { + return Ok(Some(proof.proof_bytes().to_vec())); + } + self.block_store.load_proof(block_num).await.map_err(Into::into) + } + + /// Loads serialized block proving inputs from the block store. + pub async fn load_proving_inputs( + &self, + block_num: BlockNumber, + ) -> std::io::Result>> { + self.block_store.load_proving_inputs(block_num).await + } +} diff --git a/crates/store/src/state/tip.rs b/crates/store/src/state/tip.rs new file mode 100644 index 000000000..fee824047 --- /dev/null +++ b/crates/store/src/state/tip.rs @@ -0,0 +1,45 @@ +//! Live chain-tip queries and tip subscriptions. +//! +//! Both tips are published through watch channels by their single writers (the block writer for +//! the committed tip, the proof scheduler or proof sync for the proven tip). Everything here +//! reads or subscribes to those channels; none of it touches state snapshots, so these values +//! advance independently of any [`StateView`](super::StateView). + +use miden_protocol::block::BlockNumber; +use tokio::sync::watch; + +use super::State; + +// TIP QUERIES & SUBSCRIPTIONS +// ================================================================================================ + +impl State { + /// Returns the latest committed (but not necessarily proven) block number. + /// + /// This is a live value: it advances independently of any [`StateView`](super::StateView). + /// Reads that must be consistent with data must use a view's tip instead. + /// + /// The committed tip is published after the corresponding state snapshot, so it never reports + /// a block that a freshly created view cannot serve. + pub fn committed_tip(&self) -> BlockNumber { + *self.committed_tip_tx.borrow() + } + + /// Returns the latest block number proven in an unbroken sequence from genesis. + /// + /// This is a live value published by the proof scheduler (sequencer mode) or the proof sync + /// loop (full-node mode); it is always at or behind the committed tip. + pub fn proven_tip(&self) -> BlockNumber { + self.proven_tip.read() + } + + /// Returns a watch receiver that wakes every time a new block is committed. + pub fn subscribe_committed_tip(&self) -> watch::Receiver { + self.committed_tip_tx.subscribe() + } + + /// Returns a watch receiver that wakes every time the proven-in-sequence tip advances. + pub fn subscribe_proven_tip(&self) -> watch::Receiver { + self.proven_tip.subscribe() + } +} diff --git a/crates/store/src/state/view/account/mod.rs b/crates/store/src/state/view/account/mod.rs new file mode 100644 index 000000000..f03238fa1 --- /dev/null +++ b/crates/store/src/state/view/account/mod.rs @@ -0,0 +1,419 @@ +use std::collections::HashSet; + +use miden_node_proto::domain::account::{ + AccountDetailRequest, + AccountDetails, + AccountRequest, + AccountResponse, + AccountStorageDetails, + AccountStorageMapDetails, + AccountStorageRequest, + AccountVaultDetails, + SlotData, + StorageMapEntries, + StorageMapRequest, +}; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::account::{AccountId, AccountStorageHeader, StorageSlotName, StorageSlotType}; +use miden_protocol::block::BlockNumber; +use miden_protocol::block::account_tree::AccountWitness; + +use super::{ScopedBlockNum, StateView}; +use crate::COMPONENT; +use crate::account_state_forest::AccountStorageMapResult; +use crate::errors::{DatabaseError, GetAccountError}; + +impl StateView { + /// Returns an account witness and optionally account details at a specific block. + /// + /// The witness is a Merkle proof of inclusion in the account tree, proving the account's + /// state commitment. If `details` is requested, the method also returns the account's code, + /// vault assets, and storage data. Account details are only available for public accounts. + /// + /// If `block_num` is provided, returns the state at that historical block; otherwise, returns + /// the latest state. Note that historical states are only available for recent blocks close + /// to the chain tip. + #[miden_instrument( + target = COMPONENT, + skip_all, + )] + pub async fn get_account( + &self, + account_request: AccountRequest, + ) -> Result { + let AccountRequest { block_num, account_id, details } = account_request; + + if details.is_some() && !account_id.is_public() { + return Err(GetAccountError::AccountNotPublic(account_id)); + } + + let (block_num, witness) = self.get_account_witness(block_num, account_id).await?; + + let details = if let Some(request) = details { + Some( + self.fetch_public_account_details(account_id, block_num, &witness, request) + .await?, + ) + } else { + None + }; + + Ok(AccountResponse { block_num, witness, details }) + } + + /// Returns an account witness (Merkle proof of inclusion in the account tree). + /// + /// If `block_num` is provided, returns the witness at that historical block; + /// otherwise, returns the witness at the latest block. + #[miden_instrument( + target = COMPONENT, + skip_all, + )] + async fn get_account_witness( + &self, + block_num: Option, + account_id: AccountId, + ) -> Result<(BlockNumber, AccountWitness), GetAccountError> { + self.with_inner_read_blocking(|inner_state| { + // Determine which block to query + let (block_num, witness) = if let Some(requested_block) = block_num { + // Historical query: use the account tree with history + let witness = inner_state + .account_tree + .open_at(account_id, requested_block) + .ok_or_else(|| { + let latest_block = inner_state.account_tree.block_number_latest(); + if requested_block > latest_block { + GetAccountError::UnknownBlock(requested_block) + } else { + GetAccountError::BlockPruned(requested_block) + } + })?; + (requested_block, witness) + } else { + // Latest query: use the latest state + let block_num = inner_state.account_tree.block_number_latest(); + let witness = inner_state.account_tree.open_latest(account_id); + (block_num, witness) + }; + + Ok((block_num, witness)) + }) + } + + /// Returns storage map details from the forest for a specific account and storage slot. + /// + /// The forest can only be used if all hashed keys in the storage map are known in the + /// reverse-key LRU cache. If any hashed key is unknown, the method returns `Ok(None)` to signal + /// that the caller should fall back to reconstructing the storage map details from the + /// database. + #[miden_instrument( + target = COMPONENT, + skip_all, + )] + fn get_storage_map_details_from_forest( + &self, + account_id: AccountId, + slot_name: &StorageSlotName, + block_num: BlockNumber, + ) -> Result, DatabaseError> { + self.with_forest_read_blocking(|forest| { + match forest + .get_storage_map_details_for_all_entries(account_id, slot_name.clone(), block_num) + .map_err(DatabaseError::MerkleError)? + { + AccountStorageMapResult::NotFound => Err(DatabaseError::StorageRootNotFound { + account_id, + slot_name: slot_name.to_string(), + block_num, + }), + AccountStorageMapResult::Details(details) => Ok(Some(details)), + AccountStorageMapResult::CannotReconstructKeysFromCache => Ok(None), + } + }) + } + + /// Returns vault details by reconstructing the vault from the database. + async fn reconstruct_vault_details_from_db( + &self, + account_id: AccountId, + block_num: ScopedBlockNum, + ) -> Result { + let assets = self.db().select_account_vault_at_block(account_id, block_num).await?; + + if assets.len() > AccountVaultDetails::MAX_RETURN_ENTRIES { + return Ok(AccountVaultDetails::LimitExceeded); + } + + let keys = assets.iter().map(miden_protocol::asset::Asset::id); + + // The reverse-key caches are shared between the writer and all snapshots, so caching via + // the current snapshot's forest is visible everywhere. + self.with_forest_read_blocking(|forest| { + forest + .vault_key_cache + .put_many(keys.into_iter().map(|raw_key| (raw_key.hash(), raw_key))); + }); + + Ok(AccountVaultDetails::from_assets(assets)) + } + + /// Returns storage map details by reconstructing the storage map from the database. + async fn reconstruct_storage_map_details_from_db( + &self, + account_id: AccountId, + slot_name: StorageSlotName, + block_num: ScopedBlockNum, + ) -> Result { + let details = self + .db() + .reconstruct_storage_map_from_db( + account_id, + slot_name, + block_num, + Some(AccountStorageMapDetails::MAX_RETURN_ENTRIES), + ) + .await?; + + if let StorageMapEntries::AllEntries(entries) = &details.entries { + self.with_forest_read_blocking(|forest| { + forest.cache_storage_map_keys(entries.iter().map(|(raw_key, _)| *raw_key)); + }); + } + + Ok(details) + } + + /// Fetches the account details (code, vault, storage) for a public account at the specified + /// block. + /// + /// This method queries the database to fetch the account state and processes the detail + /// request to return only the requested information. + /// + /// For specific key queries (`SlotData::MapKeys`), the forest is used to provide SMT proofs. + /// Returns an error if the forest doesn't have data for the requested slot. + /// All-entries queries (`SlotData::All`) use the forest when all hashed keys are known in the + /// reverse-key LRU cache, otherwise they fall back to database reconstruction. + #[miden_instrument( + target = COMPONENT, + skip_all, + )] + async fn fetch_public_account_details( + &self, + account_id: AccountId, + block_num: BlockNumber, + witness: &AccountWitness, + detail_request: AccountDetailRequest, + ) -> Result { + let AccountDetailRequest { + code_commitment, + asset_vault_commitment, + storage_request, + } = detail_request; + + if !account_id.is_public() { + return Err(GetAccountError::AccountNotPublic(account_id)); + } + + // Validate block exists in the blockchain before querying the database. The view's tip is + // the same snapshot the witness was resolved against, so the witness and the DB reads below + // observe a single consistent block height. + let scoped_block = + self.scope_block(block_num).ok_or(GetAccountError::UnknownBlock(block_num))?; + + // Query account header and storage header together in a single DB call + let (account_header, storage_header) = self + .db() + .select_account_header_with_storage_header_at_block(account_id, scoped_block) + .await? + .ok_or(GetAccountError::AccountNotFound(account_id, block_num))?; + + let should_apply_response_budget = + matches!(&storage_request, AccountStorageRequest::AllStorageMaps); + let storage_requests = expand_account_storage_request(storage_request, &storage_header); + + let account_code = match code_commitment { + Some(commitment) if commitment == account_header.code_commitment() => None, + Some(_) => { + self.db() + .select_account_code_by_commitment(account_header.code_commitment()) + .await? + }, + None => None, + }; + + // Query account state forest for vault details on commitment mismatch. + // + // The forest can only reconstruct the vault if all hashed vault keys are known in the + // reverse-key LRU cache. If any hashed key is unknown, the forest returns `None` and we + // fall back to reconstructing the vault details from the database. + let vault_details = match asset_vault_commitment { + Some(commitment) if commitment == account_header.vault_root() => { + AccountVaultDetails::empty() + }, + Some(_) => { + let forest_details = self.with_forest_read_blocking(|forest| { + forest.get_vault_details(account_id, block_num).map_err(|err| { + DatabaseError::DataCorrupted(format!( + "failed to reconstruct vault for account {account_id} at block {block_num}: {err}" + )) + }) + })?; + + match forest_details { + Some(details) => details, + None => { + self.reconstruct_vault_details_from_db(account_id, scoped_block).await? + }, + } + }, + None => AccountVaultDetails::empty(), + }; + + // Split storage map requests into two categories: + // - slots with explicit keys (including proofs) + // - slots with "all entries" + let mut storage_map_details = + Vec::::with_capacity(storage_requests.len()); + let mut map_keys_requests = Vec::new(); + let mut all_entries_requests = Vec::new(); + let mut storage_request_slots = Vec::with_capacity(storage_requests.len()); + + for (index, StorageMapRequest { slot_name, slot_data }) in + storage_requests.into_iter().enumerate() + { + storage_request_slots.push(slot_name.clone()); + match slot_data { + SlotData::MapKeys(keys) => { + map_keys_requests.push((index, slot_name, keys)); + }, + SlotData::All => { + all_entries_requests.push((index, slot_name)); + }, + } + } + + let mut storage_map_details_by_index = vec![None; storage_request_slots.len()]; + + // Handle slots with explicit key requests + if !map_keys_requests.is_empty() { + self.with_forest_read_blocking(|forest| { + for (index, slot_name, keys) in map_keys_requests { + let details = forest + .get_storage_map_details_for_keys( + account_id, + slot_name.clone(), + block_num, + &keys, + ) + .ok_or_else(|| DatabaseError::StorageRootNotFound { + account_id, + slot_name: slot_name.to_string(), + block_num, + })? + .map_err(DatabaseError::MerkleError)?; + storage_map_details_by_index[index] = Some(details); + } + Ok::<(), DatabaseError>(()) + })?; + } + + // Handle slots with "all entries" requests + for (index, slot_name) in all_entries_requests { + let details = match self + .get_storage_map_details_from_forest(account_id, &slot_name, block_num)? + { + Some(details) => details, + None => { + self.reconstruct_storage_map_details_from_db( + account_id, + slot_name, + scoped_block, + ) + .await? + }, + }; + storage_map_details_by_index[index] = Some(details); + } + + for (details, slot_name) in + storage_map_details_by_index.into_iter().zip(storage_request_slots.iter()) + { + let details = details.ok_or_else(|| DatabaseError::StorageRootNotFound { + account_id, + slot_name: slot_name.to_string(), + block_num, + })?; + storage_map_details.push(details); + } + + // In case of an "all storage maps" request we have to be careful: even with the per-slot + // limit of [`AccountStorageMapDetails::MAX_RETURN_ENTRIES`] we might go over the response + // size limit. Here we make sure that we're within that limit by potentially truncating the + // response. + if should_apply_response_budget { + return Ok(apply_all_storage_maps_response_budget( + block_num, + witness, + account_header, + account_code, + vault_details, + storage_header, + storage_map_details, + storage_request_slots, + MAX_ALL_STORAGE_MAPS_RESPONSE_PAYLOAD_WITH_BUDGET_RESERVED_FOR_LIMIT_EXCEEDED_SLOTS, + )); + } + + Ok(AccountDetails { + account_header, + account_code, + vault_details, + storage_details: AccountStorageDetails { + header: storage_header, + map_details: storage_map_details, + }, + }) + } +} + +// HELPERS +// ================================================================================================ + +/// Expand [`AccountStorageRequest`] to a vector of slot requests. +fn expand_account_storage_request( + storage_request: AccountStorageRequest, + storage_header: &AccountStorageHeader, +) -> Vec { + match storage_request { + AccountStorageRequest::None => Vec::new(), + AccountStorageRequest::Explicit(requests) => requests, + AccountStorageRequest::AllStorageMaps => storage_header + .slots() + .filter(|slot| slot.slot_type() == StorageSlotType::Map) + .map(|slot| StorageMapRequest { + slot_name: slot.name().clone(), + slot_data: SlotData::All, + }) + .collect(), + } +} + +mod response_budget; +use response_budget::{ + MAX_ALL_STORAGE_MAPS_RESPONSE_PAYLOAD_WITH_BUDGET_RESERVED_FOR_LIMIT_EXCEEDED_SLOTS, + apply_all_storage_maps_response_budget, +}; + +// NETWORK ACCOUNT CLASSIFICATION +// ================================================================================================ + +impl StateView { + /// Filters `account_ids` down to the subset classified as network accounts. + pub async fn filter_network_accounts( + &self, + account_ids: &[AccountId], + ) -> Result, DatabaseError> { + self.db().select_network_accounts_subset(account_ids.to_vec()).await + } +} diff --git a/crates/store/src/state/account.rs b/crates/store/src/state/view/account/response_budget.rs similarity index 50% rename from crates/store/src/state/account.rs rename to crates/store/src/state/view/account/response_budget.rs index 347c43937..b8ecaa699 100644 --- a/crates/store/src/state/account.rs +++ b/crates/store/src/state/view/account/response_budget.rs @@ -1,409 +1,25 @@ +//! Response-size budgeting for `all-storage-maps` account detail responses. +//! +//! Even with the per-slot entry limit, an `all-storage-maps` request can exceed the response +//! payload cap. The budget below keeps the encoded response within the cap by replacing map +//! contents with "limit exceeded" markers once the budget is spent, reserving space for those +//! markers up front. + use miden_node_proto::domain::account::{ - AccountDetailRequest, AccountDetails, - AccountRequest, AccountResponse, AccountStorageDetails, AccountStorageMapDetails, - AccountStorageRequest, AccountVaultDetails, - SlotData, StorageMapEntries, - StorageMapRequest, }; use miden_node_proto::generated as proto; use miden_node_proto::prost::Message as _; use miden_node_proto::prost::encoding::{encoded_len_varint, key_len}; use miden_node_utils::limiter::MAX_RESPONSE_PAYLOAD_BYTES; -use miden_node_utils::tracing::miden_instrument; -use miden_protocol::account::{ - AccountHeader, - AccountId, - AccountStorageHeader, - StorageSlotName, - StorageSlotType, -}; +use miden_protocol::account::{AccountHeader, AccountStorageHeader, StorageSlotName}; use miden_protocol::block::BlockNumber; use miden_protocol::block::account_tree::AccountWitness; -use tracing::Instrument; - -use super::State; -use crate::COMPONENT; -use crate::account_state_forest::AccountStorageMapResult; -use crate::errors::{DatabaseError, GetAccountError}; - -impl State { - /// Returns an account witness and optionally account details at a specific block. - /// - /// The witness is a Merkle proof of inclusion in the account tree, proving the account's - /// state commitment. If `details` is requested, the method also returns the account's code, - /// vault assets, and storage data. Account details are only available for public accounts. - /// - /// If `block_num` is provided, returns the state at that historical block; otherwise, returns - /// the latest state. Note that historical states are only available for recent blocks close - /// to the chain tip. - #[miden_instrument( - target = COMPONENT, - skip_all, - )] - pub async fn get_account( - &self, - account_request: AccountRequest, - ) -> Result { - let AccountRequest { block_num, account_id, details } = account_request; - - if details.is_some() && !account_id.is_public() { - return Err(GetAccountError::AccountNotPublic(account_id)); - } - - let (block_num, witness) = self.get_account_witness(block_num, account_id).await?; - - let details = if let Some(request) = details { - Some( - self.fetch_public_account_details(account_id, block_num, &witness, request) - .await?, - ) - } else { - None - }; - - Ok(AccountResponse { block_num, witness, details }) - } - - /// Returns an account witness (Merkle proof of inclusion in the account tree). - /// - /// If `block_num` is provided, returns the witness at that historical block; - /// otherwise, returns the witness at the latest block. - #[miden_instrument( - target = COMPONENT, - skip_all, - )] - async fn get_account_witness( - &self, - block_num: Option, - account_id: AccountId, - ) -> Result<(BlockNumber, AccountWitness), GetAccountError> { - self.with_inner_read_blocking(|inner_state| { - // Determine which block to query - let (block_num, witness) = if let Some(requested_block) = block_num { - // Historical query: use the account tree with history - let witness = inner_state - .account_tree - .open_at(account_id, requested_block) - .ok_or_else(|| { - let latest_block = inner_state.account_tree.block_number_latest(); - if requested_block > latest_block { - GetAccountError::UnknownBlock(requested_block) - } else { - GetAccountError::BlockPruned(requested_block) - } - })?; - (requested_block, witness) - } else { - // Latest query: use the latest state - let block_num = inner_state.account_tree.block_number_latest(); - let witness = inner_state.account_tree.open_latest(account_id); - (block_num, witness) - }; - - Ok((block_num, witness)) - }) - } - - /// Returns storage map details from the forest for a specific account and storage slot. - /// - /// The forest can only be used if all hashed keys in the storage map are known in the - /// reverse-key LRU cache. If any hashed key is unknown, the method returns `Ok(None)` to signal - /// that the caller should fall back to reconstructing the storage map details from the - /// database. - #[miden_instrument( - target = COMPONENT, - skip_all, - )] - fn get_storage_map_details_from_forest( - &self, - account_id: AccountId, - slot_name: &StorageSlotName, - block_num: BlockNumber, - ) -> Result, DatabaseError> { - self.with_forest_read_blocking(|forest| { - match forest - .get_storage_map_details_for_all_entries(account_id, slot_name.clone(), block_num) - .map_err(DatabaseError::MerkleError)? - { - AccountStorageMapResult::NotFound => Err(DatabaseError::StorageRootNotFound { - account_id, - slot_name: slot_name.to_string(), - block_num, - }), - AccountStorageMapResult::Details(details) => Ok(Some(details)), - AccountStorageMapResult::CannotReconstructKeysFromCache => Ok(None), - } - }) - } - - /// Returns vault details by reconstructing the vault from the database. - async fn reconstruct_vault_details_from_db( - &self, - account_id: AccountId, - block_num: BlockNumber, - ) -> Result { - let assets = self.db.select_account_vault_at_block(account_id, block_num).await?; - - if assets.len() > AccountVaultDetails::MAX_RETURN_ENTRIES { - return Ok(AccountVaultDetails::LimitExceeded); - } - - let keys = assets.iter().map(miden_protocol::asset::Asset::id); - - let forest = self.forest.write().await; - - forest - .vault_key_cache - .put_many(keys.into_iter().map(|raw_key| (raw_key.hash(), raw_key))); - - Ok(AccountVaultDetails::from_assets(assets)) - } - - /// Returns storage map details by reconstructing the storage map from the database. - async fn reconstruct_storage_map_details_from_db( - &self, - account_id: AccountId, - slot_name: StorageSlotName, - block_num: BlockNumber, - ) -> Result { - let details = self - .db - .reconstruct_storage_map_from_db( - account_id, - slot_name, - block_num, - Some(AccountStorageMapDetails::MAX_RETURN_ENTRIES), - ) - .await?; - - if let StorageMapEntries::AllEntries(entries) = &details.entries { - self.forest - .write() - .await - .cache_storage_map_keys(entries.iter().map(|(raw_key, _)| *raw_key)); - } - - Ok(details) - } - - /// Fetches the account details (code, vault, storage) for a public account at the specified - /// block. - /// - /// This method queries the database to fetch the account state and processes the detail - /// request to return only the requested information. - /// - /// For specific key queries (`SlotData::MapKeys`), the forest is used to provide SMT proofs. - /// Returns an error if the forest doesn't have data for the requested slot. - /// All-entries queries (`SlotData::All`) use the forest when all hashed keys are known in the - /// reverse-key LRU cache, otherwise they fall back to database reconstruction. - #[miden_instrument( - target = COMPONENT, - skip_all, - )] - async fn fetch_public_account_details( - &self, - account_id: AccountId, - block_num: BlockNumber, - witness: &AccountWitness, - detail_request: AccountDetailRequest, - ) -> Result { - let AccountDetailRequest { - code_commitment, - asset_vault_commitment, - storage_request, - } = detail_request; - - if !account_id.is_public() { - return Err(GetAccountError::AccountNotPublic(account_id)); - } - - // Validate block exists in the blockchain before querying the database - { - let inner = self.inner.read().instrument(tracing::info_span!("acquire_inner")).await; - let latest_block_num = inner.latest_block_num(); - - if block_num > latest_block_num { - return Err(GetAccountError::UnknownBlock(block_num)); - } - } - - // Query account header and storage header together in a single DB call - let (account_header, storage_header) = self - .db - .select_account_header_with_storage_header_at_block(account_id, block_num) - .await? - .ok_or(GetAccountError::AccountNotFound(account_id, block_num))?; - - let should_apply_response_budget = - matches!(&storage_request, AccountStorageRequest::AllStorageMaps); - let storage_requests = expand_account_storage_request(storage_request, &storage_header); - - let account_code = match code_commitment { - Some(commitment) if commitment == account_header.code_commitment() => None, - Some(_) => { - self.db - .select_account_code_by_commitment(account_header.code_commitment()) - .await? - }, - None => None, - }; - - // Query account state forest for vault details on commitment mismatch. - // - // The forest can only reconstruct the vault if all hashed vault keys are known in the - // reverse-key LRU cache. If any hashed key is unknown, the forest returns `None` and we - // fall back to reconstructing the vault details from the database. - let vault_details = match asset_vault_commitment { - Some(commitment) if commitment == account_header.vault_root() => { - AccountVaultDetails::empty() - }, - Some(_) => { - let forest_details = self.with_forest_read_blocking(|forest| { - forest.get_vault_details(account_id, block_num).map_err(|err| { - DatabaseError::DataCorrupted(format!( - "failed to reconstruct vault for account {account_id} at block {block_num}: {err}" - )) - }) - })?; - - match forest_details { - Some(details) => details, - None => self.reconstruct_vault_details_from_db(account_id, block_num).await?, - } - }, - None => AccountVaultDetails::empty(), - }; - - // Split storage map requests into two categories: - // - slots with explicit keys (including proofs) - // - slots with "all entries" - let mut storage_map_details = - Vec::::with_capacity(storage_requests.len()); - let mut map_keys_requests = Vec::new(); - let mut all_entries_requests = Vec::new(); - let mut storage_request_slots = Vec::with_capacity(storage_requests.len()); - - for (index, StorageMapRequest { slot_name, slot_data }) in - storage_requests.into_iter().enumerate() - { - storage_request_slots.push(slot_name.clone()); - match slot_data { - SlotData::MapKeys(keys) => { - map_keys_requests.push((index, slot_name, keys)); - }, - SlotData::All => { - all_entries_requests.push((index, slot_name)); - }, - } - } - - let mut storage_map_details_by_index = vec![None; storage_request_slots.len()]; - - // Handle slots with explicit key requests - if !map_keys_requests.is_empty() { - self.with_forest_read_blocking(|forest| { - for (index, slot_name, keys) in map_keys_requests { - let details = forest - .get_storage_map_details_for_keys( - account_id, - slot_name.clone(), - block_num, - &keys, - ) - .ok_or_else(|| DatabaseError::StorageRootNotFound { - account_id, - slot_name: slot_name.to_string(), - block_num, - })? - .map_err(DatabaseError::MerkleError)?; - storage_map_details_by_index[index] = Some(details); - } - Ok::<(), DatabaseError>(()) - })?; - } - - // Handle slots with "all entries" requests - for (index, slot_name) in all_entries_requests { - let details = match self - .get_storage_map_details_from_forest(account_id, &slot_name, block_num)? - { - Some(details) => details, - None => { - self.reconstruct_storage_map_details_from_db(account_id, slot_name, block_num) - .await? - }, - }; - storage_map_details_by_index[index] = Some(details); - } - - for (details, slot_name) in - storage_map_details_by_index.into_iter().zip(storage_request_slots.iter()) - { - let details = details.ok_or_else(|| DatabaseError::StorageRootNotFound { - account_id, - slot_name: slot_name.to_string(), - block_num, - })?; - storage_map_details.push(details); - } - - // In case of an "all storage maps" request we have to be careful: even with the per-slot - // limit of [`AccountStorageMapDetails::MAX_RETURN_ENTRIES`] we might go over the response - // size limit. Here we make sure that we're within that limit by potentially truncating the - // response. - if should_apply_response_budget { - return Ok(apply_all_storage_maps_response_budget( - block_num, - witness, - account_header, - account_code, - vault_details, - storage_header, - storage_map_details, - storage_request_slots, - MAX_ALL_STORAGE_MAPS_RESPONSE_PAYLOAD_WITH_BUDGET_RESERVED_FOR_LIMIT_EXCEEDED_SLOTS, - )); - } - - Ok(AccountDetails { - account_header, - account_code, - vault_details, - storage_details: AccountStorageDetails { - header: storage_header, - map_details: storage_map_details, - }, - }) - } -} - -// HELPERS -// ================================================================================================ - -/// Expand [`AccountStorageRequest`] to a vector of slot requests. -fn expand_account_storage_request( - storage_request: AccountStorageRequest, - storage_header: &AccountStorageHeader, -) -> Vec { - match storage_request { - AccountStorageRequest::None => Vec::new(), - AccountStorageRequest::Explicit(requests) => requests, - AccountStorageRequest::AllStorageMaps => storage_header - .slots() - .filter(|slot| slot.slot_type() == StorageSlotType::Map) - .map(|slot| StorageMapRequest { - slot_name: slot.name().clone(), - slot_data: SlotData::All, - }) - .collect(), - } -} // This is intentionally conservative. Storage slot names can be up to u8::MAX bytes, and a // `limit_exceeded` map detail stores only the slot name plus the `too_many_entries` flag. @@ -411,7 +27,7 @@ const STORAGE_MAP_LIMIT_EXCEEDED_FIELD_MAX_LEN: usize = 263; // A conservative limit that makes sure that limit exceeded messages can be appended for all slots // in the response. -const MAX_ALL_STORAGE_MAPS_RESPONSE_PAYLOAD_WITH_BUDGET_RESERVED_FOR_LIMIT_EXCEEDED_SLOTS: usize = +pub(super) const MAX_ALL_STORAGE_MAPS_RESPONSE_PAYLOAD_WITH_BUDGET_RESERVED_FOR_LIMIT_EXCEEDED_SLOTS: usize = MAX_RESPONSE_PAYLOAD_BYTES - 256 * STORAGE_MAP_LIMIT_EXCEEDED_FIELD_MAX_LEN - 8192; // Conservative max length for storage map entries: key-value pairs, each one is four `fixed64` @@ -451,7 +67,7 @@ fn estimate_storage_map_details_field_len(details: &AccountStorageMapDetails) -> /// We reserve space for the "limit exceeded" responses in advance so we're safe to start appending /// "limit exceeded" at any point during iteration. #[expect(clippy::too_many_arguments)] -fn apply_all_storage_maps_response_budget( +pub(super) fn apply_all_storage_maps_response_budget( block_num: BlockNumber, witness: &AccountWitness, account_header: AccountHeader, @@ -508,7 +124,6 @@ fn apply_all_storage_maps_response_budget( }, } } - // TESTS // ================================================================================================ @@ -540,7 +155,8 @@ mod tests { use miden_protocol::testing::account_id::AccountIdBuilder; use miden_protocol::{EMPTY_WORD, Felt, Word}; - use super::{apply_all_storage_maps_response_budget, expand_account_storage_request}; + use super::super::expand_account_storage_request; + use super::apply_all_storage_maps_response_budget; fn storage_header() -> AccountStorageHeader { AccountStorageHeader::new(vec![ diff --git a/crates/store/src/state/view/batch_inputs.rs b/crates/store/src/state/view/batch_inputs.rs new file mode 100644 index 000000000..c1e6a5c40 --- /dev/null +++ b/crates/store/src/state/view/batch_inputs.rs @@ -0,0 +1,129 @@ +//! Batch input query for the block producer. +//! +//! Combines in-memory snapshot data (partial MMR) with database lookups, scoping the latter by +//! the view's tip so both sources describe the same block height. + +use std::collections::BTreeSet; + +use miden_node_proto::domain::batch::BatchInputs; +use miden_protocol::Word; +use miden_protocol::block::BlockNumber; +use miden_protocol::transaction::PartialBlockchain; + +use super::StateView; +use crate::errors::GetBatchInputsError; + +impl StateView { + /// Fetches the inputs for a transaction batch from the database. + /// + /// ## Inputs + /// + /// The function takes as input: + /// - The tx reference blocks are the set of blocks referenced by transactions in the batch. + /// - The unauthenticated note commitments are the set of commitments of unauthenticated notes + /// consumed by all transactions in the batch. For these notes, we attempt to find inclusion + /// proofs. Not all notes will exist in the DB necessarily, as some notes can be created and + /// consumed within the same batch. + /// + /// ## Outputs + /// + /// The function will return: + /// - A block inclusion proof for all tx reference blocks and for all blocks which are + /// referenced by a note inclusion proof. + /// - Note inclusion proofs for all notes that were found in the DB. + /// - The block header that the batch should reference, i.e. the latest known block. + pub async fn get_batch_inputs( + &self, + tx_reference_blocks: BTreeSet, + unauthenticated_note_commitments: BTreeSet, + ) -> Result { + if tx_reference_blocks.is_empty() { + return Err(GetBatchInputsError::TransactionBlockReferencesEmpty); + } + + // First we grab note inclusion proofs for the known notes. These proofs only prove that the + // note was included in a given block. We then also need to prove that each of those blocks + // is included in the chain. + let note_proofs = self + .db() + .select_note_inclusion_proofs(unauthenticated_note_commitments) + .await + .map_err(GetBatchInputsError::SelectNoteInclusionProofError)?; + + // The set of blocks that the notes are included in. + let note_blocks = note_proofs.values().map(|proof| proof.location().block_num()); + + // Collect all blocks we need to query without duplicates, which is: + // - all blocks for which we need to prove note inclusion. + // - all blocks referenced by transactions in the batch. + let mut blocks: BTreeSet = tx_reference_blocks; + blocks.extend(note_blocks); + + let latest_block_num = self.tip(); + + let highest_block_num = + *blocks.last().expect("we should have checked for empty block references"); + if highest_block_num > latest_block_num { + return Err(GetBatchInputsError::UnknownTransactionBlockReference { + highest_block_num, + latest_block_num, + }); + } + + // Remove the latest block from the to-be-tracked blocks as it will be the reference block + // for the batch itself and thus added to the MMR within the batch kernel, so there is no + // need to prove its inclusion. + blocks.remove(&latest_block_num); + + // SAFETY: + // - The latest block num was retrieved from the view's blockchain from which we will + // also retrieve the proofs, so it is guaranteed to exist in that chain. + // - We have checked that no block number in the blocks set is greater than latest block + // number *and* latest block num was removed from the set. Therefore only block + // numbers smaller than latest block num remain in the set. Therefore all the block + // numbers are guaranteed to exist in the chain state at latest block num. + let partial_mmr = + self.blockchain().partial_mmr_from_blocks(&blocks, latest_block_num).expect( + "latest block num should exist and all blocks in set should be < than latest block", + ); + + let batch_reference_block = latest_block_num; + + // Fetch the reference block of the batch as part of this query, so we can avoid looking it + // up in a separate DB access. + let mut headers = self + .db() + .select_block_headers(blocks.into_iter().chain(std::iter::once(batch_reference_block))) + .await + .map_err(GetBatchInputsError::SelectBlockHeaderError)?; + + // Find and remove the batch reference block as we don't want to add it to the chain MMR. + let header_index = headers + .iter() + .enumerate() + .find_map(|(index, header)| { + (header.block_num() == batch_reference_block).then_some(index) + }) + .expect("DB should have returned the header of the batch reference block"); + + // The order doesn't matter for PartialBlockchain::new, so swap remove is fine. + let batch_reference_block_header = headers.swap_remove(header_index); + + // SAFETY: This should not error because: + // - we're passing exactly the block headers that we've added to the partial MMR, + // - so none of the block headers block numbers should exceed the chain length of the + // partial MMR, + // - and we've added blocks to a BTreeSet, so there can be no duplicates. + // + // We construct headers and partial MMR in concert, so they are consistent. This is why we + // can call the unchecked constructor. + let partial_block_chain = PartialBlockchain::new_unchecked(partial_mmr, headers) + .expect("partial mmr and block headers should be consistent"); + + Ok(BatchInputs { + batch_reference_block_header, + note_proofs, + partial_block_chain, + }) + } +} diff --git a/crates/store/src/state/view/block.rs b/crates/store/src/state/view/block.rs new file mode 100644 index 000000000..a2ae77b8f --- /dev/null +++ b/crates/store/src/state/view/block.rs @@ -0,0 +1,49 @@ +//! Block header reads. + +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::block::{BlockHeader, BlockNumber}; +use miden_protocol::crypto::merkle::mmr::MmrProof; + +use super::StateView; +use crate::COMPONENT; +use crate::errors::GetBlockHeaderError; + +impl StateView { + /// Queries a [BlockHeader] from the database, and returns it alongside its inclusion proof. + /// + /// If [None] is given as the value of `block_num`, the data for the latest [BlockHeader] is + /// returned. + #[miden_instrument( + level = "debug", + target = COMPONENT, + skip_all, + err, + )] + pub async fn get_block_header( + &self, + block_num: Option, + include_mmr_proof: bool, + ) -> Result<(Option, Option), GetBlockHeaderError> { + // Resolve "latest" against the view's snapshot rather than the DB: mid-apply, the DB may + // already contain a block that the snapshot's blockchain cannot prove yet. Scoping the DB + // query by the view's tip keeps the header and MMR proof consistent. + let latest_block_num = self.tip(); + let block_num = block_num.unwrap_or(latest_block_num); + if block_num > latest_block_num { + return Ok((None, None)); + } + + let block_header = self.db().select_block_header_by_block_num(Some(block_num)).await?; + if let Some(header) = block_header { + let mmr_proof = if include_mmr_proof { + let mmr_proof = self.blockchain().open(header.block_num())?; + Some(mmr_proof) + } else { + None + }; + Ok((Some(header), mmr_proof)) + } else { + Ok((None, None)) + } + } +} diff --git a/crates/store/src/state/view/block_inputs.rs b/crates/store/src/state/view/block_inputs.rs new file mode 100644 index 000000000..e4175f075 --- /dev/null +++ b/crates/store/src/state/view/block_inputs.rs @@ -0,0 +1,155 @@ +//! Block input query for the block producer. +//! +//! Combines in-memory snapshot data (tree witnesses, partial MMR) with database lookups, scoping +//! the latter by the view's tip so both sources describe the same block height. + +use std::collections::{BTreeMap, BTreeSet}; + +use miden_protocol::Word; +use miden_protocol::account::AccountId; +use miden_protocol::block::account_tree::AccountWitness; +use miden_protocol::block::nullifier_tree::NullifierWitness; +use miden_protocol::block::{BlockInputs, BlockNumber}; +use miden_protocol::crypto::merkle::mmr::PartialMmr; +use miden_protocol::note::Nullifier; +use miden_protocol::transaction::PartialBlockchain; + +use super::StateView; +use crate::errors::GetBlockInputsError; + +type BlockInputWitnesses = ( + BlockNumber, + BTreeMap, + BTreeMap, + PartialMmr, +); + +impl StateView { + /// Returns data needed by the block producer to construct and prove the next block. + pub async fn get_block_inputs( + &self, + account_ids: Vec, + nullifiers: Vec, + unauthenticated_note_commitments: BTreeSet, + reference_blocks: BTreeSet, + ) -> Result { + // Get the note inclusion proofs from the DB. We do this first so we have to acquire the + // lock to the state just once. There we need the reference blocks of the note proofs to get + // their authentication paths in the chain MMR. + let unauthenticated_note_proofs = self + .db() + .select_note_inclusion_proofs(unauthenticated_note_commitments) + .await + .map_err(GetBlockInputsError::SelectNoteInclusionProofError)?; + + // The set of blocks that the notes are included in. + let note_proof_reference_blocks = + unauthenticated_note_proofs.values().map(|proof| proof.location().block_num()); + + // Collect all blocks we need to prove inclusion for, without duplicates. + let mut blocks = reference_blocks; + blocks.extend(note_proof_reference_blocks); + + let (latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr) = + self.get_block_inputs_witnesses(&mut blocks, &account_ids, &nullifiers)?; + + // Fetch the block headers for all blocks in the partial MMR plus the latest one which will + // be used as the previous block header of the block being built. + let mut headers = self + .db() + .select_block_headers(blocks.into_iter().chain(std::iter::once(latest_block_number))) + .await + .map_err(GetBlockInputsError::SelectBlockHeaderError)?; + + // Find and remove the latest block as we must not add it to the chain MMR, since it is not + // yet in the chain. + let latest_block_header_index = headers + .iter() + .enumerate() + .find_map(|(index, header)| { + (header.block_num() == latest_block_number).then_some(index) + }) + .expect("DB should have returned the header of the latest block header"); + + // The order doesn't matter for PartialBlockchain::new, so swap remove is fine. + let latest_block_header = headers.swap_remove(latest_block_header_index); + + // SAFETY: This should not error because: + // - we're passing exactly the block headers that we've added to the partial MMR, + // - so none of the block header's block numbers should exceed the chain length of the + // partial MMR, + // - and we've added blocks to a BTreeSet, so there can be no duplicates. + // + // We construct headers and partial MMR in concert, so they are consistent. This is why we + // can call the unchecked constructor. + let partial_block_chain = PartialBlockchain::new_unchecked(partial_mmr, headers) + .expect("partial mmr and block headers should be consistent"); + + Ok(BlockInputs::new( + latest_block_header, + partial_block_chain, + account_witnesses, + nullifier_witnesses, + unauthenticated_note_proofs, + )) + } + + /// Get account and nullifier witnesses for the requested account IDs and nullifier as well as + /// the [`PartialMmr`] for the given blocks. The MMR won't contain the latest block and its + /// number is removed from `blocks` and returned separately. + fn get_block_inputs_witnesses( + &self, + blocks: &mut BTreeSet, + account_ids: &[AccountId], + nullifiers: &[Nullifier], + ) -> Result { + self.with_inner_read_blocking(|inner| { + let latest_block_number = inner.latest_block_num(); + + // If `blocks` is empty, use the latest block number which will never trigger the error. + let highest_block_number = blocks.last().copied().unwrap_or(latest_block_number); + if highest_block_number > latest_block_number { + return Err(GetBlockInputsError::UnknownBatchBlockReference { + highest_block_number, + latest_block_number, + }); + } + + // The latest block is not yet in the chain MMR, so we can't (and don't need to) prove + // its inclusion in the chain. + blocks.remove(&latest_block_number); + + // Fetch the partial MMR at the state of the latest block with authentication paths for + // the provided set of blocks. + // + // SAFETY: + // - The latest block num was retrieved from the inner blockchain from which we will + // also retrieve the proofs, so it is guaranteed to exist in that chain. + // - We have checked that no block number in the blocks set is greater than latest block + // number *and* latest block num was removed from the set. Therefore only block + // numbers smaller than latest block num remain in the set. Therefore all the block + // numbers are guaranteed to exist in the chain state at latest block num. + let partial_mmr = + inner.blockchain.partial_mmr_from_blocks(blocks, latest_block_number).expect( + "latest block num should exist and all blocks in set should be < than latest block", + ); + + // Fetch witnesses for all accounts. + let account_witnesses = account_ids + .iter() + .copied() + .map(|account_id| (account_id, inner.account_tree.open_latest(account_id))) + .collect::>(); + + // Fetch witnesses for all nullifiers. We don't check whether the nullifiers are spent + // or not as this is done as part of proposing the block. + let nullifier_witnesses: BTreeMap = nullifiers + .iter() + .copied() + .map(|nullifier| (nullifier, inner.nullifier_tree.open(&nullifier))) + .collect(); + + Ok((latest_block_number, account_witnesses, nullifier_witnesses, partial_mmr)) + }) + } +} diff --git a/crates/store/src/state/view/mod.rs b/crates/store/src/state/view/mod.rs new file mode 100644 index 000000000..5dcbe1b14 --- /dev/null +++ b/crates/store/src/state/view/mod.rs @@ -0,0 +1,141 @@ +//! Request-scoped, consistent read view of the store. +//! +//! All store reads go through [`StateView`]: it pins one state snapshot for its whole lifetime, +//! and every database query it exposes is scoped by that snapshot's block height. This makes it +//! impossible to implement a read whose tree and database halves observe different chain tips — +//! mid-apply, the database may already contain rows for a block the snapshot cannot prove yet. +//! +//! The submodules hold the read endpoints, all `impl StateView`; the snapshot internals +//! ([`StateSnapshot`]) are only visible within this module tree, so no other part of the store +//! can reach the trees directly. + +use std::ops::RangeInclusive; +use std::sync::Arc; + +use miden_protocol::block::{BlockNumber, Blockchain}; +use tracing::Span; + +use crate::account_state_forest::{AccountStateForest, AccountStateForestBackendReader}; +use crate::db::Db; +use crate::errors::RangeBeyondTip; +use crate::state::State; + +mod scoped; +pub use scoped::{ScopedBlockNum, ScopedBlockRange}; + +mod snapshot; +pub(in crate::state) use snapshot::{SNAPSHOTS_LIVE_WARN_THRESHOLD, SnapshotGuard, StateSnapshot}; + +mod account; +mod batch_inputs; +mod block; +mod block_inputs; +mod note; +mod sync; + +mod transaction_inputs; +pub use transaction_inputs::TransactionInputs; + +// STATE VIEW +// ================================================================================================ + +/// A consistent read view of the store, pinned at its snapshot's block height. +/// +/// Obtained from [`State::view`]; create one per request and drop it when the request completes. +/// Holding a view pins a snapshot generation (and thereby the `RocksDB` snapshots backing the +/// trees), so it must not be stored in long-lived structs; leaked or slow readers are reported by +/// the store's snapshot-lifetime warnings. +/// +/// Reads that are technically not block-scoped (e.g. content-addressed note scripts) also live +/// here so that every read path flows through a single, consistently-scoped type. +pub struct StateView { + snapshot: Arc, + db: Arc, +} + +impl State { + /// Returns a read view pinned at the current chain tip (wait-free, no lock required). + /// + /// The view is frozen: it is unaffected if the writer publishes a new snapshot while it is + /// held. + pub fn view(&self) -> StateView { + StateView { + snapshot: self.latest_snapshot.load_full(), + db: Arc::clone(&self.db), + } + } +} + +impl StateView { + /// The chain tip this view is pinned at. + pub fn tip(&self) -> BlockNumber { + self.snapshot.latest_block_num() + } + + /// Returns the pinned snapshot's blockchain MMR. + /// + /// The MMR is the only part of the snapshot that is purely in-memory and therefore safe to + /// access directly on an async worker thread. The account and nullifier trees may be backed + /// by `RocksDB` and are deliberately not reachable here — they must be accessed through + /// [`Self::with_inner_read_blocking`]. + fn blockchain(&self) -> &Blockchain { + &self.snapshot.blockchain + } + + /// Returns the database handle. + /// + /// Queries whose results depend on the chain tip must be scoped by [`Self::tip`] (or a block + /// number validated against it), never by a tip obtained elsewhere. + fn db(&self) -> &Db { + &self.db + } + + /// Returns this view's tip as a scoped block number for tip-bounded database queries. + fn scoped_tip(&self) -> ScopedBlockNum { + ScopedBlockNum::new(self.tip()) + } + + /// Validates that `block_num` does not exceed this view's chain tip, returning the scoped block + /// number required by block-bounded database queries. + fn scope_block(&self, block_num: BlockNumber) -> Option { + (block_num <= self.tip()).then(|| ScopedBlockNum::new(block_num)) + } + + /// Validates that `range` does not extend beyond this view's chain tip, returning the scoped + /// range required by range-bounded database queries. + /// + /// Every range-scoped read on this type calls this before touching the database, so callers + /// never need to pre-validate ranges themselves. + fn scope_range( + &self, + range: RangeInclusive, + ) -> Result { + let tip = self.tip(); + if *range.end() > tip { + return Err(RangeBeyondTip { chain_tip: tip, block_to: *range.end() }); + } + Ok(ScopedBlockRange::new(range)) + } + + /// Runs a synchronous read-only operation over the pinned state snapshot on Tokio's blocking + /// path. + /// + /// The account and nullifier trees may be backed by `RocksDB`, so tree access must not run on + /// an async worker thread directly. This helper preserves the current tracing span while + /// moving the closure body into `block_in_place`. + fn with_inner_read_blocking(&self, f: impl FnOnce(&StateSnapshot) -> R) -> R { + let span = Span::current(); + tokio::task::block_in_place(|| span.in_scope(|| f(&self.snapshot))) + } + + /// Runs a synchronous read-only operation over the account state forest snapshot on Tokio's + /// blocking path. + /// + /// See [`Self::with_inner_read_blocking`] for why this uses `block_in_place`. + fn with_forest_read_blocking( + &self, + f: impl FnOnce(&AccountStateForest) -> R, + ) -> R { + self.with_inner_read_blocking(|snapshot| f(&snapshot.forest)) + } +} diff --git a/crates/store/src/state/view/note.rs b/crates/store/src/state/view/note.rs new file mode 100644 index 000000000..82958b1a3 --- /dev/null +++ b/crates/store/src/state/view/note.rs @@ -0,0 +1,32 @@ +//! Note reads. +//! +//! These are content-addressed lookups and technically not block-scoped, but they live on +//! [`StateView`] so that every read path flows through the same type. + +use miden_protocol::Word; +use miden_protocol::note::{NoteId, NoteScript}; + +use super::StateView; +use crate::db::NoteRecord; +use crate::errors::DatabaseError; + +impl StateView { + /// Queries a list of notes from the database. + /// + /// If the provided list of [`NoteId`] given is empty or no note matches the provided + /// [`NoteId`] an empty list is returned. + pub async fn get_notes_by_id( + &self, + note_ids: Vec, + ) -> Result, DatabaseError> { + self.db().select_notes_by_id(note_ids).await + } + + /// Returns the script for a note by its root. + pub async fn get_note_script_by_root( + &self, + root: Word, + ) -> Result, DatabaseError> { + self.db().select_note_script_by_root(root).await + } +} diff --git a/crates/store/src/state/view/scoped.rs b/crates/store/src/state/view/scoped.rs new file mode 100644 index 000000000..85e49a923 --- /dev/null +++ b/crates/store/src/state/view/scoped.rs @@ -0,0 +1,62 @@ +//! Proof-of-validation block-number types issued by [`StateView`](super::StateView). +//! +//! Tip-scoped database queries take these types instead of raw block numbers, so a query whose +//! bound was not validated against a state view's tip is not expressible: the only constructors +//! live on [`StateView`](super::StateView), which checks the bound against its pinned snapshot. +//! +//! Chain tips are monotonic, so a value issued by an older view remains valid for any later +//! state — holding one across view lifetimes is sound, if pointless. + +use std::ops::RangeInclusive; + +use miden_protocol::block::BlockNumber; + +/// A block number proven to be at or below the issuing view's chain tip. +#[derive(Debug, Clone, Copy)] +pub struct ScopedBlockNum(BlockNumber); + +impl ScopedBlockNum { + /// Issued by [`StateView`](super::StateView) after validating the bound against its tip. + pub(super) fn new(block_num: BlockNumber) -> Self { + Self(block_num) + } + + /// Constructs a scoped block number without validation. + /// + /// Test-only: lets database tests exercise scoped queries without a running state. + #[cfg(test)] + pub(crate) fn new_unchecked(block_num: BlockNumber) -> Self { + Self(block_num) + } + + /// Returns the validated block number. + pub(crate) fn get(self) -> BlockNumber { + self.0 + } +} + +/// A block range whose upper bound is proven to be at or below the issuing view's chain tip. +#[derive(Debug, Clone)] +pub struct ScopedBlockRange(RangeInclusive); + +impl ScopedBlockRange { + /// Issued by [`StateView`](super::StateView) after validating the range against its tip. + pub(super) fn new(range: RangeInclusive) -> Self { + Self(range) + } + + /// Returns the start of the validated range. + pub(crate) fn start(&self) -> BlockNumber { + *self.0.start() + } + + /// Returns the end of the validated range. + pub(crate) fn end(&self) -> BlockNumber { + *self.0.end() + } + + /// Returns the validated range. + pub(crate) fn into_inner(self) -> RangeInclusive { + self.0 + } +} diff --git a/crates/store/src/state/view/snapshot.rs b/crates/store/src/state/view/snapshot.rs new file mode 100644 index 000000000..b0d34f1ac --- /dev/null +++ b/crates/store/src/state/view/snapshot.rs @@ -0,0 +1,136 @@ +//! In-memory snapshot machinery for lock-free reads. +//! +//! Readers access the store's tree state through immutable [`StateSnapshot`] snapshots published +//! by the block writer after each committed block. [`SnapshotGuard`] tracks how many snapshot +//! generations are pinned by readers, since each generation pins a `RocksDB` snapshot. + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use miden_protocol::block::nullifier_tree::NullifierTree; +use miden_protocol::block::{BlockNumber, Blockchain}; +use miden_protocol::crypto::merkle::smt::LargeSmt; + +use crate::COMPONENT; +use crate::account_state_forest::{AccountStateForest, AccountStateForestBackendReader}; +use crate::accounts::AccountTreeWithHistory; +use crate::state::loader::TreeStorageReader; + +/// Snapshot lifetime above which [`SnapshotGuard`] logs a warning on release. +/// +/// Readers are expected to be request-scoped, so a snapshot outliving several block intervals +/// indicates a slow or leaked reader pinning a `RocksDB` snapshot (see [`SnapshotGuard`]). +const SNAPSHOT_LIFETIME_WARN_THRESHOLD: Duration = Duration::from_secs(10); + +/// Number of live snapshot generations above which the block writer logs a warning after +/// publishing a new snapshot. +/// +/// Steady state is 1-2 generations: the just-published snapshot plus predecessors briefly pinned +/// by in-flight requests. A sustained higher count means slow or leaked readers are holding old +/// generations alive (see [`SnapshotGuard`]). +pub(in crate::state) const SNAPSHOTS_LIVE_WARN_THRESHOLD: u64 = 4; + +// SNAPSHOT GUARD +// ================================================================================================ + +/// RAII member of [`StateSnapshot`] that tracks the number of live snapshot generations. +/// +/// [`StateSnapshot`] is dropped exactly when the last [`Arc`] reference to it is released, so the +/// shared counter reports how many distinct snapshot generations are currently pinned by readers. +/// A sustained count above 1-2 means slow readers are holding old generations alive. Each +/// generation pins a `RocksDB` snapshot, which delays garbage collection of superseded key +/// versions during compaction (compaction itself keeps running); the retained garbage grows with +/// write churn for as long as the snapshot is held and is reclaimed once it is released. +/// +/// Readers are expected to be request-scoped, so snapshot lifetimes should be well under a block +/// interval. A lifetime exceeding [`SNAPSHOT_LIFETIME_WARN_THRESHOLD`] is logged at warn level. +pub(in crate::state) struct SnapshotGuard { + live: Arc, + created_at: Instant, + block_num: BlockNumber, +} + +impl SnapshotGuard { + pub(in crate::state) fn new(live: Arc, block_num: BlockNumber) -> Self { + live.fetch_add(1, Ordering::Relaxed); + Self { + live, + created_at: Instant::now(), + block_num, + } + } +} + +impl Drop for SnapshotGuard { + fn drop(&mut self) { + let remaining = self.live.fetch_sub(1, Ordering::Relaxed) - 1; + let lifetime = self.created_at.elapsed(); + let lifetime_ms = u64::try_from(lifetime.as_millis()).unwrap_or(u64::MAX); + let block_num = self.block_num.as_u32(); + if lifetime > SNAPSHOT_LIFETIME_WARN_THRESHOLD { + tracing::warn!( + target: COMPONENT, + block_num, + snapshot.lifetime_ms = lifetime_ms, + snapshots.live = remaining, + "state snapshot held for excessive time", + ); + } else { + tracing::debug!( + target: COMPONENT, + block_num, + snapshot.lifetime_ms = lifetime_ms, + snapshots.live = remaining, + "state snapshot released", + ); + } + } +} + +// STATE SNAPSHOT +// ================================================================================================ + +/// Immutable snapshot of the store's tree state published after each committed block. +/// +/// The trees are backed by read-only snapshot storage ([`TreeStorageReader`] / +/// [`AccountStateForestBackendReader`]), so any number of readers can access the data concurrently +/// without holding a lock and without blocking the writer. +/// +/// The writer and lifecycle can *construct* snapshots via [`Self::new`], but the fields are only +/// readable within the view module tree: every read outside it must go through +/// [`StateView`](super::StateView). +pub(in crate::state) struct StateSnapshot { + pub(super) nullifier_tree: NullifierTree>, + pub(super) blockchain: Blockchain, + pub(super) account_tree: AccountTreeWithHistory, + pub(super) forest: AccountStateForest, + /// Keeps the live-snapshot count accurate; see [`SnapshotGuard`]. + _guard: SnapshotGuard, +} + +impl StateSnapshot { + /// Assembles a snapshot from reader views of the trees and the guard tracking its generation. + pub(in crate::state) fn new( + nullifier_tree: NullifierTree>, + blockchain: Blockchain, + account_tree: AccountTreeWithHistory, + forest: AccountStateForest, + guard: SnapshotGuard, + ) -> Self { + Self { + nullifier_tree, + blockchain, + account_tree, + forest, + _guard: guard, + } + } + + /// Returns the latest block number. + pub(super) fn latest_block_num(&self) -> BlockNumber { + self.blockchain + .chain_tip() + .expect("chain should always have at least the genesis block") + } +} diff --git a/crates/store/src/state/sync_state.rs b/crates/store/src/state/view/sync.rs similarity index 69% rename from crates/store/src/state/sync_state.rs rename to crates/store/src/state/view/sync.rs index f7dbc5ae6..89fbb9bff 100644 --- a/crates/store/src/state/sync_state.rs +++ b/crates/store/src/state/view/sync.rs @@ -5,7 +5,7 @@ use miden_protocol::account::AccountId; use miden_protocol::block::{BlockHeader, BlockNumber, BlockSignatures}; use miden_protocol::crypto::merkle::mmr::{Forest, MmrDelta, MmrProof}; -use super::State; +use super::StateView; use crate::COMPONENT; use crate::db::models::queries::StorageMapValuesPage; use crate::db::{AccountVaultValue, NoteSyncUpdate, NullifierInfo}; @@ -14,18 +14,25 @@ use crate::errors::{DatabaseError, NoteSyncError, StateSyncError}; // STATE SYNCHRONIZATION ENDPOINTS // ================================================================================================ -impl State { +impl StateView { /// Returns the complete transaction records for the specified accounts within the specified /// block range, including state commitments and note IDs. + /// + /// Returns [`RangeBeyondTip`](crate::errors::RangeBeyondTip) if the range extends beyond this + /// view's chain tip. pub async fn sync_transactions( &self, account_ids: Vec, block_range: RangeInclusive, ) -> Result<(BlockNumber, Vec), DatabaseError> { - self.db.select_transactions_records(account_ids, block_range).await + let block_range = self.scope_range(block_range)?; + self.db().select_transactions_records(account_ids, block_range).await } /// Returns the chain MMR delta and the `block_to` block header for the specified block range. + /// + /// Returns [`RangeBeyondTip`](crate::errors::RangeBeyondTip) if the range extends beyond this + /// view's chain tip. #[miden_instrument( level = "debug", target = COMPONENT, @@ -36,13 +43,15 @@ impl State { &self, block_range: RangeInclusive, ) -> Result<(MmrDelta, BlockHeader, BlockSignatures), StateSyncError> { - let block_from = *block_range.start(); - let block_to = *block_range.end(); + let block_range = self.scope_range(block_range)?; + + let block_from = block_range.start(); + let block_to = block_range.end(); - // SAFETY: block_to has been validated to be <= the effective tip (chain tip or latest - // proven block) by the caller, so it must exist in the database. + // SAFETY: block_to <= this view's tip (checked above), so it is committed and must exist in + // the database. let (block_header, signatures) = self - .db + .db() .select_block_header_and_signatures_by_block_num(block_to) .await? .expect("block_to should exist in the database"); @@ -71,10 +80,7 @@ impl State { let to_forest = block_to.as_usize(); let mmr_delta = self - .inner - .read() - .await - .blockchain + .blockchain() .as_mmr() .get_delta( Forest::new(from_forest).expect("from_forest fits in u32"), @@ -93,6 +99,9 @@ impl State { /// /// Also returns the last block number checked. If this equals `block_range.end()`, the /// sync is complete. + /// + /// Returns [`RangeBeyondTip`](crate::errors::RangeBeyondTip) if the range extends beyond this + /// view's chain tip. #[miden_instrument( level = "debug", target = COMPONENT, @@ -104,24 +113,22 @@ impl State { note_tags: Vec, block_range: RangeInclusive, ) -> Result<(Vec<(NoteSyncUpdate, MmrProof)>, BlockNumber), NoteSyncError> { - let block_end = *block_range.end(); + let block_range = self.scope_range(block_range)?; + + let block_end = block_range.end(); // The MMR at forest N contains proofs for blocks 0..N-1, so we use block_end + 1 to include - // the proof for block_end. SAFETY: it is ensured that block_end <= chain_tip, and the - // blockchain MMR always has at least chain_tip + 1 leaves. + // the proof for block_end. SAFETY: block_end <= this view's tip (checked above), and the + // view's blockchain MMR always has at least tip + 1 leaves. let mmr_checkpoint = block_end + 1; - let note_syncs = self.db.get_note_sync_multi(block_range, note_tags.into()).await?; + let note_syncs = self.db().get_note_sync_multi(block_range, note_tags.into()).await?; let mut results = Vec::new(); - { - let inner = self.inner.read().await; - - for note_sync in note_syncs { - let mmr_proof = - inner.blockchain.open_at(note_sync.block_header.block_num(), mmr_checkpoint)?; - results.push((note_sync, mmr_proof)); - } + for note_sync in note_syncs { + let mmr_proof = + self.blockchain().open_at(note_sync.block_header.block_num(), mmr_checkpoint)?; + results.push((note_sync, mmr_proof)); } // if results is empty, return `block_end` since the sync is complete. @@ -131,13 +138,18 @@ impl State { Ok((results, last_block_checked)) } + /// Returns nullifiers matching the given prefixes that were created within a block range. + /// + /// Returns [`RangeBeyondTip`](crate::errors::RangeBeyondTip) if the range extends beyond this + /// view's chain tip. pub async fn sync_nullifiers( &self, prefix_len: u32, nullifier_prefixes: Vec, block_range: RangeInclusive, ) -> Result<(Vec, BlockNumber), DatabaseError> { - self.db + let block_range = self.scope_range(block_range)?; + self.db() .select_nullifiers_by_prefix(prefix_len, nullifier_prefixes, block_range) .await } @@ -146,20 +158,28 @@ impl State { // -------------------------------------------------------------------------------------------- /// Returns account vault updates for specified account within a block range. + /// + /// Returns [`RangeBeyondTip`](crate::errors::RangeBeyondTip) if the range extends beyond this + /// view's chain tip. pub async fn sync_account_vault( &self, account_id: AccountId, block_range: RangeInclusive, ) -> Result<(BlockNumber, Vec), DatabaseError> { - self.db.get_account_vault_sync(account_id, block_range).await + let block_range = self.scope_range(block_range)?; + self.db().get_account_vault_sync(account_id, block_range).await } /// Returns storage map values for syncing within a block range. + /// + /// Returns [`RangeBeyondTip`](crate::errors::RangeBeyondTip) if the range extends beyond this + /// view's chain tip. pub async fn sync_account_storage_maps( &self, account_id: AccountId, block_range: RangeInclusive, ) -> Result { - self.db.select_storage_map_sync_values(account_id, block_range, None).await + let block_range = self.scope_range(block_range)?; + self.db().select_storage_map_sync_values(account_id, block_range, None).await } } diff --git a/crates/store/src/state/view/transaction_inputs.rs b/crates/store/src/state/view/transaction_inputs.rs new file mode 100644 index 000000000..01aaf7ad2 --- /dev/null +++ b/crates/store/src/state/view/transaction_inputs.rs @@ -0,0 +1,93 @@ +//! Transaction input query for the block producer's transaction validation. + +use std::collections::HashSet; +use std::ops::ControlFlow; + +use miden_node_utils::formatting::format_array; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::Word; +use miden_protocol::account::AccountId; +use miden_protocol::note::Nullifier; + +use super::StateView; +use crate::COMPONENT; +use crate::db::NullifierInfo; +use crate::errors::DatabaseError; + +/// Store-level inputs for validating a proven transaction. +#[derive(Debug, Default)] +pub struct TransactionInputs { + pub account_commitment: Word, + pub nullifiers: Vec, + pub found_unauthenticated_notes: HashSet, + pub new_account_id_prefix_is_unique: Option, +} + +impl StateView { + /// Returns data needed by the block producer to verify transactions validity. + #[miden_instrument( + target = COMPONENT, + skip_all, + fields( + account.id=%account_id, + nullifiers = %format_array(nullifiers), + ), + )] + pub async fn get_transaction_inputs( + &self, + account_id: AccountId, + nullifiers: &[Nullifier], + unauthenticated_note_commitments: Vec, + ) -> Result { + let tree_inputs = self.with_inner_read_blocking(|inner| { + let account_commitment = inner.account_tree.get_latest_commitment(account_id); + + let new_account_id_prefix_is_unique = if account_commitment.is_empty() { + Some(!inner.account_tree.contains_account_id_prefix_in_latest(account_id.prefix())) + } else { + None + }; + + // Non-unique account Id prefixes for new accounts are not allowed, so the transaction + // cannot be valid and the response is already complete. + if let Some(false) = new_account_id_prefix_is_unique { + return ControlFlow::Break(TransactionInputs { + new_account_id_prefix_is_unique, + ..Default::default() + }); + } + + let nullifiers = nullifiers + .iter() + .map(|nullifier| NullifierInfo { + nullifier: *nullifier, + block_num: inner.nullifier_tree.get_block_num(nullifier).unwrap_or_default(), + }) + .collect(); + + ControlFlow::Continue((account_commitment, nullifiers, new_account_id_prefix_is_unique)) + }); + // `Break` carries a complete response (duplicate account ID prefix), so it is returned + // as-is without the note lookup below; `Continue` carries the tree reads needed to build + // the full response. + let (account_commitment, nullifiers, new_account_id_prefix_is_unique) = match tree_inputs { + ControlFlow::Continue(inputs) => inputs, + ControlFlow::Break(response) => return Ok(response), + }; + + // Scope the note lookup by the view's tip so the result is consistent with the tree reads + // above: mid-apply, the DB may already contain notes from a block the snapshot does not + // include yet. + let found_unauthenticated_notes = self + .db() + .select_existing_note_commitments(unauthenticated_note_commitments, self.scoped_tip()) + .await?; + + Ok(TransactionInputs { + account_commitment, + nullifiers, + found_unauthenticated_notes, + new_account_id_prefix_is_unique, + }) + } +} diff --git a/crates/store/src/state/writer/apply_block.rs b/crates/store/src/state/writer/apply_block.rs new file mode 100644 index 000000000..9806316fe --- /dev/null +++ b/crates/store/src/state/writer/apply_block.rs @@ -0,0 +1,54 @@ +use miden_node_proto::domain::proof_request::BlockProofRequest; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::batch::OrderedBatches; +use miden_protocol::block::{BlockInputs, BlockNumber, SignedBlock}; +use miden_protocol::utils::serde::Serializable; + +use crate::COMPONENT; +use crate::errors::ApplyBlockWithProvingInputsError; +use crate::state::BlockWriter; + +impl BlockWriter { + /// Saves proving inputs for a signed block and applies it to the state. + /// + /// Used by the in-process block producer after it has built and signed a block. + #[miden_instrument( + target = COMPONENT, + skip_all, + err, + )] + pub async fn apply_block_with_proving_inputs( + &self, + ordered_batches: OrderedBatches, + block_inputs: BlockInputs, + signed_block: SignedBlock, + ) -> Result<(), ApplyBlockWithProvingInputsError> { + let block_header = signed_block.header().clone(); + let block_num = block_header.block_num(); + + let proving_inputs = BlockProofRequest { + tx_batches: ordered_batches, + block_header, + block_inputs, + }; + + self.save_proving_inputs(block_num, &proving_inputs) + .await + .map_err(ApplyBlockWithProvingInputsError::SaveProvingInputs)?; + + self.apply_block(signed_block) + .await + .map_err(ApplyBlockWithProvingInputsError::ApplyBlock) + } + + /// Saves the proving inputs for the given block to the block store. + pub async fn save_proving_inputs( + &self, + block_num: BlockNumber, + proving_inputs: &BlockProofRequest, + ) -> std::io::Result<()> { + self.block_store + .save_proving_inputs(block_num, &proving_inputs.to_bytes()) + .await + } +} diff --git a/crates/store/src/state/apply_proof.rs b/crates/store/src/state/writer/apply_proof.rs similarity index 82% rename from crates/store/src/state/apply_proof.rs rename to crates/store/src/state/writer/apply_proof.rs index e6db3e0dd..ecd4e5af5 100644 --- a/crates/store/src/state/apply_proof.rs +++ b/crates/store/src/state/writer/apply_proof.rs @@ -4,9 +4,9 @@ use miden_protocol::block::{BlockNumber, BlockProof}; use miden_protocol::utils::serde::Deserializable; use crate::COMPONENT; -use crate::state::{Finality, ProofNotification, State}; +use crate::state::{ProofNotification, ProofWriter}; -impl State { +impl ProofWriter { /// Saves a block proof, advances the proven-in-sequence tip, and notifies replica subscribers. /// /// # Errors @@ -26,13 +26,13 @@ impl State { block_num: BlockNumber, proof_bytes: Vec, ) -> anyhow::Result<()> { - let expected = self.proven_tip.read().child(); + let expected = self.state.proven_tip().child(); ensure!( block_num == expected, "out-of-sequence proof: expected block {expected}, got {block_num}", ); - let committed_tip = self.chain_tip(Finality::Committed).await; + let committed_tip = self.state.committed_tip(); ensure!( block_num <= committed_tip, "proof for uncommitted block {block_num} exceeds committed tip {committed_tip}", @@ -40,11 +40,12 @@ impl State { verify_block_proof(block_num, &proof_bytes)?; - self.block_store.commit_proof(block_num, &proof_bytes).await?; - self.proof_cache + self.state.block_store.commit_proof(block_num, &proof_bytes).await?; + self.state + .proof_cache .push(block_num, ProofNotification::new(block_num, proof_bytes)) .expect("proof cache receives sequential block numbers"); - self.proven_tip.advance(block_num); + self.state.proven_tip.advance(block_num); Ok(()) } } diff --git a/crates/store/src/state/writer/mod.rs b/crates/store/src/state/writer/mod.rs new file mode 100644 index 000000000..0ae34d5a7 --- /dev/null +++ b/crates/store/src/state/writer/mod.rs @@ -0,0 +1,139 @@ +//! Serialized block-write path for the store state. +//! +//! A single [`WriteWorker`] task owns the mutable trees and processes incoming [`WriteRequest`]s +//! one at a time via an mpsc channel. After each successful commit it publishes a new +//! [`StateSnapshot`] snapshot via an [`ArcSwap`], making the updated trees immediately visible to +//! wait-free readers. +//! +//! The [`BlockWriter`] and [`ProofWriter`] capabilities defined here are the only handles able +//! to feed this worker and to commit proofs; the submodules hold their entry points. + +mod apply_block; +mod apply_proof; + +mod worker; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; + +use miden_node_utils::ErrorReport; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::block::SignedBlock; +use tokio::sync::{mpsc, oneshot}; +pub(in crate::state) use worker::WriteWorker; + +use crate::COMPONENT; +use crate::blocks::BlockStore; +use crate::errors::ApplyBlockError; + +// WRITE CAPABILITIES +// ================================================================================================ + +/// The store's block-write capability. +/// +/// Only handle able to apply blocks; obtained exactly once from [`LoadedState::start`](crate::state::LoadedState::start) and +/// deliberately not cloneable, so granting it to a single task (the block builder in sequencer +/// mode, the block sync loop in full-node mode) statically prevents every other component from +/// writing blocks. +/// +/// Exposes no read access: holders that also need to query the store receive the +/// [`Arc`](crate::state::State) returned alongside this capability by +/// [`LoadedState::start`](crate::state::LoadedState::start). +pub struct BlockWriter { + /// The block store, used to persist proving inputs alongside applied blocks. + pub(super) block_store: Arc, + /// Sender for block-write requests to the [`WriteWorker`] task. Never cloned out of this + /// struct: the writer exits once it is dropped. + pub(super) write_tx: mpsc::Sender, +} + +/// The store's proof-write capability. +/// +/// Only handle able to commit block proofs and advance the proven tip; obtained exactly once from +/// [`LoadedState::start`](crate::state::LoadedState::start) and deliberately not cloneable, so granting it to a single task (the +/// proof scheduler in sequencer mode, the proof sync loop in full-node mode) statically prevents +/// every other component from writing proofs. +/// +/// Exposes no read access: the held state is only used internally to commit proofs and advance +/// the proven tip. Holders that also need to query the store receive the +/// [`Arc`](crate::state::State) returned alongside this capability by +/// [`LoadedState::start`](crate::state::LoadedState::start). +pub struct ProofWriter { + pub(super) state: Arc, +} + +// WRITER TASK +// ================================================================================================ + +/// Handle of the store's write worker task, returned by [`LoadedState::start`](crate::state::LoadedState::start). +/// +/// Awaiting it resolves once the writer has exited and released the tree storage it owns; a join +/// error carries a writer panic. The newtype ensures [`BlockWriter::stop`] can only be given the +/// store's own writer task, and deliberately does not expose [`tokio::task::JoinHandle::abort`]: +/// aborting the writer mid-write could leave the trees lagging the committed database state, +/// voiding the guarantee that an in-flight block write always completes. +#[must_use = "await the writer task to observe its exit, or pass it to `BlockWriter::stop`"] +pub struct WriterTask(pub(super) tokio::task::JoinHandle<()>); + +impl Future for WriterTask { + type Output = Result<(), tokio::task::JoinError>; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + Pin::new(&mut self.0).poll(cx) + } +} + +// WRITE REQUEST +// ================================================================================================ + +/// A request to apply a block, paired with a one-shot channel for the result. +pub(super) struct WriteRequest { + signed_block: SignedBlock, + result_tx: oneshot::Sender>, +} + +impl BlockWriter { + /// Stops the store, waiting until the write worker has released the tree storage it owns. + /// + /// Consumes the capability — closing the write channel the write worker listens on — and then + /// joins the writer task returned by [`LoadedState::start`](crate::state::LoadedState::start). The drop must precede the join + /// or the write worker never observes the closed channel; doing both here keeps that ordering + /// out of caller hands. Read-only [`State`](crate::state::State) references may outlive the + /// stop. + /// + /// Callers that need the storage released deterministically must use this method instead of + /// dropping: the node's `recover` command stops the store before the process exits, and the + /// stress-test's store seeding stops it so the same data directory can be re-loaded (or its + /// temporary directory deleted) immediately afterwards. The running node does not use this + /// method — its writer exits via the shutdown token passed to + /// [`State::load`](crate::state::State::load) and is joined through the node's task set. + /// + /// # Panics + /// + /// Panics if the writer task panicked. + pub async fn stop(self, writer_task: WriterTask) { + drop(self); + writer_task.await.expect("write worker task should not panic"); + } + + /// Apply changes of a new block to the DB and in-memory data structures. + /// + /// Blocks are forwarded to the store's write worker task, which processes them one at a + /// time. + /// Readers are unaffected while a block is being applied: they keep reading from the previous + /// in-memory snapshot until the writer atomically publishes the new one. + #[miden_instrument( + target = COMPONENT, + skip_all, + err, + )] + pub async fn apply_block(&self, signed_block: SignedBlock) -> Result<(), ApplyBlockError> { + let (result_tx, result_rx) = oneshot::channel(); + self.write_tx + .send(WriteRequest { signed_block, result_tx }) + .await + .map_err(|e| ApplyBlockError::WriterTaskSendFailed(e.as_report()))?; + result_rx.await? + } +} diff --git a/crates/store/src/state/writer/worker.rs b/crates/store/src/state/writer/worker.rs new file mode 100644 index 000000000..26639612c --- /dev/null +++ b/crates/store/src/state/writer/worker.rs @@ -0,0 +1,513 @@ +//! The write worker: single-task owner of the store's mutable trees. + +use std::fmt::Display; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use arc_swap::ArcSwap; +use miden_node_utils::ErrorReport; +use miden_node_utils::shutdown::CancellationToken; +use miden_node_utils::tracing::{miden_instrument, miden_span_record}; +use miden_protocol::Word; +use miden_protocol::account::AccountUpdateDetails; +use miden_protocol::block::account_tree::AccountMutationSet; +use miden_protocol::block::nullifier_tree::{NullifierMutationSet, NullifierTree}; +use miden_protocol::block::{BlockBody, BlockHeader, BlockNumber, Blockchain, SignedBlock}; +use miden_protocol::crypto::merkle::smt::LargeSmt; +use miden_protocol::note::{NoteDetails, Nullifier}; +use miden_protocol::transaction::OutputNote; +use miden_protocol::utils::serde::Serializable; +use tokio::sync::{mpsc, watch}; + +use super::WriteRequest; +use crate::account_state_forest::{ + AccountStateForest, + AccountStateForestBackend, + PreparedAccountStateForestBlockUpdate, +}; +use crate::accounts::AccountTreeWithHistory; +use crate::blocks::BlockStore; +use crate::db::{Db, NoteRecord}; +use crate::errors::{ApplyBlockError, InvalidBlockError}; +use crate::state::block_lifecycle::{BlockLifecycle, lifecycle_events_enabled}; +use crate::state::loader::TreeStorage; +use crate::state::view::{SNAPSHOTS_LIVE_WARN_THRESHOLD, SnapshotGuard, StateSnapshot}; +use crate::state::{BlockCache, BlockNotification}; +use crate::{COMPONENT, HistoricalError, LOG_TARGET}; + +// WRITE WORKER +// ================================================================================================ + +/// Single-task owner of the mutable trees. Processes [`WriteRequest`]s serially. +/// +/// The writer owns the writable trees directly, so no locks are held at any point: validation and +/// mutation-computation read the owned trees, the DB commit runs without touching them, and the +/// new [`StateSnapshot`] snapshot is published atomically at the end. +pub(in crate::state) struct WriteWorker { + db: Arc, + block_store: Arc, + /// Atomically swappable pointer through which new snapshots are published. + latest_snapshot: Arc>, + committed_tip_tx: Arc>, + block_cache: BlockCache, + rx: mpsc::Receiver, + /// Token signalling node shutdown; the writer stops accepting new requests once cancelled. + shutdown: CancellationToken, + /// The mutable nullifier tree owned by this writer. + nullifier_tree: NullifierTree>, + /// The mutable account tree owned by this writer. + account_tree: AccountTreeWithHistory, + /// The blockchain MMR owned by this writer. + blockchain: Blockchain, + /// The mutable account state forest owned by this writer. + forest: AccountStateForest, + /// Shared counter of live snapshot generations, for observability. + snapshots_live: Arc, +} + +/// Note records and state mutations computed from a validated block, before any modifications. +struct PreparedBlockUpdate { + notes: Vec<(NoteRecord, Option)>, + nullifier_tree_update: NullifierMutationSet, + account_tree_update: AccountMutationSet, + account_forest_update: PreparedAccountStateForestBlockUpdate, +} + +impl WriteWorker { + /// Assembles the write worker from the loaded trees and the store's shared infrastructure. + /// + /// Only construction is exposed outside this module: once assembled, the worker's internals + /// are reachable solely through [`Self::run`]. + #[expect(clippy::too_many_arguments)] + pub(in crate::state) fn new( + db: Arc, + block_store: Arc, + latest_snapshot: Arc>, + committed_tip_tx: Arc>, + block_cache: BlockCache, + rx: mpsc::Receiver, + shutdown: CancellationToken, + nullifier_tree: NullifierTree>, + account_tree: AccountTreeWithHistory, + blockchain: Blockchain, + forest: AccountStateForest, + snapshots_live: Arc, + ) -> Self { + Self { + db, + block_store, + latest_snapshot, + committed_tip_tx, + block_cache, + rx, + shutdown, + nullifier_tree, + account_tree, + blockchain, + forest, + snapshots_live, + } + } + + /// Runs the writer loop, processing requests until shutdown is signalled or the + /// [`BlockWriter`] (holding the only request sender) is dropped. + /// + /// Cancellation is only observed between requests: an in-flight block write always runs to + /// completion, so shutdown never leaves the trees lagging the committed database state. + /// Requests still queued when cancellation fires are dropped, failing their senders. + pub async fn run(mut self) { + loop { + let req = tokio::select! { + biased; + () = self.shutdown.cancelled() => break, + req = self.rx.recv() => match req { + Some(req) => req, + None => break, + }, + }; + let result = self.write_block(req.signed_block).await; + let _ = req.result_tx.send(result); + } + } + + /// Validates and commits a signed block to all persistent and in-memory stores. + /// + /// ## Note on state consistency + /// + /// Readers access the in-memory state through frozen snapshots, so consistency is maintained + /// by ordering the commit steps rather than by locking: + /// + /// - the block is validated against the writer-owned trees and the DB prior to starting any + /// modifications. + /// - the block is saved to the block store. Such blocks are considered candidates and are not + /// yet available for reading because the latest block pointer is not updated yet. + /// - the DB transaction is committed. Concurrent readers still see the previous in-memory + /// snapshot; queries that combine DB and in-memory data are scoped by block number. + /// - the in-memory structures owned by the writer are updated. On a crash in between, the trees + /// lag the DB by one block, which is detected by the consistency checks at startup (the same + /// crash semantics as the previous lock-based implementation). + /// - the new snapshot is published atomically, making the block visible to readers. + #[miden_instrument( + target = COMPONENT, + skip_all, + err, + )] + async fn write_block(&mut self, signed_block: SignedBlock) -> Result<(), ApplyBlockError> { + let header = signed_block.header(); + let body = signed_block.body(); + + let block_num = header.block_num(); + let block_commitment = header.commitment(); + let num_transactions = body.transactions().as_slice().len(); + + miden_span_record!( + block.number = %block_num, + block.commitment = %block_commitment, + block.transactions.count = num_transactions, + ); + + self.validate_block_header(header, body).await?; + + let block_lifecycle = + lifecycle_events_enabled().then(|| BlockLifecycle::from_block_body(block_num, body)); + let unresolved_note_nullifiers = block_lifecycle + .as_ref() + .map_or_else(Vec::new, BlockLifecycle::unresolved_note_nullifiers); + + // Compute the tree and forest mutations and note records upfront, before any modifications. + // The writer is the sole forest mutator, so the precomputed forest update stays valid until + // it is applied after the DB commit below. + let PreparedBlockUpdate { + notes, + nullifier_tree_update, + account_tree_update, + account_forest_update, + } = tokio::task::block_in_place(|| self.prepare_block_update(header, body))?; + let precomputed_public_states = account_forest_update.account_states.clone(); + + // Save the block to the block store. In a case of a failed DB transaction, the in-memory + // state will be unchanged, but the file might still be written. Such blocks should be + // considered candidates, not finalized blocks. + let signed_block_bytes = signed_block.to_bytes(); + // Clone before moving into the block-save call so we can cache for replicas at commit. + let cache_bytes = signed_block_bytes.clone(); + self.block_store.save_block(block_num, &signed_block_bytes).await?; + + // Commit to the DB. Readers continue to see the previous in-memory snapshot while the DB + // commits; queries that combine DB and in-memory data are scoped by block number. + let resolved_note_ids = self + .db + .apply_block(signed_block, notes, precomputed_public_states, unresolved_note_nullifiers) + .await + .map_err(|err| ApplyBlockError::DbUpdateTaskFailed(err.as_report()))?; + + // The DB is committed at this point, so the prepared mutations must be applied and any + // failure to do so aborts the process. + let snapshot = tokio::task::block_in_place(|| { + self.apply_prepared_mutations( + block_num, + block_commitment, + nullifier_tree_update, + account_tree_update, + account_forest_update, + ) + }); + + // Atomically publish the new state. Readers that call `snapshot()` after this point will + // see the updated state. Readers holding the old snapshot continue unaffected. + self.latest_snapshot.store(snapshot); + + let snapshots_live = self.check_live_snapshots(block_num); + miden_span_record!(snapshots.live = snapshots_live); + + // Push to cache and notify replica subscribers. + self.block_cache + .push(block_num, BlockNotification::new(block_num, cache_bytes)) + .expect("block cache receives sequential block numbers"); + let _ = self.committed_tip_tx.send(block_num); + + if let Some(block_lifecycle) = block_lifecycle { + block_lifecycle.emit(&resolved_note_ids); + } + tracing::debug!(target: LOG_TARGET, "Block applied"); + + Ok(()) + } + + /// Returns the number of live snapshot generations, warning when slow readers are pinning too + /// many old generations in memory. + /// + /// The count is returned rather than recorded here because `miden_span_record!` must be used + /// within a `#[miden_instrument]` function. + fn check_live_snapshots(&self, block_num: BlockNumber) -> u64 { + let snapshots_live = self.snapshots_live.load(Ordering::Relaxed) as u64; + if snapshots_live > SNAPSHOTS_LIVE_WARN_THRESHOLD { + tracing::warn!( + target: COMPONENT, + block_num = block_num.as_u32(), + snapshots.live = snapshots_live, + "too many live state snapshots; slow readers are pinning old generations", + ); + } + snapshots_live + } + + /// Computes the note records and all tree and forest mutations for a block, without mutating + /// any state. + /// + /// May block on backend I/O, so it must run on Tokio's blocking path. The returned forest + /// update is bound to the forest state observed here; it remains valid until applied because + /// the writer is the sole forest mutator. + fn prepare_block_update( + &self, + header: &BlockHeader, + body: &BlockBody, + ) -> Result { + let notes = Self::build_note_records(header, body)?; + let (nullifier_tree_update, account_tree_update) = + self.compute_tree_mutations(header, body)?; + + // Public account updates carry patches; private accounts are filtered out since they don't + // expose their state changes. + let account_patches = + body.updated_accounts().iter().filter_map(|update| match update.details() { + AccountUpdateDetails::Public(patch) => Some(patch.clone()), + AccountUpdateDetails::Private => None, + }); + let account_forest_update = self + .forest + .compute_block_update_mutations(header.block_num(), account_patches) + .map_err(ApplyBlockError::AccountStateForestPreparation)?; + + Ok(PreparedBlockUpdate { + notes, + nullifier_tree_update, + account_tree_update, + account_forest_update, + }) + } + + /// Applies the prepared mutations to the writer-owned trees and builds the new snapshot from + /// reader views of them. The reader views are point-in-time storage snapshots, so no tree data + /// is copied. + /// + /// Must only be called after the corresponding DB commit: at that point the mutations are part + /// of canonical state, so a failure to apply them leaves the trees divergent and aborts the + /// process. May block on backend I/O, so it must run on Tokio's blocking path. + fn apply_prepared_mutations( + &mut self, + block_num: BlockNumber, + block_commitment: Word, + nullifier_tree_update: NullifierMutationSet, + account_tree_update: AccountMutationSet, + account_forest_update: PreparedAccountStateForestBlockUpdate, + ) -> Arc { + self.nullifier_tree + .apply_mutations(nullifier_tree_update) + .unwrap_or_else(|error| { + Self::abort_after_post_commit_failure("nullifier tree", &error) + }); + + self.account_tree + .apply_mutations(account_tree_update) + .unwrap_or_else(|error| Self::abort_after_post_commit_failure("account tree", &error)); + + self.blockchain.push(block_commitment); + + self.forest + .apply_precomputed_block_update(block_num, account_forest_update) + .unwrap_or_else(|error| { + Self::abort_after_post_commit_failure("account-state forest", &error) + }); + + Arc::new(StateSnapshot::new( + self.nullifier_tree + .reader() + .expect("nullifier tree snapshot creation should not fail"), + self.blockchain.clone(), + self.account_tree.reader(), + self.forest.reader().expect("forest snapshot creation should not fail"), + SnapshotGuard::new(Arc::clone(&self.snapshots_live), block_num), + )) + } + + /// Terminates after a persistent state failure that occurred after the canonical DB commit. + /// + /// Returning would expose components at different block heights. Tests panic so the fatal path + /// can be asserted without terminating the test process; production aborts immediately. + fn abort_after_post_commit_failure(component: &str, error: &impl Display) -> ! { + tracing::error!( + target: LOG_TARGET, + component, + error = %error, + "persistent state update failed after database commit; aborting" + ); + + #[cfg(test)] + panic!("{component} update failed after database commit: {error}"); + + #[cfg(not(test))] + std::process::abort(); + } + + /// Validates that the block header is consistent with the block body and the current state. + #[miden_instrument( + target = COMPONENT, + skip_all, + err, + )] + async fn validate_block_header( + &self, + header: &BlockHeader, + body: &BlockBody, + ) -> Result<(), ApplyBlockError> { + // Validate that header and body match. + let tx_commitment = body.transactions().commitment(); + if header.tx_commitment() != tx_commitment { + return Err(InvalidBlockError::InvalidBlockTxCommitment { + expected: tx_commitment, + actual: header.tx_commitment(), + } + .into()); + } + + let block_num = header.block_num(); + + // Validate that the applied block is the next block in sequence. + let prev_block = self + .db + .select_block_header_by_block_num(None) + .await? + .ok_or(ApplyBlockError::DbBlockHeaderEmpty)?; + let expected_block_num = prev_block.block_num().child(); + if block_num != expected_block_num { + return Err(InvalidBlockError::NewBlockInvalidBlockNum { + expected: expected_block_num, + submitted: block_num, + } + .into()); + } + if header.prev_block_commitment() != prev_block.commitment() { + return Err(InvalidBlockError::NewBlockInvalidPrevCommitment.into()); + } + + Ok(()) + } + + /// Computes nullifier and account tree mutations, validating roots against the block header. + #[miden_instrument( + target = COMPONENT, + skip_all, + err, + )] + fn compute_tree_mutations( + &self, + header: &BlockHeader, + body: &BlockBody, + ) -> Result<(NullifierMutationSet, AccountMutationSet), ApplyBlockError> { + let block_num = header.block_num(); + + // A nullifier can only ever be created once, so the block is invalid if any of its + // nullifiers are already recorded in the tree. + let duplicate_nullifiers: Vec<_> = body + .created_nullifiers() + .iter() + .filter(|&nullifier| self.nullifier_tree.get_block_num(nullifier).is_some()) + .copied() + .collect(); + if !duplicate_nullifiers.is_empty() { + return Err(InvalidBlockError::DuplicatedNullifiers(duplicate_nullifiers).into()); + } + + // The header's chain commitment must equal the chain MMR root prior to this block. + let peaks = self.blockchain.peaks(); + if peaks.hash_peaks() != header.chain_commitment() { + return Err(InvalidBlockError::NewBlockInvalidChainCommitment.into()); + } + + // Compute the nullifier tree mutations and verify that they produce the nullifier root + // claimed in the header. + let nullifier_tree_update = self + .nullifier_tree + .compute_mutations( + body.created_nullifiers().iter().map(|nullifier| (*nullifier, block_num)), + ) + .map_err(InvalidBlockError::NewBlockNullifierAlreadySpent)?; + + if nullifier_tree_update.as_mutation_set().root() != header.nullifier_root() { + return Err(InvalidBlockError::NewBlockInvalidNullifierRoot.into()); + } + + // Compute the account tree mutations and verify that they produce the account root claimed + // in the header. + let account_tree_update = self + .account_tree + .compute_mutations( + body.updated_accounts() + .iter() + .map(|update| (update.account_id(), update.final_state_commitment())), + ) + .map_err(|e| match e { + HistoricalError::AccountTreeError(err) => { + InvalidBlockError::NewBlockDuplicateAccountIdPrefix(err) + }, + HistoricalError::MerkleError(_) => { + panic!("Unexpected MerkleError during account tree mutation computation") + }, + })?; + + if account_tree_update.as_mutation_set().root() != header.account_root() { + return Err(InvalidBlockError::NewBlockInvalidAccountRoot.into()); + } + + Ok((nullifier_tree_update, account_tree_update)) + } + + /// Builds note records with inclusion proofs from the block body. + #[miden_instrument( + target = COMPONENT, + skip_all, + err, + )] + fn build_note_records( + header: &BlockHeader, + body: &BlockBody, + ) -> Result)>, ApplyBlockError> { + let block_num = header.block_num(); + + let note_tree = body.compute_block_note_tree(); + if note_tree.root() != header.note_root() { + return Err(InvalidBlockError::NewBlockInvalidNoteRoot.into()); + } + + let notes = body + .output_notes() + .map(|(note_index, note)| { + let (details, attachments, nullifier) = match note { + OutputNote::Public(public) => ( + Some(NoteDetails::from(public.as_note())), + public.as_note().attachments().clone(), + Some(public.as_note().nullifier()), + ), + OutputNote::Private(private) => (None, private.attachments().clone(), None), + }; + + let inclusion_path = note_tree.open(note_index); + + let note_record = NoteRecord { + block_num, + note_index, + note_id: note.id().as_word(), + metadata: *note.metadata(), + details, + attachments, + inclusion_path, + }; + + Ok((note_record, nullifier)) + }) + .collect::, InvalidBlockError>>()?; + + Ok(notes) + } +} diff --git a/crates/tracing-macro/src/lib.rs b/crates/tracing-macro/src/lib.rs index 6de930b8e..8074102a0 100644 --- a/crates/tracing-macro/src/lib.rs +++ b/crates/tracing-macro/src/lib.rs @@ -81,6 +81,9 @@ const ALLOWED_FIELD_NAMES: &[&str] = &[ "reference_block.number", "request.kind", "script.root", + "snapshot.block_num", + "snapshot.lifetime_ms", + "snapshots.live", "transaction.id", "transaction.expires_at", "transaction.input_notes.count", diff --git a/crates/utils/tests/ui/tracing_macros/invalid_field_name.stderr b/crates/utils/tests/ui/tracing_macros/invalid_field_name.stderr index ab5408157..3ef514675 100644 --- a/crates/utils/tests/ui/tracing_macros/invalid_field_name.stderr +++ b/crates/utils/tests/ui/tracing_macros/invalid_field_name.stderr @@ -1,4 +1,4 @@ -error: unsupported tracing field `tx_id`; use one of: account.id, account.id.network_prefix, account.ids, account.ids.count, account.updated, batch.id, batch.account_updates.count, batch.expires_at, batch.expiration_height, batch.input_notes.count, batch.output_notes.count, batch.reference_block.commitment, batch.reference_block.number, block.batch.ids, block.batches.count, block.batches.output_notes.count, block.commitment, block.commitments.account, block.commitments.chain, block.commitments.kernel, block.commitments.note, block.commitments.nullifier, block.commitments.transaction, block.erased_note_proofs.count, block.erased_notes.count, block.from, block.nullifiers.count, block.number, block.output_notes.count, block.prev_block_commitment, block.protocol.version, block.size, block.sub_commitment, block.timestamp, block.transactions.ids, block.transactions.count, block.updated_accounts.count, block_range.from, block_range.to, current_client_block_height, cutoff_block, db.account_state_forest.size, db.account_tree.size, db.block_store.size, db.nullifier_tree.size, db.sqlite.size, db.sqlite.wal.size, dice_roll, failure_rate, finality_level, inputs_size, mempool.accounts, mempool.batches.proposed, mempool.batches.proven, mempool.nullifiers, mempool.output_notes, mempool.transactions.unbatched, mempool.transactions.uncommitted, note.id, notes.count, nullifiers, path, port, prefix_len, prefixes, proof_size, prover, prover.kind, reference_block.number, request.kind, script.root, transaction.id, transaction.expires_at, transaction.input_notes.count, transaction.output_notes.count, transaction.reference_block.commitment, transaction.reference_block.number, tip.number, transactions.count, transactions.ids, transactions.input_notes.count, transactions.output_notes.count, transactions.unauthenticated_notes.count, workers.active, workers.capacity, workers.count +error: unsupported tracing field `tx_id`; use one of: account.id, account.id.network_prefix, account.ids, account.ids.count, account.updated, batch.id, batch.account_updates.count, batch.expires_at, batch.expiration_height, batch.input_notes.count, batch.output_notes.count, batch.reference_block.commitment, batch.reference_block.number, block.batch.ids, block.batches.count, block.batches.output_notes.count, block.commitment, block.commitments.account, block.commitments.chain, block.commitments.kernel, block.commitments.note, block.commitments.nullifier, block.commitments.transaction, block.erased_note_proofs.count, block.erased_notes.count, block.from, block.nullifiers.count, block.number, block.output_notes.count, block.prev_block_commitment, block.protocol.version, block.size, block.sub_commitment, block.timestamp, block.transactions.ids, block.transactions.count, block.updated_accounts.count, block_range.from, block_range.to, current_client_block_height, cutoff_block, db.account_state_forest.size, db.account_tree.size, db.block_store.size, db.nullifier_tree.size, db.sqlite.size, db.sqlite.wal.size, dice_roll, failure_rate, finality_level, inputs_size, mempool.accounts, mempool.batches.proposed, mempool.batches.proven, mempool.nullifiers, mempool.output_notes, mempool.transactions.unbatched, mempool.transactions.uncommitted, note.id, notes.count, nullifiers, path, port, prefix_len, prefixes, proof_size, prover, prover.kind, reference_block.number, request.kind, script.root, snapshot.block_num, snapshot.lifetime_ms, snapshots.live, transaction.id, transaction.expires_at, transaction.input_notes.count, transaction.output_notes.count, transaction.reference_block.commitment, transaction.reference_block.number, tip.number, transactions.count, transactions.ids, transactions.input_notes.count, transactions.output_notes.count, transactions.unauthenticated_notes.count, workers.active, workers.capacity, workers.count --> tests/ui/tracing_macros/invalid_field_name.rs:10:9 | 10 | tx_id = %tx_id, diff --git a/crates/utils/tests/ui/tracing_macros/invalid_instrument_field_name.stderr b/crates/utils/tests/ui/tracing_macros/invalid_instrument_field_name.stderr index 11a1b8970..ef07e9cdd 100644 --- a/crates/utils/tests/ui/tracing_macros/invalid_instrument_field_name.stderr +++ b/crates/utils/tests/ui/tracing_macros/invalid_instrument_field_name.stderr @@ -1,4 +1,4 @@ -error: unsupported tracing field `tx_id`; use one of: account.id, account.id.network_prefix, account.ids, account.ids.count, account.updated, batch.id, batch.account_updates.count, batch.expires_at, batch.expiration_height, batch.input_notes.count, batch.output_notes.count, batch.reference_block.commitment, batch.reference_block.number, block.batch.ids, block.batches.count, block.batches.output_notes.count, block.commitment, block.commitments.account, block.commitments.chain, block.commitments.kernel, block.commitments.note, block.commitments.nullifier, block.commitments.transaction, block.erased_note_proofs.count, block.erased_notes.count, block.from, block.nullifiers.count, block.number, block.output_notes.count, block.prev_block_commitment, block.protocol.version, block.size, block.sub_commitment, block.timestamp, block.transactions.ids, block.transactions.count, block.updated_accounts.count, block_range.from, block_range.to, current_client_block_height, cutoff_block, db.account_state_forest.size, db.account_tree.size, db.block_store.size, db.nullifier_tree.size, db.sqlite.size, db.sqlite.wal.size, dice_roll, failure_rate, finality_level, inputs_size, mempool.accounts, mempool.batches.proposed, mempool.batches.proven, mempool.nullifiers, mempool.output_notes, mempool.transactions.unbatched, mempool.transactions.uncommitted, note.id, notes.count, nullifiers, path, port, prefix_len, prefixes, proof_size, prover, prover.kind, reference_block.number, request.kind, script.root, transaction.id, transaction.expires_at, transaction.input_notes.count, transaction.output_notes.count, transaction.reference_block.commitment, transaction.reference_block.number, tip.number, transactions.count, transactions.ids, transactions.input_notes.count, transactions.output_notes.count, transactions.unauthenticated_notes.count, workers.active, workers.capacity, workers.count +error: unsupported tracing field `tx_id`; use one of: account.id, account.id.network_prefix, account.ids, account.ids.count, account.updated, batch.id, batch.account_updates.count, batch.expires_at, batch.expiration_height, batch.input_notes.count, batch.output_notes.count, batch.reference_block.commitment, batch.reference_block.number, block.batch.ids, block.batches.count, block.batches.output_notes.count, block.commitment, block.commitments.account, block.commitments.chain, block.commitments.kernel, block.commitments.note, block.commitments.nullifier, block.commitments.transaction, block.erased_note_proofs.count, block.erased_notes.count, block.from, block.nullifiers.count, block.number, block.output_notes.count, block.prev_block_commitment, block.protocol.version, block.size, block.sub_commitment, block.timestamp, block.transactions.ids, block.transactions.count, block.updated_accounts.count, block_range.from, block_range.to, current_client_block_height, cutoff_block, db.account_state_forest.size, db.account_tree.size, db.block_store.size, db.nullifier_tree.size, db.sqlite.size, db.sqlite.wal.size, dice_roll, failure_rate, finality_level, inputs_size, mempool.accounts, mempool.batches.proposed, mempool.batches.proven, mempool.nullifiers, mempool.output_notes, mempool.transactions.unbatched, mempool.transactions.uncommitted, note.id, notes.count, nullifiers, path, port, prefix_len, prefixes, proof_size, prover, prover.kind, reference_block.number, request.kind, script.root, snapshot.block_num, snapshot.lifetime_ms, snapshots.live, transaction.id, transaction.expires_at, transaction.input_notes.count, transaction.output_notes.count, transaction.reference_block.commitment, transaction.reference_block.number, tip.number, transactions.count, transactions.ids, transactions.input_notes.count, transactions.output_notes.count, transactions.unauthenticated_notes.count, workers.active, workers.capacity, workers.count --> tests/ui/tracing_macros/invalid_instrument_field_name.rs:6:9 | 6 | tx_id = %"0x1234",