diff --git a/Cargo.lock b/Cargo.lock index a9a6a361ae..a4eded5bd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3284,7 +3284,6 @@ dependencies = [ "futures", "itertools 0.14.0", "miden-node-proto", - "miden-node-proto-build", "miden-node-store", "miden-node-utils", "miden-protocol", @@ -3299,10 +3298,7 @@ dependencies = [ "tempfile", "thiserror 2.0.18", "tokio", - "tokio-stream", "tonic", - "tonic-reflection", - "tower-http", "tracing", "url", ] @@ -3384,6 +3380,7 @@ dependencies = [ "futures", "http 1.4.0", "mediatype", + "miden-node-block-producer", "miden-node-proto", "miden-node-proto-build", "miden-node-store", @@ -3460,7 +3457,6 @@ dependencies = [ "clap", "fs-err", "futures", - "miden-node-block-producer", "miden-node-proto", "miden-node-store", "miden-node-utils", diff --git a/bin/ntx-builder/src/actor/mod.rs b/bin/ntx-builder/src/actor/mod.rs index c566b50a8c..85439fa1eb 100644 --- a/bin/ntx-builder/src/actor/mod.rs +++ b/bin/ntx-builder/src/actor/mod.rs @@ -174,16 +174,14 @@ enum ActorMode { /// based on current chain state and DB queries. /// - **Transaction Execution**: Executes selected transactions using either local or remote /// proving. -/// - **Mempool Integration**: Listens for mempool events to stay synchronized with the network -/// state and adjust behavior based on transaction confirmations. +/// - **Chain Integration**: Reacts to committed chain state persisted by the builder. /// /// ## Lifecycle /// /// 1. **Initialization**: Waits for committed account state, then checks DB for available notes. -/// 2. **Event Loop**: Continuously processes mempool events and executes transactions. +/// 2. **Event Loop**: Re-evaluates persisted state and executes transactions when notified. /// 3. **Transaction Processing**: Selects, executes, proves, and submits transactions through RPC. -/// 4. **State Updates**: Event effects are persisted to DB by the coordinator before actors are -/// notified. +/// 4. **State Updates**: Committed block effects are persisted to DB before actors are notified. /// 5. **Shutdown**: Terminates gracefully on idle timeout, or returns an error on unrecoverable /// failures. /// @@ -383,9 +381,8 @@ impl AccountActor { /// For accounts that are being created by an inflight transaction, this will idle /// until the transaction is committed. Returns `true` when the account is ready, or /// `false` if no commit arrived within [`ActorConfig::idle_timeout`] — in which case - /// the coordinator will respawn a new actor when the account reappears through - /// [`Coordinator::send_targeted`](crate::coordinator::Coordinator::send_targeted) or the - /// account loader. + /// the coordinator will respawn a new actor when the account reappears through the account + /// loader. async fn wait_for_committed_account( &self, account_id: NetworkAccountId, diff --git a/bin/ntx-builder/src/coordinator.rs b/bin/ntx-builder/src/coordinator.rs index d4243cc979..dac7b3d6b9 100644 --- a/bin/ntx-builder/src/coordinator.rs +++ b/bin/ntx-builder/src/coordinator.rs @@ -1,25 +1,13 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::sync::Arc; -use miden_node_db::DatabaseError; use miden_node_proto::domain::account::NetworkAccountId; -use miden_node_proto::domain::mempool::MempoolEvent; -use miden_protocol::account::delta::AccountUpdateDetails; use tokio::sync::{Notify, Semaphore}; use tokio::task::JoinSet; use crate::actor::{AccountActor, AccountActorContext}; use crate::db::Db; -// WRITE EVENT RESULT -// ================================================================================================ - -/// Result of writing a mempool event to the database. -pub struct WriteEventResult { - /// Accounts that should be notified of state changes. - pub accounts_to_notify: Vec, -} - // ACTOR HANDLE // ================================================================================================ @@ -75,7 +63,7 @@ impl ActorHandle { /// - Gracefully handles actor shutdown and cleanup when actors complete or fail. /// - Monitors actor tasks through a join set to detect completion or errors. /// -/// ## Event Notification +/// ## State Notification /// - Notifies actors via a shared [`Notify`] when state may have changed. /// - The DB is the source of truth: actors re-evaluate their state from DB on notification. /// - Notifications are coalesced: [`Notify`] stores at most one permit, so multiple notifications @@ -89,12 +77,11 @@ impl ActorHandle { /// - Actors that have been idle for longer than the idle timeout deactivate themselves. /// - When an actor deactivates, the coordinator checks if a notification arrived just as the actor /// timed out. If so, the actor is respawned immediately. -/// - Deactivated actors are re-spawned when [`Coordinator::send_targeted`] detects notes targeting -/// an account without an active actor. +/// - Deactivated actors may be re-spawned when later committed chain state requires them. /// /// The coordinator operates in an event-driven manner: /// 1. Network accounts are registered and actors spawned as needed. -/// 2. Mempool events are written to DB, then actors are notified. +/// 2. Actors are notified when DB state relevant to them may have changed. /// 3. Actor completion/failure events are monitored and handled. /// 4. Failed or completed actors are cleaned up from the registry. pub struct Coordinator { @@ -255,101 +242,6 @@ impl Coordinator { }, } } - - /// Notifies account actors that are affected by a `TransactionAdded` event. - /// - /// Only actors that are currently active are notified. Since event effects are already - /// persisted in the DB by `write_event()`, actors that spawn later read their state from the - /// DB and do not need predating events. - /// - /// Returns account IDs of note targets that do not have active actors (e.g. previously - /// deactivated due to sterility). The caller can use this to re-activate actors for those - /// accounts. - pub fn send_targeted(&self, event: &MempoolEvent) -> Vec { - let mut target_account_ids = HashSet::new(); - let mut inactive_targets = Vec::new(); - - if let MempoolEvent::TransactionAdded { network_notes, account_delta, .. } = event { - // We need to inform the account if it was updated. This lets it know that its own - // transaction has been applied, and in the future also resolves race conditions with - // external network transactions (once these are allowed). - if let Some(AccountUpdateDetails::Delta(delta)) = account_delta { - let account_id = delta.id(); - if account_id.is_network() { - let network_account_id = - account_id.try_into().expect("account is network account"); - if self.actor_registry.contains_key(&network_account_id) { - target_account_ids.insert(network_account_id); - } - } - } - - // Determine target actors for each note. - for note in network_notes { - let account = note.target_account_id(); - let account = NetworkAccountId::try_from(account) - .expect("network note target account should be a network account"); - - if self.actor_registry.contains_key(&account) { - target_account_ids.insert(account); - } else { - inactive_targets.push(account); - } - } - } - // Notify target actors. - for account_id in &target_account_ids { - if let Some(handle) = self.actor_registry.get(account_id) { - handle.notify(); - } - } - - inactive_targets - } - - /// Writes mempool event effects to the database. - /// - /// This must be called BEFORE sending notifications to actors. Returns a [`WriteEventResult`] - /// with the accounts to notify and cancel. - pub async fn write_event( - &self, - event: &MempoolEvent, - ) -> Result { - match event { - MempoolEvent::TransactionAdded { - id, - nullifiers, - network_notes, - account_delta, - } => { - self.db - .handle_transaction_added( - *id, - account_delta.clone(), - network_notes.clone(), - nullifiers.clone(), - ) - .await?; - Ok(WriteEventResult { accounts_to_notify: Vec::new() }) - }, - MempoolEvent::BlockCommitted { header, txs } => { - let affected_accounts = self - .db - .handle_block_committed( - txs.clone(), - header.block_num(), - header.as_ref().clone(), - ) - .await?; - Ok(WriteEventResult { accounts_to_notify: affected_accounts }) - }, - MempoolEvent::TransactionsReverted(tx_ids) => { - let affected_accounts = - self.db.handle_transactions_reverted(tx_ids.iter().copied().collect()).await?; - Ok(WriteEventResult { accounts_to_notify: affected_accounts }) - }, - } - } } #[cfg(test)] @@ -363,48 +255,11 @@ impl Coordinator { #[cfg(test)] mod tests { - use miden_node_proto::domain::mempool::MempoolEvent; - use super::*; use crate::actor::AccountActorContext; use crate::db::Db; use crate::test_utils::*; - /// Registers a dummy actor handle (no real actor task) in the coordinator's registry. - fn register_dummy_actor(coordinator: &mut Coordinator, account_id: NetworkAccountId) { - let notify = Arc::new(Notify::new()); - coordinator.actor_registry.insert(account_id, ActorHandle::new(notify)); - } - - // SEND TARGETED TESTS - // ============================================================================================ - - #[tokio::test] - async fn send_targeted_returns_inactive_targets() { - let (mut coordinator, _dir) = Coordinator::test().await; - - let active_id = mock_network_account_id(); - let inactive_id = mock_network_account_id_seeded(42); - - // Only register the active account. - register_dummy_actor(&mut coordinator, active_id); - - let note_active = mock_single_target_note(active_id, 10); - let note_inactive = mock_single_target_note(inactive_id, 20); - - let event = MempoolEvent::TransactionAdded { - id: mock_tx_id(1), - nullifiers: vec![], - network_notes: vec![note_active, note_inactive], - account_delta: None, - }; - - let inactive_targets = coordinator.send_targeted(&event); - - assert_eq!(inactive_targets.len(), 1); - assert_eq!(inactive_targets[0], inactive_id); - } - // DEACTIVATED ACCOUNTS // ============================================================================================ diff --git a/bin/ntx-builder/src/db/mod.rs b/bin/ntx-builder/src/db/mod.rs index 4c85efba0b..fffe464da9 100644 --- a/bin/ntx-builder/src/db/mod.rs +++ b/bin/ntx-builder/src/db/mod.rs @@ -186,7 +186,7 @@ impl Db { // DEAD-CODE STUBS // ============================================================================================ // - // These methods exist to keep the dead actor/coordinator modules compiling in PR 1. They are + // These methods exist to keep the dead actor module compiling in PR 1. They are // never reached because `NetworkTransactionBuilder` does not spawn the actor path. PR 2 // replaces them with their new committed-block-driven equivalents. @@ -198,37 +198,6 @@ impl Db { unimplemented!("transaction_exists is rewired in PR 2 of the ntx-builder refactor") } - #[expect(clippy::unused_async)] - pub async fn handle_transaction_added( - &self, - _tx_id: miden_protocol::transaction::TransactionId, - _account_delta: Option, - _notes: Vec, - _nullifiers: Vec, - ) -> Result<()> { - unimplemented!("handle_transaction_added is rewired in PR 2 of the ntx-builder refactor") - } - - #[expect(clippy::unused_async)] - pub async fn handle_block_committed( - &self, - _txs: Vec, - _block_num: BlockNumber, - _header: BlockHeader, - ) -> Result> { - unimplemented!("handle_block_committed is rewired in PR 2 of the ntx-builder refactor") - } - - #[expect(clippy::unused_async)] - pub async fn handle_transactions_reverted( - &self, - _tx_ids: Vec, - ) -> Result> { - unimplemented!( - "handle_transactions_reverted is rewired in PR 2 of the ntx-builder refactor" - ) - } - /// Creates a file-backed SQLite test connection with migrations applied. #[cfg(test)] pub fn test_conn() -> (diesel::SqliteConnection, tempfile::TempDir) { diff --git a/bin/ntx-builder/src/test_utils.rs b/bin/ntx-builder/src/test_utils.rs index d2f8cece3a..d2ce79d5ad 100644 --- a/bin/ntx-builder/src/test_utils.rs +++ b/bin/ntx-builder/src/test_utils.rs @@ -14,7 +14,6 @@ use miden_protocol::testing::account_id::{ ACCOUNT_ID_REGULAR_NETWORK_ACCOUNT_IMMUTABLE_CODE, AccountIdBuilder, }; -use miden_protocol::transaction::TransactionId; use miden_standards::note::{AccountTargetNetworkNote, NetworkAccountTarget, NoteExecutionHint}; use miden_standards::testing::note::NoteBuilder; use rand_chacha::ChaCha20Rng; @@ -27,25 +26,6 @@ pub fn mock_network_account_id() -> NetworkAccountId { NetworkAccountId::try_from(account_id).unwrap() } -/// Creates a distinct network account ID using a seeded RNG. -pub fn mock_network_account_id_seeded(seed: u8) -> NetworkAccountId { - let account_id = AccountIdBuilder::new() - .account_type(AccountType::RegularAccountImmutableCode) - .storage_mode(AccountStorageMode::Network) - .build_with_seed([seed; 32]); - NetworkAccountId::try_from(account_id).unwrap() -} - -/// Creates a unique `TransactionId` from a seed value. -pub fn mock_tx_id(seed: u64) -> TransactionId { - use miden_protocol::testing::account_id::ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET; - - let w = |n: u64| Word::try_from([n, 0, 0, 0]).unwrap(); - let faucet_id = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap(); - let fee = miden_protocol::asset::FungibleAsset::new(faucet_id, 0).unwrap(); - TransactionId::new(w(seed), w(seed + 1), w(seed + 2), w(seed + 3), fee) -} - /// Creates a `AccountTargetNetworkNote` targeting the given network account. pub fn mock_single_target_note( network_account_id: NetworkAccountId, diff --git a/bin/stress-test/Cargo.toml b/bin/stress-test/Cargo.toml index 0743e2b751..2650244db1 100644 --- a/bin/stress-test/Cargo.toml +++ b/bin/stress-test/Cargo.toml @@ -17,17 +17,16 @@ version.workspace = true workspace = true [dependencies] -clap = { features = ["derive"], workspace = true } -fs-err = { workspace = true } -futures = { workspace = true } -miden-node-block-producer = { workspace = true } -miden-node-proto = { workspace = true } -miden-node-store = { workspace = true } -miden-node-utils = { workspace = true } -miden-protocol = { workspace = true } -miden-standards = { workspace = true } -rand = { workspace = true } -rayon = { workspace = true } -tokio = { workspace = true } -tonic = { default-features = true, workspace = true } -url = { workspace = true } +clap = { features = ["derive"], workspace = true } +fs-err = { workspace = true } +futures = { workspace = true } +miden-node-proto = { workspace = true } +miden-node-store = { workspace = true } +miden-node-utils = { workspace = true } +miden-protocol = { workspace = true } +miden-standards = { workspace = true } +rand = { workspace = true } +rayon = { workspace = true } +tokio = { workspace = true } +tonic = { default-features = true, workspace = true } +url = { workspace = true } diff --git a/bin/stress-test/src/seeding/mod.rs b/bin/stress-test/src/seeding/mod.rs index a846587389..1ae9b21dfa 100644 --- a/bin/stress-test/src/seeding/mod.rs +++ b/bin/stress-test/src/seeding/mod.rs @@ -1,12 +1,13 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::time::Instant; use metrics::SeedingMetrics; -use miden_node_block_producer::store::StoreClient; use miden_node_proto::domain::batch::BatchInputs; +use miden_node_proto::domain::proof_request::BlockProofRequest; use miden_node_proto::generated::store::rpc_client::RpcClient; +use miden_node_store::state::State; use miden_node_store::{DataDirectory, GenesisState, Store, StoreMode}; use miden_node_utils::clap::{GrpcOptionsInternal, StorageOptions}; use miden_node_utils::tracing::grpc::OtelInterceptor; @@ -131,9 +132,15 @@ pub async fn seed_store( .expect("genesis block should be created"); Store::bootstrap(genesis_block, &data_directory).expect("store should bootstrap"); - // start the store - let (_, store_url) = start_store(data_directory.clone()).await; - let store_client = StoreClient::new(store_url); + let (termination_ask, _termination_signal) = tokio::sync::mpsc::channel(1); + let (state, _proven_tip) = State::load_with_database_options( + &data_directory, + StorageOptions::bench(), + miden_node_store::DatabaseOptions::default(), + termination_ask, + ) + .await + .expect("state should load"); // start generating blocks let accounts_filepath = data_directory.join(ACCOUNTS_FILENAME); @@ -145,7 +152,7 @@ pub async fn seed_store( public_accounts_percentage, faucet, genesis_header, - &store_client, + &state, data_directory, accounts_filepath, &signer, @@ -171,7 +178,7 @@ async fn generate_blocks( public_accounts_percentage: u8, mut faucet: Account, genesis_block: SignedBlock, - store_client: &StoreClient, + state: &State, data_directory: DataDirectory, accounts_filepath: PathBuf, signer: &EcdsaSecretKey, @@ -260,11 +267,10 @@ async fn generate_blocks( .collect(); // create the block and send it to the store - let block_inputs = get_block_inputs(store_client, &batches, &mut metrics).await; + let block_inputs = get_block_inputs(state, &batches, &mut metrics).await; // update blocks - prev_block_header = - apply_block(batches, block_inputs, store_client, &mut metrics, signer).await; + prev_block_header = apply_block(batches, block_inputs, state, &mut metrics, signer).await; account_states .extend(pending_consumed_accounts.into_iter().map(|account| (account.id(), account))); if current_anchor_header.block_epoch() != prev_block_header.block_epoch() { @@ -272,8 +278,7 @@ async fn generate_blocks( } // create the consume notes txs to be used in the next block - let batch_inputs = - get_batch_inputs(store_client, &prev_block_header, ¬es, &mut metrics).await; + let batch_inputs = get_batch_inputs(state, &prev_block_header, ¬es, &mut metrics).await; (pending_consumed_accounts, consume_notes_txs) = create_consume_note_txs( &prev_block_header, accounts, @@ -317,18 +322,16 @@ async fn generate_blocks( .map(|txs| create_batch(txs, &prev_block_header)) .collect(); - let block_inputs = get_block_inputs(store_client, &batches, &mut metrics).await; + let block_inputs = get_block_inputs(state, &batches, &mut metrics).await; - prev_block_header = - apply_block(batches, block_inputs, store_client, &mut metrics, signer).await; + prev_block_header = apply_block(batches, block_inputs, state, &mut metrics, signer).await; account_states .extend(pending_consumed_accounts.into_iter().map(|account| (account.id(), account))); if current_anchor_header.block_epoch() != prev_block_header.block_epoch() { current_anchor_header = prev_block_header.clone(); } - let batch_inputs = - get_batch_inputs(store_client, &prev_block_header, ¬es, &mut metrics).await; + let batch_inputs = get_batch_inputs(state, &prev_block_header, ¬es, &mut metrics).await; let accounts = selected_account_ids .iter() .filter_map(|account_id| account_states.get(account_id).cloned()) @@ -365,10 +368,11 @@ async fn generate_blocks( async fn apply_block( batches: Vec, block_inputs: BlockInputs, - store_client: &StoreClient, + state: &State, metrics: &mut SeedingMetrics, signer: &EcdsaSecretKey, ) -> BlockHeader { + let proving_block_inputs = block_inputs.clone(); let proposed_block = ProposedBlock::new(block_inputs, batches).unwrap(); let (header, body) = proposed_block.clone().into_header_and_body().unwrap(); let block_size: usize = header.to_bytes().len() + body.to_bytes().len(); @@ -376,13 +380,22 @@ async fn apply_block( // SAFETY: The header, body, and signature are known to correspond to each other. let signed_block = SignedBlock::new_unchecked(header, body, signature); let ordered_batches = proposed_block.batches().clone(); + let block_header = signed_block.header().clone(); let start = Instant::now(); - store_client.apply_block(&ordered_batches, &signed_block).await.unwrap(); + let proving_inputs = BlockProofRequest { + tx_batches: ordered_batches, + block_header: block_header.clone(), + block_inputs: proving_block_inputs, + }; + state + .save_proving_inputs(block_header.block_num(), &proving_inputs) + .await + .unwrap(); + state.apply_block(signed_block).await.unwrap(); metrics.track_block_insertion(start.elapsed(), block_size); - let (header, ..) = signed_block.into_parts(); - header + block_header } // HELPER FUNCTIONS @@ -792,7 +805,7 @@ fn create_emit_note_tx( /// Gets the batch inputs from the store and tracks the query time on the metrics. async fn get_batch_inputs( - store_client: &StoreClient, + state: &State, block_ref: &BlockHeader, notes: &[Note], metrics: &mut SeedingMetrics, @@ -800,10 +813,10 @@ async fn get_batch_inputs( let start = Instant::now(); // Mark every note as unauthenticated, so that the store returns the inclusion proofs for all of // them - let batch_inputs = store_client + let batch_inputs = state .get_batch_inputs( - vec![(block_ref.block_num(), block_ref.commitment())].into_iter(), - notes.iter().map(Note::commitment), + BTreeSet::from([block_ref.block_num()]), + notes.iter().map(Note::commitment).collect(), ) .await .unwrap(); @@ -813,22 +826,25 @@ async fn get_batch_inputs( /// Gets the block inputs from the store and tracks the query time on the metrics. async fn get_block_inputs( - store_client: &StoreClient, + state: &State, batches: &[ProvenBatch], metrics: &mut SeedingMetrics, ) -> BlockInputs { let start = Instant::now(); - let inputs = store_client + let inputs = state .get_block_inputs( - batches.iter().flat_map(ProvenBatch::updated_accounts), - batches.iter().flat_map(ProvenBatch::created_nullifiers), - batches.iter().flat_map(|batch| { - batch - .input_notes() - .into_iter() - .filter_map(|note| note.header().map(NoteHeader::to_commitment)) - }), - batches.iter().map(ProvenBatch::reference_block_num), + batches.iter().flat_map(ProvenBatch::updated_accounts).collect(), + batches.iter().flat_map(ProvenBatch::created_nullifiers).collect(), + batches + .iter() + .flat_map(|batch| { + batch + .input_notes() + .into_iter() + .filter_map(|note| note.header().map(NoteHeader::to_commitment)) + }) + .collect(), + batches.iter().map(ProvenBatch::reference_block_num).collect(), ) .await .unwrap(); @@ -848,20 +864,13 @@ pub async fn start_store( let rpc_listener = TcpListener::bind("127.0.0.1:0") .await .expect("Failed to bind store RPC gRPC endpoint"); - let block_producer_listener = TcpListener::bind("127.0.0.1:0") - .await - .expect("Failed to bind store block-producer gRPC endpoint"); let store_addr = rpc_listener.local_addr().expect("Failed to get store RPC address"); - let store_block_producer_addr = block_producer_listener - .local_addr() - .expect("Failed to get store block-producer address"); let dir = data_directory.clone(); task::spawn(async move { Store { rpc_listener, mode: StoreMode::BlockProducer { - block_producer_listener, block_prover_url: None, max_concurrent_proofs: miden_node_store::DEFAULT_MAX_CONCURRENT_PROOFS, }, @@ -881,7 +890,7 @@ pub async fn start_store( .await .expect("Failed to connect to store"); - // SAFETY: The store_block_producer_addr is always valid as it is created from a `SocketAddr`. - let store_url = Url::parse(&format!("http://{store_block_producer_addr}")).unwrap(); + // SAFETY: The store_addr is always valid as it is created from a `SocketAddr`. + let store_url = Url::parse(&format!("http://{store_addr}")).unwrap(); (RpcClient::with_interceptor(channel, OtelInterceptor), store_url) } diff --git a/crates/block-producer/Cargo.toml b/crates/block-producer/Cargo.toml index 5509826e74..d149ad2402 100644 --- a/crates/block-producer/Cargo.toml +++ b/crates/block-producer/Cargo.toml @@ -26,25 +26,20 @@ anyhow = { workspace = true } futures = { workspace = true } itertools = { workspace = true } miden-node-proto = { workspace = true } -miden-node-proto-build = { features = ["internal"], workspace = true } +miden-node-store = { workspace = true } miden-node-utils = { features = ["testing"], workspace = true } miden-protocol = { default-features = true, workspace = true } miden-remote-prover-client = { features = ["batch-prover", "block-prover"], workspace = true } -miden-standards = { workspace = true } miden-tx-batch-prover = { workspace = true } rand = { workspace = true } thiserror = { workspace = true } tokio = { features = ["macros", "net", "rt-multi-thread"], workspace = true } -tokio-stream = { features = ["net"], workspace = true } tonic = { default-features = true, features = ["transport"], workspace = true } -tonic-reflection = { workspace = true } -tower-http = { features = ["util"], workspace = true } tracing = { workspace = true } url = { workspace = true } [dev-dependencies] assert_matches = { workspace = true } -miden-node-store = { workspace = true } miden-node-utils = { features = ["testing"], workspace = true } miden-protocol = { default-features = true, features = ["testing"], workspace = true } miden-standards = { features = ["testing"], workspace = true } diff --git a/crates/block-producer/src/batch_builder/mod.rs b/crates/block-producer/src/batch_builder/mod.rs index a18024cfdb..32ace02109 100644 --- a/crates/block-producer/src/batch_builder/mod.rs +++ b/crates/block-producer/src/batch_builder/mod.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeSet; use std::num::NonZeroUsize; use std::ops::Deref; use std::sync::Arc; @@ -5,6 +6,7 @@ use std::time::Duration; use futures::TryFutureExt; use miden_node_proto::domain::batch::BatchInputs; +use miden_node_store::state::State; use miden_node_utils::spawn::spawn_blocking_in_current_span; use miden_node_utils::tracing::OpenTelemetrySpanExt; use miden_protocol::MIN_PROOF_SECURITY_LEVEL; @@ -19,9 +21,8 @@ use url::Url; use crate::domain::batch::SelectedBatch; use crate::domain::transaction::AuthenticatedTransaction; -use crate::errors::BuildBatchError; +use crate::errors::{BuildBatchError, StoreError}; use crate::mempool::SharedMempool; -use crate::store::StoreClient; use crate::{COMPONENT, TelemetryInjectorExt}; // BATCH BUILDER @@ -49,7 +50,7 @@ pub struct BatchBuilder { /// /// Note: this _must_ be sign positive and less than 1.0. failure_rate: f64, - store: StoreClient, + state: Arc, } impl BatchBuilder { @@ -58,7 +59,7 @@ impl BatchBuilder { /// /// If no batch prover URL is provided, a local batch prover is used instead. pub fn new( - store: StoreClient, + state: Arc, num_workers: NonZeroUsize, batch_prover_url: Option, batch_interval: Duration, @@ -75,7 +76,7 @@ impl BatchBuilder { worker_pool, failure_rate: 0.0, batch_prover, - store, + state, } } @@ -110,7 +111,7 @@ impl BatchBuilder { let job = BatchJob { failure_rate: self.failure_rate, - store: self.store.clone(), + state: Arc::clone(&self.state), mempool, batch_prover: self.batch_prover.clone(), }; @@ -161,7 +162,7 @@ struct BatchJob { /// /// Note: this _must_ be sign positive and less than 1.0. failure_rate: f64, - store: StoreClient, + state: Arc, batch_prover: BatchProver, mempool: SharedMempool, } @@ -212,20 +213,24 @@ impl BatchJob { &self, batch: SelectedBatch, ) -> Result<(SelectedBatch, BatchInputs), BuildBatchError> { - let block_references = batch + let block_references: BTreeSet<_> = batch .transactions() .iter() .map(Deref::deref) - .map(AuthenticatedTransaction::reference_block); - let unauthenticated_notes = batch + .map(AuthenticatedTransaction::reference_block) + .map(|(block_num, _)| block_num) + .collect(); + let unauthenticated_notes: BTreeSet<_> = batch .transactions() .iter() .map(Deref::deref) - .flat_map(AuthenticatedTransaction::unauthenticated_note_commitments); + .flat_map(AuthenticatedTransaction::unauthenticated_note_commitments) + .collect(); - self.store + self.state .get_batch_inputs(block_references, unauthenticated_notes) .await + .map_err(StoreError::from) .map_err(BuildBatchError::FetchBatchInputsFailed) .map(|inputs| (batch, inputs)) } diff --git a/crates/block-producer/src/block_builder/mod.rs b/crates/block-producer/src/block_builder/mod.rs index 00611ebcd2..19a38061d2 100644 --- a/crates/block-producer/src/block_builder/mod.rs +++ b/crates/block-producer/src/block_builder/mod.rs @@ -1,7 +1,10 @@ +use std::collections::BTreeSet; use std::ops::Deref; use std::sync::Arc; use anyhow::Context; +use miden_node_proto::domain::proof_request::BlockProofRequest; +use miden_node_store::state::State; use miden_node_utils::spawn::spawn_blocking_in_current_span; use miden_node_utils::tracing::OpenTelemetrySpanExt; use miden_protocol::batch::{OrderedBatches, ProvenBatch}; @@ -11,9 +14,8 @@ use miden_protocol::transaction::TransactionHeader; use tokio::time::Duration; use tracing::{Span, instrument}; -use crate::errors::BuildBlockError; +use crate::errors::{BuildBlockError, StoreError}; use crate::mempool::SharedMempool; -use crate::store::StoreClient; use crate::validator::BlockProducerValidatorClient; use crate::{COMPONENT, TelemetryInjectorExt}; @@ -29,19 +31,17 @@ pub struct BlockBuilder { /// Note: this _must_ be sign positive and less than 1.0. pub failure_rate: f64, - /// The store RPC client for committing blocks. - pub store: StoreClient, + /// Shared store state used to build and commit blocks. + pub state: Arc, /// The validator RPC client for validating blocks. pub validator: BlockProducerValidatorClient, } impl BlockBuilder { - /// Creates a new [`BlockBuilder`] with the given [`StoreClient`] and optional block prover URL. - /// - /// If the block prover URL is not set, the block builder will use the local block prover. + /// Creates a new [`BlockBuilder`] using the shared store [`State`]. pub fn new( - store: StoreClient, + state: Arc, validator: BlockProducerValidatorClient, block_interval: Duration, ) -> Self { @@ -49,7 +49,7 @@ impl BlockBuilder { block_interval, // Note: The range cannot be empty. failure_rate: 0.0, - store, + state, validator, } } @@ -181,15 +181,21 @@ impl BlockBuilder { let created_nullifiers_iter = batch_iter.map(Deref::deref).flat_map(ProvenBatch::created_nullifiers); + let account_ids = account_ids_iter.collect(); + let nullifiers = created_nullifiers_iter.collect(); + let unauthenticated_note_commitments = unauthenticated_notes_iter.collect(); + let reference_blocks = block_references_iter.collect(); + let inputs = self - .store + .state .get_block_inputs( - account_ids_iter, - created_nullifiers_iter, - unauthenticated_notes_iter, - block_references_iter, + account_ids, + nullifiers, + unauthenticated_note_commitments, + reference_blocks, ) .await + .map_err(StoreError::from) .map_err(BuildBlockError::GetBlockInputsFailed)?; // Check that the latest committed block in the store matches our expectations. @@ -265,17 +271,67 @@ impl BlockBuilder { ordered_batches: OrderedBatches, signed_block: SignedBlock, ) -> Result<(), BuildBlockError> { - self.store - .apply_block(&ordered_batches, &signed_block) + let block_num = signed_block.header().block_num(); + let header = signed_block.header().clone(); + let block_inputs = self.block_inputs_from_ordered_batches(&ordered_batches).await?; + let proving_inputs = BlockProofRequest { + tx_batches: ordered_batches, + block_header: header.clone(), + block_inputs, + }; + + self.state + .save_proving_inputs(block_num, &proving_inputs) + .await + .map_err(|err| { + BuildBlockError::StoreApplyBlockFailed(StoreError::SaveProvingInputsFailed(err)) + })?; + + self.state + .apply_block(signed_block) .await + .map_err(StoreError::from) .map_err(BuildBlockError::StoreApplyBlockFailed)?; - let (header, ..) = signed_block.into_parts(); - mempool.lock().map_err(BuildBlockError::MempoolPoisoned)?.commit_block(header); + mempool.lock().map_err(BuildBlockError::MempoolPoisoned)?.commit_block(&header); Ok(()) } + /// Retrieves block proving inputs for the ordered batches that were actually committed. + async fn block_inputs_from_ordered_batches( + &self, + batches: &OrderedBatches, + ) -> Result { + let mut account_ids = BTreeSet::new(); + let mut nullifiers = Vec::new(); + let mut unauthenticated_note_commitments = BTreeSet::new(); + let mut reference_blocks = BTreeSet::new(); + + for batch in batches.as_slice() { + account_ids.extend(batch.updated_accounts()); + nullifiers.extend(batch.created_nullifiers()); + reference_blocks.insert(batch.reference_block_num()); + + for note in batch.input_notes().iter() { + if let Some(header) = note.header() { + unauthenticated_note_commitments.insert(header.to_commitment()); + } + } + } + + self.state + .get_block_inputs( + account_ids.into_iter().collect(), + nullifiers, + unauthenticated_note_commitments, + reference_blocks, + ) + .await + .map_err(StoreError::from) + .map_err(BuildBlockError::GetBlockInputsFailed) + } + #[instrument(target = COMPONENT, name = "block_builder.rollback_block", skip_all)] fn rollback_block(mempool: &SharedMempool, block: BlockNumber) -> Result<(), BuildBlockError> { mempool.lock().map_err(BuildBlockError::MempoolPoisoned)?.rollback_block(block); diff --git a/crates/block-producer/src/domain/transaction.rs b/crates/block-producer/src/domain/transaction.rs index 74b179b656..55994df022 100644 --- a/crates/block-producer/src/domain/transaction.rs +++ b/crates/block-producer/src/domain/transaction.rs @@ -164,18 +164,21 @@ impl AuthenticatedTransaction { } /// Overrides the authentication height with the given value. + #[must_use] pub fn with_authentication_height(mut self, height: BlockNumber) -> Self { self.authentication_height = height; self } /// Overrides the store state with the given value. + #[must_use] pub fn with_store_state(mut self, state: Word) -> Self { self.store_account_state = Some(state); self } /// Unsets the store state. + #[must_use] pub fn with_empty_store_state(mut self) -> Self { self.store_account_state = None; self diff --git a/crates/block-producer/src/errors.rs b/crates/block-producer/src/errors.rs index 862a1a2ce9..9ce6f39481 100644 --- a/crates/block-producer/src/errors.rs +++ b/crates/block-producer/src/errors.rs @@ -1,6 +1,7 @@ use core::error::Error as CoreError; -use miden_node_proto::errors::{ConversionError, GrpcError}; +use miden_node_proto::errors::GrpcError; +use miden_node_store::{ApplyBlockError, DatabaseError, GetBatchInputsError, GetBlockInputsError}; use miden_protocol::Word; use miden_protocol::account::AccountId; use miden_protocol::block::BlockNumber; @@ -181,21 +182,19 @@ impl BuildBlockError { // Store errors // ================================================================================================= -/// Errors returned by the [`StoreClient`](crate::store::StoreClient). +/// Errors returned when querying or updating store state. #[derive(Debug, Error)] pub enum StoreError { #[error("account Id prefix already exists: {0}")] DuplicateAccountIdPrefix(AccountId), - #[error("gRPC client error")] - GrpcClientError(#[from] Box), - #[error("malformed response from store: {0}")] - MalformedResponse(String), - #[error("failed to parse response")] - DeserializationError(#[from] ConversionError), -} - -impl From for StoreError { - fn from(value: tonic::Status) -> Self { - StoreError::GrpcClientError(value.into()) - } + #[error("failed to retrieve transaction inputs from store")] + GetTransactionInputsFailed(#[from] DatabaseError), + #[error("failed to retrieve batch inputs from store")] + GetBatchInputsFailed(#[from] GetBatchInputsError), + #[error("failed to retrieve block inputs from store")] + GetBlockInputsFailed(#[from] GetBlockInputsError), + #[error("failed to save block proving inputs")] + SaveProvingInputsFailed(#[source] std::io::Error), + #[error("failed to apply block to store")] + ApplyBlockFailed(#[from] ApplyBlockError), } diff --git a/crates/block-producer/src/lib.rs b/crates/block-producer/src/lib.rs index 955aa23565..aaea9be590 100644 --- a/crates/block-producer/src/lib.rs +++ b/crates/block-producer/src/lib.rs @@ -9,13 +9,25 @@ mod batch_builder; mod block_builder; mod domain; mod mempool; -pub mod store; +mod store; mod validator; #[cfg(feature = "testing")] pub mod errors; #[cfg(not(feature = "testing"))] mod errors; +pub use domain::transaction::AuthenticatedTransaction; +pub use errors::{MempoolSubmissionError, StateConflict, StoreError}; +pub use mempool::{ + BatchBudget, + BlockBudget, + Mempool, + MempoolConfig, + MempoolPoisonError, + MempoolStats, + SharedMempool, +}; +pub use store::TransactionInputs; pub mod server; pub use server::BlockProducer; @@ -47,9 +59,6 @@ const SERVER_MEMPOOL_STATE_RETENTION: NonZeroUsize = NonZeroUsize::new(5).unwrap /// This rejects transactions which would likely expire before making it into a block. const SERVER_MEMPOOL_EXPIRATION_SLACK: u32 = 2; -/// The interval at which to update the cached mempool statistics. -const CACHED_MEMPOOL_STATS_UPDATE_INTERVAL: Duration = Duration::from_secs(5); - /// How often a block is created. pub const DEFAULT_BLOCK_INTERVAL: Duration = Duration::from_secs(3); diff --git a/crates/block-producer/src/mempool/mod.rs b/crates/block-producer/src/mempool/mod.rs index 9ec79c417b..02dc35c629 100644 --- a/crates/block-producer/src/mempool/mod.rs +++ b/crates/block-producer/src/mempool/mod.rs @@ -50,18 +50,16 @@ //! Recently committed batches are retained in `committed_blocks` according to the configured //! `state_retention`, giving the mempool enough local history to validate newly authenticated //! transactions even if the store and block producer momentarily disagree on the chain tip. -use std::collections::{HashSet, VecDeque}; +use std::collections::VecDeque; use std::num::NonZeroUsize; use std::sync::{Arc, LockResult, Mutex, MutexGuard}; -use miden_node_proto::domain::mempool::MempoolEvent; +use miden_node_proto::generated as proto; use miden_node_utils::ErrorReport; use miden_protocol::batch::{BatchId, ProvenBatch}; use miden_protocol::block::{BlockHeader, BlockNumber}; -use miden_protocol::transaction::{TransactionHeader, TransactionId}; -use subscription::SubscriptionProvider; +use miden_protocol::transaction::TransactionHeader; use thiserror::Error; -use tokio::sync::mpsc; use tracing::instrument; use crate::block_builder::SelectedBlock; @@ -80,7 +78,6 @@ mod budget; pub use budget::{BatchBudget, BlockBudget}; mod graph; -mod subscription; #[cfg(test)] mod tests; @@ -95,6 +92,29 @@ pub struct SharedMempool(Arc>); #[error("shared mempool lock is poisoned")] pub struct MempoolPoisonError; +/// Snapshot of the mempool's current state. +#[derive(Clone, Copy, Default)] +pub struct MempoolStats { + /// The mempool's current view of the chain tip height. + pub chain_tip: BlockNumber, + /// Number of transactions currently in the mempool waiting to be batched. + pub unbatched_transactions: u64, + /// Number of batches currently being proven. + pub proposed_batches: u64, + /// Number of proven batches waiting for block inclusion. + pub proven_batches: u64, +} + +impl From for proto::rpc::MempoolStats { + fn from(stats: MempoolStats) -> Self { + proto::rpc::MempoolStats { + unbatched_transactions: stats.unbatched_transactions, + proposed_batches: stats.proposed_batches, + proven_batches: stats.proven_batches, + } + } +} + #[derive(Debug, Clone, PartialEq)] pub struct MempoolConfig { /// The constraints each proposed block must adhere to. @@ -157,10 +177,41 @@ impl SharedMempool { /// Callers should minimise the amount of work performed while holding the lock to reduce /// contention with other subsystems that need to access the pool. #[instrument(target = COMPONENT, name = "mempool.lock", skip_all, err)] - pub fn lock(&self) -> Result, MempoolPoisonError> { + pub(crate) fn lock(&self) -> Result, MempoolPoisonError> { let result: LockResult> = self.0.lock(); result.map_err(|_| MempoolPoisonError) } + + /// Adds an authenticated transaction to the mempool. + pub fn add_transaction( + &self, + tx: Arc, + ) -> Result { + self.lock() + .map_err(MempoolSubmissionError::MempoolPoisoned)? + .add_transaction(tx) + } + + /// Adds a user-provided batch of authenticated transactions to the mempool. + pub fn add_user_batch( + &self, + txs: &[Arc], + ) -> Result { + self.lock() + .map_err(MempoolSubmissionError::MempoolPoisoned)? + .add_user_batch(txs) + } + + /// Returns a snapshot of the current mempool statistics. + pub fn stats(&self) -> Result { + let mempool = self.lock()?; + Ok(MempoolStats { + chain_tip: mempool.chain_tip(), + unbatched_transactions: mempool.unbatched_transactions_count() as u64, + proposed_batches: mempool.proposed_batches_count() as u64, + proven_batches: mempool.proven_batches_count() as u64, + }) + } } // MEMPOOL @@ -183,7 +234,6 @@ pub struct Mempool { committed_chain_tip: BlockNumber, config: MempoolConfig, - subscription: subscription::SubscriptionProvider, } impl Mempool { @@ -199,7 +249,6 @@ impl Mempool { Self { config, committed_chain_tip: chain_tip, - subscription: SubscriptionProvider::new(chain_tip), transactions: graph::TransactionGraph::default(), batches: graph::BatchGraph::default(), pending_block: None, @@ -221,12 +270,6 @@ impl Mempool { /// Adds a transaction to the mempool. /// - /// Sends a [`MempoolEvent::TransactionAdded`] event to subscribers. - /// - /// # Returns - /// - /// Returns the current block height. - /// /// # Errors /// /// Returns an error if the transaction would exceed the mempool capacity or if its initial @@ -236,7 +279,7 @@ impl Mempool { reason = "Not impactful, and we may want ownership in the future" )] #[instrument(target = COMPONENT, name = "mempool.add_transaction", skip_all, fields(tx=%tx.id()))] - pub fn add_transaction( + pub(crate) fn add_transaction( &mut self, tx: Arc, ) -> Result { @@ -251,14 +294,13 @@ impl Mempool { self.transactions .append(Arc::clone(&tx)) .map_err(MempoolSubmissionError::StateConflict)?; - self.subscription.transaction_added(&tx); self.inject_telemetry(); Ok(self.committed_chain_tip) } #[instrument(target = COMPONENT, name = "mempool.add_user_batch", skip_all)] - pub fn add_user_batch( + pub(crate) fn add_user_batch( &mut self, txs: &[Arc], ) -> Result { @@ -286,9 +328,6 @@ impl Mempool { .append_user_batch(txs) .map_err(MempoolSubmissionError::StateConflict)?; - for tx in txs { - self.subscription.transaction_added(tx); - } self.inject_telemetry(); Ok(self.committed_chain_tip) @@ -300,7 +339,7 @@ impl Mempool { /// /// Returns `None` if no transactions are available. #[instrument(target = COMPONENT, name = "mempool.select_batch", skip_all)] - pub fn select_batch(&mut self) -> Option { + pub(crate) fn select_batch(&mut self) -> Option { let batch = self.transactions.select_batch(self.config.batch_budget)?; if let Err(err) = self.batches.append(batch.clone()) { panic!("failed to append batch to dependency graph: {}", err.as_report()); @@ -315,7 +354,7 @@ impl Mempool { /// transactions have their failure count incremented, reverting them if they now exceed the /// failure limit. #[instrument(target = COMPONENT, name = "mempool.rollback_batch", skip_all)] - pub fn rollback_batch(&mut self, batch: BatchId) { + pub(crate) fn rollback_batch(&mut self, batch: BatchId) { // Guards against bugs in the proof scheduler where a retry results in multiple results // coming back for the same batch. If the batch previously succeeded, then yanking it would // corrupt the mempool since the batch might be in a block. @@ -341,8 +380,7 @@ impl Mempool { // could check this precondition above. if let Some(batch) = reverted_batches.iter().find(|reverted| reverted.id() == batch) { let failed_txs = batch.transactions().iter().map(|tx| tx.id()); - let reverted_txs = self.transactions.increment_failure_count(failed_txs); - self.subscription.txs_reverted(reverted_txs); + self.transactions.increment_failure_count(failed_txs); } self.inject_telemetry(); @@ -350,7 +388,7 @@ impl Mempool { /// Marks a batch as proven if it exists. #[instrument(target = COMPONENT, name = "mempool.commit_batch", skip_all)] - pub fn commit_batch(&mut self, proof: Arc) { + pub(crate) fn commit_batch(&mut self, proof: Arc) { self.batches.submit_proof(proof); self.inject_telemetry(); } @@ -365,7 +403,7 @@ impl Mempool { /// /// Panics if there is already a block in flight. #[instrument(target = COMPONENT, name = "mempool.select_block", skip_all)] - pub fn select_block(&mut self) -> SelectedBlock { + pub(crate) fn select_block(&mut self) -> SelectedBlock { assert!( self.pending_block.is_none(), "block {} is already in progress", @@ -385,39 +423,26 @@ impl Mempool { /// The pool will mark the associated batches and transactions as committed, and prune stale /// committed data, and purge transactions that are now considered expired. /// - /// Sends a [`MempoolEvent::BlockCommitted`] event to subscribers, as well as a - /// [`MempoolEvent::TransactionsReverted`] for transactions that are now considered expired. - /// /// On success the internal state is updated in place: the chain tip advances, expired data is - /// pruned, and subscribers are notified about the committed block and any reverted - /// transactions. + /// pruned, and transactions that are no longer valid are reverted. /// /// # Panics /// /// Panics if there is no matching block in flight. #[instrument(target = COMPONENT, name = "mempool.commit_block", skip_all)] - pub fn commit_block(&mut self, block_header: BlockHeader) { + pub(crate) fn commit_block(&mut self, block_header: &BlockHeader) { assert_eq!(self.committed_chain_tip.child(), block_header.block_num()); let block = self .pending_block .take_if(|pending| pending.block_number == block_header.block_num()) .expect("block must be in progress to commit"); - let tx_ids = block - .batches - .iter() - .flat_map(|batch| batch.transactions().as_slice().iter()) - .map(miden_protocol::transaction::TransactionHeader::id) - .collect(); - self.committed_chain_tip = self.committed_chain_tip.child(); - self.subscription.block_committed(block_header, tx_ids); self.committed_blocks.push_back(block); self.prune_oldest_block(); - let reverted_tx_ids = self.revert_expired(); - self.subscription.txs_reverted(reverted_tx_ids); + self.revert_expired(); self.inject_telemetry(); } @@ -427,13 +452,11 @@ impl Mempool { /// Additionally, the transactions from this block have their failure count incremented, /// potentially reverting them if they exceed the failure limit. /// - /// Sends a [`MempoolEvent::TransactionsReverted`] event to subscribers. - /// /// # Panics /// /// Panics if there is no matching block in flight. #[instrument(target = COMPONENT, name = "mempool.rollback_block", skip_all)] - pub fn rollback_block(&mut self, block: BlockNumber) { + pub(crate) fn rollback_block(&mut self, block: BlockNumber) { // FIXME: We should consider a more robust check here to identify the block by a hash. // If multiple jobs are possible, then so are multiple variants with the same // block number. @@ -456,24 +479,11 @@ impl Mempool { .batches .iter() .flat_map(|batch| batch.transactions().as_slice().iter().map(TransactionHeader::id)); - let reverted_txs = self.transactions.increment_failure_count(failed_txs); + self.transactions.increment_failure_count(failed_txs); - self.subscription.txs_reverted(reverted_txs); self.inject_telemetry(); } - // EVENTS & SUBSCRIPTIONS - // -------------------------------------------------------------------------------------------- - - /// Creates a subscription to [`MempoolEvent`] which will be emitted in the order they - /// occur. - /// - /// Only emits events which occurred after the current committed block. - #[instrument(target = COMPONENT, name = "mempool.subscribe", skip_all)] - pub fn subscribe(&mut self) -> mpsc::Receiver { - self.subscription.subscribe() - } - // STATS & INSPECTION // -------------------------------------------------------------------------------------------- @@ -549,12 +559,12 @@ impl Mempool { /// /// Transactions from batches are requeued. Expired transactions and their descendants are then /// reverted as well. - fn revert_expired(&mut self) -> HashSet { + fn revert_expired(&mut self) { let batches = self.batches.revert_expired(self.chain_tip()); for batch in batches { self.transactions.requeue_transactions(&batch); } - self.transactions.revert_expired(self.chain_tip()) + self.transactions.revert_expired(self.chain_tip()); } /// Rejects authentication heights that fall outside the overlap guaranteed by the locally diff --git a/crates/block-producer/src/mempool/subscription.rs b/crates/block-producer/src/mempool/subscription.rs deleted file mode 100644 index d38ea0fe0f..0000000000 --- a/crates/block-producer/src/mempool/subscription.rs +++ /dev/null @@ -1,213 +0,0 @@ -use std::collections::{BTreeMap, HashSet}; -use std::ops::Mul; - -use miden_node_proto::domain::mempool::MempoolEvent; -use miden_protocol::block::{BlockHeader, BlockNumber}; -use miden_protocol::transaction::{OutputNote, TransactionId}; -use miden_standards::note::NetworkNoteExt; -use tokio::sync::mpsc; - -use crate::domain::transaction::AuthenticatedTransaction; - -/// Coordinates mempool event delivery for a single subscriber. -/// -/// Retains the active subscriber channel (if any) and an in-memory queue of uncommitted -/// transaction events so new subscriptions can immediately replay pending updates. -#[derive(Clone, Debug)] -pub(crate) struct SubscriptionProvider { - /// The latest event subscription, if any. - /// - /// The only current interested party is the network transaction builder, so one subscription - /// is enough. - subscription: Option>, - - /// The latest committed block number. - /// - /// This is used to ensure synchronicity with new subscribers. - chain_tip: BlockNumber, - - /// Tracks all uncommitted transaction events. These events must be resent on start - /// of a new subscription since the subscriber will only have data up to the latest - /// committed block and would otherwise miss these uncommitted transactions. - /// - /// The size is bounded by removing events as they are committed or reverted, and as - /// such this is always bound to the current amount of inflight transactions. - inflight_txs: InflightTransactions, -} - -impl PartialEq for SubscriptionProvider { - fn eq(&self, other: &Self) -> bool { - self.chain_tip == other.chain_tip && self.inflight_txs == other.inflight_txs - } -} - -impl SubscriptionProvider { - pub fn new(chain_tip: BlockNumber) -> Self { - Self { - chain_tip, - subscription: None, - inflight_txs: InflightTransactions::default(), - } - } - - /// Creates a new [`MempoolEvent`] subscription. - /// - /// This replaces any existing subscription. - /// Any previous subscriber is dropped and must resubscribe to continue receiving events. - pub fn subscribe(&mut self) -> mpsc::Receiver { - // We should leave enough space to at least send the uncommitted events (plus some extra). - let capacity = self.inflight_txs.len().mul(2).max(1024); - let (tx, rx) = mpsc::channel(capacity); - self.subscription.replace(tx); - - // Send each uncommitted tx event in chronological order. - // - // The ordering is guaranteed by the tracker. - // - // We don't clear the queue so that they're available for other new subscriptions. - // The queue size is managed by instead removing events once they're committed or reverted. - for tx in self.inflight_txs.iter() { - Self::send_event(&mut self.subscription, tx.clone()); - } - - rx - } - - /// Records a newly added transaction in the inflight queue and forwards the event to the - /// subscriber. - pub(super) fn transaction_added(&mut self, tx: &AuthenticatedTransaction) { - let id = tx.id(); - let nullifiers = tx.nullifiers().collect(); - let network_notes = tx - .output_notes() - .filter_map(|note| match note { - OutputNote::Public(inner) => { - inner.clone().into_note().into_account_target_network_note().ok() - }, - OutputNote::Private(_) => None, - }) - .collect(); - let account_delta = - tx.account_id().is_network().then(|| tx.account_update().details().clone()); - let event = MempoolEvent::TransactionAdded { - id, - nullifiers, - network_notes, - account_delta, - }; - - self.inflight_txs.insert(event.clone()); - Self::send_event(&mut self.subscription, event); - } - - /// Records a committed block, prunes replayed transactions, and forwards the event so future - /// subscribers continue from the latest chain tip. - pub(super) fn block_committed(&mut self, header: BlockHeader, txs: Vec) { - self.chain_tip = header.block_num(); - for tx in &txs { - self.inflight_txs.remove(tx); - } - - Self::send_event( - &mut self.subscription, - MempoolEvent::BlockCommitted { header: Box::new(header), txs }, - ); - } - - /// Removes reverted transactions from the inflight queue and notifies the subscriber so they - /// can drop or retry the affected items. - pub(super) fn txs_reverted(&mut self, txs: HashSet) { - for tx in &txs { - self.inflight_txs.remove(tx); - } - Self::send_event(&mut self.subscription, MempoolEvent::TransactionsReverted(txs)); - } - - /// Sends a [`MempoolEvent`] to the subscriber, if any. - /// - /// If the send fails, the subscription is cancelled and the event is dropped, so callers must - /// resubscribe to continue receiving updates. - /// - /// This function does not take `&self` to work-around borrowing issues - /// where both the sender and inflight events need to be borrowed at the same time. - fn send_event(subscription: &mut Option>, event: MempoolEvent) { - let Some(sender) = subscription else { - return; - }; - - // If sending fails, end the subscription to prevent desync. - if let Err(error) = sender.try_send(event) { - tracing::warn!(%error, "mempool subscription failed, cancelling subscription"); - subscription.take(); - } - } -} - -/// Maintains an ordered index of [`MempoolEvent::TransactionAdded`] events which can be efficiently -/// added and removed. -/// -/// This is used to track events which need to be sent on fresh subscriptions. -/// -/// The events can be iterated over in chronological order. -#[derive(Default, Clone, Debug, PartialEq)] -struct InflightTransactions { - /// [`MempoolEvent::TransactionAdded`] events which are still inflight i.e. have not been - /// committed or reverted. - /// - /// These events need to be transmitted when a subscription is started, since the subscriber - /// only has the committed state. - /// - /// A [`BTreeMap`] is used to maintain event ordering while allowing for efficient removals of - /// committed or reverted transactions. - /// - /// The key is auto-incremented on each new insert to support this event ordering. - /// - /// A reverse lookup index is maintained in `index`. - txs: BTreeMap, - - /// A reverse lookup index for `txs` which allows for efficient removal of committed or reverted - /// events. - index: BTreeMap, -} - -impl InflightTransactions { - /// Adds a new transaction event to the tracker. - /// - /// # Panics - /// - /// Panics if: - /// - the event is not a [`MempoolEvent::TransactionAdded`], or - /// - the event already exists - fn insert(&mut self, tx: MempoolEvent) { - let MempoolEvent::TransactionAdded { id, .. } = &tx else { - panic!("Cannot submit a non-tx event to inflight transaction event tracker"); - }; - - let idx = self.txs.last_key_value().map(|(&k, _v)| k + 1).unwrap_or_default(); - assert!( - self.index.insert(*id, idx).is_none(), - "transaction event already exists in tracker" - ); - self.txs.insert(idx, tx); - } - - /// Removes a transaction from the tracker. - /// - /// # Panics - /// - /// Panics if the transaction was not being tracked. - fn remove(&mut self, tx: &TransactionId) { - let idx = self.index.remove(tx).expect("transaction to remove should be tracked"); - self.txs.remove(&idx); - } - - /// An iterator over all transaction events in the order they were added. - fn iter(&self) -> impl Iterator { - self.txs.values() - } - - /// The number of transaction events. - fn len(&self) -> usize { - self.txs.len() - } -} diff --git a/crates/block-producer/src/mempool/tests.rs b/crates/block-producer/src/mempool/tests.rs index 1eaf725d52..b5eb66e5bf 100644 --- a/crates/block-producer/src/mempool/tests.rs +++ b/crates/block-producer/src/mempool/tests.rs @@ -167,7 +167,7 @@ fn block_commit_reverts_expired_txns() { // Create and commit the block which should revert the above tx. let block = uut.select_block(); let arb_header = BlockHeader::mock(block.block_number, None, None, &[], Word::empty()); - uut.commit_block(arb_header.clone()); + uut.commit_block(&arb_header); // A reverted transaction behaves as if it never existed. reference.add_transaction(tx_to_commit.clone()).unwrap(); @@ -176,7 +176,7 @@ fn block_commit_reverts_expired_txns() { tx_to_commit.raw_proven_transaction() ]))); reference.select_block(); - reference.commit_block(arb_header); + reference.commit_block(&arb_header); assert_eq!(uut, reference); } @@ -188,7 +188,7 @@ fn empty_block_commitment() { for _ in 0..3 { let block = uut.select_block(); let arb_header = BlockHeader::mock(block.block_number, None, None, &[], Word::empty()); - uut.commit_block(arb_header); + uut.commit_block(&arb_header); } } @@ -233,11 +233,11 @@ fn pruned_committed_notes_are_authenticated_for_inflight_descendants() { let block = uut.select_block(); let header = BlockHeader::mock(block.block_number, None, None, &[], Word::empty()); - uut.commit_block(header); + uut.commit_block(&header); let block = uut.select_block(); let header = BlockHeader::mock(block.block_number, None, None, &[], Word::empty()); - uut.commit_block(header); + uut.commit_block(&header); let child_batch = uut.select_batch().unwrap(); @@ -251,7 +251,7 @@ fn pruned_committed_notes_are_authenticated_for_inflight_descendants() { #[should_panic] fn block_commitment_is_rejected_if_no_block_is_in_flight() { let arb_header = BlockHeader::mock(0, None, None, &[], Word::empty()); - Mempool::for_tests().0.commit_block(arb_header); + Mempool::for_tests().0.commit_block(&arb_header); } #[test] diff --git a/crates/block-producer/src/mempool/tests/add_transaction.rs b/crates/block-producer/src/mempool/tests/add_transaction.rs index ae70c5f612..dbb82677ad 100644 --- a/crates/block-producer/src/mempool/tests/add_transaction.rs +++ b/crates/block-producer/src/mempool/tests/add_transaction.rs @@ -70,7 +70,7 @@ mod tx_expiration { for _ in 0..slack + 10 { let block = uut.select_block(); let header = BlockHeader::mock(block.block_number, None, None, &[], Word::default()); - uut.commit_block(header); + uut.commit_block(&header); } uut @@ -144,7 +144,7 @@ mod authentication_height { for _ in 0..retention + 10 { let block = uut.select_block(); let header = BlockHeader::mock(block.block_number, None, None, &[], Word::default()); - uut.commit_block(header); + uut.commit_block(&header); } uut diff --git a/crates/block-producer/src/server/mod.rs b/crates/block-producer/src/server/mod.rs index 0e9876d59d..c0bf66d843 100644 --- a/crates/block-producer/src/server/mod.rs +++ b/crates/block-producer/src/server/mod.rs @@ -1,54 +1,28 @@ use std::collections::HashMap; -use std::net::SocketAddr; use std::num::NonZeroUsize; use std::sync::Arc; use std::time::Duration; -use anyhow::{Context, Result}; -use futures::StreamExt; -use miden_node_proto::domain::mempool::MempoolEvent; -use miden_node_proto::generated::block_producer::api_server; -use miden_node_proto::generated::{self as proto}; -use miden_node_proto_build::block_producer_api_descriptor; -use miden_node_utils::clap::GrpcOptionsInternal; -use miden_node_utils::formatting::{format_input_notes, format_output_notes}; -use miden_node_utils::panic::{CatchPanicLayer, catch_panic_layer_fn}; -use miden_node_utils::tracing::grpc::grpc_trace_fn; -use miden_protocol::batch::ProposedBatch; -use miden_protocol::block::BlockNumber; -use miden_protocol::transaction::ProvenTransaction; -use miden_protocol::utils::serde::Deserializable; -use tokio::net::TcpListener; -use tokio::sync::{Mutex, RwLock}; -use tokio_stream::wrappers::{ReceiverStream, TcpListenerStream}; -use tonic::Status; -use tower_http::trace::TraceLayer; -use tracing::{debug, error, info, instrument}; +use miden_node_store::state::{Finality, State}; +use tracing::info; use url::Url; use crate::batch_builder::BatchBuilder; use crate::block_builder::BlockBuilder; -use crate::domain::transaction::AuthenticatedTransaction; -use crate::errors::{BlockProducerError, MempoolSubmissionError, StoreError}; -use crate::mempool::{BatchBudget, BlockBudget, Mempool, MempoolConfig, SharedMempool}; -use crate::store::StoreClient; +use crate::errors::BlockProducerError; +use crate::mempool::{BatchBudget, BlockBudget, Mempool, MempoolConfig}; use crate::validator::BlockProducerValidatorClient; -use crate::{CACHED_MEMPOOL_STATS_UPDATE_INTERVAL, COMPONENT, SERVER_NUM_BATCH_BUILDERS}; +use crate::{COMPONENT, SERVER_NUM_BATCH_BUILDERS}; #[cfg(test)] mod tests; -/// The block producer server. +/// The block producer component. /// -/// Specifies how to connect to the store, batch prover, and block prover components. -/// The connection to the store is established at startup and retried with exponential backoff -/// until the store becomes available. Once the connection is established, the block producer -/// will start serving requests. +/// Specifies the shared store state and how to connect to validator and prover components. pub struct BlockProducer { - /// The address of the block producer component. - pub block_producer_address: SocketAddr, - /// The address of the store component. - pub store_url: Url, + /// Shared store state used by the batch and block builders. + pub state: Arc, /// The address of the validator component. pub validator_url: Url, /// The address of the batch prover component. @@ -61,9 +35,6 @@ pub struct BlockProducer { pub max_txs_per_batch: usize, /// The maximum number of batches per block. pub max_batches_per_block: usize, - /// Server-side gRPC options. - pub grpc_options: GrpcOptionsInternal, - /// The maximum number of inflight transactions allowed in the mempool at once. pub mempool_tx_capacity: NonZeroUsize, } @@ -72,53 +43,22 @@ pub struct BlockProducer { // ================================================================================================ impl BlockProducer { - /// Serves the block-producer RPC API, the batch-builder and the block-builder. + /// Runs the batch-builder and block-builder. /// /// Executes in place (i.e. not spawned) and will run indefinitely until a fatal error is /// encountered. pub async fn serve(self) -> anyhow::Result<()> { - info!(target: COMPONENT, endpoint=?self.block_producer_address, store=%self.store_url, "Initializing server"); - let store = StoreClient::new(self.store_url.clone()); + info!(target: COMPONENT, "Initializing block producer"); let validator = BlockProducerValidatorClient::new(self.validator_url.clone()); - // Retry fetching the chain tip from the store until it succeeds. - let mut retries_counter = 0; - let chain_tip = loop { - match store.latest_header().await { - Err(StoreError::GrpcClientError(err)) => { - // exponential backoff with base 500ms and max 30s - let backoff = Duration::from_millis(500) - .saturating_mul(1 << retries_counter) - .min(Duration::from_secs(30)); - - error!( - store = %self.store_url, - ?backoff, - %retries_counter, - %err, - "store connection failed while fetching chain tip, retrying" - ); - - retries_counter += 1; - tokio::time::sleep(backoff).await; - }, - Ok(header) => break header.block_num(), - Err(e) => { - error!(target: COMPONENT, %e, "failed to fetch chain tip from store"); - return Err(e.into()); - }, - } - }; - - let listener = TcpListener::bind(self.block_producer_address) - .await - .context("failed to bind to block producer address")?; + let chain_tip = self.state.chain_tip(Finality::Committed).await; - info!(target: COMPONENT, "Server initialized"); + info!(target: COMPONENT, chain_tip = %chain_tip, "Block producer initialized"); - let block_builder = BlockBuilder::new(store.clone(), validator, self.block_interval); + let block_builder = + BlockBuilder::new(Arc::clone(&self.state), validator, self.block_interval); let batch_builder = BatchBuilder::new( - store.clone(), + Arc::clone(&self.state), SERVER_NUM_BATCH_BUILDERS, self.batch_prover_url, self.batch_interval, @@ -134,26 +74,12 @@ impl BlockProducer { }; let mempool = Mempool::shared(chain_tip, mempool); - // Spawn rpc server and batch and block provers. - // - // These communicate indirectly via a shared mempool. + // Spawn batch and block builders. These communicate indirectly via a shared mempool. // // These should run forever, so we combine them into a joinset so that if // any complete or fail, we can shutdown the rest (somewhat) gracefully. let mut tasks = tokio::task::JoinSet::new(); - // Launch the gRPC server. - let rpc_id = tasks - .spawn({ - let mempool = mempool.clone(); - async move { - BlockProducerRpcServer::new(mempool, store) - .serve(listener, self.grpc_options) - .await - } - }) - .id(); - let batch_builder_id = tasks .spawn({ let mempool = mempool.clone(); @@ -170,7 +96,6 @@ impl BlockProducer { let task_ids = HashMap::from([ (batch_builder_id, "batch-builder"), (block_builder_id, "block-builder"), - (rpc_id, "rpc"), ]); // Wait for any task to end. They should run indefinitely, so this is an unexpected result. @@ -195,302 +120,3 @@ impl BlockProducer { .and_then(|x| x)? } } - -// BLOCK PRODUCER RPC SERVER -// ================================================================================================ - -/// Serves the block producer's RPC [api](api_server::Api). -struct BlockProducerRpcServer { - /// The mutex effectively rate limits incoming transactions into the mempool by forcing them - /// through a queue. - /// - /// This gives mempool users such as the batch and block builders equal footing with __all__ - /// incoming transactions combined. Without this incoming transactions would greatly restrict - /// the block-producers usage of the mempool. - mempool: Mutex, - - store: StoreClient, - - /// Cached mempool statistics that are updated periodically to avoid locking the mempool for - /// each status request. - cached_mempool_stats: Arc>, -} - -impl BlockProducerRpcServer { - pub fn new(mempool: SharedMempool, store: StoreClient) -> Self { - Self { - mempool: Mutex::new(mempool), - store, - cached_mempool_stats: Arc::new(RwLock::new(MempoolStats::default())), - } - } - - // SERVER STARTUP - // -------------------------------------------------------------------------------------------- - - async fn serve( - self, - listener: TcpListener, - grpc_options: GrpcOptionsInternal, - ) -> anyhow::Result<()> { - // Start background task to periodically update cached mempool stats - self.spawn_mempool_stats_updater().await; - - let reflection_service = tonic_reflection::server::Builder::configure() - .register_file_descriptor_set(block_producer_api_descriptor()) - .build_v1() - .context("failed to build reflection service")?; - - // Build the gRPC server with the API service and trace layer. - - tonic::transport::Server::builder() - .accept_http1(true) - .timeout(grpc_options.request_timeout) - .layer(CatchPanicLayer::custom(catch_panic_layer_fn)) - .layer(TraceLayer::new_for_grpc().make_span_with(grpc_trace_fn)) - .add_service(api_server::ApiServer::new(self)) - .add_service(reflection_service) - .serve_with_incoming(TcpListenerStream::new(listener)) - .await - .context("failed to serve block producer API") - } - - /// Starts a background task that periodically updates the cached mempool statistics. - /// - /// This prevents the need to lock the mempool for each status request. - async fn spawn_mempool_stats_updater(&self) { - let cached_mempool_stats = Arc::clone(&self.cached_mempool_stats); - let mempool = self.mempool.lock().await.clone(); - - tokio::spawn(async move { - let mut interval = tokio::time::interval(CACHED_MEMPOOL_STATS_UPDATE_INTERVAL); - - loop { - interval.tick().await; - - let (chain_tip, unbatched_transactions, proposed_batches, proven_batches) = { - let Ok(mempool) = mempool.lock() else { - tracing::error!("mempool lock poisoned, stopping mempool stats updater"); - return; - }; - ( - mempool.chain_tip(), - mempool.unbatched_transactions_count() as u64, - mempool.proposed_batches_count() as u64, - mempool.proven_batches_count() as u64, - ) - }; - - let mut cache = cached_mempool_stats.write().await; - *cache = MempoolStats { - chain_tip, - unbatched_transactions, - proposed_batches, - proven_batches, - }; - } - }); - } - - // RPC ENDPOINTS - // -------------------------------------------------------------------------------------------- - - #[instrument( - target = COMPONENT, - name = "block_producer.server.submit_proven_tx", - skip_all, - err - )] - #[expect(clippy::let_and_return)] - async fn submit_proven_tx( - &self, - request: proto::transaction::ProvenTransaction, - ) -> Result { - debug!(target: COMPONENT, ?request); - - let tx = ProvenTransaction::read_from_bytes(&request.transaction) - .map_err(MempoolSubmissionError::DeserializationFailed)?; - - let tx_id = tx.id(); - - debug!( - target: COMPONENT, - tx_id = %tx_id.to_hex(), - account_id = %tx.account_id().to_hex(), - initial_state_commitment = %tx.account_update().initial_state_commitment(), - final_state_commitment = %tx.account_update().final_state_commitment(), - input_notes = %format_input_notes(tx.input_notes()), - output_notes = %format_output_notes(tx.output_notes()), - ref_block_commitment = %tx.ref_block_commitment(), - "Deserialized transaction" - ); - debug!(target: COMPONENT, proof = ?tx.proof()); - - let inputs = self - .store - .get_tx_inputs(&tx) - .await - .map_err(MempoolSubmissionError::StoreConnectionFailed)?; - - // SAFETY: we assume that the rpc component has verified the transaction proof already. - let tx = AuthenticatedTransaction::new_unchecked(Arc::new(tx), inputs) - .map(Arc::new) - .map_err(MempoolSubmissionError::StateConflict)?; - - let shared_mempool = self.mempool.lock().await; - // We need the let binding here to avoid E0597 `shared_mempool` does not live long enough - let result = shared_mempool - .lock() - .map_err(MempoolSubmissionError::MempoolPoisoned)? - .add_transaction(tx) - .map(Into::into); - result - } - - #[instrument( - target = COMPONENT, - name = "block_producer.server.submit_proven_tx_batch", - skip_all, - err - )] - #[expect(clippy::let_and_return)] - async fn submit_proven_tx_batch( - &self, - request: proto::transaction::TransactionBatch, - ) -> Result { - let proposed = request - .proposed_batch - .expect("proposed batch existence is enforced by RPC component"); - let batch = ProposedBatch::read_from_bytes(&proposed) - .map_err(MempoolSubmissionError::DeserializationFailed)?; - - // We assume that the rpc component has verified everything, including the transaction - // proofs. - - let mut txs = Vec::with_capacity(batch.transactions().len()); - for tx in batch.transactions() { - let inputs = self - .store - .get_tx_inputs(tx) - .await - .map_err(MempoolSubmissionError::StoreConnectionFailed)?; - - // SAFETY: We assume that the rpc component has verified the transaction proofs, as well - // as the batch integrity itself. - let tx = AuthenticatedTransaction::new_unchecked(Arc::clone(tx), inputs) - .map(Arc::new) - .map_err(MempoolSubmissionError::StateConflict)?; - txs.push(tx); - } - - let shared_mempool = self.mempool.lock().await; - // We need the let binding here to avoid E0597 `shared_mempool` does not live long enough - let result = shared_mempool - .lock() - .map_err(MempoolSubmissionError::MempoolPoisoned)? - .add_user_batch(&txs) - .map(Into::into); - result - } -} - -#[tonic::async_trait] -impl api_server::Api for BlockProducerRpcServer { - type MempoolSubscriptionStream = MempoolEventSubscription; - - async fn submit_proven_tx( - &self, - request: tonic::Request, - ) -> Result, Status> { - self.submit_proven_tx(request.into_inner()) - .await - .map(tonic::Response::new) - // This Status::from mapping takes care of hiding internal errors. - .map_err(Into::into) - } - - async fn submit_proven_tx_batch( - &self, - request: tonic::Request, - ) -> Result, Status> { - self.submit_proven_tx_batch(request.into_inner()) - .await - .map(tonic::Response::new) - // This Status::from mapping takes care of hiding internal errors. - .map_err(Into::into) - } - - async fn status( - &self, - _request: tonic::Request<()>, - ) -> Result, Status> { - let mempool_stats = *self.cached_mempool_stats.read().await; - - Ok(tonic::Response::new(proto::rpc::BlockProducerStatus { - version: env!("CARGO_PKG_VERSION").to_string(), - status: "connected".to_string(), - chain_tip: mempool_stats.chain_tip.as_u32(), - mempool_stats: Some(mempool_stats.into()), - })) - } - - async fn mempool_subscription( - &self, - _request: tonic::Request<()>, - ) -> Result, tonic::Status> { - let shared_mempool = self.mempool.lock().await; - let subscription = shared_mempool - .lock() - .map_err(|err| tonic::Status::internal(err.to_string()))? - .subscribe(); - let subscription = ReceiverStream::new(subscription); - - Ok(tonic::Response::new(MempoolEventSubscription { inner: subscription })) - } -} - -// MEMPOOL SUBSCRIPTION -// ================================================================================================ - -struct MempoolEventSubscription { - inner: ReceiverStream, -} - -impl tokio_stream::Stream for MempoolEventSubscription { - type Item = Result; - - fn poll_next( - mut self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - self.inner - .poll_next_unpin(cx) - .map(|x| x.map(proto::block_producer::MempoolEvent::from).map(Result::Ok)) - } -} - -// MEMPOOL STATISTICS -// ================================================================================================ - -/// Mempool statistics that are updated periodically to avoid locking the mempool. -#[derive(Clone, Copy, Default)] -struct MempoolStats { - /// The mempool's current view of the chain tip height. - chain_tip: BlockNumber, - /// Number of transactions currently in the mempool waiting to be batched. - unbatched_transactions: u64, - /// Number of batches currently being proven. - proposed_batches: u64, - /// Number of proven batches waiting for block inclusion. - proven_batches: u64, -} - -impl From for proto::rpc::MempoolStats { - fn from(stats: MempoolStats) -> Self { - proto::rpc::MempoolStats { - unbatched_transactions: stats.unbatched_transactions, - proposed_batches: stats.proposed_batches, - proven_batches: stats.proven_batches, - } - } -} diff --git a/crates/block-producer/src/server/tests.rs b/crates/block-producer/src/server/tests.rs index e5a7a91291..008a321191 100644 --- a/crates/block-producer/src/server/tests.rs +++ b/crates/block-producer/src/server/tests.rs @@ -1,59 +1,23 @@ use std::num::NonZeroUsize; +use std::path::Path; +use std::sync::Arc; use std::time::Duration; -use miden_node_proto::generated::block_producer::api_client as block_producer_client; -use miden_node_store::{DEFAULT_MAX_CONCURRENT_PROOFS, GenesisState, Store, StoreMode}; +use miden_node_store::state::State; +use miden_node_store::{DatabaseOptions, GenesisState, Store}; use miden_node_utils::clap::{GrpcOptionsInternal, StorageOptions}; use miden_node_utils::fee::test_fee_params; use miden_protocol::testing::random_secret_key::random_secret_key; use miden_validator::{Validator, ValidatorSigner}; use tokio::net::TcpListener; +use tokio::task; use tokio::time::sleep; -use tokio::{runtime, task}; -use tonic::transport::{Channel, Endpoint}; use url::Url; use crate::{BlockProducer, DEFAULT_MAX_BATCHES_PER_BLOCK, DEFAULT_MAX_TXS_PER_BATCH}; -/// A wrapper around the store runtime and data directory. -/// -/// Guarantees that the store runtime is shut down _before_ the data directory is dropped and thus removed. -struct TestStore { - runtime: Option, - _data_directory: tempfile::TempDir, -} - -impl Drop for TestStore { - fn drop(&mut self) { - if let Some(runtime) = self.runtime.take() { - std::thread::spawn(move || { - runtime.shutdown_timeout(Duration::from_millis(500)); - }) - .join() - .expect("store runtime shutdown thread should complete"); - } - } -} - -/// Tests that the block producer starts up correctly even when the store is not initially -/// available. The block producer should retry with exponential backoff until the store becomes -/// available, then start serving requests. -#[tokio::test] -async fn block_producer_startup_is_robust_to_network_failures() { - // get the addresses for the store and block producer - let store_addr = { - let store_listener = - TcpListener::bind("127.0.0.1:0").await.expect("store should bind a port"); - store_listener.local_addr().expect("store should get a local address") - }; - let block_producer_addr = { - let block_producer_listener = - TcpListener::bind("127.0.0.1:0").await.expect("failed to bind block-producer"); - block_producer_listener - .local_addr() - .expect("Failed to get block-producer address") - }; - +#[tokio::test(flavor = "multi_thread")] +async fn block_producer_starts_with_shared_state() { let validator_addr = { let validator_listener = TcpListener::bind("127.0.0.1:0").await.expect("failed to bind validator"); @@ -78,22 +42,19 @@ async fn block_producer_startup_is_robust_to_network_failures() { .unwrap(); }); - // start the block producer BEFORE the store is available this tests the exponential backoff - // behavior - let store_url = Url::parse(&format!("http://{store_addr}")).expect("Failed to parse store URL"); + let data_directory = tempfile::tempdir().expect("tempdir should be created"); + let state = bootstrap_and_load_state(data_directory.path()).await; let validator_url = Url::parse(&format!("http://{validator_addr}")).expect("Failed to parse validator URL"); - task::spawn(async move { + let block_producer = task::spawn(async move { BlockProducer { - block_producer_address: block_producer_addr, - store_url, + state, validator_url, batch_prover_url: None, batch_interval: Duration::from_millis(500), block_interval: Duration::from_millis(500), max_txs_per_batch: DEFAULT_MAX_TXS_PER_BATCH, max_batches_per_block: DEFAULT_MAX_BATCHES_PER_BLOCK, - grpc_options, mempool_tx_capacity: NonZeroUsize::new(100).unwrap(), } .serve() @@ -101,94 +62,31 @@ async fn block_producer_startup_is_robust_to_network_failures() { .unwrap(); }); - // test: connecting to the block producer should fail because the store is not yet started (and - // therefore the block producer is not yet listening) - let block_producer_endpoint = - Endpoint::try_from(format!("http://{block_producer_addr}")).expect("valid url"); - let block_producer_client = - block_producer_client::ApiClient::connect(block_producer_endpoint.clone()).await; + sleep(Duration::from_secs(2)).await; assert!( - block_producer_client.is_err(), - "Block producer should not be available before store is started" + !block_producer.is_finished(), + "block producer should keep running with shared state" ); - - // start the store - let _store = start_store(store_addr).await; - - // wait for the block producer's exponential backoff to connect to the store use a retry loop - // since CI environments may be slower - let block_producer_client = { - let mut attempts = 0; - loop { - attempts += 1; - match block_producer_client::ApiClient::connect(block_producer_endpoint.clone()).await { - Ok(client) => break client, - Err(_) if attempts < 30 => { - sleep(Duration::from_millis(200)).await; - }, - Err(e) => panic!( - "block producer client should connect after store is started (after {attempts} attempts): {e}" - ), - } - } - }; - - // test: status request against block-producer should succeed - let response = send_status_request(block_producer_client).await; - assert!(response.is_ok(), "Status request should succeed, got: {:?}", response.err()); - - // verify the response contains expected data - let status = response.unwrap().into_inner(); - assert_eq!(status.status, "connected"); } -/// Starts the store with a fresh genesis state and returns the runtime handle. -async fn start_store(store_addr: std::net::SocketAddr) -> TestStore { - let data_directory = tempfile::tempdir().expect("tempdir should be created"); +async fn bootstrap_and_load_state(data_directory: &Path) -> Arc { let signer = random_secret_key(); let genesis_state = GenesisState::new(vec![], test_fee_params(), 1, 1, signer.public_key()); let genesis_block = genesis_state .clone() .into_block(&signer) .expect("genesis block should be created"); - Store::bootstrap(genesis_block, data_directory.path()).expect("store should bootstrap"); - - let dir = data_directory.path().to_path_buf(); - let rpc_listener = - TcpListener::bind("127.0.0.1:0").await.expect("store should bind the RPC port"); - let block_producer_listener = TcpListener::bind(store_addr) - .await - .expect("store should bind the block-producer port"); - - // Use a separate runtime so we can kill all store tasks later - let store_runtime = - runtime::Builder::new_multi_thread().enable_time().enable_io().build().unwrap(); - store_runtime.spawn(async move { - Store { - rpc_listener, - mode: StoreMode::BlockProducer { - block_producer_listener, - block_prover_url: None, - max_concurrent_proofs: DEFAULT_MAX_CONCURRENT_PROOFS, - }, - data_directory: dir, - database_options: miden_node_store::DatabaseOptions::default(), - grpc_options: GrpcOptionsInternal::bench(), - storage_options: StorageOptions::bench(), - } - .serve() - .await - .expect("store should start serving"); - }); - TestStore { - runtime: Some(store_runtime), - _data_directory: data_directory, - } -} - -/// Sends a status request to the block producer to verify connectivity. -async fn send_status_request( - mut client: block_producer_client::ApiClient, -) -> Result, tonic::Status> { - client.status(()).await + Store::bootstrap(genesis_block, data_directory).expect("store should bootstrap"); + + let (termination_ask, _termination_signal) = tokio::sync::mpsc::channel(1); + let (state, _proven_tip) = State::load_with_database_options( + data_directory, + StorageOptions::bench(), + DatabaseOptions::default(), + termination_ask, + ) + .await + .expect("state should load"); + + Arc::new(state) } diff --git a/crates/block-producer/src/store/mod.rs b/crates/block-producer/src/store/mod.rs index 92c78cfe4d..dd4fa330b5 100644 --- a/crates/block-producer/src/store/mod.rs +++ b/crates/block-producer/src/store/mod.rs @@ -3,24 +3,11 @@ use std::fmt::{Display, Formatter}; use std::num::NonZeroU32; use itertools::Itertools; -use miden_node_proto::clients::{Builder, StoreBlockProducerClient}; -use miden_node_proto::decode::{ConversionResultExt, GrpcDecodeExt}; -use miden_node_proto::domain::batch::BatchInputs; -use miden_node_proto::errors::ConversionError; -use miden_node_proto::{AccountState, decode, generated as proto}; use miden_node_utils::formatting::format_opt; use miden_protocol::Word; use miden_protocol::account::AccountId; -use miden_protocol::batch::OrderedBatches; -use miden_protocol::block::{BlockHeader, BlockInputs, BlockNumber, SignedBlock}; +use miden_protocol::block::BlockNumber; use miden_protocol::note::Nullifier; -use miden_protocol::transaction::ProvenTransaction; -use miden_protocol::utils::serde::Serializable; -use tracing::{debug, info, instrument}; -use url::Url; - -use crate::COMPONENT; -use crate::errors::StoreError; // TRANSACTION INPUTS // ================================================================================================ @@ -66,186 +53,3 @@ impl Display for TransactionInputs { )) } } - -impl TryFrom for TransactionInputs { - type Error = ConversionError; - - fn try_from(response: proto::store::TransactionInputs) -> Result { - let decoder = response.decoder(); - let AccountState { account_id, account_commitment } = - decode!(decoder, response.account_state)?; - - let mut nullifiers = HashMap::new(); - for nullifier_record in response.nullifiers { - let decoder = nullifier_record.decoder(); - let nullifier = decode!(decoder, nullifier_record.nullifier)?; - - // Note that this intentionally maps 0 to None as this is the definition used in - // protobuf. - nullifiers.insert(nullifier, NonZeroU32::new(nullifier_record.block_num)); - } - - let found_unauthenticated_notes = response - .found_unauthenticated_notes - .into_iter() - .map(Word::try_from) - .collect::>() - .context("found_unauthenticated_notes")?; - - let current_block_height = response.block_height.into(); - - Ok(Self { - account_id, - account_commitment, - nullifiers, - found_unauthenticated_notes, - current_block_height, - }) - } -} - -// STORE CLIENT -// ================================================================================================ - -/// Interface to the store's block-producer gRPC API. -/// -/// Essentially just a thin wrapper around the generated gRPC client which improves type safety. -#[derive(Clone, Debug)] -pub struct StoreClient { - client: StoreBlockProducerClient, -} - -impl StoreClient { - /// Creates a new store client with a lazy connection. - pub fn new(store_url: Url) -> Self { - info!(target: COMPONENT, store_endpoint = %store_url, "Initializing store client"); - - let store = Builder::new(store_url) - .without_tls() - .without_timeout() - .without_metadata_version() - .without_metadata_genesis() - .with_otel_context_injection() - .connect_lazy::(); - - Self { client: store } - } - - /// Returns the latest block's header from the store. - #[instrument(target = COMPONENT, name = "store.client.latest_header", skip_all, err)] - pub async fn latest_header(&self) -> Result { - let response = self - .client - .clone() - .get_block_header_by_number(tonic::Request::new( - proto::rpc::BlockHeaderByNumberRequest::default(), - )) - .await? - .into_inner() - .block_header - .ok_or_else(|| { - StoreError::DeserializationError(ConversionError::missing_field::< - miden_node_proto::generated::blockchain::BlockHeader, - >("block_header")) - })?; - - BlockHeader::try_from(response).map_err(StoreError::DeserializationError) - } - - #[instrument(target = COMPONENT, name = "store.client.get_tx_inputs", skip_all, err)] - pub async fn get_tx_inputs( - &self, - proven_tx: &ProvenTransaction, - ) -> Result { - let message = proto::store::TransactionInputsRequest { - account_id: Some(proven_tx.account_id().into()), - nullifiers: proven_tx.nullifiers().map(Into::into).collect(), - unauthenticated_notes: proven_tx - .unauthenticated_notes() - .map(|note| note.to_commitment().into()) - .collect(), - }; - - info!(target: COMPONENT, tx_id = %proven_tx.id().to_hex()); - debug!(target: COMPONENT, ?message); - - let request = tonic::Request::new(message); - let response = self.client.clone().get_transaction_inputs(request).await?.into_inner(); - - debug!(target: COMPONENT, ?response); - - if !response.new_account_id_prefix_is_unique.unwrap_or(true) { - debug_assert!( - proven_tx.account_update().initial_state_commitment().is_empty(), - "account id prefix uniqueness should not be validated unless transaction creates a new account" - ); - return Err(StoreError::DuplicateAccountIdPrefix(proven_tx.account_id())); - } - - let tx_inputs: TransactionInputs = response.try_into()?; - - if tx_inputs.account_id != proven_tx.account_id() { - return Err(StoreError::MalformedResponse(format!( - "incorrect account id returned from store. Got: {}, expected: {}", - tx_inputs.account_id, - proven_tx.account_id() - ))); - } - - debug!(target: COMPONENT, %tx_inputs); - - Ok(tx_inputs) - } - - #[instrument(target = COMPONENT, name = "store.client.get_block_inputs", skip_all, err)] - pub async fn get_block_inputs( - &self, - updated_accounts: impl Iterator + Send, - created_nullifiers: impl Iterator + Send, - unauthenticated_notes: impl Iterator + Send, - reference_blocks: impl Iterator + Send, - ) -> Result { - let request = tonic::Request::new(proto::store::BlockInputsRequest { - account_ids: updated_accounts.map(Into::into).collect(), - nullifiers: created_nullifiers.map(proto::primitives::Digest::from).collect(), - unauthenticated_notes: unauthenticated_notes - .map(proto::primitives::Digest::from) - .collect(), - reference_blocks: reference_blocks.map(|block_num| block_num.as_u32()).collect(), - }); - - let store_response = self.client.clone().get_block_inputs(request).await?.into_inner(); - - store_response.try_into().map_err(StoreError::DeserializationError) - } - - #[instrument(target = COMPONENT, name = "store.client.get_batch_inputs", skip_all, err)] - pub async fn get_batch_inputs( - &self, - block_references: impl Iterator + Send, - note_commitments: impl Iterator + Send, - ) -> Result { - let request = tonic::Request::new(proto::store::BatchInputsRequest { - reference_blocks: block_references.map(|(block_num, _)| block_num.as_u32()).collect(), - note_commitments: note_commitments.map(proto::primitives::Digest::from).collect(), - }); - - let store_response = self.client.clone().get_batch_inputs(request).await?.into_inner(); - - store_response.try_into().map_err(StoreError::DeserializationError) - } - - #[instrument(target = COMPONENT, name = "store.client.apply_block", skip_all, err)] - pub async fn apply_block( - &self, - ordered_batches: &OrderedBatches, - signed_block: &SignedBlock, - ) -> Result<(), StoreError> { - let request = tonic::Request::new(proto::store::ApplyBlockRequest { - ordered_batches: ordered_batches.to_bytes(), - block: Some(signed_block.into()), - }); - - self.client.clone().apply_block(request).await.map(|_| ()).map_err(Into::into) - } -} diff --git a/crates/proto/build.rs b/crates/proto/build.rs index 2cadf011c8..78e0047fa5 100644 --- a/crates/proto/build.rs +++ b/crates/proto/build.rs @@ -5,7 +5,6 @@ use std::process::Command; use codegen::{Function, Impl, Module, Trait, Type}; use fs_err as fs; use miden_node_proto_build::{ - block_producer_api_descriptor, ntx_builder_api_descriptor, remote_prover_api_descriptor, rpc_api_descriptor, @@ -29,7 +28,6 @@ fn main() -> miette::Result<()> { let descriptor_sets = [ rpc_api_descriptor(), store_api_descriptor(), - block_producer_api_descriptor(), remote_prover_api_descriptor(), validator_api_descriptor(), ntx_builder_api_descriptor(), diff --git a/crates/proto/src/clients/mod.rs b/crates/proto/src/clients/mod.rs index c1d7c00359..51e96fe69d 100644 --- a/crates/proto/src/clients/mod.rs +++ b/crates/proto/src/clients/mod.rs @@ -109,10 +109,6 @@ impl tonic::service::Interceptor for Interceptor { type InterceptedChannel = InterceptedService; type GeneratedRpcClient = generated::rpc::api_client::ApiClient; -type GeneratedBlockProducerClient = - generated::block_producer::api_client::ApiClient; -type GeneratedStoreClientForBlockProducer = - generated::store::block_producer_client::BlockProducerClient; type GeneratedStoreClientForRpc = generated::store::rpc_client::RpcClient; type GeneratedProxyStatusClient = generated::remote_prover::proxy_status_api_client::ProxyStatusApiClient; @@ -126,10 +122,6 @@ type GeneratedNtxBuilderClient = generated::ntx_builder::api_client::ApiClient &mut Self::Target { - &mut self.0 - } -} - -impl Deref for BlockProducerClient { - type Target = GeneratedBlockProducerClient; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl DerefMut for StoreBlockProducerClient { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -impl Deref for StoreBlockProducerClient { - type Target = GeneratedStoreClientForBlockProducer; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - impl DerefMut for StoreRpcClient { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.0 @@ -266,21 +230,6 @@ impl GrpcClient for RpcClient { } } -impl GrpcClient for BlockProducerClient { - fn with_interceptor(channel: Channel, interceptor: Interceptor) -> Self { - Self(GeneratedBlockProducerClient::new(InterceptedService::new(channel, interceptor))) - } -} - -impl GrpcClient for StoreBlockProducerClient { - fn with_interceptor(channel: Channel, interceptor: Interceptor) -> Self { - Self(GeneratedStoreClientForBlockProducer::new(InterceptedService::new( - channel, - interceptor, - ))) - } -} - impl GrpcClient for StoreRpcClient { fn with_interceptor(channel: Channel, interceptor: Interceptor) -> Self { Self(GeneratedStoreClientForRpc::new(InterceptedService::new(channel, interceptor))) diff --git a/crates/proto/src/domain/account.rs b/crates/proto/src/domain/account.rs index 33e31b8526..ea494cf471 100644 --- a/crates/proto/src/domain/account.rs +++ b/crates/proto/src/domain/account.rs @@ -1,6 +1,5 @@ use std::fmt::{Debug, Display, Formatter}; -use miden_node_utils::formatting::format_opt; use miden_node_utils::limiter::{QueryParamLimiter, QueryParamStorageMapKeyTotalLimit}; use miden_protocol::Word; use miden_protocol::account::{ @@ -893,60 +892,6 @@ impl From for proto::account::AccountWitness { } } -// ACCOUNT STATE -// ================================================================================================ - -/// Information needed from the store to verify account in transaction. -#[derive(Debug)] -pub struct AccountState { - /// Account ID - pub account_id: AccountId, - /// The account commitment in the store corresponding to tx's account ID - pub account_commitment: Option, -} - -impl Display for AccountState { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_fmt(format_args!( - "{{ account_id: {}, account_commitment: {} }}", - self.account_id, - format_opt(self.account_commitment.as_ref()), - )) - } -} - -impl TryFrom for AccountState { - type Error = ConversionError; - - fn try_from( - from: proto::store::transaction_inputs::AccountTransactionInputRecord, - ) -> Result { - let decoder = from.decoder(); - let account_id = decode!(decoder, from.account_id)?; - - let account_commitment = decode!(decoder, from.account_commitment)?; - - // If the commitment is equal to `Word::empty()`, it signifies that this is a new account - // which is not yet present in the Store. - let account_commitment = if account_commitment == Word::empty() { - None - } else { - Some(account_commitment) - }; - - Ok(Self { account_id, account_commitment }) - } -} - -impl From for proto::store::transaction_inputs::AccountTransactionInputRecord { - fn from(from: AccountState) -> Self { - Self { - account_id: Some(from.account_id.into()), - account_commitment: from.account_commitment.map(Into::into), - } - } -} - // ASSET // ================================================================================================ diff --git a/crates/proto/src/domain/batch.rs b/crates/proto/src/domain/batch.rs index 590ea026a4..d4f1418b66 100644 --- a/crates/proto/src/domain/batch.rs +++ b/crates/proto/src/domain/batch.rs @@ -3,11 +3,6 @@ use std::collections::BTreeMap; use miden_protocol::block::BlockHeader; use miden_protocol::note::{NoteId, NoteInclusionProof}; use miden_protocol::transaction::PartialBlockchain; -use miden_protocol::utils::serde::Serializable; - -use crate::decode::{ConversionResultExt, DecodeBytesExt, GrpcDecodeExt}; -use crate::errors::ConversionError; -use crate::generated as proto; /// Data required for a transaction batch. #[derive(Clone, Debug)] @@ -16,39 +11,3 @@ pub struct BatchInputs { pub note_proofs: BTreeMap, pub partial_block_chain: PartialBlockchain, } - -impl From for proto::store::BatchInputs { - fn from(inputs: BatchInputs) -> Self { - Self { - batch_reference_block_header: Some(inputs.batch_reference_block_header.into()), - note_proofs: inputs.note_proofs.iter().map(Into::into).collect(), - partial_block_chain: inputs.partial_block_chain.to_bytes(), - } - } -} - -impl TryFrom for BatchInputs { - type Error = ConversionError; - - fn try_from(response: proto::store::BatchInputs) -> Result { - let decoder = response.decoder(); - let result = Self { - batch_reference_block_header: crate::decode!( - decoder, - response.batch_reference_block_header - )?, - note_proofs: response - .note_proofs - .iter() - .map(<(NoteId, NoteInclusionProof)>::try_from) - .collect::>() - .context("note_proofs")?, - partial_block_chain: PartialBlockchain::decode_bytes( - &response.partial_block_chain, - "PartialBlockchain", - )?, - }; - - Ok(result) - } -} diff --git a/crates/proto/src/domain/block.rs b/crates/proto/src/domain/block.rs index 36d37cbb01..8df4e2ba30 100644 --- a/crates/proto/src/domain/block.rs +++ b/crates/proto/src/domain/block.rs @@ -1,25 +1,14 @@ -use std::collections::BTreeMap; use std::ops::RangeInclusive; use miden_protocol::account::AccountId; -use miden_protocol::block::nullifier_tree::NullifierWitness; -use miden_protocol::block::{ - BlockBody, - BlockHeader, - BlockInputs, - BlockNumber, - FeeParameters, - SignedBlock, -}; +use miden_protocol::block::{BlockBody, BlockHeader, BlockNumber, FeeParameters, SignedBlock}; use miden_protocol::crypto::dsa::ecdsa_k256_keccak::{PublicKey, Signature}; -use miden_protocol::note::{NoteId, NoteInclusionProof}; -use miden_protocol::transaction::PartialBlockchain; use miden_protocol::utils::serde::Serializable; use thiserror::Error; use crate::decode::{ConversionResultExt, DecodeBytesExt, GrpcDecodeExt}; use crate::errors::ConversionError; -use crate::{AccountWitnessRecord, NullifierWitnessRecord, decode, generated as proto}; +use crate::{decode, generated as proto}; // BLOCK NUMBER // ================================================================================================ @@ -173,88 +162,6 @@ impl TryFrom for SignedBlock { } } -// BLOCK INPUTS -// ================================================================================================ - -impl From for proto::store::BlockInputs { - fn from(inputs: BlockInputs) -> Self { - let ( - prev_block_header, - partial_block_chain, - account_witnesses, - nullifier_witnesses, - unauthenticated_note_proofs, - ) = inputs.into_parts(); - - proto::store::BlockInputs { - latest_block_header: Some(prev_block_header.into()), - account_witnesses: account_witnesses - .into_iter() - .map(|(id, witness)| AccountWitnessRecord { account_id: id, witness }.into()) - .collect(), - nullifier_witnesses: nullifier_witnesses - .into_iter() - .map(|(nullifier, witness)| { - let proof = witness.into_proof(); - NullifierWitnessRecord { nullifier, proof }.into() - }) - .collect(), - partial_block_chain: partial_block_chain.to_bytes(), - unauthenticated_note_proofs: unauthenticated_note_proofs - .iter() - .map(proto::note::NoteInclusionInBlockProof::from) - .collect(), - } - } -} - -impl TryFrom for BlockInputs { - type Error = ConversionError; - - fn try_from(response: proto::store::BlockInputs) -> Result { - let decoder = response.decoder(); - let latest_block_header: BlockHeader = decode!(decoder, response.latest_block_header)?; - - let account_witnesses = response - .account_witnesses - .into_iter() - .map(|entry| { - let witness_record: AccountWitnessRecord = entry.try_into()?; - Ok((witness_record.account_id, witness_record.witness)) - }) - .collect::, ConversionError>>() - .context("account_witnesses")?; - - let nullifier_witnesses = response - .nullifier_witnesses - .into_iter() - .map(|entry| { - let witness: NullifierWitnessRecord = entry.try_into()?; - Ok((witness.nullifier, NullifierWitness::new(witness.proof))) - }) - .collect::, ConversionError>>() - .context("nullifier_witnesses")?; - - let unauthenticated_note_proofs = response - .unauthenticated_note_proofs - .iter() - .map(<(NoteId, NoteInclusionProof)>::try_from) - .collect::>() - .context("unauthenticated_note_proofs")?; - - let partial_block_chain = - PartialBlockchain::decode_bytes(&response.partial_block_chain, "PartialBlockchain")?; - - Ok(BlockInputs::new( - latest_block_header, - partial_block_chain, - account_witnesses, - nullifier_witnesses, - unauthenticated_note_proofs, - )) - } -} - // PUBLIC KEY // ================================================================================================ diff --git a/crates/proto/src/domain/mempool.rs b/crates/proto/src/domain/mempool.rs deleted file mode 100644 index 4b278e4796..0000000000 --- a/crates/proto/src/domain/mempool.rs +++ /dev/null @@ -1,144 +0,0 @@ -use std::collections::HashSet; - -use miden_protocol::account::delta::AccountUpdateDetails; -use miden_protocol::block::BlockHeader; -use miden_protocol::note::Nullifier; -use miden_protocol::transaction::TransactionId; -use miden_protocol::utils::serde::Serializable; -use miden_standards::note::AccountTargetNetworkNote; - -use crate::decode::{ConversionResultExt, DecodeBytesExt, GrpcDecodeExt}; -use crate::errors::ConversionError; -use crate::{decode, generated as proto}; - -#[derive(Debug, Clone, PartialEq)] -pub enum MempoolEvent { - TransactionAdded { - id: TransactionId, - nullifiers: Vec, - network_notes: Vec, - account_delta: Option, - }, - BlockCommitted { - // Box'd as this struct is quite large and triggers clippy. - header: Box, - txs: Vec, - }, - TransactionsReverted(HashSet), -} - -impl MempoolEvent { - pub fn kind(&self) -> &'static str { - match self { - MempoolEvent::TransactionAdded { .. } => "TransactionAdded", - MempoolEvent::BlockCommitted { .. } => "BlockCommitted", - MempoolEvent::TransactionsReverted(_) => "TransactionsReverted", - } - } -} - -impl From for proto::block_producer::MempoolEvent { - fn from(event: MempoolEvent) -> Self { - let event = match event { - MempoolEvent::TransactionAdded { - id, - nullifiers, - network_notes, - account_delta, - } => { - let event = proto::block_producer::mempool_event::TransactionAdded { - id: Some(id.into()), - nullifiers: nullifiers.into_iter().map(Into::into).collect(), - network_notes: network_notes.into_iter().map(Into::into).collect(), - network_account_delta: account_delta - .as_ref() - .map(AccountUpdateDetails::to_bytes), - }; - - proto::block_producer::mempool_event::Event::TransactionAdded(event) - }, - MempoolEvent::BlockCommitted { header, txs } => { - proto::block_producer::mempool_event::Event::BlockCommitted( - proto::block_producer::mempool_event::BlockCommitted { - block_header: Some(header.as_ref().into()), - transactions: txs.into_iter().map(Into::into).collect(), - }, - ) - }, - MempoolEvent::TransactionsReverted(txs) => { - proto::block_producer::mempool_event::Event::TransactionsReverted( - proto::block_producer::mempool_event::TransactionsReverted { - reverted: txs.into_iter().map(Into::into).collect(), - }, - ) - }, - } - .into(); - - Self { event } - } -} - -impl TryFrom for MempoolEvent { - type Error = ConversionError; - - fn try_from(event: proto::block_producer::MempoolEvent) -> Result { - let event = event.event.ok_or(ConversionError::missing_field::< - proto::block_producer::MempoolEvent, - >("event"))?; - - match event { - proto::block_producer::mempool_event::Event::TransactionAdded(tx) => { - let decoder = tx.decoder(); - let id = decode!(decoder, tx.id)?; - let nullifiers = tx - .nullifiers - .into_iter() - .map(Nullifier::try_from) - .collect::>() - .context("nullifiers")?; - let network_notes = tx - .network_notes - .into_iter() - .map(AccountTargetNetworkNote::try_from) - .collect::>() - .context("network_notes")?; - let account_delta = tx - .network_account_delta - .as_deref() - .map(|bytes| AccountUpdateDetails::decode_bytes(bytes, "account_delta")) - .transpose()?; - - Ok(Self::TransactionAdded { - id, - nullifiers, - network_notes, - account_delta, - }) - }, - proto::block_producer::mempool_event::Event::BlockCommitted(block_committed) => { - let decoder = block_committed.decoder(); - let header = decode!(decoder, block_committed.block_header)?; - let header = Box::new(header); - let txs = block_committed - .transactions - .into_iter() - .map(TransactionId::try_from) - .collect::>() - .context("transactions")?; - - Ok(Self::BlockCommitted { header, txs }) - }, - proto::block_producer::mempool_event::Event::TransactionsReverted(txs) => { - let txs = txs - .reverted - .into_iter() - .map(TransactionId::try_from) - .collect::>() - .context("reverted")?; - - Ok(Self::TransactionsReverted(txs)) - }, - } - } -} diff --git a/crates/proto/src/domain/mod.rs b/crates/proto/src/domain/mod.rs index b078655532..e04eff7947 100644 --- a/crates/proto/src/domain/mod.rs +++ b/crates/proto/src/domain/mod.rs @@ -2,7 +2,6 @@ pub mod account; pub mod batch; pub mod block; pub mod digest; -pub mod mempool; pub mod merkle; pub mod note; pub mod nullifier; diff --git a/crates/proto/src/domain/nullifier.rs b/crates/proto/src/domain/nullifier.rs index 326cc4ebf4..e206c165b6 100644 --- a/crates/proto/src/domain/nullifier.rs +++ b/crates/proto/src/domain/nullifier.rs @@ -1,10 +1,8 @@ use miden_protocol::Word; -use miden_protocol::crypto::merkle::smt::SmtProof; use miden_protocol::note::Nullifier; -use crate::decode::GrpcDecodeExt; use crate::errors::ConversionError; -use crate::{decode, generated as proto}; +use crate::generated as proto; // FROM NULLIFIER // ================================================================================================ @@ -32,35 +30,3 @@ impl TryFrom for Nullifier { Ok(Nullifier::from_raw(digest)) } } - -// NULLIFIER WITNESS RECORD -// ================================================================================================ - -#[derive(Clone, Debug)] -pub struct NullifierWitnessRecord { - pub nullifier: Nullifier, - pub proof: SmtProof, -} - -impl TryFrom for NullifierWitnessRecord { - type Error = ConversionError; - - fn try_from( - nullifier_witness_record: proto::store::block_inputs::NullifierWitness, - ) -> Result { - let decoder = nullifier_witness_record.decoder(); - Ok(Self { - nullifier: decode!(decoder, nullifier_witness_record.nullifier)?, - proof: decode!(decoder, nullifier_witness_record.opening)?, - }) - } -} - -impl From for proto::store::block_inputs::NullifierWitness { - fn from(value: NullifierWitnessRecord) -> Self { - Self { - nullifier: Some(value.nullifier.into()), - opening: Some(value.proof.into()), - } - } -} diff --git a/crates/proto/src/generated/mod.rs b/crates/proto/src/generated/mod.rs index 63dc1dfa2c..e559675415 100644 --- a/crates/proto/src/generated/mod.rs +++ b/crates/proto/src/generated/mod.rs @@ -1,6 +1,5 @@ #![expect( clippy::pedantic, - clippy::large_enum_variant, clippy::allow_attributes, reason = "generated by build.rs and tonic" )] diff --git a/crates/proto/src/lib.rs b/crates/proto/src/lib.rs index 1ec05672cf..615f516a41 100644 --- a/crates/proto/src/lib.rs +++ b/crates/proto/src/lib.rs @@ -9,8 +9,7 @@ pub mod generated; // RE-EXPORTS // ================================================================================================ -pub use domain::account::{AccountState, AccountWitnessRecord}; -pub use domain::nullifier::NullifierWitnessRecord; +pub use domain::account::AccountWitnessRecord; pub use domain::proof_request::BlockProofRequest; pub use domain::{convert, try_convert}; pub use prost; diff --git a/crates/rpc/Cargo.toml b/crates/rpc/Cargo.toml index ed3d8ff903..06eee7fc35 100644 --- a/crates/rpc/Cargo.toml +++ b/crates/rpc/Cargo.toml @@ -18,27 +18,29 @@ workspace = true doctest = false [dependencies] -anyhow = { workspace = true } -futures = { workspace = true } -http = { workspace = true } -mediatype = { version = "0.21" } -miden-node-proto = { workspace = true } -miden-node-proto-build = { workspace = true } -miden-node-utils = { workspace = true } -miden-protocol = { default-features = true, workspace = true } -miden-tx = { features = ["concurrent"], workspace = true } -miden-tx-batch-prover = { workspace = true } -semver = { version = "1.0" } -thiserror = { workspace = true } -tokio = { features = ["macros", "net", "rt-multi-thread"], workspace = true } -tokio-stream = { features = ["net"], workspace = true } -tonic = { default-features = true, features = ["tls-native-roots", "tls-ring"], workspace = true } -tonic-reflection = { workspace = true } -tonic-web = { workspace = true } -tower = { workspace = true } -tower-http = { features = ["trace"], workspace = true } -tracing = { workspace = true } -url = { workspace = true } +anyhow = { workspace = true } +futures = { workspace = true } +http = { workspace = true } +mediatype = { version = "0.21" } +miden-node-block-producer = { workspace = true } +miden-node-proto = { workspace = true } +miden-node-proto-build = { workspace = true } +miden-node-store = { workspace = true } +miden-node-utils = { workspace = true } +miden-protocol = { default-features = true, workspace = true } +miden-tx = { features = ["concurrent"], workspace = true } +miden-tx-batch-prover = { workspace = true } +semver = { version = "1.0" } +thiserror = { workspace = true } +tokio = { features = ["macros", "net", "rt-multi-thread"], workspace = true } +tokio-stream = { features = ["net"], workspace = true } +tonic = { default-features = true, features = ["tls-native-roots", "tls-ring"], workspace = true } +tonic-reflection = { workspace = true } +tonic-web = { workspace = true } +tower = { workspace = true } +tower-http = { features = ["trace"], workspace = true } +tracing = { workspace = true } +url = { workspace = true } [dev-dependencies] miden-node-store = { features = ["rocksdb"], workspace = true } diff --git a/crates/rpc/src/lib.rs b/crates/rpc/src/lib.rs index 368cb63de4..69b2eca888 100644 --- a/crates/rpc/src/lib.rs +++ b/crates/rpc/src/lib.rs @@ -2,7 +2,7 @@ mod server; #[cfg(test)] mod tests; -pub use server::Rpc; +pub use server::{Rpc, RpcSubmissionMode}; // CONSTANTS // ================================================================================================= diff --git a/crates/rpc/src/server/api.rs b/crates/rpc/src/server/api.rs index 2aa5ea9cc1..bec54b1d2f 100644 --- a/crates/rpc/src/server/api.rs +++ b/crates/rpc/src/server/api.rs @@ -1,12 +1,18 @@ -use std::num::NonZeroUsize; -use std::sync::LazyLock; +use std::collections::HashMap; +use std::num::{NonZeroU32, NonZeroUsize}; +use std::sync::{Arc, LazyLock}; use std::time::Duration; use anyhow::Context; +use miden_node_block_producer::{ + AuthenticatedTransaction, + SharedMempool, + TransactionInputs as AuthenticatedTransactionInputs, +}; use miden_node_proto::clients::{ - BlockProducerClient, Builder, NtxBuilderClient, + RpcClient, StoreRpcClient, ValidatorClient, }; @@ -17,6 +23,7 @@ use miden_node_proto::generated::rpc::MempoolStats; use miden_node_proto::generated::rpc::api_server::{self, Api}; use miden_node_proto::generated::{self as proto}; use miden_node_proto::try_convert; +use miden_node_store::state::{Finality, State, TransactionInputs as StateTransactionInputs}; use miden_node_utils::ErrorReport; use miden_node_utils::limiter::{ QueryParamAccountIdLimit, @@ -30,6 +37,7 @@ use miden_node_utils::lru_cache::LruCache; use miden_node_utils::tracing::OpenTelemetrySpanExt; use miden_protocol::batch::{ProposedBatch, ProvenBatch}; use miden_protocol::block::{BlockHeader, BlockNumber}; +use miden_protocol::note::NoteHeader; use miden_protocol::transaction::{ OutputNote, ProvenTransaction, @@ -45,24 +53,36 @@ use tracing::{Span, debug, info, info_span}; use url::Url; use crate::COMPONENT; +use crate::server::RpcSubmissionMode; // RPC SERVICE // ================================================================================================ pub struct RpcService { store: StoreRpcClient, - block_producer: Option, - validator: ValidatorClient, + submission: SubmissionTarget, ntx_builder: Option, genesis_commitment: Option, block_commitment_cache: LruCache, } +enum SubmissionTarget { + ReadOnly, + Full { + source_rpc_url: Url, + source: Option, + }, + Sequencer { + validator: ValidatorClient, + state: Arc, + mempool: SharedMempool, + }, +} + impl RpcService { pub(super) fn new( store_url: Url, - block_producer_url: Option, - validator_url: Url, + submission: RpcSubmissionMode, ntx_builder_url: Option, commitment_cache_capacity: NonZeroUsize, ) -> Self { @@ -77,34 +97,34 @@ impl RpcService { .connect_lazy::() }; - let block_producer = block_producer_url.map(|block_producer_url| { - info!( - target: COMPONENT, - block_producer_endpoint = %block_producer_url, - "Initializing block producer client", - ); - Builder::new(block_producer_url) - .without_tls() - .without_timeout() - .without_metadata_version() - .without_metadata_genesis() - .with_otel_context_injection() - .connect_lazy::() - }); - - let validator = { - info!( - target: COMPONENT, - validator_endpoint = %validator_url, - "Initializing validator client", - ); - Builder::new(validator_url) - .without_tls() - .without_timeout() - .without_metadata_version() - .without_metadata_genesis() - .with_otel_context_injection() - .connect_lazy::() + let submission = match submission { + RpcSubmissionMode::ReadOnly => SubmissionTarget::ReadOnly, + RpcSubmissionMode::Full { source_rpc_url } => { + info!( + target: COMPONENT, + source_rpc_endpoint = %source_rpc_url, + "Initializing source RPC submission target", + ); + SubmissionTarget::Full { source_rpc_url, source: None } + }, + RpcSubmissionMode::Sequencer { validator_url, state, mempool } => { + let validator = { + info!( + target: COMPONENT, + validator_endpoint = %validator_url, + "Initializing validator client", + ); + Builder::new(validator_url) + .without_tls() + .without_timeout() + .without_metadata_version() + .without_metadata_genesis() + .with_otel_context_injection() + .connect_lazy::() + }; + + SubmissionTarget::Sequencer { validator, state, mempool } + }, }; let ntx_builder = ntx_builder_url.map(|ntx_builder_url| { @@ -124,8 +144,7 @@ impl RpcService { Self { store, - block_producer, - validator, + submission, ntx_builder, genesis_commitment: None, block_commitment_cache: LruCache::new(commitment_cache_capacity), @@ -140,6 +159,17 @@ impl RpcService { if self.genesis_commitment.is_some() { return Err(anyhow::anyhow!("genesis commitment already set")); } + if let SubmissionTarget::Full { source_rpc_url, source } = &mut self.submission { + *source = Some( + Builder::new(source_rpc_url.clone()) + .without_tls() + .without_timeout() + .with_metadata_version(env!("CARGO_PKG_VERSION").to_string()) + .with_metadata_genesis(commitment.to_hex()) + .with_otel_context_injection() + .connect_lazy::(), + ); + } self.genesis_commitment = Some(commitment); Ok(()) } @@ -237,6 +267,67 @@ impl RpcService { Ok(()) } + + async fn authenticate_transaction( + state: &State, + tx: Arc, + ) -> Result, Status> { + let inputs = Self::get_transaction_inputs(state, &tx).await?; + + AuthenticatedTransaction::new_unchecked(tx, inputs) + .map(Arc::new) + .map_err(|err| Status::invalid_argument(err.to_string())) + } + + async fn get_transaction_inputs( + state: &State, + tx: &ProvenTransaction, + ) -> Result { + let nullifiers = tx.nullifiers().collect::>(); + let unauthenticated_notes = + tx.unauthenticated_notes().map(NoteHeader::to_commitment).collect::>(); + + let inputs = state + .get_transaction_inputs(tx.account_id(), &nullifiers, unauthenticated_notes) + .await + .map_err(|err| Status::internal(err.as_report()))?; + + if !inputs.new_account_id_prefix_is_unique.unwrap_or(true) { + return Err(Status::invalid_argument(format!( + "account Id prefix already exists: {}", + tx.account_id() + ))); + } + + let current_block_height = state.chain_tip(Finality::Committed).await; + Ok(Self::convert_transaction_inputs(tx, inputs, current_block_height)) + } + + fn convert_transaction_inputs( + tx: &ProvenTransaction, + inputs: StateTransactionInputs, + current_block_height: BlockNumber, + ) -> AuthenticatedTransactionInputs { + let account_commitment = if inputs.account_commitment.is_empty() { + None + } else { + Some(inputs.account_commitment) + }; + + let nullifiers = inputs + .nullifiers + .into_iter() + .map(|nullifier| (nullifier.nullifier, NonZeroU32::new(nullifier.block_num.as_u32()))) + .collect::>(); + + AuthenticatedTransactionInputs { + account_id: tx.account_id(), + account_commitment, + nullifiers, + found_unauthenticated_notes: inputs.found_unauthenticated_notes, + current_block_height, + } + } } // API IMPLEMENTATION @@ -461,20 +552,14 @@ impl api_server::Api for RpcService { // -- Transaction submission -------------------------------------------------------------- /// Deserializes and rebuilds the transaction with MAST decorators stripped from output note - /// scripts, verifies the transaction proof, optionally re-executes via the validator if - /// transaction inputs are provided, then forwards the transaction to the block producer. + /// scripts, verifies the transaction proof, then either forwards it to the source RPC or + /// re-executes it via the validator and submits it directly to the mempool. async fn submit_proven_tx( &self, request: Request, ) -> Result, Status> { debug!(target: COMPONENT, request = ?request.get_ref()); - let Some(block_producer) = &self.block_producer else { - return Err(Status::unavailable( - "Transaction submission not available in read-only mode", - )); - }; - let request = request.into_inner(); let tx = ProvenTransaction::read_from_bytes(&request.transaction).map_err(|err| { @@ -535,27 +620,42 @@ impl api_server::Api for RpcService { )) })?; - // Transaction inputs must be provided in order to allow for transaction re-execution via - // the Validator. - if request.transaction_inputs.is_some() { - self.validator.clone().submit_proven_transaction(request.clone()).await?; - } else { + if request.transaction_inputs.is_none() { return Err(Status::invalid_argument("Transaction inputs must be provided")); } - block_producer.clone().submit_proven_tx(request).await + match &self.submission { + SubmissionTarget::ReadOnly => { + Err(Status::unavailable("Transaction submission not available in read-only mode")) + }, + SubmissionTarget::Full { source: Some(source), .. } => { + source.clone().submit_proven_tx(request).await + }, + SubmissionTarget::Full { source: None, .. } => { + Err(Status::unavailable("Source RPC submission target is not initialized")) + }, + SubmissionTarget::Sequencer { validator, state, mempool } => { + // Transaction inputs must be provided in order to allow for transaction + // re-execution via the Validator. + validator.clone().submit_proven_transaction(request.clone()).await?; + + let tx = Self::authenticate_transaction(state, Arc::new(rebuilt_tx)).await?; + + mempool + .add_transaction(tx) + .map(Into::into) + .map(Response::new) + .map_err(Into::into) + }, + } } /// Deserializes the batch, strips MAST decorators from full output note scripts, rebuilds the - /// batch, then forwards it to the block producer. + /// batch, then either forwards it to the source RPC or submits it directly to the mempool. async fn submit_proven_tx_batch( &self, request: tonic::Request, ) -> Result, Status> { - let Some(block_producer) = &self.block_producer else { - return Err(Status::unavailable("Batch submission not available in read-only mode")); - }; - let request = request.into_inner(); let proven_batch = ProvenBatch::read_from_bytes(&request.batch_proof).map_err(|err| { @@ -623,18 +723,39 @@ impl api_server::Api for RpcService { return Err(Status::invalid_argument("batch proof did not match proposed batch")); } - // Submit each transaction to the validator. - // - // SAFETY: We checked earlier that the two iterators are the same length. - for (tx, inputs) in proposed_batch.transactions().iter().zip(&request.transaction_inputs) { - let request = proto::transaction::ProvenTransaction { - transaction: tx.to_bytes(), - transaction_inputs: inputs.clone().into(), - }; - self.validator.clone().submit_proven_transaction(request).await?; + match &self.submission { + SubmissionTarget::ReadOnly => { + Err(Status::unavailable("Batch submission not available in read-only mode")) + }, + SubmissionTarget::Full { source: Some(source), .. } => { + source.clone().submit_proven_tx_batch(request).await + }, + SubmissionTarget::Full { source: None, .. } => { + Err(Status::unavailable("Source RPC submission target is not initialized")) + }, + SubmissionTarget::Sequencer { validator, state, mempool } => { + // Submit each transaction to the validator. + // + // SAFETY: We checked earlier that the two iterators are the same length. + let mut txs = Vec::with_capacity(proposed_batch.transactions().len()); + for (tx, inputs) in + proposed_batch.transactions().iter().zip(&request.transaction_inputs) + { + let tx_request = proto::transaction::ProvenTransaction { + transaction: tx.to_bytes(), + transaction_inputs: inputs.clone().into(), + }; + validator.clone().submit_proven_transaction(tx_request).await?; + txs.push(Self::authenticate_transaction(state, Arc::clone(tx)).await?); + } + + mempool + .add_user_batch(&txs) + .map(Into::into) + .map(Response::new) + .map_err(Into::into) + }, } - - block_producer.clone().submit_proven_tx_batch(request).await } // -- Status & utility endpoints ---------------------------------------------------------- @@ -670,15 +791,17 @@ impl api_server::Api for RpcService { let store_status = self.store.clone().status(Request::new(())).await.map(Response::into_inner).ok(); - let block_producer_status = if let Some(block_producer) = &self.block_producer { - block_producer - .clone() - .status(Request::new(())) - .await - .map(Response::into_inner) - .ok() - } else { - None + let block_producer_status = match &self.submission { + SubmissionTarget::Sequencer { mempool, .. } => mempool + .stats() + .map(|stats| proto::rpc::BlockProducerStatus { + status: "connected".to_string(), + version: env!("CARGO_PKG_VERSION").to_string(), + chain_tip: stats.chain_tip.as_u32(), + mempool_stats: Some(stats.into()), + }) + .ok(), + SubmissionTarget::ReadOnly | SubmissionTarget::Full { .. } => None, }; Ok(Response::new(proto::rpc::RpcStatus { diff --git a/crates/rpc/src/server/mod.rs b/crates/rpc/src/server/mod.rs index 6a8c3acc98..4d489ed83d 100644 --- a/crates/rpc/src/server/mod.rs +++ b/crates/rpc/src/server/mod.rs @@ -1,9 +1,12 @@ use std::num::NonZeroUsize; +use std::sync::Arc; use accept::AcceptHeaderLayer; use anyhow::Context; +use miden_node_block_producer::SharedMempool; use miden_node_proto::generated::rpc::api_server; use miden_node_proto_build::rpc_api_descriptor; +use miden_node_store::state::State; use miden_node_utils::clap::GrpcOptionsExternal; use miden_node_utils::cors::cors_for_grpc_web_layer; use miden_node_utils::grpc; @@ -28,27 +31,50 @@ mod health; /// The RPC server component. /// /// On startup, binds to the provided listener and starts serving the RPC API. -/// It connects lazily to the store, validator and block producer components as needed. +/// It connects lazily to the store and optional submission source as needed. /// Requests will fail if the components are not available. pub struct Rpc { pub listener: TcpListener, pub store_url: Url, - pub block_producer_url: Option, - pub validator_url: Url, + pub submission: RpcSubmissionMode, pub ntx_builder_url: Option, pub grpc_options: GrpcOptionsExternal, } +/// Controls how the RPC handles transaction and batch submissions. +pub enum RpcSubmissionMode { + /// Submission endpoints are disabled. + ReadOnly, + /// Full-node mode: validate locally, then forward submissions to the source RPC. + Full { source_rpc_url: Url }, + /// Sequencer mode: validate locally, re-execute via the validator, then submit to the mempool. + Sequencer { + validator_url: Url, + state: Arc, + mempool: SharedMempool, + }, +} + +impl RpcSubmissionMode { + pub(super) fn as_str(&self) -> &'static str { + match self { + Self::ReadOnly => "read-only", + Self::Full { .. } => "full", + Self::Sequencer { .. } => "sequencer", + } + } +} + impl Rpc { /// Serves the RPC API. /// /// Note: Executes in place (i.e. not spawned) and will run indefinitely until /// a fatal error is encountered. pub async fn serve(self) -> anyhow::Result<()> { + let submission_mode = self.submission.as_str(); let mut api = api::RpcService::new( self.store_url.clone(), - self.block_producer_url.clone(), - self.validator_url, + self.submission, self.ntx_builder_url.clone(), NonZeroUsize::new(1_000_000).unwrap(), ); @@ -66,7 +92,7 @@ impl Rpc { .build_v1() .context("failed to build reflection service")?; - info!(target: COMPONENT, endpoint=?self.listener, store=%self.store_url, block_producer=?self.block_producer_url, "Server initialized"); + info!(target: COMPONENT, endpoint=?self.listener, store=%self.store_url, submission_mode, "Server initialized"); let rpc_version = env!("CARGO_PKG_VERSION"); let rpc_version = diff --git a/crates/rpc/src/tests.rs b/crates/rpc/src/tests.rs index 2c48617cf8..89f5098938 100644 --- a/crates/rpc/src/tests.rs +++ b/crates/rpc/src/tests.rs @@ -43,7 +43,7 @@ use tokio::task; use tokio::time::sleep; use url::Url; -use crate::Rpc; +use crate::{Rpc, RpcSubmissionMode}; /// A wrapper around the store runtime and data directory. /// @@ -71,10 +71,10 @@ impl TestStore { self.data_directory.take().expect("data_directory should be set") } - async fn start(store_listener: TcpListener) -> Self { + fn start(store_listener: TcpListener) -> Self { let data_directory = tempfile::tempdir().expect("tempdir should be created"); let genesis_commitment = Self::bootstrap(data_directory.path()); - Self::start_without_bootstrap(data_directory, genesis_commitment, store_listener).await + Self::start_without_bootstrap(data_directory, genesis_commitment, store_listener) } fn bootstrap(path: &std::path::Path) -> Word { @@ -92,7 +92,7 @@ impl TestStore { genesis_commitment } - async fn start_without_bootstrap( + fn start_without_bootstrap( data_directory: TempDir, genesis_commitment: Word, store_listener: TcpListener, @@ -101,8 +101,6 @@ impl TestStore { let store_addr = store_listener.local_addr().expect("store listener should get a local address"); let rpc_listener = store_listener; - let block_producer_listener = - TcpListener::bind("127.0.0.1:0").await.expect("store should bind a port"); // In order to later kill the store, we need to spawn a new runtime and run the store on it. // That allows us to kill all the tasks spawned by the store when we kill the runtime. @@ -112,7 +110,6 @@ impl TestStore { Store { rpc_listener, mode: StoreMode::BlockProducer { - block_producer_listener, block_prover_url: None, max_concurrent_proofs: DEFAULT_MAX_CONCURRENT_PROOFS, }, @@ -222,7 +219,7 @@ fn build_test_proven_tx( async fn rpc_server_accepts_requests_without_accept_header() { // Start the RPC. let (_, rpc_addr, store_listener) = start_rpc().await; - let _store = TestStore::start(store_listener).await; + let _store = TestStore::start(store_listener); // Override the client so that the ACCEPT header is not set. let mut rpc_client = { @@ -250,7 +247,7 @@ async fn rpc_rate_limits_per_ip() { ..GrpcOptionsExternal::test() }; let (_, rpc_addr, store_listener) = start_rpc_with_options(grpc_options).await; - let _store = TestStore::start(store_listener).await; + let _store = TestStore::start(store_listener); let url = rpc_addr.to_string(); let url = Url::parse(format!("http://{}", &url).as_str()).unwrap(); @@ -277,7 +274,7 @@ async fn rpc_rate_limits_per_ip() { async fn rpc_server_accepts_requests_with_accept_header() { // Start the RPC. let (mut rpc_client, _, store_listener) = start_rpc().await; - let _store = TestStore::start(store_listener).await; + let _store = TestStore::start(store_listener); // Send any request to the RPC. let response = send_request(&mut rpc_client).await; @@ -291,7 +288,7 @@ async fn rpc_server_rejects_requests_with_accept_header_invalid_version() { for version in ["1.9.0", "0.8.1", "0.8.0", "0.999.0", "99.0.0"] { // Start the RPC. let (_, rpc_addr, store_listener) = start_rpc().await; - let _store = TestStore::start(store_listener).await; + let _store = TestStore::start(store_listener); // Recreate the RPC client with an invalid version. let url = rpc_addr.to_string(); @@ -330,7 +327,7 @@ async fn rpc_startup_is_robust_to_network_failures() { assert!(response.is_err()); // Start the store. - let store = TestStore::start(store_listener).await; + let store = TestStore::start(store_listener); // Test: send request against RPC api and should succeed let response = send_request_until_success(&mut rpc_client).await; @@ -346,8 +343,7 @@ async fn rpc_startup_is_robust_to_network_failures() { // Test: restart the store and request should succeed let store_listener = TcpListener::bind(store_addr).await.expect("Failed to bind store"); let _store = - TestStore::start_without_bootstrap(data_directory, genesis_commitment, store_listener) - .await; + TestStore::start_without_bootstrap(data_directory, genesis_commitment, store_listener); let response = send_request_until_success(&mut rpc_client).await; assert_eq!(response.unwrap().into_inner().block_header.unwrap().block_num, 0); } @@ -356,7 +352,7 @@ async fn rpc_startup_is_robust_to_network_failures() { async fn rpc_server_has_web_support() { // Start server let (_, rpc_addr, store_listener) = start_rpc().await; - let _store = TestStore::start(store_listener).await; + let _store = TestStore::start(store_listener); // Send a status request let client = reqwest::Client::new(); @@ -398,7 +394,7 @@ async fn rpc_server_has_web_support() { async fn rpc_server_rejects_proven_transactions_with_invalid_commitment() { // Start the RPC. let (_, rpc_addr, store_listener) = start_rpc().await; - let store = TestStore::start(store_listener).await; + let store = TestStore::start(store_listener); let genesis = store.genesis_commitment(); // Wait for the store to be ready before sending requests. @@ -450,7 +446,7 @@ async fn rpc_server_rejects_proven_transactions_with_invalid_commitment() { async fn rpc_server_rejects_proven_transactions_with_invalid_reference_block() { // Start the RPC. let (_, rpc_addr, store_listener) = start_rpc().await; - let store = TestStore::start(store_listener).await; + let store = TestStore::start(store_listener); let genesis = store.genesis_commitment(); // Wait for the store to be ready before sending requests. @@ -493,7 +489,7 @@ async fn rpc_server_rejects_proven_transactions_with_invalid_reference_block() { async fn rpc_server_rejects_tx_submissions_without_genesis() { // Start the RPC. let (_, rpc_addr, store_listener) = start_rpc().await; - let store = TestStore::start(store_listener).await; + let store = TestStore::start(store_listener); let genesis = store.genesis_commitment(); // Override the client so that the ACCEPT header is not set. @@ -581,29 +577,16 @@ async fn start_rpc_with_options( ) -> (RpcClient, std::net::SocketAddr, TcpListener) { let store_listener = TcpListener::bind("127.0.0.1:0").await.expect("store should bind a port"); let store_addr = store_listener.local_addr().expect("store should get a local address"); - let block_producer_addr = { - let block_producer_listener = - TcpListener::bind("127.0.0.1:0").await.expect("Failed to bind block-producer"); - block_producer_listener - .local_addr() - .expect("Failed to get block-producer address") - }; - // Start the rpc component. let rpc_listener = TcpListener::bind("127.0.0.1:0").await.expect("Failed to bind rpc"); let rpc_addr = rpc_listener.local_addr().expect("Failed to get rpc address"); task::spawn(async move { // SAFETY: The store_addr is always valid as it is created from a `SocketAddr`. let store_url = Url::parse(&format!("http://{store_addr}")).unwrap(); - // SAFETY: The block_producer_addr is always valid as it is created from a `SocketAddr`. - let block_producer_url = Url::parse(&format!("http://{block_producer_addr}")).unwrap(); - // SAFETY: Using dummy validator URL for test - not actually contacted in this test - let validator_url = Url::parse("http://127.0.0.1:0").unwrap(); Rpc { listener: rpc_listener, store_url, - block_producer_url: Some(block_producer_url), - validator_url, + submission: RpcSubmissionMode::ReadOnly, ntx_builder_url: None, grpc_options, } @@ -623,7 +606,7 @@ async fn start_rpc_with_options( async fn get_limits_endpoint() { // Start the RPC and store let (mut rpc_client, _rpc_addr, store_listener) = start_rpc().await; - let _store = TestStore::start(store_listener).await; + let _store = TestStore::start(store_listener); // Call the get_limits endpoint let response = rpc_client.get_limits(()).await.expect("get_limits should succeed"); @@ -688,7 +671,7 @@ async fn get_limits_endpoint() { #[tokio::test] async fn sync_chain_mmr_returns_delta() { let (mut rpc_client, _rpc_addr, store_listener) = start_rpc().await; - let _store = TestStore::start(store_listener).await; + let _store = TestStore::start(store_listener); let request = proto::rpc::SyncChainMmrRequest { current_client_block_height: 0, diff --git a/crates/store/README.md b/crates/store/README.md index 5331a0df6e..2ff5ded38e 100644 --- a/crates/store/README.md +++ b/crates/store/README.md @@ -38,17 +38,13 @@ Without the environment variables above, `librocksdb-sys` compiles RocksDB from ## API overview -The full gRPC API can be found [here](../../proto/proto/store.proto). +The full gRPC API can be found [here](../../proto/proto/internal/store.proto). -- [ApplyBlock](#applyblock) - [GetAccount](#getaccount) - [GetBlockByNumber](#getblockbynumber) - [GetBlockHeaderByNumber](#getblockheaderbynumber) -- [GetBlockInputs](#getblockinputs) -- [GetNoteAuthenticationInfo](#getnoteauthenticationinfo) - [GetNotesById](#getnotesbyid) -- [GetTransactionInputs](#gettransactioninputs) - [GetNoteScriptByRoot](#getnotescriptbyroot) - [SyncNullifiers](#syncnullifiers) - [SyncAccountVault](#syncaccountvault) @@ -60,12 +56,6 @@ The full gRPC API can be found [here](../../proto/proto/store.proto). --- -### ApplyBlock - -Applies changes of a new block to the DB and in-memory data structures. Raw block data is also stored as a flat file. - ---- - ### GetAccount Returns an account witness (Merkle proof of inclusion in the account tree) and optionally account details. @@ -89,20 +79,6 @@ authenticate the block's inclusion. --- -### GetBlockInputs - -Used by the `block-producer` to query state required to prove the next block. - ---- - -### GetNoteAuthenticationInfo - -Returns a list of Note inclusion proofs for the specified Note IDs. - -This is used by the `block-producer` as part of the batch proving process. - ---- - ### GetNotesById Returns a list of notes matching the provided note IDs. @@ -121,12 +97,6 @@ When note retrieval fails, detailed error information is provided through gRPC s --- -### GetTransactionInputs - -Used by the `block-producer` to query state required to verify a submitted transaction. - ---- - ### GetNoteScriptByRoot Returns the script for a note by its root. diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index a2a30d4c84..61d4e7320b 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -13,7 +13,13 @@ pub use accounts::PersistentAccountTree; pub use accounts::{AccountTreeWithHistory, HistoricalError, InMemoryAccountTree}; pub use db::Db; pub use db::models::conv::SqlTypeConvert; -pub use errors::DatabaseError; +pub use errors::{ + ApplyBlockError, + DatabaseError, + GetBatchInputsError, + GetBlockHeaderError, + GetBlockInputsError, +}; pub use genesis::GenesisState; pub use server::block_prover_client::BlockProver; pub use server::proof_scheduler::DEFAULT_MAX_CONCURRENT_PROOFS; diff --git a/crates/store/src/server/api.rs b/crates/store/src/server/api.rs index c5d393eb65..4ec4a3abdf 100644 --- a/crates/store/src/server/api.rs +++ b/crates/store/src/server/api.rs @@ -1,20 +1,12 @@ -use std::collections::BTreeSet; use std::sync::Arc; -use miden_node_proto::decode::ConversionResultExt; -use miden_node_proto::errors::ConversionError; use miden_node_proto::generated as proto; -use miden_node_utils::ErrorReport; -use miden_protocol::Word; -use miden_protocol::batch::OrderedBatches; -use miden_protocol::block::{BlockInputs, BlockNumber}; -use miden_protocol::note::Nullifier; +use miden_protocol::block::BlockNumber; use tokio::sync::{Semaphore, watch}; use tonic::{Request, Response, Status}; -use tracing::{info, instrument}; +use tracing::info; use crate::COMPONENT; -use crate::errors::GetBlockInputsError; use crate::state::{BlockCache, ProofCache, State}; // STORE API @@ -77,40 +69,6 @@ impl StoreApi { mmr_path: mmr_proof.map(|p| Into::into(p.merkle_path())), })) } - - /// Retrieves block inputs from state based on the contents of the supplied ordered batches. - pub(crate) async fn block_inputs_from_ordered_batches( - &self, - batches: &OrderedBatches, - ) -> Result { - // Construct fields required to retrieve block inputs. - let mut account_ids = BTreeSet::new(); - let mut nullifiers = Vec::new(); - let mut unauthenticated_note_commitments = BTreeSet::new(); - let mut reference_blocks = BTreeSet::new(); - - for batch in batches.as_slice() { - account_ids.extend(batch.updated_accounts()); - nullifiers.extend(batch.created_nullifiers()); - reference_blocks.insert(batch.reference_block_num()); - - for note in batch.input_notes().iter() { - if let Some(header) = note.header() { - unauthenticated_note_commitments.insert(header.to_commitment()); - } - } - } - - // Retrieve block inputs from the store. - self.state - .get_block_inputs( - account_ids.into_iter().collect(), - nullifiers, - unauthenticated_note_commitments, - reference_blocks, - ) - .await - } } // UTILITIES @@ -120,58 +78,3 @@ impl StoreApi { pub fn internal_error(err: E) -> Status { Status::internal(err.to_string()) } - -/// Formats an "Invalid argument" error -pub fn invalid_argument(err: E) -> Status { - Status::invalid_argument(err.to_string()) -} - -/// Converts `ConversionError` to Status for nullifier validation -pub fn conversion_error_to_status(value: &ConversionError) -> Status { - invalid_argument(value.as_report_context("Invalid nullifier format")) -} - -#[instrument( - level = "debug", - target = COMPONENT, - skip_all, - fields(nullifiers = nullifiers.len()), - err -)] -pub fn validate_nullifiers(nullifiers: &[proto::primitives::Digest]) -> Result, E> -where - E: From + std::fmt::Display, -{ - nullifiers - .iter() - .copied() - .map(Nullifier::try_from) - .collect::>() - .context("nullifiers") - .map_err(Into::into) -} - -#[instrument( - level = "debug", - target = COMPONENT, - skip_all, - fields(notes = notes.len()), - err -)] -pub fn validate_note_commitments(notes: &[proto::primitives::Digest]) -> Result, Status> { - notes - .iter() - .map(Word::try_from) - .collect::, _>>() - .map_err(|_| invalid_argument("Digest field is not in the modulus range")) -} - -#[instrument( - level = "debug", - target = COMPONENT, - skip_all, - fields(block_numbers = block_numbers.len()) -)] -pub fn read_block_numbers(block_numbers: &[u32]) -> BTreeSet { - BTreeSet::from_iter(block_numbers.iter().map(|raw_number| BlockNumber::from(*raw_number))) -} diff --git a/crates/store/src/server/block_producer.rs b/crates/store/src/server/block_producer.rs deleted file mode 100644 index 7f971b8ecf..0000000000 --- a/crates/store/src/server/block_producer.rs +++ /dev/null @@ -1,258 +0,0 @@ -use std::convert::Infallible; - -use miden_crypto::dsa::ecdsa_k256_keccak::Signature; -use miden_node_proto::decode::{GrpcDecodeExt, read_account_id, read_account_ids}; -use miden_node_proto::domain::proof_request::BlockProofRequest; -use miden_node_proto::errors::ConversionError; -use miden_node_proto::generated::store::block_producer_server; -use miden_node_proto::generated::{self as proto}; -use miden_node_proto::{decode, try_convert}; -use miden_node_utils::ErrorReport; -use miden_node_utils::tracing::OpenTelemetrySpanExt; -use miden_protocol::Word; -use miden_protocol::batch::OrderedBatches; -use miden_protocol::block::{BlockBody, BlockHeader, BlockNumber, SignedBlock}; -use miden_protocol::utils::serde::Deserializable; -use tokio::sync::watch; -use tonic::{Request, Response, Status}; -use tracing::{Instrument, error}; - -use crate::errors::ApplyBlockError; -use crate::server::api::{ - StoreApi, - conversion_error_to_status, - read_block_numbers, - validate_note_commitments, - validate_nullifiers, -}; -use crate::state::Finality; - -// BLOCK PRODUCER API -// ================================================================================================ - -/// Extends [`StoreApi`] with the proof-scheduler notification channel, which is only required by -/// the `BlockProducer` gRPC service. Not used in replica mode. -#[derive(Clone)] -pub(super) struct BlockProducerApi { - pub(super) inner: StoreApi, - /// Notifies the proof scheduler of the latest committed block number after each `apply_block`. - pub(super) chain_tip_sender: watch::Sender, -} - -// BLOCK PRODUCER ENDPOINTS -// ================================================================================================ - -#[tonic::async_trait] -impl block_producer_server::BlockProducer for BlockProducerApi { - /// Returns block header for the specified block number. - /// - /// If the block number is not provided, block header for the latest block is returned. - async fn get_block_header_by_number( - &self, - request: Request, - ) -> Result, Status> { - self.inner.get_block_header_by_number_inner(request).await - } - - /// Updates the local DB by inserting a new block header and the related data. - async fn apply_block( - &self, - request: Request, - ) -> Result, Status> { - let request = request.into_inner(); - // Read ordered batches. - let ordered_batches = - OrderedBatches::read_from_bytes(&request.ordered_batches).map_err(|err| { - Status::invalid_argument( - err.as_report_context("failed to deserialize ordered batches"), - ) - })?; - // Read block. - let block = request - .block - .ok_or(ConversionError::missing_field::("block"))?; - // Decode block fields. - let decoder = block.decoder(); - let header: BlockHeader = decode!(decoder, block.header)?; - let body: BlockBody = decode!(decoder, block.body)?; - let signature: Signature = decode!(decoder, block.signature)?; - - // Get block inputs from ordered batches. - let block_inputs = - self.inner.block_inputs_from_ordered_batches(&ordered_batches).await.map_err( - |err| { - Status::invalid_argument( - err.as_report_context("failed to get block inputs from ordered batches"), - ) - }, - )?; - - let span = tracing::Span::current(); - span.set_attribute("block.number", header.block_num()); - span.set_attribute("block.commitment", header.commitment()); - span.set_attribute("block.accounts.count", body.updated_accounts().len()); - span.set_attribute("block.output_notes.count", body.output_notes().count()); - span.set_attribute("block.nullifiers.count", body.created_nullifiers().len()); - - // Construct block proof request to be stored alongside the block for deferred block - // proving. - let proving_inputs = BlockProofRequest { - tx_batches: ordered_batches, - block_header: header.clone(), - block_inputs, - }; - let block_num = header.block_num(); - self.inner - .state - .save_proving_inputs(block_num, &proving_inputs) - .await - .map_err(|err| Status::new(tonic::Code::Internal, err.as_report()))?; - - // We perform the apply block work in a separate task. This prevents the caller - // cancelling the request and thereby cancelling the task at an arbitrary point of - // execution. - // - // Normally this shouldn't be a problem, however our apply_block isn't quite ACID compliant - // so things get a bit messy. This is more a temporary hack-around to minimize this risk. - let this = self.clone(); - tokio::spawn( - async move { - let signed_block = SignedBlock::new(header, body, signature) - .map_err(|err| Status::new(tonic::Code::Internal, err.as_report()))?; - // Note: This is an internal endpoint, so its safe to expose the full error report. - this.inner - .state - .apply_block(signed_block) - .await - .inspect(|_| { - if let Err(err) = this.chain_tip_sender.send(block_num) { - error!("Failed to send chain tip: {:?}", err); - } - }) - .map_err(|err| { - span.set_error(&err); - let code = match err { - ApplyBlockError::InvalidBlockError(_) => tonic::Code::InvalidArgument, - _ => tonic::Code::Internal, - }; - Status::new(code, err.as_report()) - }) - } - .in_current_span(), - ) - .await - .map_err(|err| { - tonic::Status::internal(err.as_report_context("joining apply_block task failed")) - }) - .flatten()?; - Ok(Response::new(())) - } - - /// Returns data needed by the block producer to construct and prove the next block. - async fn get_block_inputs( - &self, - request: Request, - ) -> Result, Status> { - let request = request.into_inner(); - - let account_ids = read_account_ids::(request.account_ids)?; - let nullifiers = validate_nullifiers(&request.nullifiers) - .map_err(|err| conversion_error_to_status(&err))?; - let unauthenticated_note_commitments = - validate_note_commitments(&request.unauthenticated_notes)?; - let reference_blocks = read_block_numbers(&request.reference_blocks); - let unauthenticated_note_commitments = - unauthenticated_note_commitments.into_iter().collect(); - - self.inner - .state - .get_block_inputs( - account_ids, - nullifiers, - unauthenticated_note_commitments, - reference_blocks, - ) - .await - .map(proto::store::BlockInputs::from) - .map(Response::new) - .inspect_err(|err| tracing::Span::current().set_error(err)) - .map_err(|err| tonic::Status::internal(err.as_report())) - } - - /// Fetches the inputs for a transaction batch from the database. - /// - /// See [`State::get_batch_inputs`] for details. - async fn get_batch_inputs( - &self, - request: Request, - ) -> Result, Status> { - let request = request.into_inner(); - - let note_commitments: Vec = try_convert(request.note_commitments) - .collect::>() - .map_err(|err| Status::invalid_argument(format!("Invalid note commitment: {err}")))?; - - let reference_blocks: Vec = - try_convert::<_, Infallible, _, _>(request.reference_blocks) - .collect::, _>>() - .expect("operation should be infallible"); - let reference_blocks = reference_blocks.into_iter().map(BlockNumber::from).collect(); - - self.inner - .state - .get_batch_inputs(reference_blocks, note_commitments.into_iter().collect()) - .await - .map(Into::into) - .map(Response::new) - .inspect_err(|err| tracing::Span::current().set_error(err)) - .map_err(|err| tonic::Status::internal(err.as_report())) - } - - async fn get_transaction_inputs( - &self, - request: Request, - ) -> Result, Status> { - let request = request.into_inner(); - - let account_id = - read_account_id::(request.account_id)?; - let nullifiers = validate_nullifiers(&request.nullifiers) - .map_err(|err| conversion_error_to_status(&err))?; - let unauthenticated_note_commitments = - validate_note_commitments(&request.unauthenticated_notes)?; - - let tx_inputs = self - .inner - .state - .get_transaction_inputs(account_id, &nullifiers, unauthenticated_note_commitments) - .await - .inspect_err(|err| tracing::Span::current().set_error(err)) - .map_err(|err| tonic::Status::internal(err.as_report()))?; - - let block_height = self.inner.state.chain_tip(Finality::Committed).await.as_u32(); - - Ok(Response::new(proto::store::TransactionInputs { - account_state: Some(proto::store::transaction_inputs::AccountTransactionInputRecord { - account_id: Some(account_id.into()), - account_commitment: Some(tx_inputs.account_commitment.into()), - }), - nullifiers: tx_inputs - .nullifiers - .into_iter() - .map(|nullifier| { - proto::store::transaction_inputs::NullifierTransactionInputRecord { - nullifier: Some(nullifier.nullifier.into()), - block_num: nullifier.block_num.as_u32(), - } - }) - .collect(), - found_unauthenticated_notes: tx_inputs - .found_unauthenticated_notes - .into_iter() - .map(Into::into) - .collect(), - new_account_id_prefix_is_unique: tx_inputs.new_account_id_prefix_is_unique, - block_height, - })) - } -} diff --git a/crates/store/src/server/mod.rs b/crates/store/src/server/mod.rs index 98c21ef84e..7db7ba37ea 100644 --- a/crates/store/src/server/mod.rs +++ b/crates/store/src/server/mod.rs @@ -13,7 +13,6 @@ use miden_node_utils::spawn::spawn_blocking_in_span; use miden_node_utils::tracing::OpenTelemetrySpanExt; use miden_node_utils::tracing::grpc::grpc_trace_fn; use tokio::net::TcpListener; -use tokio::sync::watch; use tokio::task::JoinSet; use tokio_stream::wrappers::TcpListenerStream; use tower_http::trace::TraceLayer; @@ -30,7 +29,6 @@ use crate::state::{ProofCache, State}; use crate::{BlockProver, COMPONENT}; mod api; -mod block_producer; pub mod block_prover_client; mod replica_sync; @@ -41,17 +39,13 @@ mod rpc_api; /// Determines how the store receives new blocks. /// -/// The two modes are mutually exclusive: a store either accepts blocks from a block producer -/// via its `BlockProducer` gRPC service, or it syncs blocks from an upstream store instance. -/// The services exposed on the network differ between modes accordingly. +/// The two modes are mutually exclusive: a store either runs alongside a block producer sharing +/// the same [`State`], or it syncs blocks from an upstream store instance. pub enum StoreMode { - /// Accepts blocks from a block producer via the `BlockProducer` gRPC service. + /// Runs in the sequencer process alongside the block producer. /// - /// Exposes the `Rpc` and `BlockProducer` gRPC services and runs the proof scheduler to - /// generate block proofs. + /// Exposes the `Rpc` gRPC service and runs the proof scheduler to generate block proofs. BlockProducer { - /// Listener for the block producer gRPC endpoint. - block_producer_listener: TcpListener, /// URL of the remote block prover. Uses a local prover if `None`. block_prover_url: Option, /// Maximum number of blocks proven concurrently by the proof scheduler. @@ -152,21 +146,15 @@ impl Store { let _disk_monitor_task = Self::spawn_disk_monitor(self.data_directory.clone()); let ModeSetup { mut grpc_servers, mode_task } = match self.mode { - StoreMode::BlockProducer { - block_producer_listener, - block_prover_url, - max_concurrent_proofs, - } => { + StoreMode::BlockProducer { block_prover_url, max_concurrent_proofs } => { Self::setup_block_producer_mode( state, - block_producer_listener, block_prover_url, max_concurrent_proofs, tx_proven_tip, self.grpc_options, self.rpc_listener, - ) - .await? + )? }, StoreMode::Replica { upstream_url } => { Self::setup_replica_mode(state, upstream_url, self.grpc_options, self.rpc_listener)? @@ -193,43 +181,29 @@ impl Store { } } - async fn setup_block_producer_mode( + fn setup_block_producer_mode( state: State, - block_producer_listener: TcpListener, block_prover_url: Option, max_concurrent_proofs: NonZeroUsize, tx_proven_tip: ProvenTipWriter, grpc_options: GrpcOptionsInternal, rpc_listener: TcpListener, ) -> anyhow::Result { - info!(target: COMPONENT, - block_producer_endpoint=?block_producer_listener.local_addr()?, - "Starting in block-producer mode"); + info!(target: COMPONENT, "Starting in block-producer mode"); let proof_cache = state.proof_cache.clone(); - let (proof_scheduler_task, chain_tip_sender) = Self::spawn_proof_scheduler( + let proof_scheduler_task = Self::spawn_proof_scheduler( &state, block_prover_url, max_concurrent_proofs, tx_proven_tip, proof_cache, - ) - .await; + ); let state = Arc::new(state); let store_api = api::StoreApi::new(state); - let block_producer_api = block_producer::BlockProducerApi { - inner: store_api.clone(), - chain_tip_sender, - }; - let join_set = Self::spawn_block_producer_grpc_servers( - store_api, - block_producer_api, - grpc_options, - rpc_listener, - block_producer_listener, - )?; + let join_set = Self::spawn_rpc_grpc_server(store_api, grpc_options, rpc_listener)?; Ok(ModeSetup { grpc_servers: join_set, @@ -256,7 +230,7 @@ impl Store { }); let store_api = api::StoreApi::new(state); - let join_set = Self::spawn_replica_grpc_servers(store_api, grpc_options, rpc_listener)?; + let join_set = Self::spawn_rpc_grpc_server(store_api, grpc_options, rpc_listener)?; Ok(ModeSetup { grpc_servers: join_set, @@ -266,89 +240,34 @@ impl Store { /// Initializes the block prover client and spawns the proof scheduler as a background task. /// - /// Returns the scheduler task handle and the chain tip sender (needed by the block-producer - /// gRPC service to notify the scheduler of new blocks). - async fn spawn_proof_scheduler( + /// Returns the scheduler task handle. + fn spawn_proof_scheduler( state: &State, block_prover_url: Option, max_concurrent_proofs: NonZeroUsize, proven_tip: ProvenTipWriter, proof_cache: ProofCache, - ) -> ( - tokio::task::JoinHandle>, - watch::Sender, - ) { + ) -> tokio::task::JoinHandle> { let block_prover = if let Some(url) = block_prover_url { Arc::new(BlockProver::remote(url)) } else { Arc::new(BlockProver::local()) }; - let chain_tip = state.chain_tip(crate::state::Finality::Committed).await; - let (chain_tip_tx, chain_tip_rx) = watch::channel(chain_tip); + let chain_tip_rx = state.subscribe_committed_tip(); - let handle = proof_scheduler::spawn( + proof_scheduler::spawn( block_prover, state.block_store(), chain_tip_rx, proven_tip, max_concurrent_proofs, proof_cache, - ); - - (handle, chain_tip_tx) - } - - /// Spawns the gRPC servers for block-producer mode. - /// - /// Starts two listeners: `Rpc` and `BlockProducer`. - fn spawn_block_producer_grpc_servers( - store_api: api::StoreApi, - block_producer_api: block_producer::BlockProducerApi, - grpc_options: GrpcOptionsInternal, - rpc_listener: TcpListener, - block_producer_listener: TcpListener, - ) -> anyhow::Result>> { - let mut join_set = JoinSet::new(); - - let rpc_service = store::rpc_server::RpcServer::new(store_api); - let block_producer_service = - store::block_producer_server::BlockProducerServer::new(block_producer_api); - - let reflection_service = tonic_reflection::server::Builder::configure() - .register_file_descriptor_set(store_api_descriptor()) - .build_v1() - .context("failed to build reflection service")?; - - let make_server = || { - tonic::transport::Server::builder() - .timeout(grpc_options.request_timeout) - .layer(CatchPanicLayer::custom(catch_panic_layer_fn)) - .layer(TraceLayer::new_for_grpc().make_span_with(grpc_trace_fn)) - }; - - join_set.spawn( - make_server() - .add_service(rpc_service) - .add_service(reflection_service.clone()) - .serve_with_incoming(TcpListenerStream::new(rpc_listener)), - ); - - join_set.spawn( - make_server() - .accept_http1(true) - .add_service(block_producer_service) - .add_service(reflection_service) - .serve_with_incoming(TcpListenerStream::new(block_producer_listener)), - ); - - Ok(join_set) + ) } - /// Spawns the gRPC servers for replica mode. - /// - /// Only the `Rpc` service is exposed — no `BlockProducer` or proof scheduler. - fn spawn_replica_grpc_servers( + /// Spawns the store RPC gRPC server. + fn spawn_rpc_grpc_server( store_api: api::StoreApi, grpc_options: GrpcOptionsInternal, rpc_listener: TcpListener, diff --git a/proto/Cargo.toml b/proto/Cargo.toml index dcdbbe5f66..6904f73b58 100644 --- a/proto/Cargo.toml +++ b/proto/Cargo.toml @@ -12,8 +12,7 @@ rust-version.workspace = true version.workspace = true [features] -# Enables the gRPC file descriptors for the internal component APIs, -# i.e. the `store` and `block-producer` gRPC services. +# Enables the gRPC file descriptors for the internal component APIs. internal = [] [lints] diff --git a/proto/proto/internal/block_producer.proto b/proto/proto/internal/block_producer.proto deleted file mode 100644 index 407054cd89..0000000000 --- a/proto/proto/internal/block_producer.proto +++ /dev/null @@ -1,84 +0,0 @@ -// Specification of the user facing gRPC API. -syntax = "proto3"; -package block_producer; - -import "google/protobuf/empty.proto"; -import "rpc.proto"; -import "types/blockchain.proto"; -import "types/note.proto"; -import "types/primitives.proto"; -import "types/transaction.proto"; - -// BLOCK PRODUCER SERVICE -// ================================================================================================ - -service Api { - // Returns the status info. - rpc Status(google.protobuf.Empty) returns (rpc.BlockProducerStatus) {} - - // Submits proven transaction to the Miden network. Returns the node's current block height. - rpc SubmitProvenTx(transaction.ProvenTransaction) returns (blockchain.BlockNumber) {} - - // Submits a batch of transactions to the Miden network. - // - // All transactions in this batch will be considered atomic, and be committed together or not all. - // - // Returns the node's current block height. - rpc SubmitProvenTxBatch(transaction.TransactionBatch) returns (blockchain.BlockNumber) {} - - // Subscribe to mempool events. - // - // The event stream will contain all events after the current chain tip. This includes all - // currently inflight events that have not yet been committed to the chain. - // - // Currently only a single active subscription is supported. Subscription requests will cancel - // the active subscription, if any. - rpc MempoolSubscription(google.protobuf.Empty) returns (stream MempoolEvent) {} -} - -// MEMPOOL SUBSCRIPTION -// ================================================================================================ - -// Request to subscribe to mempool events. -message MempoolSubscriptionRequest {} - -// Event from the mempool. -message MempoolEvent { - // A block was committed. - // - // This event is sent when a block is committed to the chain. - message BlockCommitted { - blockchain.BlockHeader block_header = 1; - repeated transaction.TransactionId transactions = 2; - } - - // A transaction was added to the mempool. - // - // This event is sent when a transaction is added to the mempool. - message TransactionAdded { - // The ID of the transaction. - transaction.TransactionId id = 1; - // Nullifiers consumed by the transaction. - repeated primitives.Digest nullifiers = 2; - // Network notes created by the transaction. - repeated note.NetworkNote network_notes = 3; - // Changes to a network account, if any. This includes creation of new network accounts. - // - // The account delta is encoded using [miden_serde_utils::Serializable] implementation - // for [miden_protocol::account::delta::AccountDelta]. - optional bytes network_account_delta = 4; - } - - // A set of transactions was reverted and dropped from the mempool. - // - // This event is sent when a set of transactions are reverted and dropped from the mempool. - message TransactionsReverted { - repeated transaction.TransactionId reverted = 1; - } - - oneof event { - TransactionAdded transaction_added = 1; - BlockCommitted block_committed = 2; - TransactionsReverted transactions_reverted = 3; - } -} diff --git a/proto/proto/internal/store.proto b/proto/proto/internal/store.proto index afa058ca69..54f862967d 100644 --- a/proto/proto/internal/store.proto +++ b/proto/proto/internal/store.proto @@ -3,11 +3,8 @@ syntax = "proto3"; package store; import "google/protobuf/empty.proto"; -import "types/account.proto"; import "types/blockchain.proto"; -import "types/transaction.proto"; import "types/note.proto"; -import "types/primitives.proto"; import "rpc.proto"; // RPC STORE API @@ -69,170 +66,3 @@ service Rpc { // On lag, the stream is closed with a DATA_LOSS error. rpc ProofSubscription(rpc.ProofSubscriptionRequest) returns (stream rpc.ProofSubscriptionResponse) {} } - -// BLOCK PRODUCER STORE API -// ================================================================================================ - -// Store API for the BlockProducer component -service BlockProducer { - // Applies changes of a new block to the DB and in-memory data structures. - rpc ApplyBlock(ApplyBlockRequest) returns (google.protobuf.Empty) {} - - // Retrieves block header by given block number. Optionally, it also returns the MMR path - // and current chain length to authenticate the block's inclusion. - rpc GetBlockHeaderByNumber(rpc.BlockHeaderByNumberRequest) returns (rpc.BlockHeaderByNumberResponse) {} - - // Returns data required to prove the next block. - rpc GetBlockInputs(BlockInputsRequest) returns (BlockInputs) {} - - // Returns the inputs for a transaction batch. - rpc GetBatchInputs(BatchInputsRequest) returns (BatchInputs) {} - - // Returns data required to validate a new transaction. - rpc GetTransactionInputs(TransactionInputsRequest) returns (TransactionInputs) {} -} - -// APPLY BLOCK REQUEST -// ================================================================================================ - -// Applies a block to the state. -message ApplyBlockRequest { - // Ordered batches encoded using [miden_serde_utils::Serializable] implementation for - // [miden_objects::batch::OrderedBatches]. - bytes ordered_batches = 1; - // Block signed by the Validator. - blockchain.SignedBlock block = 2; -} - -// GET BLOCK INPUTS -// ================================================================================================ - -// Returns data required to prove the next block. -message BlockInputsRequest { - // IDs of all accounts updated in the proposed block for which to retrieve account witnesses. - repeated account.AccountId account_ids = 1; - - // Nullifiers of all notes consumed by the block for which to retrieve witnesses. - // - // Due to note erasure it will generally not be possible to know the exact set of nullifiers - // a block will create, unless we pre-execute note erasure. So in practice, this set of - // nullifiers will be the set of nullifiers of all proven batches in the block, which is a - // superset of the nullifiers the block may create. - // - // However, if it is known that a certain note will be erased, it would not be necessary to - // provide a nullifier witness for it. - repeated primitives.Digest nullifiers = 2; - - // Array of note IDs for which to retrieve note inclusion proofs, **if they exist in the store**. - repeated primitives.Digest unauthenticated_notes = 3; - - // Array of block numbers referenced by all batches in the block. - repeated fixed32 reference_blocks = 4; -} - -// Represents the result of getting block inputs. -message BlockInputs { - // A nullifier returned as a response to the `GetBlockInputs`. - message NullifierWitness { - // The nullifier. - primitives.Digest nullifier = 1; - - // The SMT proof to verify the nullifier's inclusion in the nullifier tree. - primitives.SmtOpening opening = 2; - } - // The latest block header. - blockchain.BlockHeader latest_block_header = 1; - - // Proof of each requested unauthenticated note's inclusion in a block, **if it existed in - // the store**. - repeated note.NoteInclusionInBlockProof unauthenticated_note_proofs = 2; - - // The serialized chain MMR which includes proofs for all blocks referenced by the - // above note inclusion proofs as well as proofs for inclusion of the requested blocks - // referenced by the batches in the block. - bytes partial_block_chain = 3; - - // The state commitments of the requested accounts and their authentication paths. - repeated account.AccountWitness account_witnesses = 4; - - // The requested nullifiers and their authentication paths. - repeated NullifierWitness nullifier_witnesses = 5; -} - -// GET BATCH INPUTS -// ================================================================================================ - -// Returns the inputs for a transaction batch. -message BatchInputsRequest { - // List of unauthenticated note commitments to be queried from the database. - repeated primitives.Digest note_commitments = 1; - // Set of block numbers referenced by transactions. - repeated fixed32 reference_blocks = 2; -} - -// Represents the result of getting batch inputs. -message BatchInputs { - // The block header that the transaction batch should reference. - blockchain.BlockHeader batch_reference_block_header = 1; - - // Proof of each _found_ unauthenticated note's inclusion in a block. - repeated note.NoteInclusionInBlockProof note_proofs = 2; - - // The serialized chain MMR which includes proofs for all blocks referenced by the - // above note inclusion proofs as well as proofs for inclusion of the blocks referenced - // by the transactions in the batch. - bytes partial_block_chain = 3; -} - -// GET TRANSACTION INPUTS -// ================================================================================================ - -// Returns data required to validate a new transaction. -message TransactionInputsRequest { - // ID of the account against which a transaction is executed. - account.AccountId account_id = 1; - // Set of nullifiers consumed by this transaction. - repeated primitives.Digest nullifiers = 2; - // Set of unauthenticated note commitments to check for existence on-chain. - // - // These are notes which were not on-chain at the state the transaction was proven, - // but could by now be present. - repeated primitives.Digest unauthenticated_notes = 3; -} - -// Represents the result of getting transaction inputs. -message TransactionInputs { - // An account returned as a response to the `GetTransactionInputs`. - message AccountTransactionInputRecord { - // The account ID. - account.AccountId account_id = 1; - - // The latest account commitment, zero commitment if the account doesn't exist. - primitives.Digest account_commitment = 2; - } - - // A nullifier returned as a response to the `GetTransactionInputs`. - message NullifierTransactionInputRecord { - // The nullifier ID. - primitives.Digest nullifier = 1; - - // The block at which the nullifier has been consumed, zero if not consumed. - fixed32 block_num = 2; - } - - // Account state proof. - AccountTransactionInputRecord account_state = 1; - - // List of nullifiers that have been consumed. - repeated NullifierTransactionInputRecord nullifiers = 2; - - // List of unauthenticated notes that were not found in the database. - repeated primitives.Digest found_unauthenticated_notes = 3; - - // The node's current block height. - fixed32 block_height = 4; - - // Whether the account ID prefix is unique. Only relevant for account creation requests. - optional bool new_account_id_prefix_is_unique = 5; // TODO: Replace this with an error. When a general error message exists. -} -