diff --git a/.env b/.env index 7140065e..9a251637 100644 --- a/.env +++ b/.env @@ -13,13 +13,22 @@ NIGHTFALL_ADDRESS="0x68B1D87F95878fE05B998F19b66F4baba5De1aed" # Contract(0) WEBHOOK_URL= # Azure Key Vault configuration AZURE_KEY_NAME=ethereum-signing-key -AZURE_VAULT_URL=https://nightfall-kv-1760976507.vault.azure.net/ +AZURE_VAULT_URL=https://nightfall-certificate.vault.azure.net/ CLIENT_SIGNING_KEY_NAME=ethereum-signing-key CLIENT2_SIGNING_KEY_NAME=ethereum-signing-key DEPLOYER_SIGNING_KEY_NAME=ethereum-signing-key PROPOSER_SIGNING_KEY_NAME=ethereum-signing-key PROPOSER_2_SIGNING_KEY_NAME=ethereum-signing-key + # Azure Authentication (KEEP THESE SECRET!) AZURE_CLIENT_ID=your-client-id-here -AZURE_CLIENT_SECRET=your-client-secret-her +AZURE_CLIENT_SECRET=your-client-secret-here AZURE_TENANT_ID=your-tenant-id-here +NF4_NIGHTFALL_CLIENT2__WALLET_TYPE=local +NF4_NIGHTFALL_CLIENT__WALLET_TYPE=local +NF4_NIGHTFALL_CLIENT__X509_SIGNER_TYPE=local +NF4_NIGHTFALL_CLIENT2__X509_SIGNER_TYPE=local +CLIENT_X509_SIGNING_KEY_NAME=user-1-cert-key +CLIENT2_X509_SIGNING_KEY_NAME=your-client2-x509-key-name-here +NF4_NIGHTFALL_PROPOSER__WALLET_TYPE=local +NF4_NIGHTFALL_PROPOSER__X509_SIGNER_TYPE=local diff --git a/.github/workflows/on-pull-request.yml b/.github/workflows/on-pull-request.yml index 2087e096..6b9c9313 100644 --- a/.github/workflows/on-pull-request.yml +++ b/.github/workflows/on-pull-request.yml @@ -12,7 +12,15 @@ jobs: name: Check + Lint runs-on: self-hosted environment: DEV + env: + NF4_NIGHTFALL_PROPOSER__SNAPSHOT_ROOT_DIR: /tmp/proposer_snapshots/build steps: + - name: Cleanup stale proposer snapshots + shell: bash + run: | + set -euo pipefail + test -n "${NF4_NIGHTFALL_PROPOSER__SNAPSHOT_ROOT_DIR:-}" + rm -rf "$NF4_NIGHTFALL_PROPOSER__SNAPSHOT_ROOT_DIR" - name: Checkout uses: actions/checkout@v4 # Add Node so npx exists for OZ upgrades validation @@ -42,6 +50,13 @@ jobs: env: RUST_BACKTRACE: "1" RUST_MIN_STACK: "8388608" + - name: Cleanup snapshots after tests + if: always() + shell: bash + run: | + set -euo pipefail + test -n "${NF4_NIGHTFALL_PROPOSER__SNAPSHOT_ROOT_DIR:-}" + rm -rf "$NF4_NIGHTFALL_PROPOSER__SNAPSHOT_ROOT_DIR" resync: name: Synchronisation Test with Mock Prover needs: [build] @@ -49,11 +64,12 @@ jobs: env: NF4_MOCK_PROVER: "true" NF4_CONTRACTS__DEPLOY_CONTRACTS: "true" + NF4_PROPOSER_SNAPSHOT_MOUNT: proposer_snapshots_data steps: - name: Checkout uses: actions/checkout@v4 - name: Clear previous run - run: docker compose --profile sync_test down -v + run: docker compose --profile sync_test down -v --remove-orphans - name: Build run: cargo build - name: Generate keys @@ -61,14 +77,17 @@ jobs: - name: Build Docker Image run: docker compose --profile sync_test build - name: Run Integration Test - run: docker compose --profile sync_test run test + run: docker compose --profile sync_test run --rm test - name: Print logs - run: docker compose --profile sync_test logs + if: always() + run: docker compose --profile sync_test logs || true - name: Remove databases - run: docker compose --profile sync_test rm -fvs client client2 proposer db_client db_client2 db_proposer + if: always() + run: docker compose --profile sync_test rm -fvs client client2 proposer db_client db_client2 db_proposer || true - name: Shut Down Containers + if: always() run: | - docker compose --profile sync_test down -v + docker compose --profile sync_test down -v --remove-orphans || true rollup-prover: name: Rollup prover integration Test @@ -79,6 +98,7 @@ jobs: env: NF4_MOCK_PROVER: "false" NF4_CONTRACTS__DEPLOY_CONTRACTS: "true" + NF4_PROPOSER_SNAPSHOT_MOUNT: proposer_snapshots_data steps: - name: Checkout uses: actions/checkout@v4 @@ -143,7 +163,7 @@ jobs: ls -l "$CACHE_FILE" "$TARGET" fi - name: Clear previous run - run: docker compose --profile sync_test down -v + run: docker compose --profile sync_test down -v --remove-orphans - name: Build run: cargo build - name: Generate keys @@ -177,19 +197,35 @@ jobs: - name: Build Docker Image run: docker compose --profile sync_test build - name: Run Integration Test - run: docker compose --profile development run test + run: docker compose --profile sync_test run --rm test - name: Print logs - run: docker compose --profile development logs + if: always() + run: docker compose --profile sync_test logs || true - name: Shut Down Containers + if: always() run: | - docker compose --profile sync_test down -v - + docker compose --profile sync_test down -v --remove-orphans || true forge: name: Smart Contract Unit Tests needs: [build] runs-on: self-hosted + env: + NF4_NIGHTFALL_PROPOSER__SNAPSHOT_ROOT_DIR: /tmp/proposer_snapshots/forge steps: + - name: Cleanup stale proposer snapshots + shell: bash + run: | + set -euo pipefail + test -n "${NF4_NIGHTFALL_PROPOSER__SNAPSHOT_ROOT_DIR:-}" + rm -rf "$NF4_NIGHTFALL_PROPOSER__SNAPSHOT_ROOT_DIR" - name: Checkout uses: actions/checkout@v4 - name: Run Forge Tests run: forge test + - name: Cleanup snapshots after tests + if: always() + shell: bash + run: | + set -euo pipefail + test -n "${NF4_NIGHTFALL_PROPOSER__SNAPSHOT_ROOT_DIR:-}" + rm -rf "$NF4_NIGHTFALL_PROPOSER__SNAPSHOT_ROOT_DIR" diff --git a/.gitignore b/.gitignore index 72dd8e88..1e44b3a1 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ soljson* configuration/toml remappings.txt local.env +/data/proposer_snapshots/ +/nightfall_proposer/data/ +temp/ diff --git a/Cargo.lock b/Cargo.lock index 2e37be12..82c3107b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5680,6 +5680,7 @@ dependencies = [ "either", "futures", "hex", + "hmac", "itertools 0.10.5", "jf-plonk", "jf-primitives", @@ -5692,6 +5693,7 @@ dependencies = [ "nf-curves", "nightfall_bindings", "num-bigint", + "rand 0.8.5", "rayon", "serde", "serde_bytes", @@ -5999,15 +6001,14 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.75" +version = "0.10.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "bf0b434746ee2832f4f0baf10137e1cabb18cbe6912c69e2e33263c45250f542" dependencies = [ "bitflags 2.10.0", "cfg-if", "foreign-types", "libc", - "once_cell", "openssl-macros", "openssl-sys", ] @@ -6037,9 +6038,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.111" +version = "0.9.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +checksum = "158fe5b292746440aa6e7a7e690e55aeb72d41505e2804c23c6973ad0e9c9781" dependencies = [ "cc", "libc", diff --git a/README.md b/README.md index 83491834..a05cf966 100644 --- a/README.md +++ b/README.md @@ -7,3 +7,26 @@ Nightfall_4 is a ZK rollup build around the ZK Privacy of Nightfall. It enables Please note that this software should be treated as experimental. It should not be used to make significant value transactions. +## Proposer MongoDB + +For Docker-based development and sync testing, proposer MongoDB runs as a single-node replica set so the proposer can later use MongoDB transactions safely. + +- `db_proposer` uses replica set `nf4-rs-proposer` +- `db_proposer2` uses replica set `nf4-rs-proposer2` +- proposer MongoDB URIs should include both `replicaSet=...` and `directConnection=true` + +Examples: + +```text +mongodb://nf4_db_proposer:27017/?replicaSet=nf4-rs-proposer&directConnection=true +mongodb://nf4_db_proposer2:27017/?replicaSet=nf4-rs-proposer2&directConnection=true +``` + +Quick verification: + +```bash +docker exec nf4_db_proposer mongosh --quiet --eval 'rs.status()' +docker exec nf4_db_proposer2 mongosh --quiet --eval 'rs.status()' +``` + +This change is proposer-only. `db_client` and `db_client2` remain standalone for now and client-side MongoDB replica set conversion is intentionally deferred to a separate task. diff --git a/blockchain_assets/contracts/Nightfall.sol b/blockchain_assets/contracts/Nightfall.sol index b399328f..9e7b3646 100644 --- a/blockchain_assets/contracts/Nightfall.sol +++ b/blockchain_assets/contracts/Nightfall.sol @@ -447,7 +447,7 @@ contract Nightfall is uint256 secretHash, TokenType token_type ) external payable virtual onlyCertified nonReentrant { - uint256 nfTokenId = sha256_and_shift(abi.encode(ercAddress, tokenId)); + uint256 nfTokenId = sha256_and_shift(abi.encode(ercAddress, 0, tokenId)); tokenIdMapping[nfTokenId] = TokenIdValue(ercAddress, tokenId, token_type); uint256 nativeSlotId = tokenId; @@ -455,7 +455,7 @@ contract Nightfall is if (token_type == TokenType.ERC3525) { nativeSlotId = IERC3525(ercAddress).slotOf(tokenId); nfSlotId = uint256( - keccak256(abi.encode(ercAddress, nativeSlotId)) + keccak256(abi.encode(ercAddress, 1, nativeSlotId)) ) >> 4; } slotIdMapping[nfSlotId] = SlotIdValue(ercAddress, nativeSlotId, token_type); diff --git a/configuration/src/addresses.rs b/configuration/src/addresses.rs index 9bc4cb7d..1ee455f3 100644 --- a/configuration/src/addresses.rs +++ b/configuration/src/addresses.rs @@ -505,7 +505,16 @@ mod tests { use serial_test::serial; #[tokio::test] + #[serial] async fn test_checksum_validation() { + let tmp_run_mode = std::env::var("NF4_RUN_MODE").ok(); + let tmp_client_wallet_type = std::env::var("NF4_NIGHTFALL_CLIENT__WALLET_TYPE").ok(); + let tmp_proposer_wallet_type = std::env::var("NF4_NIGHTFALL_PROPOSER__WALLET_TYPE").ok(); + + std::env::set_var("NF4_RUN_MODE", "development"); + std::env::set_var("NF4_NIGHTFALL_CLIENT__WALLET_TYPE", "local"); + std::env::set_var("NF4_NIGHTFALL_PROPOSER__WALLET_TYPE", "local"); + let settings = Settings::new().expect("Could not load settings"); // checksummed address (valid) let valid_address = "0x52908400098527886E0F7030069857D2E4169EE7"; @@ -525,6 +534,19 @@ mod tests { }; addresses.ensure_nonzero(settings.mock_prover).unwrap(); + + match tmp_run_mode { + Some(val) => std::env::set_var("NF4_RUN_MODE", val), + None => std::env::remove_var("NF4_RUN_MODE"), + } + match tmp_client_wallet_type { + Some(val) => std::env::set_var("NF4_NIGHTFALL_CLIENT__WALLET_TYPE", val), + None => std::env::remove_var("NF4_NIGHTFALL_CLIENT__WALLET_TYPE"), + } + match tmp_proposer_wallet_type { + Some(val) => std::env::set_var("NF4_NIGHTFALL_PROPOSER__WALLET_TYPE", val), + None => std::env::remove_var("NF4_NIGHTFALL_PROPOSER__WALLET_TYPE"), + } } #[tokio::test] #[serial] diff --git a/configuration/src/settings.rs b/configuration/src/settings.rs index 3fd76a9f..40c5171a 100644 --- a/configuration/src/settings.rs +++ b/configuration/src/settings.rs @@ -3,7 +3,7 @@ use figment::{ Figment, }; use serde::{de, Deserialize, Deserializer, Serialize}; -use std::{env, sync::OnceLock}; +use std::{env, path::PathBuf, sync::OnceLock}; // rather than pass around what are effectively constant values, or recreate them locally, // let's use the lazy_static crate to create a global variable that can be used to consume @@ -62,12 +62,28 @@ pub enum WalletTypeConfig { EyTransactionManager, } +#[derive(Debug, Deserialize, Serialize, Default, PartialEq, Clone, Eq)] +#[serde(rename_all = "lowercase")] +pub enum X509SignerTypeConfig { + #[default] + Local, + Azure, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WalletRole { + Client, + Proposer, +} + #[derive(Debug, Deserialize, Default, Serialize)] #[allow(unused)] pub struct ClientConfig { pub url: String, pub log_level: String, pub wallet_type: WalletTypeConfig, + #[serde(default)] + pub x509_signer_type: X509SignerTypeConfig, pub db_url: String, pub max_event_listener_attempts: Option, pub webhook_url: String, @@ -80,12 +96,24 @@ pub struct ProposerConfig { pub url: String, pub log_level: String, pub wallet_type: WalletTypeConfig, + #[serde(default)] + pub x509_signer_type: X509SignerTypeConfig, pub db_url: String, pub block_assembly_max_wait_secs: u64, pub block_assembly_target_fill_ratio: f64, pub block_assembly_initial_interval_secs: u64, pub max_event_listener_attempts: Option, pub block_size: u64, + #[serde(default = "default_proposer_snapshot_root_dir")] + pub snapshot_root_dir: String, + #[serde(default = "default_snapshot_interval_l2_blocks")] + pub snapshot_interval_l2_blocks: u64, + #[serde(default = "default_snapshot_min_l1_confirmations")] + pub snapshot_min_l1_confirmations: u64, + #[serde(default = "default_snapshot_retention_count")] + pub snapshot_retention_count: u64, + #[serde(default = "default_snapshot_enabled")] + pub snapshot_enabled: bool, } #[derive(Debug, Deserialize, Default, Serialize)] @@ -146,6 +174,38 @@ fn default_rpc_rate_limit() -> u32 { 0 // 0 = unlimited } +fn default_proposer_snapshot_root_dir() -> String { + normalize_snapshot_root_dir("./data/proposer_snapshots") + .expect("default proposer snapshot root dir should resolve to an absolute path") +} + +fn default_snapshot_interval_l2_blocks() -> u64 { + 100 +} + +fn default_snapshot_min_l1_confirmations() -> u64 { + 12 +} + +fn default_snapshot_retention_count() -> u64 { + 5 +} + +fn default_snapshot_enabled() -> bool { + true +} + +fn normalize_snapshot_root_dir(path: &str) -> Result { + let path = PathBuf::from(path); + if path.is_absolute() { + return Ok(path.to_string_lossy().into_owned()); + } + + let current_dir = env::current_dir() + .map_err(|error| format!("Could not resolve current working directory: {error}"))?; + Ok(current_dir.join(path).to_string_lossy().into_owned()) +} + #[derive(Debug, Deserialize, Serialize, Default)] #[allow(unused)] pub struct Settings { @@ -154,8 +214,10 @@ pub struct Settings { pub swap_cancel_auth_token: Option, #[serde(default)] pub azure_vault_url: String, - #[serde(default)] - pub azure_key_name: String, + pub client_azure_key_name: Option, + pub proposer_azure_key_name: Option, + pub client_x509_azure_key_name: Option, + pub proposer_x509_azure_key_name: Option, pub log_app_only: bool, pub test_x509_certificates: bool, pub mock_prover: bool, @@ -193,18 +255,93 @@ impl Settings { .select(run_mode); let mut settings: Settings = figment.extract().map_err(|e| format!("{e}"))?; - // Check the wallet type and read additional Azure-specific settings - if settings.nightfall_client.wallet_type == WalletTypeConfig::Azure - || settings.nightfall_proposer.wallet_type == WalletTypeConfig::Azure + let client_uses_azure = settings.nightfall_client.wallet_type == WalletTypeConfig::Azure; + let proposer_uses_azure = + settings.nightfall_proposer.wallet_type == WalletTypeConfig::Azure; + let client_uses_x509_azure = + settings.nightfall_client.x509_signer_type == X509SignerTypeConfig::Azure; + let proposer_uses_x509_azure = + settings.nightfall_proposer.x509_signer_type == X509SignerTypeConfig::Azure; + + if client_uses_azure + || proposer_uses_azure + || client_uses_x509_azure + || proposer_uses_x509_azure { - settings.azure_vault_url = env::var("AZURE_VAULT_URL").unwrap_or_default(); - settings.azure_key_name = env::var("PROPOSER_SIGNING_KEY_NAME") - .or_else(|_| env::var("CLIENT_SIGNING_KEY_NAME")) - .or_else(|_| env::var("AZURE_KEY_NAME")) - .unwrap_or_default(); + settings.azure_vault_url = + Self::require_env("AZURE_VAULT_URL", "Azure signer configuration")?; } + if client_uses_azure { + settings.client_azure_key_name = Some(Self::require_env( + "CLIENT_SIGNING_KEY_NAME", + "Azure client wallet", + )?); + } + if proposer_uses_azure { + settings.proposer_azure_key_name = Some(Self::require_env( + "PROPOSER_SIGNING_KEY_NAME", + "Azure proposer wallet", + )?); + } + if client_uses_x509_azure { + settings.client_x509_azure_key_name = Some(Self::require_env( + "CLIENT_X509_SIGNING_KEY_NAME", + "Azure client X509 signer", + )?); + } + if proposer_uses_x509_azure { + settings.proposer_x509_azure_key_name = Some(Self::require_env( + "PROPOSER_X509_SIGNING_KEY_NAME", + "Azure proposer X509 signer", + )?); + } + settings.nightfall_proposer.snapshot_root_dir = + normalize_snapshot_root_dir(&settings.nightfall_proposer.snapshot_root_dir)?; Ok(settings) } + + fn require_env(var_name: &str, context: &str) -> Result { + match env::var(var_name) { + Ok(value) if !value.trim().is_empty() => Ok(value), + Ok(_) | Err(_) => Err(format!("Missing {var_name} for {context}")), + } + } + + pub fn wallet_type_for_role(&self, role: WalletRole) -> &WalletTypeConfig { + match role { + WalletRole::Client => &self.nightfall_client.wallet_type, + WalletRole::Proposer => &self.nightfall_proposer.wallet_type, + } + } + + pub fn x509_signer_type_for_role(&self, role: WalletRole) -> &X509SignerTypeConfig { + match role { + WalletRole::Client => &self.nightfall_client.x509_signer_type, + WalletRole::Proposer => &self.nightfall_proposer.x509_signer_type, + } + } + + pub fn azure_key_name_for_role(&self, role: WalletRole) -> Result<&str, String> { + match role { + WalletRole::Client => self.client_azure_key_name.as_deref().ok_or_else(|| { + "Missing CLIENT_SIGNING_KEY_NAME for Azure client wallet".to_string() + }), + WalletRole::Proposer => self.proposer_azure_key_name.as_deref().ok_or_else(|| { + "Missing PROPOSER_SIGNING_KEY_NAME for Azure proposer wallet".to_string() + }), + } + } + + pub fn x509_azure_key_name_for_role(&self, role: WalletRole) -> Result<&str, String> { + match role { + WalletRole::Client => self.client_x509_azure_key_name.as_deref().ok_or_else(|| { + "Missing CLIENT_X509_SIGNING_KEY_NAME for Azure client X509 signer".to_string() + }), + WalletRole::Proposer => self.proposer_x509_azure_key_name.as_deref().ok_or_else(|| { + "Missing PROPOSER_X509_SIGNING_KEY_NAME for Azure proposer X509 signer".to_string() + }), + } + } } #[cfg(test)] @@ -325,4 +462,255 @@ mod tests { None => env::remove_var("NF4_RUN_MODE"), } } + + #[test] + #[serial] + fn test_wallet_type_is_resolved_per_role() { + let tmp_run_mode = env::var("NF4_RUN_MODE").ok(); + let tmp_client_wallet_type = env::var("NF4_NIGHTFALL_CLIENT__WALLET_TYPE").ok(); + let tmp_proposer_wallet_type = env::var("NF4_NIGHTFALL_PROPOSER__WALLET_TYPE").ok(); + let tmp_proposer_signing_key_name = env::var("PROPOSER_SIGNING_KEY_NAME").ok(); + let tmp_azure_vault_url = env::var("AZURE_VAULT_URL").ok(); + + env::set_var("NF4_RUN_MODE", "development"); + env::set_var("NF4_NIGHTFALL_CLIENT__WALLET_TYPE", "local"); + env::set_var("NF4_NIGHTFALL_PROPOSER__WALLET_TYPE", "azure"); + env::set_var("PROPOSER_SIGNING_KEY_NAME", "proposer-key"); + env::set_var("AZURE_VAULT_URL", "https://example.vault.azure.net"); + + let s = Settings::new().unwrap(); + + assert_eq!( + s.wallet_type_for_role(WalletRole::Client), + &WalletTypeConfig::Local + ); + assert_eq!( + s.wallet_type_for_role(WalletRole::Proposer), + &WalletTypeConfig::Azure + ); + + match tmp_run_mode { + Some(val) => env::set_var("NF4_RUN_MODE", val), + None => env::remove_var("NF4_RUN_MODE"), + } + match tmp_client_wallet_type { + Some(val) => env::set_var("NF4_NIGHTFALL_CLIENT__WALLET_TYPE", val), + None => env::remove_var("NF4_NIGHTFALL_CLIENT__WALLET_TYPE"), + } + match tmp_proposer_wallet_type { + Some(val) => env::set_var("NF4_NIGHTFALL_PROPOSER__WALLET_TYPE", val), + None => env::remove_var("NF4_NIGHTFALL_PROPOSER__WALLET_TYPE"), + } + match tmp_proposer_signing_key_name { + Some(val) => env::set_var("PROPOSER_SIGNING_KEY_NAME", val), + None => env::remove_var("PROPOSER_SIGNING_KEY_NAME"), + } + match tmp_azure_vault_url { + Some(val) => env::set_var("AZURE_VAULT_URL", val), + None => env::remove_var("AZURE_VAULT_URL"), + } + } + + #[test] + #[serial] + fn test_missing_client_azure_key_name_fails_fast() { + let tmp_run_mode = env::var("NF4_RUN_MODE").ok(); + let tmp_client_wallet_type = env::var("NF4_NIGHTFALL_CLIENT__WALLET_TYPE").ok(); + let tmp_proposer_wallet_type = env::var("NF4_NIGHTFALL_PROPOSER__WALLET_TYPE").ok(); + let tmp_client_signing_key_name = env::var("CLIENT_SIGNING_KEY_NAME").ok(); + let tmp_azure_vault_url = env::var("AZURE_VAULT_URL").ok(); + + env::set_var("NF4_RUN_MODE", "development"); + env::set_var("NF4_NIGHTFALL_CLIENT__WALLET_TYPE", "azure"); + env::set_var("NF4_NIGHTFALL_PROPOSER__WALLET_TYPE", "local"); + env::remove_var("CLIENT_SIGNING_KEY_NAME"); + env::set_var("AZURE_VAULT_URL", "https://example.vault.azure.net"); + + let err = Settings::new().unwrap_err(); + assert_eq!( + err, + "Missing CLIENT_SIGNING_KEY_NAME for Azure client wallet" + ); + + match tmp_run_mode { + Some(val) => env::set_var("NF4_RUN_MODE", val), + None => env::remove_var("NF4_RUN_MODE"), + } + match tmp_client_wallet_type { + Some(val) => env::set_var("NF4_NIGHTFALL_CLIENT__WALLET_TYPE", val), + None => env::remove_var("NF4_NIGHTFALL_CLIENT__WALLET_TYPE"), + } + match tmp_proposer_wallet_type { + Some(val) => env::set_var("NF4_NIGHTFALL_PROPOSER__WALLET_TYPE", val), + None => env::remove_var("NF4_NIGHTFALL_PROPOSER__WALLET_TYPE"), + } + match tmp_client_signing_key_name { + Some(val) => env::set_var("CLIENT_SIGNING_KEY_NAME", val), + None => env::remove_var("CLIENT_SIGNING_KEY_NAME"), + } + match tmp_azure_vault_url { + Some(val) => env::set_var("AZURE_VAULT_URL", val), + None => env::remove_var("AZURE_VAULT_URL"), + } + } + + #[test] + #[serial] + fn test_missing_proposer_azure_key_name_fails_fast() { + let tmp_run_mode = env::var("NF4_RUN_MODE").ok(); + let tmp_client_wallet_type = env::var("NF4_NIGHTFALL_CLIENT__WALLET_TYPE").ok(); + let tmp_proposer_wallet_type = env::var("NF4_NIGHTFALL_PROPOSER__WALLET_TYPE").ok(); + let tmp_proposer_signing_key_name = env::var("PROPOSER_SIGNING_KEY_NAME").ok(); + let tmp_azure_vault_url = env::var("AZURE_VAULT_URL").ok(); + + env::set_var("NF4_RUN_MODE", "development"); + env::set_var("NF4_NIGHTFALL_CLIENT__WALLET_TYPE", "local"); + env::set_var("NF4_NIGHTFALL_PROPOSER__WALLET_TYPE", "azure"); + env::remove_var("PROPOSER_SIGNING_KEY_NAME"); + env::set_var("AZURE_VAULT_URL", "https://example.vault.azure.net"); + + let err = Settings::new().unwrap_err(); + assert_eq!( + err, + "Missing PROPOSER_SIGNING_KEY_NAME for Azure proposer wallet" + ); + + match tmp_run_mode { + Some(val) => env::set_var("NF4_RUN_MODE", val), + None => env::remove_var("NF4_RUN_MODE"), + } + match tmp_client_wallet_type { + Some(val) => env::set_var("NF4_NIGHTFALL_CLIENT__WALLET_TYPE", val), + None => env::remove_var("NF4_NIGHTFALL_CLIENT__WALLET_TYPE"), + } + match tmp_proposer_wallet_type { + Some(val) => env::set_var("NF4_NIGHTFALL_PROPOSER__WALLET_TYPE", val), + None => env::remove_var("NF4_NIGHTFALL_PROPOSER__WALLET_TYPE"), + } + match tmp_proposer_signing_key_name { + Some(val) => env::set_var("PROPOSER_SIGNING_KEY_NAME", val), + None => env::remove_var("PROPOSER_SIGNING_KEY_NAME"), + } + match tmp_azure_vault_url { + Some(val) => env::set_var("AZURE_VAULT_URL", val), + None => env::remove_var("AZURE_VAULT_URL"), + } + } + + #[test] + #[serial] + fn test_x509_signer_type_is_resolved_per_role() { + let tmp_run_mode = env::var("NF4_RUN_MODE").ok(); + let tmp_client_x509_signer_type = env::var("NF4_NIGHTFALL_CLIENT__X509_SIGNER_TYPE").ok(); + let tmp_proposer_x509_signer_type = + env::var("NF4_NIGHTFALL_PROPOSER__X509_SIGNER_TYPE").ok(); + let tmp_proposer_x509_signing_key_name = env::var("PROPOSER_X509_SIGNING_KEY_NAME").ok(); + let tmp_azure_vault_url = env::var("AZURE_VAULT_URL").ok(); + + env::set_var("NF4_RUN_MODE", "development"); + env::set_var("NF4_NIGHTFALL_CLIENT__X509_SIGNER_TYPE", "local"); + env::set_var("NF4_NIGHTFALL_PROPOSER__X509_SIGNER_TYPE", "azure"); + env::set_var("PROPOSER_X509_SIGNING_KEY_NAME", "proposer-x509-key"); + env::set_var("AZURE_VAULT_URL", "https://example.vault.azure.net"); + + let s = Settings::new().unwrap(); + + assert_eq!( + s.x509_signer_type_for_role(WalletRole::Client), + &X509SignerTypeConfig::Local + ); + assert_eq!( + s.x509_signer_type_for_role(WalletRole::Proposer), + &X509SignerTypeConfig::Azure + ); + + match tmp_run_mode { + Some(val) => env::set_var("NF4_RUN_MODE", val), + None => env::remove_var("NF4_RUN_MODE"), + } + match tmp_client_x509_signer_type { + Some(val) => env::set_var("NF4_NIGHTFALL_CLIENT__X509_SIGNER_TYPE", val), + None => env::remove_var("NF4_NIGHTFALL_CLIENT__X509_SIGNER_TYPE"), + } + match tmp_proposer_x509_signer_type { + Some(val) => env::set_var("NF4_NIGHTFALL_PROPOSER__X509_SIGNER_TYPE", val), + None => env::remove_var("NF4_NIGHTFALL_PROPOSER__X509_SIGNER_TYPE"), + } + match tmp_proposer_x509_signing_key_name { + Some(val) => env::set_var("PROPOSER_X509_SIGNING_KEY_NAME", val), + None => env::remove_var("PROPOSER_X509_SIGNING_KEY_NAME"), + } + match tmp_azure_vault_url { + Some(val) => env::set_var("AZURE_VAULT_URL", val), + None => env::remove_var("AZURE_VAULT_URL"), + } + } + + #[test] + #[serial] + fn test_missing_client_x509_azure_key_name_fails_fast() { + let tmp_run_mode = env::var("NF4_RUN_MODE").ok(); + let tmp_client_x509_signer_type = env::var("NF4_NIGHTFALL_CLIENT__X509_SIGNER_TYPE").ok(); + let tmp_client_x509_signing_key_name = env::var("CLIENT_X509_SIGNING_KEY_NAME").ok(); + let tmp_azure_vault_url = env::var("AZURE_VAULT_URL").ok(); + + env::set_var("NF4_RUN_MODE", "development"); + env::set_var("NF4_NIGHTFALL_CLIENT__X509_SIGNER_TYPE", "azure"); + env::remove_var("CLIENT_X509_SIGNING_KEY_NAME"); + env::set_var("AZURE_VAULT_URL", "https://example.vault.azure.net"); + + let err = Settings::new().unwrap_err(); + assert_eq!( + err, + "Missing CLIENT_X509_SIGNING_KEY_NAME for Azure client X509 signer" + ); + + match tmp_run_mode { + Some(val) => env::set_var("NF4_RUN_MODE", val), + None => env::remove_var("NF4_RUN_MODE"), + } + match tmp_client_x509_signer_type { + Some(val) => env::set_var("NF4_NIGHTFALL_CLIENT__X509_SIGNER_TYPE", val), + None => env::remove_var("NF4_NIGHTFALL_CLIENT__X509_SIGNER_TYPE"), + } + match tmp_client_x509_signing_key_name { + Some(val) => env::set_var("CLIENT_X509_SIGNING_KEY_NAME", val), + None => env::remove_var("CLIENT_X509_SIGNING_KEY_NAME"), + } + match tmp_azure_vault_url { + Some(val) => env::set_var("AZURE_VAULT_URL", val), + None => env::remove_var("AZURE_VAULT_URL"), + } + } + + #[test] + #[serial] + fn test_relative_snapshot_root_dir_is_normalized_to_absolute() { + let tmp_run_mode = env::var("NF4_RUN_MODE").ok(); + let tmp_snapshot_root_dir = env::var("NF4_NIGHTFALL_PROPOSER__SNAPSHOT_ROOT_DIR").ok(); + + env::set_var("NF4_RUN_MODE", "development"); + env::set_var( + "NF4_NIGHTFALL_PROPOSER__SNAPSHOT_ROOT_DIR", + "relative/snapshots", + ); + + let settings = Settings::new().unwrap(); + let configured_path = PathBuf::from(&settings.nightfall_proposer.snapshot_root_dir); + let expected_path = env::current_dir() + .expect("current directory should exist for settings test") + .join("relative/snapshots"); + + assert!(configured_path.is_absolute()); + assert_eq!(configured_path, expected_path); + + match tmp_run_mode { + Some(val) => env::set_var("NF4_RUN_MODE", val), + None => env::remove_var("NF4_RUN_MODE"), + } + match tmp_snapshot_root_dir { + Some(val) => env::set_var("NF4_NIGHTFALL_PROPOSER__SNAPSHOT_ROOT_DIR", val), + None => env::remove_var("NF4_NIGHTFALL_PROPOSER__SNAPSHOT_ROOT_DIR"), + } + } } diff --git a/doc/nf_4.md b/doc/nf_4.md index a62bf25b..48957ad6 100644 --- a/doc/nf_4.md +++ b/doc/nf_4.md @@ -30,11 +30,14 @@ - [Nightfall\_4 containers](#nightfall_4-containers) - [Client Webhook](#client-webhook) - [APIs](#apis) + - [Transfer Receipt Workflow](#transfer-receipt-workflow) - [Client APIs](#client-apis) - [X509 Certificates for Client](#x509-certificates-for-client) - [Value transactions](#value-transactions) + - [Transfer Receipt Resolution](#transfer-receipt-resolution) - [Proposer APIs](#proposer-apis) - [X509 Certificates for Proposer](#x509-certificates-for-proposer) + - [Transfer Receipt Submission](#transfer-receipt-submission) - [Test UI: Using the Menu Application](#test-ui-using-the-menu-application) - [1. Build the Menu Application](#1-build-the-menu-application) - [2. Prepare Environment Variables](#2-prepare-environment-variables) @@ -180,16 +183,34 @@ Do not confuse NF4_RUN_MODE, which selects the top-level section of `nightfall.t Additonally, any configuration item can also be overridden via an enviroment variable by naming an environment variable `NF4_`. You do not need to include the main section name: that is taken from the environment variable `NF4_RUN_MODE`. It defaults to `development` if `NF4_RUN_MODE` is not set. Some configuration variables are in sub-categories, for example under `[development.nightfall_deployer]`. Use a double __ to identify a variable within a sub-category (replacing the dot that would be used in the equivalent Rust struct). For example, the `log_level` variable in `nightfall.toml` that is under `[development]` under `[development.nightfall_deployer]` would be overridden by the environment variable `NF4_NIGHTFALL_DEPLOYER__LOG_LEVEL`, provided of course that `NF4_RUN_MODE` is set to `development` (or left unset). Finally, secret items, such as keys, do not appear in `nightfall.toml` (except for one signing key used for basic testing - which will be removed soon). Depending on the wallet type being used, keys should instead be set via environment variables, a key-store or an HSM. Test keys are currently provided via a `.env` file in the project root, using the LocalWallet type, that sets its keys as environment variables. -NF_4 also supports the AzureWallet type, allowing keys to be securely stored in Azure Key Vault. To use this feature, you can register an NF_4 application in Azure App Registration and grant it access to the stored keys. When enabling this functionality, make sure to set the following environment variables: -AZURE_VAULT_URL= -DEPLOYER_SIGNING_KEY_NAME= -PROPOSER_SIGNING_KEY_NAME= -PROPOSER_2_SIGNING_KEY_NAME= -CLIENT_SIGNING_KEY_NAME= -CLIENT2_SIGNING_KEY_NAME= -AZURE_CLIENT_ID= -AZURE_CLIENT_SECRET= -AZURE_TENANT_ID= . +NF_4 also supports Azure Key Vault-backed signing. To use this feature, register an NF_4 application in Azure App Registration and grant it access to the required keys in Key Vault. + +For Azure-backed signing, the configuration is split by signing surface: + +```env +# Ethereum L1 signing +NF4_NIGHTFALL_CLIENT__WALLET_TYPE=azure +CLIENT_SIGNING_KEY_NAME= + +NF4_NIGHTFALL_PROPOSER__WALLET_TYPE=azure +PROPOSER_SIGNING_KEY_NAME= + +# X.509 proof-of-possession signing +NF4_NIGHTFALL_CLIENT__X509_SIGNER_TYPE=azure +CLIENT_X509_SIGNING_KEY_NAME= + +NF4_NIGHTFALL_PROPOSER__X509_SIGNER_TYPE=azure +PROPOSER_X509_SIGNING_KEY_NAME= + +# Shared Azure config +AZURE_VAULT_URL=https://.vault.azure.net/ +AZURE_CLIENT_ID=... +AZURE_CLIENT_SECRET=... +AZURE_TENANT_ID=... +``` + +In Azure mode, the Ethereum L1 signing key should be an EC `P-256K` / `secp256k1` key in Key Vault. +The X.509 proof-of-possession key should be the RSA private key corresponding to the user certificate already accepted by Nightfall. ### Advisory swap cancellation auth @@ -459,6 +480,88 @@ docker compose --profile development up Don't forget to down any containers and volumes from previous tests first and wait until the `test` container has successfully exited before trying any API calls. +### Transfer Receipt Workflow + +Transfer receipts are a cross-component wallet workflow rather than a proposer-only feature. + +At a high level the flow is: + +1. The sender submits a transfer with `POST /v1/transfer` and receives a `202 Accepted` response plus an `X-Request-ID`. +2. Once the transfer request reaches `Submitted`, the sender wallet calls `GET /v1/request/{uuid}` on the client to obtain: + - the canonical proposer `tx_hash` for the transfer + - the `receipt_token` authorizing receipt creation for that transaction +3. The sender wallet creates the encrypted receipt payload locally, typically in a WASM library, using the recipient's Baby JubJub public key and the receipt-specific KEM-DEM scheme. +4. The sender submits the opaque ciphertext, together with `tx_hash` and `receipt_token`, to the **client** via `POST /v1/transfer-receipts`. The client fans out the submission concurrently to all registered proposers, ensuring every proposer stores the same ciphertext under the same deterministic `receipt_id`. +5. The proposer stores only the ciphertext and metadata. It never decrypts or interprets the receipt payload. +6. The sender shares the returned `receipt_id` with the intended receiver out of band. +7. The receiver wallet retrieves the ciphertext from the proposer using `GET /v1/transfer-receipts/{receipt_id}` and decrypts it locally. +8. The receiver wallet verifies that the decrypted receipt is intended for the local wallet before displaying it. + +The logical wallet payload is token-type-specific. + +ERC721 payload: + +```json +{ + "transfer": { + "senderZkpPublicKey": "0x...", + "ercAddress": "0x...", + "tokenType": "ERC721", + "tokenId": "0x00", + "receiverCommitment": "0x..." + } +} +``` + +ERC20 payload: + +```json +{ + "transfer": { + "senderZkpPublicKey": "0x...", + "ercAddress": "0x...", + "tokenType": "ERC20", + "value": "0x01", + "receiverCommitment": "0x..." + } +} +``` + +This logical payload is serialized into a 7-field plaintext before encryption: + +- `sender_public_key_x` +- `sender_public_key_y` +- `erc_address` +- `token_type` (`0` = ERC20, `2` = ERC721) +- `token_id_or_value` +- `receiver_commitment` +- reserved zero field + +The resulting ciphertext is version `1` and is exactly **576 hex characters** (9 × 32-byte BN254 field elements): 7 encrypted plaintext fields followed by the ephemeral public key y-coordinate and x-sign flag. + +**Receiver-side verification** + +After decrypting locally, the receiver wallet should verify that the receipt is actually meant for the local wallet before displaying it. + +The proposer derives the stored receipt `status` from the referenced transaction lifecycle. In the current implementation this means a receipt can be returned as either `pending` or `included_l2`: + +- `pending`: the transaction is known to the proposer but does not yet have an L2 block assignment. +- `included_l2`: the proposer already sees the referenced transaction as included in a Layer 2 block. + +This applies both when the sender first submits `POST /v1/transfer-receipts` and when the receiver later calls the proposer `GET` endpoints. In other words, `POST /v1/transfer-receipts` does **not** always return `pending`; if the transfer has already reached L2 by the time the receipt is created, the response can already be `included_l2`. + +**API endpoints used in this workflow** + +Client-side discovery and receipt submission: + +- `GET /v1/request/{uuid}` — returns request status and, once the transfer is `Submitted`, the canonical `tx_hash` and `receipt_token` needed for receipt creation. +- `POST /v1/transfer-receipts` — sender submits encrypted receipt ciphertext; the client fans it out to all registered proposers concurrently. + +Proposer-side retrieval: + +- `GET /v1/transfer-receipts/{receipt_id}` — receiver fetches the full stored receipt ciphertext. +- `GET /v1/transfer-receipts/{receipt_id}/status` — lightweight status check for the stored receipt. + ### Client APIs Note that transactions will fail unless you have first had your x509 certificate validated on chain by the X509.sol contract. See the `X509 certificates` API for how to do that. @@ -907,14 +1010,103 @@ GET /v1/request/{:uuid} curl -i 'http://localhost:3000/v1/request/16cf74ad-e28c-421e-a125-78bed5e1c435 ``` -Returns the status of a deposit/transfer or withdraw request when provided with the `X-Request-ID` header value that was submitted with the request. The status can be one of: +Returns the status of a deposit, transfer, withdraw, or swap request when provided with the `X-Request-ID` header value that was submitted with the request. + +For transfer receipts, this endpoint is the sender wallet's source of truth for the canonical proposer transaction reference. Once a transfer request has reached `Submitted`, the response may also contain: + +- `tx_hash`: the canonical proposer-side transaction hash as a 64-character lowercase hex string +- `receipt_token`: the capability token required to create a transfer receipt for that transaction + +The status can be one of: - Queued: The transaction is waiting to be processed by the client. +- Processing: The client has taken the transaction out of the queue and is actively working on it. - Submitted: The Client has succesfully processed the transaction and handed off the result, either to the blockchain, in the case of a deposit escrow, or to a Proposer, in the case of a transfer or withdraw transaction. - Failed: The hand off to the next stage did not succeed. +- ProposerUnreachable: The client could not reach a proposer for a transfer or withdraw style request. +- Confirmed: The transaction lifecycle is complete and the relevant commitments are on-chain. +- Expired: A swap request passed its deadline without on-chain confirmation. *(Swap only — does not apply to transfer or withdraw requests.)* Note that internal failures of the client will cause the request state to be unreliable so the request status is not an alternative to error logs. +Example response after a transfer has been submitted to a proposer: + +```json +{ + "status": "Submitted", + "uuid": "16cf74ad-e28c-421e-a125-78bed5e1c435", + "tx_hash": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "receipt_token": "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" +} +``` + +Example response while the request is still being processed: + +```json +{ + "status": "Processing", + "uuid": "16cf74ad-e28c-421e-a125-78bed5e1c435" +} +``` + +Example `400 BAD REQUEST` response: + +```text +Invalid request id +``` + +Example `404 NOT FOUND` response: + +```text +No such request +``` + +*** + +#### Transfer Receipt Resolution + +Receipt resolution is part of the end-to-end transfer receipt flow described in [Transfer Receipt Workflow](#transfer-receipt-workflow). + +The relevant receiver-facing endpoints are hosted by the proposer (`localhost:3001` in the examples): + +- `GET /v1/transfer-receipts/{receipt_id}` +- `GET /v1/transfer-receipts/{receipt_id}/status` + +Use these endpoints to fetch the stored ciphertext by `receipt_id` and then decrypt it locally in the wallet. See the workflow section for payload semantics and receiver-side verification rules. + +Example `GET /v1/transfer-receipts/{receipt_id}` success response (`200 OK`): + +```json +{ + "receipt_id": "5d8e9d08a0c88fbbcb2f0f4a7f9d0efb0c8a6d488ce9a3f39d95f2ce3d8d8a71", + "tx_hash": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "ciphertext": "0a0b0c...<576 hex chars total>...ff", + "version": 1, + "status": "included_l2", + "created_at_unix": 1716200000, + "updated_at_unix": 1716200050 +} +``` + +Example `GET /v1/transfer-receipts/{receipt_id}/status` success response (`200 OK`): + +```json +{ + "receipt_id": "5d8e9d08a0c88fbbcb2f0f4a7f9d0efb0c8a6d488ce9a3f39d95f2ce3d8d8a71", + "status": "pending" +} +``` + +Example `404 Not Found` response from the proposer for either endpoint: + +```text +Transfer receipt not found +``` + +Notes: +- The proposer currently validates `receipt_id` format and existence together as `404 Not Found`. +- The receipt `status` returned by either `GET` endpoint is refreshed from the underlying transaction lifecycle before the response is sent. + *** GET /v1/queue @@ -1016,6 +1208,88 @@ This URL is used by clients to send a JSON encoded `ClientTransaction

` struct *** +#### Transfer Receipt Submission + +Receipt submission is part of the end-to-end transfer receipt flow described in [Transfer Receipt Workflow](#transfer-receipt-workflow). + +The sender submits the encrypted receipt payload to the **client** (not directly to a proposer): + +- `POST /v1/transfer-receipts` (client — `localhost:3000`) + +This endpoint accepts the canonical `tx_hash`, the opaque receipt `ciphertext`, an optional `version` (currently `1`), and the `receipt_token` returned by `GET /v1/request/{uuid}`. The client fans the submission out concurrently to every registered proposer so that the ciphertext is replicated across all of them. On success it returns a `receipt_id`, which the sender can share with the receiver out of band. Because `receipt_id` is derived deterministically (`HMAC-SHA256(receipt_token, tx_hash)`), it is identical on every proposer, and the receiver can query any available proposer to retrieve the receipt. + +Status summary: + +- `200 OK` — idempotent replay with identical payload +- `201 Created` — receipt stored successfully by at least one proposer +- `400 BAD REQUEST` — malformed request, unsupported version, invalid ciphertext, or transaction not yet known to any proposer +- `401 Unauthorized` — invalid or missing `receipt_token` +- `409 Conflict` — receipt already exists for this transaction with a different payload +- `503 Service Unavailable` — all proposers unreachable + +Example success response (`201 Created` while the proposer transaction is still pending): + +```json +{ + "receipt_id": "5d8e9d08a0c88fbbcb2f0f4a7f9d0efb0c8a6d488ce9a3f39d95f2ce3d8d8a71", + "status": "pending" +} +``` + +Example success response (`201 Created` or `200 OK` when the referenced transaction is already in L2): + +```json +{ + "receipt_id": "5d8e9d08a0c88fbbcb2f0f4a7f9d0efb0c8a6d488ce9a3f39d95f2ce3d8d8a71", + "status": "included_l2" +} +``` + +Example idempotent replay (`200 OK`): + +```json +{ + "receipt_id": "5d8e9d08a0c88fbbcb2f0f4a7f9d0efb0c8a6d488ce9a3f39d95f2ce3d8d8a71", + "status": "pending" +} +``` + +Example `400 BAD REQUEST` responses from the client: + +```text +Transaction not found on proposer +``` + +```text +Transfer receipt request failed validation +``` + +Example `401 Unauthorized` response from the client: + +```text +Invalid or missing receipt_token +``` + +Example `409 Conflict` response from the client: + +```text +Receipt already exists with different ciphertext +``` + +Example `503 Service Unavailable` responses from the client: + +```text +Failed to get list of Proposers +``` + +```text +Receipt submission failed: no proposer accepted the receipt +``` + +See the workflow section for the full meaning of the encrypted payload and how wallets are expected to use the returned `receipt_id`. + +*** + GET /v1/health ```sh diff --git a/doc/proposer_snapshot_restore_design.md b/doc/proposer_snapshot_restore_design.md new file mode 100644 index 00000000..860b309c --- /dev/null +++ b/doc/proposer_snapshot_restore_design.md @@ -0,0 +1,569 @@ +# Proposer Snapshot Restore Design + +## Goal + +Define a robust proposer snapshot restore flow that remains safe even though MongoDB does not support running `renameCollection` inside a multi-document transaction. The restore flow must therefore rely on: + +- startup/readiness gating +- shadow collections +- backup collections +- a persistent `restore_journal` +- resumable crash recovery + +This note covers restore design only. It does **not** implement restore code. + +## Design Invariants + +The restore design relies on the following invariants. + +1. Canonical snapshot boundary + +`ClientTransactions` is outside the canonical proposer snapshot boundary. + +Canonical restore correctness must depend only on the snapshotted proposer state: + +- proposer trees +- `StoredBlock` +- `sync_state` +- other explicitly snapshotted canonical collections + +`ClientTransactions` must therefore be treated as live auxiliary state during restore. + +2. Crash-idempotent non-snapshot cleanup + +The non-snapshot cleanup performed after `swap_complete` must be crash-idempotent. + +A crash after restored live collections are already in place, but before `restore_journal` is +deleted, must not allow a second cleanup pass to delete or corrupt transactions that were +intentionally preserved by the first pass. + +3. Journaled cleanup progress + +If non-snapshot cleanup is not intrinsically safe to run multiple times, its progress must be +recorded explicitly in `restore_journal`. + +Recovery must be able to distinguish unambiguously between: + +- cleanup not yet committed, in which case it must be rerun +- cleanup already committed, in which case it must be skipped on resume + +The safest design is to commit cleanup progress in the same Mongo transaction as the non-snapshot +cleanup writes. + +4. Post-restore lifecycle reconciliation + +Because `ClientTransactions` is outside the canonical snapshot boundary, active client transaction +lifecycle must be reconciled explicitly against canonical proposer state after restore and replay. + +In particular: + +- `Selected { block_l2 }` entries above restored `sync_state.last_applied_l2_block` must not + remain trusted after restore +- `Included { block_l2 }` entries above restored `sync_state.last_applied_l2_block` must not + remain trusted after restore without explicit reconciliation against canonical `StoredBlock` + contents + +Restore may normalize obviously non-canonical lifecycle state, but final lifecycle correctness for +active transactions must be derived from canonical replayed state rather than assumed from +pre-restore live documents. + +5. Destructive replay only after restore-journal state is cleared + +Destructive replay fallback is only allowed after restore-journal recovery has either cleared the +`restore_journal` and related `restore_backup__*` / `restore_shadow__*` state, or confirmed that no +such state exists. + +If restore-journal recovery fails and journal or backup/shadow state may still be present, recovery +must fail closed for manual intervention instead of falling through to destructive reset. + +6. Startup cleanup must preserve tree/state coherence + +Startup cleanup must leave `sync_state`, `StoredBlock`, `PendingBlock`, and proposer tree state +mutually coherent. + +If startup cleanup removes speculative `StoredBlock` or `PendingBlock` state beyond `sync_state`, it +must also roll back or reset the corresponding Merkle tree state to a value coherent with +`sync_state` before startup can continue. + +Deleting speculative block state without rolling back trees can leave proposer roots ahead of +`sync_state`. + +7. Startup replay reset must be crash-resumable + +If startup detects speculative proposer state that cannot be repaired in place and therefore falls +back to "clear canonical local state, reset trees, and let listener replay rebuild canonically", +that reset sequence must itself be crash-resumable. + +A crash after canonical state has been deleted, but before proposer trees have been fully reset and +reinitialized, must not leave the next startup unable to distinguish: + +- a legitimate empty proposer state ready for replay, from +- a partially applied startup replay reset that still requires recovery work + +Therefore startup replay reset must persist an explicit marker before destructive reset begins, and +must clear that marker only after all canonical collections, proposer trees, and auxiliary cleanup +have reached their intended replay-baseline state. + +Bootstrap must check for that marker before ordinary startup validation and must resume or complete +the reset rather than fail closed on the resulting mixed state. + +## Journal Model + +`restore_journal` is a single-document Mongo collection. + +Recommended document shape: + +```json +{ + "_id": "proposer", + "schema_version": 1, + "snapshot_id": "proposer-l2-123-1715440000000", + "snapshot_dir": "/path/to/snapshots/proposer-l2-123-1715440000000", + "manifest_overall_sha256": "...", + "phase": "loading_shadow | swap_in_progress | swap_complete", + "current_index": 0, + "current_step": "backup_pending | backup_created | non_snapshot_cleanup_pending | non_snapshot_cleanup_applied | rollback_pending | rollback_started | rollback_applied", + "collections": [ + { + "live": "Commitments_nodes", + "shadow": "restore_shadow__Commitments_nodes", + "backup": "restore_backup__Commitments_nodes" + } + ], + "started_at": "BSON datetime", + "updated_at": "BSON datetime" +} +``` + +`idle` is represented by **absence** of the journal document. That keeps the steady state simple: if no restore is in progress or pending cleanup, there is no journal. + +## Startup Replay Reset Marker + +Startup replay reset should use its own persistent single-document marker instead of overloading +`restore_journal`, because the reset flow is logically separate from snapshot restore and may run +even when no snapshot restore was attempted. + +Recommended document shape: + +```json +{ + "_id": "proposer", + "schema_version": 1, + "phase": "db_cleanup_pending | db_cleanup_applied | tree_reset_pending | auxiliary_cleanup_pending", + "started_at": "BSON datetime", + "updated_at": "BSON datetime" +} +``` + +Recommended collection name: + +- `startup_replay_reset_marker` + +`idle` is represented by **absence** of the marker document. + +Phase meaning: + +- `db_cleanup_pending`: startup chose the replay-reset path, but canonical collections may still be + intact +- `db_cleanup_applied`: `sync_state`, `StoredBlock`, and `PendingBlock` have been durably cleared; + trees may still reflect speculative state +- `tree_reset_pending`: retained as the same semantic boundary as `db_cleanup_applied`; an + implementation may collapse these two phases if the phase write and DB cleanup commit are made + together +- `auxiliary_cleanup_pending`: proposer trees have been reset to replay baseline; auxiliary live + state cleanup (for example reserved deposits and active transaction normalization) may still be + incomplete + +## Naming + +Shadow collections: + +- `restore_shadow__{live_collection_name}` + +Backup collections: + +- `restore_backup__{live_collection_name}` + +These names are deterministic, easy to inspect manually, and clearly distinct from live collections. + +## Phases And Transitions + +Allowed phase transitions: + +- `idle` -> `loading_shadow` +- `loading_shadow` -> `swap_in_progress` +- `swap_in_progress` -> `swap_complete` +- `swap_complete` -> `idle` + +Illegal transitions: + +- `swap_in_progress` -> `loading_shadow` +- `swap_complete` -> `loading_shadow` +- direct `loading_shadow` -> `idle` after live swap has begun + +Phase meaning: + +- `loading_shadow`: snapshot files are being validated and imported into shadow collections. Live collections remain untouched. +- `swap_in_progress`: live collections are being replaced one by one using `live -> backup`, then `shadow -> live`. +- `swap_complete`: all live collections now point to restored data; only cleanup remains (`backup` and leftover `shadow` removal, then journal removal). + +## Exact Operation Order + +### 1. Enter restore mode + +- proposer remains **not READY** +- create journal with `phase=loading_shadow` +- drop any stale `restore_shadow__*` and `restore_backup__*` collections from a previous interrupted attempt +- load `manifest.json` +- validate `schema_version`, `storage_format`, per-collection checksums, and `overall_sha256` + +### 2. Load shadow collections + +- import each snapshot file into its matching `restore_shadow__*` collection +- verify imported document counts match the manifest +- keep live collections and live `sync_state` untouched during this phase +- when all shadow collections are loaded, update journal to `phase=swap_in_progress`, `current_index=0`, `current_step=backup_pending` + +### 3. Swap live collections + +Collection order must be deterministic. Recommended order: + +- all tree and block collections first +- `sync_state` **last** + +For each collection entry at index `i`: + +1. ensure `restore_backup__*` does not already exist for this collection +2. rename `live -> backup` +3. update journal: `current_index=i`, `current_step=backup_created` +4. rename `shadow -> live` +5. update journal: advance to the next collection with `current_step=backup_pending` + +`sync_state` is swapped last so it never advertises the restored snapshot before the other restored collections are already live. + +### 4. Cleanup after swap + +- update journal to `phase=swap_complete` +- drop all `restore_backup__*` collections +- drop any leftover `restore_shadow__*` collections +- delete the `restore_journal` document, returning to `idle` + +### 4a. Non-snapshot cleanup invariants + +`ClientTransactions` is intentionally kept outside the canonical proposer snapshot boundary. + +That means restore correctness does not depend on snapshotting client transaction lifecycle, but it +also means any live-only recovery state must be normalized explicitly after the canonical snapshot +has been swapped into live collections. + +The non-snapshot cleanup phase includes at least: + +- deleting persisted `PendingBlock` state +- clearing reserved deposit selections +- deleting ordinary mempool client transactions +- normalizing active client transaction lifecycle that is ahead of the restored `sync_state` + +In particular: + +- `Selected { block_l2 }` entries with `block_l2 > restored_sync_state.last_applied_l2_block` + must not remain selected after restore +- `Included { block_l2 }` entries with `block_l2 > restored_sync_state.last_applied_l2_block` + must not be trusted as canonical after restore without explicit reconciliation + +This cleanup must be crash-idempotent. + +A crash after live collections have been restored but before `restore_journal` is deleted must not +allow a second cleanup pass to delete transactions that were intentionally preserved by the first +pass. + +Therefore, non-snapshot cleanup must either: + +- be safe to execute multiple times without changing the final result, or +- persist explicit cleanup progress in `restore_journal` and skip already-committed cleanup work on + resume + +The safest design is to treat non-snapshot cleanup as a journaled step of `swap_complete`, and to +update that journal step in the same Mongo transaction as the non-snapshot cleanup writes. + +## Crash Recovery On Startup + +### `idle` (journal absent) + +- normal startup path +- validate live `sync_state` as usual + +### `loading_shadow` + +Live state was never modified. Recovery should: + +- drop all `restore_shadow__*` +- drop all `restore_backup__*` +- delete the journal +- restart restore from scratch if restore is still desired + +This phase is restartable and should not try to resume partially imported shadow collections. + +### `startup_replay_reset_marker` + +If the startup replay reset marker exists, bootstrap must complete that reset before ordinary live +state validation. + +Recommended recovery logic: + +- if `phase=db_cleanup_pending`: + - ensure the canonical replay-reset decision is still valid + - clear `sync_state`, `StoredBlock`, and `PendingBlock` in one Mongo transaction + - advance the marker to `db_cleanup_applied` in the same transaction +- if `phase=db_cleanup_applied`: + - reset and reinitialize `CommitmentTree` + - reset and reinitialize `HistoricRootTree`, including the zero historic root leaf + - reset and reinitialize `NullifierTree`, including indexed leaves baseline state + - advance the marker to `auxiliary_cleanup_pending` +- if `phase=auxiliary_cleanup_pending`: + - restore `Selected` / `Included` client transaction lifecycle to replay-safe baseline + - clear reserved deposit state + - verify that live proposer state now matches the intended replay baseline: + - no `sync_state` + - no `StoredBlock` + - no `PendingBlock` + - empty commitment tree + - historic root tree containing only the zero leaf + - empty nullifier tree baseline + - delete the marker document, returning to `idle` + +The reset must fail closed if the marker exists but the required reset step cannot be completed +cleanly. However, bootstrap must not treat "marker exists + mixed local state" as an unexplained +manual-recovery case; it must first attempt to resume the reset described by the marker. + +### `swap_in_progress` + +Default rule: **resume forward**, not rollback. + +Reason: backups and shadows are specifically kept so the swap can continue safely while the proposer remains gated and non-READY. + +Recovery logic: + +- read `current_index` and `current_step` +- for already completed indices `< current_index`, keep their existing `backup` collections +- for the current index: + - if `current_step=backup_pending`, no swap started for that collection yet; continue with `live -> backup` + - if `current_step=backup_created`, `live -> backup` already happened; continue with `shadow -> live` + +Automatic rollback is only used if startup finds broken invariants, for example: + +- the journal says `backup_created` but the backup collection is missing +- the shadow collection needed for resume is missing +- namespace layout no longer matches the journal + +In that case, rollback should: + +- restore previously completed collections in reverse order using `backup -> live` +- restore the current collection from backup if needed +- drop remaining shadow collections +- keep the proposer non-READY and fail closed if rollback cannot be completed cleanly + +Implementation note and deferred follow-up: + +- when recovery resumes a `rollback_in_progress` journal at an **optional** collection where the journal step is `rollback_started`, `backup` is absent, and `live` is present, recovery now drops `live` only if it can still verify that `live` matches the restored snapshot state; otherwise it fails closed and requires manual intervention +- when recovery resumes a `rollback_in_progress` journal at a **required** collection where the journal step is `rollback_started`, `backup` is absent, and `live` is present, the current implementation still treats this as the backup having already been renamed back to live +- that assumption is correct for the legitimate crash window between `rename_collection(backup, live)` and the `rollback_applied` journal write +- it is **not** correct if a required backup collection was lost externally (for example storage corruption or manual deletion) while the journal was at `rollback_started` +- in that case recovery may incorrectly mark the collection as rolled back and may boot from a mixed old/new state +- if external backup loss is suspected, manual intervention is required before restart +- a robust fix would persist a per-collection fingerprint in the journal at the moment `live -> backup` succeeds, then verify `live` against that fingerprint when resuming rollback; this is deferred as a follow-up + +### `swap_complete` + +Live collections already point to restored data. Recovery should: + +- validate live `sync_state` against the restored `StoredBlock` +- complete any journaled non-snapshot cleanup that was not durably committed before the crash +- skip non-snapshot cleanup if the journal shows that cleanup was already committed +- only after cleanup and validation succeed, finish removal of `restore_backup__*` and `restore_shadow__*`, then delete the journal +- if invalid and backups still exist, attempt rollback from backups +- if invalid and backups are gone, fail closed and require manual intervention + +## Rollback vs Resume Rules + +During `swap_in_progress`: + +- **resume** is the default startup behavior +- **rollback** is a fallback only when journal/namespace invariants are broken badly enough that resume is unsafe + +This keeps the happy path simple while still preserving a safety net for partial namespace damage. + +## Startup Replay Reset Crash Windows + +The startup replay reset path must explicitly cover the following crash windows: + +1. Crash before the marker is created + +- no replay reset has started +- ordinary startup validation still applies + +2. Crash after marker creation but before canonical DB cleanup commits + +- live `sync_state`, `StoredBlock`, and `PendingBlock` may still be present +- resume by re-running the transactional DB cleanup and advancing the marker only if that + transaction commits + +3. Crash after canonical DB cleanup commits but before any tree reset completes + +- `sync_state`, `StoredBlock`, and `PendingBlock` are gone +- proposer trees may still be speculative and ahead of replay baseline +- ordinary startup validation would see "no sync_state, but non-empty trees" +- marker-guided recovery must therefore reset trees instead of failing closed + +4. Crash during commitment / historic-root / nullifier tree reset + +- some trees may already be at replay baseline while others still reflect speculative state +- marker-guided recovery must treat tree reset as repeatable and converge all trees to the same + replay baseline before allowing startup to continue + +5. Crash after tree reset but before auxiliary cleanup completes + +- canonical collections and trees are already at replay baseline +- auxiliary live state may still contain stale reserved deposits or active transaction lifecycle +- recovery must resume auxiliary cleanup and only then clear the marker + +6. Crash after auxiliary cleanup but before marker deletion + +- the target replay-baseline state may already be fully established +- resume must verify that the baseline state is already satisfied and then delete the stale marker + +## Startup Replay Reset Edge Cases + +- Marker exists, but `restore_journal` also exists: + - `restore_journal` recovery must run first + - startup replay reset is only evaluated after snapshot restore recovery has either completed or + failed closed + +- Marker exists, but canonical proposer state has partially reappeared: + - recovery must treat the marker as authoritative and re-run DB cleanup before tree validation + +- Marker exists, but one tree reset helper is not idempotent: + - that helper must be made idempotent or split into separately journaled sub-steps + - bootstrap must never assume a partially reset tree can be interpreted safely without finishing + the reset + +- Marker exists, but auxiliary cleanup has already been applied once: + - cleanup operations must be replay-safe or the marker needs a finer-grained committed step + +- Marker exists, but state no longer matches any valid replay baseline after external tampering: + - fail closed and require manual intervention + +## `sync_state` Handling + +- live `sync_state` is **not** deleted at restore start +- snapshot `sync_state` is imported into `restore_shadow__sync_state` +- live `sync_state` is swapped only as the **final** collection in `swap_in_progress` +- after restore cleanup, the live `sync_state` must be the one from the snapshot + +This ensures `sync_state` only points to the restored snapshot after the corresponding restored collections are already live. + +## When The Proposer Becomes READY Again + +The proposer must stay **not READY** for the entire restore flow, including journal cleanup. + +The proposer can become READY again only after all of the following are true: + +- `restore_journal` is back to `idle` (document absent) +- live collections have passed restore validation +- live `sync_state` validates against the restored `StoredBlock` +- post-restore replay from `sync_state.l1_ref.block_number` has completed and the proposer is back at the current L1/L2 tip + +Restore completion alone is not enough to become READY; replay catch-up must also succeed. + +## Deferred Follow-Up: Client Transaction Lifecycle Reconciliation + +Because `ClientTransactions` is outside the canonical snapshot boundary, restore cannot assume that +live `Selected` or `Included` lifecycle entries above the restored `sync_state` remain correct. + +A later replay may reconstruct the same canonical inclusion, or it may replay a different canonical +block sequence after desync recovery, block-hash mismatch, or similar divergence handling. + +As a result, lifecycle state for active client transactions should be reconciled explicitly after +restore and replay against canonical `StoredBlock` contents. + +A robust reconciliation rule is: + +- if the canonical `StoredBlock(block_l2)` contains the transaction commitments, classify the + transaction as `Included { block_l2 }` +- if `block_l2 <= sync_state.last_applied_l2_block` but the canonical block does not contain the + transaction, classify it as `Mempool` +- otherwise classify it as `Selected { block_l2 }` + +This keeps `ClientTransactions` outside the snapshot while still restoring lifecycle consistency +against canonical proposer state. + +This reconciliation should run at the end of listener historical replay catch-up, in the same +finalization path that currently reconciles orphaned selected transactions, before +`apply_listener_caught_up_runtime_state()` marks the proposer synchronised again. + +## Deferred Follow-Up: Tree State Upper Bounds Beyond HistoricRoot + +Current startup and post-restore validation enforce an exact `HistoricRootTree` progression against +`sync_state.last_applied_l2_block`. + +A symmetric "tree is ahead of sync_state" upper-bound check is not yet implemented for +`CommitmentTree` and `NullifierTree`. + +- for `CommitmentTree`, an exact bound may be derivable from canonical stored block contents, but + this is not yet enforced by the current implementation +- for `NullifierTree`, the current persisted canonical state does not include enough information to + derive a robust exact upper bound during startup or restore validation + +As a result, current validation detects empty or missing tree state and `HistoricRootTree` +incoherence, but it does not yet reject every case where commitment/nullifier tree state has +advanced beyond the persisted `sync_state`. + +If either tree has advanced beyond `sync_state`, replay may try to re-apply state that is already +materialized locally. For commitments, that can shift local tree progression away from the +canonical replay path and produce divergent roots. For nullifiers, replay may attempt to reinsert +already-present leaves, causing replay failures or inconsistent local state. Any later proof built +against those wrong roots would then be invalid. + +A robust follow-up should either: + +- persist sufficient canonical per-block nullifier accounting to derive an exact expected bound, or +- persist explicit canonical tree progress counters alongside `sync_state` for startup and restore + validation + +## Deferred Follow-Up: Destructive Replay Auxiliary Cleanup + +Destructive replay currently reuses the journaled `startup_replay_reset_marker` flow for canonical +state and proposer tree reset, but ordinary proposer mempool client transaction cleanup still runs +after that flow completes. + +This ordinary mempool cleanup is currently best-effort. A crash or DB failure after the +`startup_replay_reset_marker` is cleared but before ordinary mempool cleanup completes can leave +stale proposer mempool transactions with no automatic resume path. + +The intended follow-up is: + +- move ordinary proposer mempool client transaction cleanup into the + `AuxiliaryCleanupPending` phase +- make that cleanup fail-closed and return `Result` +- clear the `startup_replay_reset_marker` only after canonical state reset, tree reset, reserved + deposit cleanup, active transaction lifecycle cleanup, and ordinary mempool cleanup have all + succeeded + +This would make destructive replay recovery fully crash-resumable across both canonical and +auxiliary proposer state. + +## Deferred Follow-Up: Local Docker Snapshot Mount Ownership + +Local `docker-compose` currently defaults proposer snapshots to a host bind mount under +`./data/proposer_snapshots`. + +Because the proposer container runs as root, this can leave root-owned snapshot files in the local +workspace. + +The intended follow-up is one of: + +- use a named Docker volume as the default snapshot mount, or +- run the proposer container as a non-root user + +This is operational rather than protocol-critical, but it would avoid local workspace pollution and +reduce cleanup friction on developer machines or self-hosted runners that use the bind-mount +default. diff --git a/docker-compose.yml b/docker-compose.yml index f1ec7800..154d9d1e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,9 +45,11 @@ services: - NF4_NETWORK__LOG_CHUNK_SIZE=${NF4_NETWORK__LOG_CHUNK_SIZE:-} - NF4_SIGNING_KEY=${CLIENT_SIGNING_KEY} - NF4_NIGHTFALL_CLIENT__DB_URL=mongodb://nf4_db_client:27017 + - NF4_NIGHTFALL_CLIENT__WALLET_TYPE=${NF4_NIGHTFALL_CLIENT__WALLET_TYPE:-local} + - NF4_NIGHTFALL_CLIENT__X509_SIGNER_TYPE=${NF4_NIGHTFALL_CLIENT__X509_SIGNER_TYPE:-local} - AZURE_VAULT_URL=${AZURE_VAULT_URL} - - AZURE_KEY_NAME=${CLIENT_SIGNING_KEY_NAME} - - CLIENT_SIGNING_KEY_NAME=${CLIENT_SIGNING_KEY_NAME} + - CLIENT_SIGNING_KEY_NAME=${CLIENT_SIGNING_KEY_NAME} + - CLIENT_X509_SIGNING_KEY_NAME=${CLIENT_X509_SIGNING_KEY_NAME} - AZURE_CLIENT_ID=${AZURE_CLIENT_ID} - AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET} - AZURE_TENANT_ID=${AZURE_TENANT_ID} @@ -130,9 +132,11 @@ services: - NF4_SIGNING_KEY=${CLIENT2_SIGNING_KEY} - NF4_NIGHTFALL_CLIENT__DB_URL=mongodb://nf4_db_client2:27017 - NF4_NIGHTFALL_CLIENT__LOG_LEVEL=info # Mostly the client2 logs are a repeat of client1 logs, so we can reduce the log level to error - - AZURE_VAULT_URL=${AZURE_VAULT_URL} - - AZURE_KEY_NAME=${CLIENT_SIGNING_KEY_NAME} + - NF4_NIGHTFALL_CLIENT__WALLET_TYPE=${NF4_NIGHTFALL_CLIENT2__WALLET_TYPE:-local} + - NF4_NIGHTFALL_CLIENT__X509_SIGNER_TYPE=${NF4_NIGHTFALL_CLIENT2__X509_SIGNER_TYPE:-local} + - AZURE_VAULT_URL=${AZURE_VAULT_URL} - CLIENT_SIGNING_KEY_NAME=${CLIENT2_SIGNING_KEY_NAME} + - CLIENT_X509_SIGNING_KEY_NAME=${CLIENT2_X509_SIGNING_KEY_NAME} - AZURE_CLIENT_ID=${AZURE_CLIENT_ID} - AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET} - AZURE_TENANT_ID=${AZURE_TENANT_ID} @@ -165,6 +169,7 @@ services: target: /app/configuration/toml read_only: true - ./configuration/bin/keys:/app/configuration/bin/keys:ro + - ${NF4_PROPOSER_SNAPSHOT_MOUNT:-./data/proposer_snapshots}:/app/data/proposer_snapshots depends_on: db_proposer: condition: service_healthy @@ -184,8 +189,12 @@ services: - NF4_NIGHTFALL_PROPOSER__BLOCK_ASSEMBLY_MAX_WAIT_SECS=30 - NF4_NIGHTFALL_PROPOSER__BLOCK_ASSEMBLY_TARGET_FILL_RATIO=0.10 - NF4_NIGHTFALL_PROPOSER__BLOCK_ASSEMBLY_INITIAL_INTERVAL_SECS=5 + - NF4_NIGHTFALL_PROPOSER__SNAPSHOT_ROOT_DIR=/app/data/proposer_snapshots/${NF4_RUN_MODE:-development}/proposer + - NF4_NIGHTFALL_PROPOSER__WALLET_TYPE=${NF4_NIGHTFALL_PROPOSER__WALLET_TYPE:-local} + - NF4_NIGHTFALL_PROPOSER__X509_SIGNER_TYPE=${NF4_NIGHTFALL_PROPOSER__X509_SIGNER_TYPE:-local} - AZURE_VAULT_URL=${AZURE_VAULT_URL} - PROPOSER_SIGNING_KEY_NAME=${PROPOSER_SIGNING_KEY_NAME} + - PROPOSER_X509_SIGNING_KEY_NAME=${PROPOSER_X509_SIGNING_KEY_NAME} - AZURE_CLIENT_ID=${AZURE_CLIENT_ID} - AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET} - AZURE_TENANT_ID=${AZURE_TENANT_ID} @@ -210,6 +219,8 @@ services: start_period: 0s networks: - nightfall_network + volumes: + - ${NF4_PROPOSER_SNAPSHOT_MOUNT:-./data/proposer_snapshots}:/app/data/proposer_snapshots depends_on: db_proposer: condition: service_healthy @@ -220,6 +231,7 @@ services: - RUST_BACKTRACE=${RUST_BACKTRACE:-0} - NF4_SIGNING_KEY=${PROPOSER_SIGNING_KEY} - NF4_MOCK_PROVER=${NF4_MOCK_PROVER:-false} + - NF4_NIGHTFALL_PROPOSER__SNAPSHOT_ROOT_DIR=/app/data/proposer_snapshots/${NF4_RUN_MODE:-base_sepolia}/proposer - NF4_ETHEREUM_CLIENT_URL - NF4_CONTRACTS__DEPLOY_CONTRACTS=${NF4_CONTRACTS__DEPLOY_CONTRACTS:-false} @@ -249,6 +261,7 @@ services: target: /app/configuration/toml read_only: true - ./configuration/bin/keys:/app/configuration/bin/keys:ro + - ${NF4_PROPOSER_SNAPSHOT_MOUNT:-./data/proposer_snapshots}:/app/data/proposer_snapshots depends_on: db_proposer2: condition: service_healthy @@ -265,13 +278,17 @@ services: - NF4_NIGHTFALL_PROPOSER__BLOCK_ASSEMBLY_MAX_WAIT_SECS=30 - NF4_NIGHTFALL_PROPOSER__BLOCK_ASSEMBLY_TARGET_FILL_RATIO=0.10 - NF4_NIGHTFALL_PROPOSER__BLOCK_ASSEMBLY_INITIAL_INTERVAL_SECS=5 + - NF4_NIGHTFALL_PROPOSER__SNAPSHOT_ROOT_DIR=/app/data/proposer_snapshots/${NF4_RUN_MODE:-development}/proposer2 + - NF4_NIGHTFALL_PROPOSER__WALLET_TYPE=${NF4_NIGHTFALL_PROPOSER__WALLET_TYPE:-local} + - NF4_NIGHTFALL_PROPOSER__X509_SIGNER_TYPE=${NF4_NIGHTFALL_PROPOSER__X509_SIGNER_TYPE:-local} - AZURE_VAULT_URL=${AZURE_VAULT_URL} - PROPOSER_SIGNING_KEY_NAME=${PROPOSER_2_SIGNING_KEY_NAME} + - PROPOSER_X509_SIGNING_KEY_NAME=${PROPOSER_X509_SIGNING_KEY_NAME} - AZURE_CLIENT_ID=${AZURE_CLIENT_ID} - AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET} - AZURE_TENANT_ID=${AZURE_TENANT_ID} - NF4_NIGHTFALL_PROPOSER__URL=http://proposer2:3000 - - NF4_NIGHTFALL_PROPOSER__DB_URL=mongodb://nf4_db_proposer2:27017 + - NF4_NIGHTFALL_PROPOSER__DB_URL=mongodb://nf4_db_proposer2:27017/?replicaSet=nf4-rs-proposer2&directConnection=true db_client: container_name: nf4_db_client @@ -326,6 +343,7 @@ services: db_proposer: container_name: nf4_db_proposer + hostname: nf4_db_proposer stop_grace_period: 3s profiles: - proposer @@ -338,12 +356,16 @@ services: ports: - "27018:27017" healthcheck: - test: echo 'db.runCommand("ping").ok' | mongosh localhost:27017 --quiet || exit 1 + test: + [ + "CMD-SHELL", + "echo 'try { rs.status(); } catch (e) { rs.initiate({_id: \"nf4-rs-proposer\", members: [{ _id: 0, host: \"nf4_db_proposer:27017\" }]}); } const hello = db.hello(); if (!(hello.isWritablePrimary || hello.ismaster)) { quit(1); }' | mongosh localhost:27017 --quiet || exit 1", + ] interval: 10s timeout: 90s retries: 10 start_period: 0s - command: mongod --quiet --logpath /dev/null + command: mongod --replSet nf4-rs-proposer --bind_ip_all --quiet --logpath /dev/null networks: - nightfall_network # volumes: @@ -353,6 +375,7 @@ services: db_proposer2: container_name: nf4_db_proposer2 + hostname: nf4_db_proposer2 stop_grace_period: 3s profiles: - sync_test @@ -360,12 +383,16 @@ services: ports: - "27019:27017" healthcheck: - test: echo 'db.runCommand("ping").ok' | mongosh localhost:27017 --quiet || exit 1 + test: + [ + "CMD-SHELL", + "echo 'try { rs.status(); } catch (e) { rs.initiate({_id: \"nf4-rs-proposer2\", members: [{ _id: 0, host: \"nf4_db_proposer2:27017\" }]}); } const hello = db.hello(); if (!(hello.isWritablePrimary || hello.ismaster)) { quit(1); }' | mongosh localhost:27017 --quiet || exit 1", + ] interval: 10s timeout: 90s retries: 10 start_period: 0s - command: mongod --quiet --logpath /dev/null + command: mongod --replSet nf4-rs-proposer2 --bind_ip_all --quiet --logpath /dev/null networks: - nightfall_network environment: @@ -542,7 +569,6 @@ services: - CLIENT_ADDRESS=${CLIENT_ADDRESS} - NF4_LARGE_BLOCK_TEST=${NF4_LARGE_BLOCK_TEST:-false} # Set to true to run large block tests - AZURE_VAULT_URL=${AZURE_VAULT_URL} - - AZURE_KEY_NAME=${CLIENT_SIGNING_KEY_NAME} - AZURE_CLIENT_ID=${AZURE_CLIENT_ID} - AZURE_CLIENT_SECRET=${AZURE_CLIENT_SECRET} - AZURE_TENANT_ID=${AZURE_TENANT_ID} @@ -593,6 +619,7 @@ volumes: address_data: key_data: toml_data: + proposer_snapshots_data: networks: nightfall_network: diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 33e77c19..dbd750e6 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -20,7 +20,7 @@ async-trait = "0.1.88" hex = "0.4.3" dotenv = "0.15.0" x509-parser = "0.16.0" -openssl = "0.10.72" +openssl = "0.10.79" k256 = { version = "0.13", features = ["ecdsa", "sha256"] } alloy = { version = "1.0.23", features = ["full", "json-rpc"] } tokio = { version = "1.45.0", features = ["full"] } diff --git a/lib/src/blockchain_client.rs b/lib/src/blockchain_client.rs index feb1adbe..cfe630a5 100644 --- a/lib/src/blockchain_client.rs +++ b/lib/src/blockchain_client.rs @@ -3,9 +3,9 @@ use crate::wallets::WalletType; use alloy::primitives::{Address, U256}; use alloy::providers::Provider; use alloy::pubsub::PubSubConnect; -use alloy::signers::local::PrivateKeySigner; use alloy::signers::Signer; use async_trait::async_trait; +use configuration::settings::WalletRole; use serde::Deserialize; use std::{marker::Sync, sync::Arc}; use url::Url; @@ -30,15 +30,14 @@ pub trait BlockchainClientConnection: Clone + Send + Sync { fn get_wallet_type(&self) -> &WalletType; - fn get_signer(&self) -> Arc - where - Self: Sized; - async fn new(url: Url, wallet: Self::W) -> Result where Self: Sized; - async fn try_from_settings(settings: &Self::S) -> Result + async fn try_from_settings( + settings: &Self::S, + role: WalletRole, + ) -> Result where Self: Sized; } diff --git a/lib/src/build_transfer_inputs.rs b/lib/src/build_transfer_inputs.rs index de347175..1486328e 100644 --- a/lib/src/build_transfer_inputs.rs +++ b/lib/src/build_transfer_inputs.rs @@ -3,9 +3,9 @@ use crate::{ derive_key::ZKPKeys, hex_conversion::HexConvertible, nf_client_proof::{PrivateInputs, PublicInputs}, - nf_token_id::to_nf_token_id_from_str, + nf_token_id::{to_nf_slot_id_from_str, to_nf_token_id_from_str}, secret_hash::SecretHash, - shared_entities::{DepositData, DepositSecret, Preimage, Salt}, + shared_entities::{DepositData, DepositSecret, Preimage, Salt, TokenType}, }; use alloy::primitives::{hex, keccak256}; use ark_bn254::Fr as Fr254; @@ -93,7 +93,13 @@ pub fn build_valid_transfer_inputs(rng: &mut impl Rng) -> (PublicInputs, Private let token_id_string = Fr254::to_hex_string(&token_id_fr); let nf_token_id = to_nf_token_id_from_str(&erc_address_string, &token_id_string).unwrap(); - let nf_slot_id = nf_token_id; + let nf_slot_id = to_nf_slot_id_from_str( + &erc_address_string, + &token_id_string, + &token_id_string, + TokenType::ERC20, + ) + .unwrap(); // generate a 'random' fee token ID (we just use the keccak hash of 1) let fee_token_id = Fr254::from(BigUint::from_bytes_be(keccak256([1]).as_slice()) >> 4); @@ -192,6 +198,7 @@ pub fn build_valid_transfer_inputs(rng: &mut impl Rng) -> (PublicInputs, Private let private_inputs = PrivateInputs::new() .value_a(value) + .spent_nf_token_ids([nf_token_id, nf_token_id]) .nf_token_a_id(nf_token_id) .nf_slot_id(nf_slot_id) .ephemeral_key(ephemeral_key) diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 5611ad99..ff53aa7b 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -50,20 +50,45 @@ pub fn get_fee_token_id() -> Fr254 { pub mod initialisation { use crate::{blockchain_client::BlockchainClientConnection, wallets::LocalWsClient}; - use configuration::settings::get_settings; + use configuration::settings::{get_settings, WalletRole}; use tokio::sync::{OnceCell, RwLock}; + /// This function is used to provide a singleton blockchain client connection across the entire application. pub async fn get_blockchain_client_connection() -> &'static RwLock { - static BLOCKCHAIN_CLIENT_CONNECTION: OnceCell> = + get_blockchain_client_connection_for_role(WalletRole::Client).await + } + + pub async fn get_blockchain_client_connection_for_role( + role: WalletRole, + ) -> &'static RwLock { + static CLIENT_BLOCKCHAIN_CLIENT_CONNECTION: OnceCell> = OnceCell::const_new(); - BLOCKCHAIN_CLIENT_CONNECTION - .get_or_init(|| async { - RwLock::new( - LocalWsClient::try_from_settings(get_settings()) - .await - .expect("Could not create blockchain client connection"), - ) - }) - .await + static PROPOSER_BLOCKCHAIN_CLIENT_CONNECTION: OnceCell> = + OnceCell::const_new(); + + match role { + WalletRole::Client => { + CLIENT_BLOCKCHAIN_CLIENT_CONNECTION + .get_or_init(|| async { + RwLock::new( + LocalWsClient::try_from_settings(get_settings(), WalletRole::Client) + .await + .expect("Could not create blockchain client connection"), + ) + }) + .await + } + WalletRole::Proposer => { + PROPOSER_BLOCKCHAIN_CLIENT_CONNECTION + .get_or_init(|| async { + RwLock::new( + LocalWsClient::try_from_settings(get_settings(), WalletRole::Proposer) + .await + .expect("Could not create proposer blockchain client connection"), + ) + }) + .await + } + } } } diff --git a/lib/src/merkle_trees/indexed.rs b/lib/src/merkle_trees/indexed.rs index 734f5b54..2f402ddf 100644 --- a/lib/src/merkle_trees/indexed.rs +++ b/lib/src/merkle_trees/indexed.rs @@ -12,7 +12,7 @@ use jf_primitives::{ }, }; use log::{debug, error}; -use mongodb::bson::doc; +use mongodb::{bson::doc, ClientSession}; use std::convert::TryFrom; // already in prelude, but explicit is fine @@ -145,17 +145,38 @@ where &self, inner_leaf_values: &[F], tree_id: &str, + ) -> Result>::Error> { + self.insert_leaves_with_session(inner_leaf_values, tree_id, None) + .await + } + + async fn insert_leaves_with_session( + &self, + inner_leaf_values: &[F], + tree_id: &str, + mut session: Option<&mut ClientSession>, ) -> Result>::Error> { // if we're given an empty list of leaves then we should return the current root if inner_leaf_values.is_empty() { - return >::get_root(self, tree_id).await; + return >::get_root_with_session( + self, + tree_id, + session.as_deref_mut(), + ) + .await; } // check that the leaves are not already in the tree //skip the zero leaves for leaf in inner_leaf_values { - if >::get_leaf(self, Some(*leaf), None, tree_id) - .await? - .is_some() + if >::get_leaf_with_session( + self, + Some(*leaf), + None, + tree_id, + session.as_deref_mut(), + ) + .await? + .is_some() && (!leaf.is_zero()) { error!("Leaf already exists {leaf:?}"); @@ -164,38 +185,58 @@ where } let hasher = >::TreeHasher::new(); // find the low nullifiers for the leaves but do it asynchronously - let low_nullifiers_getters = inner_leaf_values - .iter() - .map(|leaf| >::get_low_leaf(self, leaf, tree_id)) - .collect::>(); - let mut low_nullifiers = join_all(low_nullifiers_getters) - .await - .into_iter() - .collect::>, _>>()? - .iter() - .filter_map(|opt| opt.map(|indexed_leaf| indexed_leaf.value)) - .collect::>(); + let mut low_nullifiers = if session.is_some() { + let mut values = Vec::with_capacity(inner_leaf_values.len()); + for leaf in inner_leaf_values { + let low_leaf = >::get_low_leaf_with_session( + self, + leaf, + tree_id, + session.as_deref_mut(), + ) + .await?; + if let Some(indexed_leaf) = low_leaf { + values.push(indexed_leaf.value); + } + } + values + } else { + let low_nullifiers_getters = inner_leaf_values + .iter() + .map(|leaf| >::get_low_leaf(self, leaf, tree_id)) + .collect::>(); + join_all(low_nullifiers_getters) + .await + .into_iter() + .collect::>, _>>()? + .iter() + .filter_map(|opt| opt.map(|indexed_leaf| indexed_leaf.value)) + .collect::>() + }; low_nullifiers.sort(); low_nullifiers.dedup(); let collection_name = format!("{}_{}", tree_id, "metadata"); let db = self.database(>::MUT_DB_NAME); let collection = db.collection::>(&collection_name); - let metadata = collection - .find_one(doc! {}) - .await - .map_err(MerkleTreeError::DatabaseError)? - .ok_or(MerkleTreeError::ItemNotFound)?; + let metadata = if let Some(session) = session.as_deref_mut() { + collection.find_one(doc! {}).session(session).await + } else { + collection.find_one(doc! {}).await + } + .map_err(MerkleTreeError::DatabaseError)? + .ok_or(MerkleTreeError::ItemNotFound)?; let mut insert_index = metadata.sub_tree_count as u32 * (1u32 << metadata.sub_tree_height); // We cannot store the leaves concurrently because the index gets updated each time we store one and we'll have // a data race. We could probably do something clever with a mutex but it's easier to just store them sequentially. for inner_leaf in inner_leaf_values.iter() { if !inner_leaf.is_zero() { - let res = >::store_leaf( + let res = >::store_leaf_with_session( self, *inner_leaf, Some(insert_index.into()), tree_id, + session.as_deref_mut(), ) .await?; if res.is_none() { @@ -209,23 +250,43 @@ where } // add the leaves to the tree - let indexed_leaves_getters = inner_leaf_values - .iter() - .map(|leaf| >::get_leaf(self, Some(*leaf), None, tree_id)) - .collect::>(); - let indexed_leaves = join_all(indexed_leaves_getters) - .await - .into_iter() - .collect::>>, >::Error>>()? - .into_iter() - .enumerate() - .map(|(index, opt)| { - opt.ok_or(MerkleTreeError::Error(format!( + let indexed_leaves = if session.is_some() { + let mut leaves = Vec::with_capacity(inner_leaf_values.len()); + for (index, leaf) in inner_leaf_values.iter().enumerate() { + let indexed_leaf = >::get_leaf_with_session( + self, + Some(*leaf), + None, + tree_id, + session.as_deref_mut(), + ) + .await? + .ok_or(MerkleTreeError::Error(format!( "failed to get IndexedLeaf struct with value {}", inner_leaf_values[index] - ))) - }) - .collect::>, _>>()?; + )))?; + leaves.push(indexed_leaf); + } + leaves + } else { + let indexed_leaves_getters = inner_leaf_values + .iter() + .map(|leaf| >::get_leaf(self, Some(*leaf), None, tree_id)) + .collect::>(); + join_all(indexed_leaves_getters) + .await + .into_iter() + .collect::>>, >::Error>>()? + .into_iter() + .enumerate() + .map(|(index, opt)| { + opt.ok_or(MerkleTreeError::Error(format!( + "failed to get IndexedLeaf struct with value {}", + inner_leaf_values[index] + ))) + }) + .collect::>, _>>()? + }; let leaf_values = indexed_leaves .into_iter() .map(|indexed_leaf| { @@ -242,39 +303,72 @@ where }) .collect::, _>>()?; - >::append_sub_trees(self, &leaf_values, true, tree_id).await?; + >::append_sub_trees_with_session( + self, + &leaf_values, + true, + tree_id, + session.as_deref_mut(), + ) + .await?; - let update_info_getters = low_nullifiers - .iter() - .map(|value| >::get_leaf(self, Some(*value), None, tree_id)) - .collect::>(); - let update_info = join_all(update_info_getters) - .await - .into_iter() - .collect::>>, _>>()? - .ok_or(MerkleTreeError::ItemNotFound)? - .into_iter() - .map(|leaf| { + let update_info = if session.is_some() { + let mut info = Vec::with_capacity(low_nullifiers.len()); + for value in &low_nullifiers { + let leaf = >::get_leaf_with_session( + self, + Some(*value), + None, + tree_id, + session.as_deref_mut(), + ) + .await? + .ok_or(MerkleTreeError::ItemNotFound)?; let leaf_value: F = leaf.value; let leaf_next_index = F::from(leaf.next_index); let leaf_next_value: F = leaf.next_value; - Ok(( + info.push(( hasher.hash(&[leaf_value, leaf_next_index, leaf_next_value])?, leaf._id as usize, - )) - }) - .collect::, MerkleTreeError>>()?; + )); + } + info + } else { + let update_info_getters = low_nullifiers + .iter() + .map(|value| { + >::get_leaf(self, Some(*value), None, tree_id) + }) + .collect::>(); + join_all(update_info_getters) + .await + .into_iter() + .collect::>>, _>>()? + .ok_or(MerkleTreeError::ItemNotFound)? + .into_iter() + .map(|leaf| { + let leaf_value: F = leaf.value; + let leaf_next_index = F::from(leaf.next_index); + let leaf_next_value: F = leaf.next_value; + Ok(( + hasher.hash(&[leaf_value, leaf_next_index, leaf_next_value])?, + leaf._id as usize, + )) + }) + .collect::, MerkleTreeError>>()? + }; let mut root = F::zero(); for info in update_info.into_iter() { let leaf_value = info.0; let ln_index = info.1 as u64; - root = >::update_sub_tree( + root = >::update_sub_tree_with_session( self, ln_index, &[leaf_value], true, tree_id, + session.as_deref_mut(), ) .await?; } @@ -513,9 +607,24 @@ impl IndexedLeaves for mongodb::Client { leaf: F, index: Option, tree_id: &str, + ) -> Result, Self::Error> { + self.store_leaf_with_session(leaf, index, tree_id, None) + .await + } + + async fn store_leaf_with_session( + &self, + leaf: F, + index: Option, + tree_id: &str, + mut session: Option<&mut ClientSession>, ) -> Result, Self::Error> { // If the new leaf is already in the db then we shouldn't store it. - if self.get_leaf(Some(leaf), None, tree_id).await?.is_some() { + if self + .get_leaf_with_session(Some(leaf), None, tree_id, session.as_deref_mut()) + .await? + .is_some() + { debug!("Leaf already exists {}", leaf.to_string_rep()); return Err(MerkleTreeError::LeafExists); } @@ -523,7 +632,10 @@ impl IndexedLeaves for mongodb::Client { let db = self.database(>::DB); let collection = db.collection::>(&collection_name); // If the new leaf is not in the db then we should update the next value of its low nullifier. - let low_leaf = if let Some(low_leaf) = self.get_low_leaf(&leaf, tree_id).await? { + let low_leaf = if let Some(low_leaf) = self + .get_low_leaf_with_session(&leaf, tree_id, session.as_deref_mut()) + .await? + { low_leaf } else { debug!("Could not find low leaf for leaf {}", leaf.to_string_rep()); @@ -536,10 +648,12 @@ impl IndexedLeaves for mongodb::Client { } else { // if the index is not provided then we search for the maximum index in the db. If there are no // maximal leaves in the db then we set the index to 1. Then we add 1 to the result. - collection - .count_documents(doc! {}) - .await - .map_err(MerkleTreeError::DatabaseError)? + if let Some(session) = session.as_deref_mut() { + collection.count_documents(doc! {}).session(session).await + } else { + collection.count_documents(doc! {}).await + } + .map_err(MerkleTreeError::DatabaseError)? }; // Create a new leaf entry let entry = IndexedLeaf:: { @@ -557,20 +671,37 @@ impl IndexedLeaves for mongodb::Client { let bson_index = u64_to_i64_checked(index)?; let bson_next_index = u64_to_i64_checked(low_leaf.next_index)?; - let updates_result = collection - .update_one( - doc! { "_id": bson_index }, - doc! { - "$set": { - "value": padded_leaf, - "next_index": bson_next_index, - "next_value": padded_next_value, - } - }, - ) - .upsert(true) - .await - .map_err(MerkleTreeError::DatabaseError)?; + let updates_result = if let Some(session) = session.as_deref_mut() { + collection + .update_one( + doc! { "_id": bson_index }, + doc! { + "$set": { + "value": padded_leaf, + "next_index": bson_next_index, + "next_value": padded_next_value, + } + }, + ) + .upsert(true) + .session(session) + .await + } else { + collection + .update_one( + doc! { "_id": bson_index }, + doc! { + "$set": { + "value": padded_leaf, + "next_index": bson_next_index, + "next_value": padded_next_value, + } + }, + ) + .upsert(true) + .await + } + .map_err(MerkleTreeError::DatabaseError)?; if updates_result.matched_count == 0 && updates_result.upserted_id.is_none() { return Err(MerkleTreeError::Error( @@ -578,7 +709,7 @@ impl IndexedLeaves for mongodb::Client { )); } let low_leaf_value: F = low_leaf.value; - self.update_leaf(low_leaf_value, index, leaf, tree_id) + self.update_leaf_with_session(low_leaf_value, index, leaf, tree_id, session) .await?; Ok(Some(())) } @@ -588,6 +719,17 @@ impl IndexedLeaves for mongodb::Client { value: Option, next_value: Option, tree_id: &str, + ) -> Result>, Self::Error> { + self.get_leaf_with_session(value, next_value, tree_id, None) + .await + } + + async fn get_leaf_with_session( + &self, + value: Option, + next_value: Option, + tree_id: &str, + session: Option<&mut ClientSession>, ) -> Result>, Self::Error> { let collection_name = format!("{}_{}", tree_id, "indexed_leaves"); let db = self.database(>::DB); @@ -618,36 +760,69 @@ impl IndexedLeaves for mongodb::Client { _ => doc! {}, }; - collection - .find_one(query) - .await - .map_err(MerkleTreeError::DatabaseError) + if let Some(session) = session { + collection.find_one(query).session(session).await + } else { + collection.find_one(query).await + } + .map_err(MerkleTreeError::DatabaseError) } async fn get_low_leaf( &self, leaf_value: &F, tree_id: &str, + ) -> Result>, Self::Error> { + self.get_low_leaf_with_session(leaf_value, tree_id, None) + .await + } + + async fn get_low_leaf_with_session( + &self, + leaf_value: &F, + tree_id: &str, + session: Option<&mut ClientSession>, ) -> Result>, Self::Error> { let collection_name = format!("{}_{}", tree_id, "indexed_leaves"); let db = self.database(>::DB); let collection = db.collection::>(&collection_name); let padded_hex = fr_to_bson_padded(leaf_value)?; - let mut cursor = collection - .find(doc! {"value": {"$lt": padded_hex}}) - .sort(doc! {"value": -1}) - .limit(1) - .await - .map_err(MerkleTreeError::DatabaseError)?; + if let Some(session) = session { + let mut cursor = collection + .find(doc! {"value": {"$lt": padded_hex}}) + .sort(doc! {"value": -1}) + .limit(1) + .session(&mut *session) + .await + .map_err(MerkleTreeError::DatabaseError)?; - if let Some(result) = cursor - .try_next() - .await - .map_err(MerkleTreeError::DatabaseError)? - { - Ok(Some(result)) + if let Some(result) = cursor + .next(session) + .await + .transpose() + .map_err(MerkleTreeError::DatabaseError)? + { + Ok(Some(result)) + } else { + Ok(None) + } } else { - Ok(None) + let mut cursor = collection + .find(doc! {"value": {"$lt": padded_hex}}) + .sort(doc! {"value": -1}) + .limit(1) + .await + .map_err(MerkleTreeError::DatabaseError)?; + + if let Some(result) = cursor + .try_next() + .await + .map_err(MerkleTreeError::DatabaseError)? + { + Ok(Some(result)) + } else { + Ok(None) + } } } @@ -657,6 +832,18 @@ impl IndexedLeaves for mongodb::Client { new_next_index: u64, new_next_value: F, tree_id: &str, + ) -> Result<(), Self::Error> { + self.update_leaf_with_session(leaf, new_next_index, new_next_value, tree_id, None) + .await + } + + async fn update_leaf_with_session( + &self, + leaf: F, + new_next_index: u64, + new_next_value: F, + tree_id: &str, + session: Option<&mut ClientSession>, ) -> Result<(), Self::Error> { let collection_name = format!("{}_{}", tree_id, "indexed_leaves"); let db = self.database(>::DB); @@ -667,10 +854,12 @@ impl IndexedLeaves for mongodb::Client { let bson_next_index = u64_to_i64_checked(new_next_index)?; let update = doc! {"$set": {"next_index": bson_next_index, "next_value": padded_next_value}}; - let result = collection - .update_one(query, update) - .await - .map_err(MerkleTreeError::DatabaseError)?; + let result = if let Some(session) = session { + collection.update_one(query, update).session(session).await + } else { + collection.update_one(query, update).await + } + .map_err(MerkleTreeError::DatabaseError)?; if result.matched_count == 0 && result.upserted_id.is_none() { return Err(MerkleTreeError::Error( "Failed to update or upsert the node in the database".to_string(), diff --git a/lib/src/merkle_trees/mutable.rs b/lib/src/merkle_trees/mutable.rs index 4fe30cae..c3f478a0 100644 --- a/lib/src/merkle_trees/mutable.rs +++ b/lib/src/merkle_trees/mutable.rs @@ -18,6 +18,7 @@ use log::debug; use mongodb::{ bson::{doc, to_bson}, options::{UpdateOneModel, WriteModel}, + ClientSession, }; use serde::{Deserialize, Serialize}; @@ -80,29 +81,58 @@ where } async fn get_root(&self, tree_id: &str) -> Result { + self.get_root_with_session(tree_id, None).await + } + + async fn get_root_with_session( + &self, + tree_id: &str, + session: Option<&mut ClientSession>, + ) -> Result { let metadata_collection_name = format!("{}_{}", tree_id, "metadata"); let metadata_collection = self .database(>::MUT_DB_NAME) .collection::>(&metadata_collection_name); - let metadata = metadata_collection - .find_one(doc! {"_id": 0}) - .await - .map_err(MerkleTreeError::DatabaseError)? - .ok_or(MerkleTreeError::TreeNotFound)?; + let metadata = if let Some(session) = session { + metadata_collection + .find_one(doc! {"_id": 0}) + .session(session) + .await + } else { + metadata_collection.find_one(doc! {"_id": 0}).await + } + .map_err(MerkleTreeError::DatabaseError)? + .ok_or(MerkleTreeError::TreeNotFound)?; Ok(metadata.root) } async fn get_node(&self, index: u64, tree_id: &str) -> Result { + self.get_node_with_session(index, tree_id, None).await + } + + async fn get_node_with_session( + &self, + index: u64, + tree_id: &str, + mut session: Option<&mut ClientSession>, + ) -> Result { // first, check if the node is in the temporary cache. This is used for when we don't want to write to the db let bson_index = to_bson(&index).map_err(|e| MerkleTreeError::DatabaseError(e.into()))?; let cache_collection_name = format!("{}_{}", tree_id, "cache"); let cache_collection = self .database(>::MUT_DB_NAME) .collection::>(&cache_collection_name); - let node = cache_collection - .find_one(doc! {"_id": bson_index.clone()}) - .await - .map_err(MerkleTreeError::DatabaseError)?; + let node = if let Some(session) = session.as_deref_mut() { + cache_collection + .find_one(doc! {"_id": bson_index.clone()}) + .session(session) + .await + } else { + cache_collection + .find_one(doc! {"_id": bson_index.clone()}) + .await + } + .map_err(MerkleTreeError::DatabaseError)?; match node { // if the node is in the cache, return it Some(node) => Ok(node.value), @@ -112,10 +142,15 @@ where let node_collection = self .database(>::MUT_DB_NAME) .collection::>(&node_collection_name); - let node = node_collection - .find_one(doc! {"_id": bson_index}) - .await - .map_err(MerkleTreeError::DatabaseError)?; + let node = if let Some(session) = session { + node_collection + .find_one(doc! {"_id": bson_index}) + .session(session) + .await + } else { + node_collection.find_one(doc! {"_id": bson_index}).await + } + .map_err(MerkleTreeError::DatabaseError)?; // nodes that aren't in the database are returned as zero match node { Some(node) => Ok(node.value), @@ -126,17 +161,20 @@ where } async fn flush_cache(&self, tree_id: &str) -> Result<(), Self::Error> { + >::flush_cache_with_session(self, tree_id, None).await + } + + async fn flush_cache_with_session( + &self, + tree_id: &str, + mut session: Option<&mut ClientSession>, + ) -> Result<(), Self::Error> { let cache_collection_name = format!("{}_{}", tree_id, "cache"); let cache_collection = self .database(>::MUT_DB_NAME) .collection::>(&cache_collection_name); // Stream all cached nodes - let mut cache_cursor = cache_collection - .find(doc! {}) - .await - .map_err(MerkleTreeError::DatabaseError)?; - // Target collection for persisted nodes let node_collection_name = format!("{}_{}", tree_id, "nodes"); let node_collection = self @@ -147,34 +185,74 @@ where let mut models: Vec = Vec::new(); let mut cached_entries: u64 = 0; - while let Some(node) = cache_cursor - .try_next() - .await - .map_err(MerkleTreeError::DatabaseError)? - { - cached_entries += 1; - - let bson_id = - to_bson(&node._id).map_err(|e| MerkleTreeError::DatabaseError(e.into()))?; - let value_padded_hex = fr_to_bson_padded(&node.value)?; + if let Some(session) = session.as_deref_mut() { + let mut cache_cursor = cache_collection + .find(doc! {}) + .session(&mut *session) + .await + .map_err(MerkleTreeError::DatabaseError)?; - // Build an UpdateOneModel for this node - let update_model = UpdateOneModel::builder() - .namespace(node_collection.namespace()) - .filter(doc! { "_id": bson_id }) - .update(doc! { "$set": { "value": value_padded_hex } }) - .upsert(true) - .build(); + while let Some(node) = cache_cursor + .next(session) + .await + .transpose() + .map_err(MerkleTreeError::DatabaseError)? + { + cached_entries += 1; + + let bson_id = + to_bson(&node._id).map_err(|e| MerkleTreeError::DatabaseError(e.into()))?; + let value_padded_hex = fr_to_bson_padded(&node.value)?; + + let update_model = UpdateOneModel::builder() + .namespace(node_collection.namespace()) + .filter(doc! { "_id": bson_id }) + .update(doc! { "$set": { "value": value_padded_hex } }) + .upsert(true) + .build(); + + models.push(WriteModel::UpdateOne(update_model)); + } + } else { + let mut cache_cursor = cache_collection + .find(doc! {}) + .await + .map_err(MerkleTreeError::DatabaseError)?; - models.push(WriteModel::UpdateOne(update_model)); + while let Some(node) = cache_cursor + .try_next() + .await + .map_err(MerkleTreeError::DatabaseError)? + { + cached_entries += 1; + + let bson_id = + to_bson(&node._id).map_err(|e| MerkleTreeError::DatabaseError(e.into()))?; + let value_padded_hex = fr_to_bson_padded(&node.value)?; + + let update_model = UpdateOneModel::builder() + .namespace(node_collection.namespace()) + .filter(doc! { "_id": bson_id }) + .update(doc! { "$set": { "value": value_padded_hex } }) + .upsert(true) + .build(); + + models.push(WriteModel::UpdateOne(update_model)); + } } - // If there is nothing to flush, we can safely drop the cache and return. + // If there is nothing to flush, we can safely clear the cache and return. if models.is_empty() { - cache_collection - .drop() - .await - .map_err(MerkleTreeError::DatabaseError)?; + if let Some(session) = session.as_deref_mut() { + cache_collection + .delete_many(doc! {}) + .session(session) + .await + .map(|_| ()) + } else { + cache_collection.drop().await + } + .map_err(MerkleTreeError::DatabaseError)?; return Ok(()); } @@ -186,18 +264,13 @@ where )) })?; - if expected < 1 { - return Err(MerkleTreeError::Error( - "Invalid cached_entries count: must be positive".to_string(), - )); - } - // Execute ordered bulk write so that on the first error, remaining operations are not applied. - let result = self - .bulk_write(models) - .ordered(true) // stop on first failure - .await - .map_err(MerkleTreeError::DatabaseError)?; + let result = if let Some(session) = session.as_deref_mut() { + self.bulk_write(models).ordered(true).session(session).await + } else { + self.bulk_write(models).ordered(true).await + } + .map_err(MerkleTreeError::DatabaseError)?; // For updates, "success" is counted as matched + upserted. let applied = result.matched_count + result.upserted_count; @@ -209,11 +282,17 @@ where ))); } - // Only now that we know all writes were acknowledged do we drop the cache. - cache_collection - .drop() - .await - .map_err(MerkleTreeError::DatabaseError)?; + // Only now that we know all writes were acknowledged do we clear the cache. + if let Some(session) = session { + cache_collection + .delete_many(doc! {}) + .session(session) + .await + .map(|_| ()) + } else { + cache_collection.drop().await + } + .map_err(MerkleTreeError::DatabaseError)?; Ok(()) } @@ -251,6 +330,18 @@ where value: F, update_tree: bool, tree_id: &str, + ) -> Result<(), Self::Error> { + self.set_node_with_session(index, value, update_tree, tree_id, None) + .await + } + + async fn set_node_with_session( + &self, + index: u64, + value: F, + update_tree: bool, + tree_id: &str, + session: Option<&mut ClientSession>, ) -> Result<(), Self::Error> { let update_value = fr_to_bson_padded(&value)?; let bson_index = to_bson(&index).map_err(|e| MerkleTreeError::DatabaseError(e.into()))?; @@ -259,14 +350,25 @@ where let cache_collection = self .database(>::MUT_DB_NAME) .collection::>(&cache_collection_name); - let update = cache_collection - .update_one( - doc! {"_id": bson_index}, - doc! {"$set": {"value": update_value}}, - ) - .upsert(true) - .await - .map_err(MerkleTreeError::DatabaseError)?; + let update = if let Some(session) = session { + cache_collection + .update_one( + doc! {"_id": bson_index}, + doc! {"$set": {"value": update_value}}, + ) + .upsert(true) + .session(session) + .await + } else { + cache_collection + .update_one( + doc! {"_id": bson_index}, + doc! {"$set": {"value": update_value}}, + ) + .upsert(true) + .await + } + .map_err(MerkleTreeError::DatabaseError)?; // Check if the update succeeded if update.matched_count == 0 && update.upserted_id.is_none() { return Err(MerkleTreeError::Error( @@ -279,14 +381,25 @@ where let node_collection = self .database(>::MUT_DB_NAME) .collection::>(&node_collection_name); - let update = node_collection - .update_one( - doc! {"_id": bson_index}, - doc! {"$set": {"value": update_value}}, - ) - .upsert(true) - .await - .map_err(MerkleTreeError::DatabaseError)?; + let update = if let Some(session) = session { + node_collection + .update_one( + doc! {"_id": bson_index}, + doc! {"$set": {"value": update_value}}, + ) + .upsert(true) + .session(session) + .await + } else { + node_collection + .update_one( + doc! {"_id": bson_index}, + doc! {"$set": {"value": update_value}}, + ) + .upsert(true) + .await + } + .map_err(MerkleTreeError::DatabaseError)?; // Check if the update succeeded if update.matched_count == 0 && update.upserted_id.is_none() { return Err(MerkleTreeError::Error( @@ -309,17 +422,33 @@ where leaf: F, update_tree: bool, tree_id: &str, + ) -> Result { + self.insert_leaf_with_session(leaf, update_tree, tree_id, None) + .await + } + + async fn insert_leaf_with_session( + &self, + leaf: F, + update_tree: bool, + tree_id: &str, + mut session: Option<&mut ClientSession>, ) -> Result { // get the tree metadata let metadata_collection_name = format!("{}_{}", tree_id, "metadata"); let metadata_collection = self .database(>::MUT_DB_NAME) .collection::>(&metadata_collection_name); - let metadata = metadata_collection - .find_one(doc! {"_id": 0}) - .await - .map_err(MerkleTreeError::DatabaseError)? - .ok_or(MerkleTreeError::TreeNotFound)?; + let metadata = if let Some(session) = session.as_deref_mut() { + metadata_collection + .find_one(doc! {"_id": 0}) + .session(session) + .await + } else { + metadata_collection.find_one(doc! {"_id": 0}).await + } + .map_err(MerkleTreeError::DatabaseError)? + .ok_or(MerkleTreeError::TreeNotFound)?; let sub_tree_count = metadata.sub_tree_count; // we'll 'add' each sub tree in turn but only write everything to the db at the end. This will be much @@ -356,24 +485,48 @@ where MerkleTreeError::Error("node index computation overflowed".to_string()) })?; // this is the index where we are going to put the sub_tree let mut node_index = sub_tree_node_index; - let mut updates = vec![self.set_node(node_index, leaf, update_tree, tree_id)]; // this will store all the hash values in the path from the leaf to the root + let mut pending_updates = vec![(node_index, leaf)]; // this will store all the hash values in the path from the leaf to the root for _i in 0..metadata.tree_height + metadata.sub_tree_height { hash = if node_index % 2 == 0 { hasher - .tree_hash(&[self.get_node(node_index - 1, tree_id).await?, hash]) + .tree_hash(&[ + self.get_node_with_session(node_index - 1, tree_id, session.as_deref_mut()) + .await?, + hash, + ]) .expect("Could not hash nodes together") } else { hasher - .tree_hash(&[hash, self.get_node(node_index + 1, tree_id).await?]) + .tree_hash(&[ + hash, + self.get_node_with_session(node_index + 1, tree_id, session.as_deref_mut()) + .await?, + ]) .expect("Could not hash nodes together") }; node_index = (node_index - 1) / 2; - updates.push(self.set_node(node_index, hash, update_tree, tree_id)); - // save the updated nodes + pending_updates.push((node_index, hash)); } - try_join_all(updates).await?; + if session.is_some() { + for (index, value) in pending_updates { + self.set_node_with_session( + index, + value, + update_tree, + tree_id, + session.as_deref_mut(), + ) + .await?; + } + } else { + let updates = pending_updates + .into_iter() + .map(|(index, value)| self.set_node(index, value, update_tree, tree_id)) + .collect::>(); + try_join_all(updates).await?; + } // store the updated sub tree count if update_tree { @@ -387,17 +540,24 @@ where _id: 0, root: hash, }; - let result = metadata_collection - .replace_one(doc! {"_id": 0}, new_metadata) - .await - .map_err(MerkleTreeError::DatabaseError)?; + let result = if let Some(session) = session.as_deref_mut() { + metadata_collection + .replace_one(doc! {"_id": 0}, new_metadata) + .session(session) + .await + } else { + metadata_collection + .replace_one(doc! {"_id": 0}, new_metadata) + .await + } + .map_err(MerkleTreeError::DatabaseError)?; if result.matched_count == 0 && result.upserted_id.is_none() { return Err(MerkleTreeError::Error( "Failed to update the tree metadata in the database".to_string(), )); } // save the cached nodes - >::flush_cache(self, tree_id).await?; + >::flush_cache_with_session(self, tree_id, session).await?; } // return the final root and the new sub tree count (from which leaf indices can be derived) Ok(hash) @@ -408,17 +568,33 @@ where leaves: &[F], update_tree: bool, tree_id: &str, + ) -> Result<(F, u64), Self::Error> { + self.append_sub_trees_with_session(leaves, update_tree, tree_id, None) + .await + } + + async fn append_sub_trees_with_session( + &self, + leaves: &[F], + update_tree: bool, + tree_id: &str, + mut session: Option<&mut ClientSession>, ) -> Result<(F, u64), Self::Error> { // get the tree metadata let metadata_collection_name = format!("{}_{}", tree_id, "metadata"); let metadata_collection = self .database(>::MUT_DB_NAME) .collection::>(&metadata_collection_name); - let metadata = metadata_collection - .find_one(doc! {"_id": 0}) - .await - .map_err(MerkleTreeError::DatabaseError)? - .ok_or(MerkleTreeError::TreeNotFound)?; + let metadata = if let Some(session) = session.as_deref_mut() { + metadata_collection + .find_one(doc! {"_id": 0}) + .session(session) + .await + } else { + metadata_collection.find_one(doc! {"_id": 0}).await + } + .map_err(MerkleTreeError::DatabaseError)? + .ok_or(MerkleTreeError::TreeNotFound)?; let mut sub_tree_count = metadata.sub_tree_count; let old_sub_tree_count = sub_tree_count; // Basic data validation @@ -472,22 +648,37 @@ where Self::Error::Error("sub tree index too large to compute node index".to_string()) })?; // this is the index where we're going to put the sub_tree let mut node_index = sub_tree_node_index; - let mut updates = vec![self.set_node(node_index, sub_tree_root, update_tree, tree_id)]; // this will store all the hash values in the path from the leaf to the root + let mut pending_updates = vec![(node_index, sub_tree_root)]; // this will store all the hash values in the path from the leaf to the root hash = sub_tree_root; // the main tree leaf value is the starting hash // hash to get the path up the tree, store the updated nodes as we go for _i in 0..metadata.tree_height { hash = if node_index % 2 == 0 { hasher - .tree_hash(&[self.get_node(node_index - 1, tree_id).await?, hash]) + .tree_hash(&[ + self.get_node_with_session( + node_index - 1, + tree_id, + session.as_deref_mut(), + ) + .await?, + hash, + ]) .expect("Could not hash nodes together") } else { hasher - .tree_hash(&[hash, self.get_node(node_index + 1, tree_id).await?]) + .tree_hash(&[ + hash, + self.get_node_with_session( + node_index + 1, + tree_id, + session.as_deref_mut(), + ) + .await?, + ]) .expect("Could not hash nodes together") }; node_index = (node_index - 1) / 2; - updates.push(self.set_node(node_index, hash, update_tree, tree_id)); - // save the updated nodes + pending_updates.push((node_index, hash)); } // for the nodes in the sub tree we count downwards, ignoring the root because we already counted that @@ -500,17 +691,28 @@ where span *= 2; for j in node_index..(node_index + span) { sub_tree_node_index += 1; - updates.push(self.set_node( - j, - sub_tree[sub_tree_node_index], + pending_updates.push((j, sub_tree[sub_tree_node_index])); + } + } + + if session.is_some() { + for (index, value) in pending_updates { + self.set_node_with_session( + index, + value, update_tree, tree_id, - )); + session.as_deref_mut(), + ) + .await?; } + } else { + let updates = pending_updates + .into_iter() + .map(|(index, value)| self.set_node(index, value, update_tree, tree_id)) + .collect::>(); + try_join_all(updates).await?; } - - // run the set functions concurrently to update the nodes we changed - try_join_all(updates).await?; sub_tree_count = sub_tree_count .checked_add(1) .ok_or_else(|| Self::Error::Error("sub_tree_count overflowed".to_string()))?; @@ -524,12 +726,19 @@ where _id: 0, root: hash, }; - metadata_collection - .replace_one(doc! {"_id": 0}, new_metadata) - .await - .map_err(MerkleTreeError::DatabaseError)?; + if let Some(session) = session.as_deref_mut() { + metadata_collection + .replace_one(doc! {"_id": 0}, new_metadata) + .session(session) + .await + } else { + metadata_collection + .replace_one(doc! {"_id": 0}, new_metadata) + .await + } + .map_err(MerkleTreeError::DatabaseError)?; // save the cached nodes - >::flush_cache(self, tree_id).await?; + >::flush_cache_with_session(self, tree_id, session).await?; } else { sub_tree_count = old_sub_tree_count; } @@ -672,17 +881,34 @@ where leaves: &[F], update_tree: bool, tree_id: &str, + ) -> Result { + self.update_sub_tree_with_session(sub_tree_index, leaves, update_tree, tree_id, None) + .await + } + + async fn update_sub_tree_with_session( + &self, + sub_tree_index: u64, + leaves: &[F], + update_tree: bool, + tree_id: &str, + mut session: Option<&mut ClientSession>, ) -> Result { // get the tree metadata let metadata_collection_name = format!("{}_{}", tree_id, "metadata"); let metadata_collection = self .database(>::MUT_DB_NAME) .collection::>(&metadata_collection_name); - let metadata = metadata_collection - .find_one(doc! {"_id": 0}) - .await - .map_err(MerkleTreeError::DatabaseError)? - .ok_or(MerkleTreeError::TreeNotFound)?; + let metadata = if let Some(session) = session.as_deref_mut() { + metadata_collection + .find_one(doc! {"_id": 0}) + .session(session) + .await + } else { + metadata_collection.find_one(doc! {"_id": 0}).await + } + .map_err(MerkleTreeError::DatabaseError)? + .ok_or(MerkleTreeError::TreeNotFound)?; // Basic data validation let total_height = metadata @@ -734,22 +960,29 @@ where Self::Error::Error("subtree index too large to compute node index".to_string()) })?; // this is the index where we're going to put the sub_tree let mut node_index = sub_tree_node_index; - let mut updates = vec![self.set_node(node_index, sub_tree_root, update_tree, tree_id)]; // this will store all the hash values in the path from the leaf to the root + let mut pending_updates = vec![(node_index, sub_tree_root)]; // this will store all the hash values in the path from the leaf to the root let mut hash = sub_tree_root; // the main tree leaf value is the starting hash // hash to get the path up the tree, store the updated nodes as we go for _i in 0..height_to_use { hash = if node_index % 2 == 0 { hasher - .tree_hash(&[self.get_node(node_index - 1, tree_id).await?, hash]) + .tree_hash(&[ + self.get_node_with_session(node_index - 1, tree_id, session.as_deref_mut()) + .await?, + hash, + ]) .expect("Could not hash nodes together") } else { hasher - .tree_hash(&[hash, self.get_node(node_index + 1, tree_id).await?]) + .tree_hash(&[ + hash, + self.get_node_with_session(node_index + 1, tree_id, session.as_deref_mut()) + .await?, + ]) .expect("Could not hash nodes together") }; node_index = (node_index - 1) / 2; - updates.push(self.set_node(node_index, hash, update_tree, tree_id)); - // save the updated nodes + pending_updates.push((node_index, hash)); } // for the nodes in the sub tree we count downwards, ignoring the root because we already counted that // i is the row in the sub tree, counting from the top @@ -762,20 +995,36 @@ where span *= 2; for j in node_index..(node_index + span) { sub_tree_node_index += 1; - updates.push(self.set_node( - j, - sub_tree[sub_tree_node_index], - update_tree, - tree_id, - )); + pending_updates.push((j, sub_tree[sub_tree_node_index])); } } }; - // run the set functions concurrently to update the nodes we changed - try_join_all(updates).await?; + if session.is_some() { + for (index, value) in pending_updates { + self.set_node_with_session( + index, + value, + update_tree, + tree_id, + session.as_deref_mut(), + ) + .await?; + } + } else { + let updates = pending_updates + .into_iter() + .map(|(index, value)| self.set_node(index, value, update_tree, tree_id)) + .collect::>(); + try_join_all(updates).await?; + } if update_tree { // save the cached nodes - >::flush_cache(self, tree_id).await? + >::flush_cache_with_session( + self, + tree_id, + session.as_deref_mut(), + ) + .await? } // save the updated root let new_metadata = TreeMetadata { @@ -785,10 +1034,17 @@ where _id: 0, root: hash, }; - metadata_collection - .replace_one(doc! {"_id": 0}, new_metadata) - .await - .map_err(MerkleTreeError::DatabaseError)?; + if let Some(session) = session { + metadata_collection + .replace_one(doc! {"_id": 0}, new_metadata) + .session(session) + .await + } else { + metadata_collection + .replace_one(doc! {"_id": 0}, new_metadata) + .await + } + .map_err(MerkleTreeError::DatabaseError)?; // return the updated root Ok(hash) } @@ -1216,4 +1472,48 @@ mod test { Err(MerkleTreeError::Error(msg)) if msg.contains("sub_tree_height too large") )); } + + #[tokio::test] + async fn flush_cache_with_session_succeeds_inside_transaction() { + let container = get_mongo().await; + let client = get_db_connection(&container).await; + let tree_id = "flush_cache_transaction_safe"; + + >::new_mutable_tree(&client, 3, 1, tree_id) + .await + .expect("create mutable tree"); + + let cache_collection = client + .database(>::MUT_DB_NAME) + .collection::>(&format!("{tree_id}_cache")); + cache_collection + .insert_one(Node { + value: Fr254::one(), + _id: 0, + }) + .await + .expect("seed cache entry"); + + let mut session = client.start_session().await.expect("start session"); + session + .start_transaction() + .and_run2(async |session| { + >::flush_cache_with_session( + &client, + tree_id, + Some(session), + ) + .await + .map_err(|error| mongodb::error::Error::custom(error.to_string()))?; + Ok::<(), mongodb::error::Error>(()) + }) + .await + .expect("flush cache in transaction should commit"); + + let remaining_cache_docs = cache_collection + .count_documents(mongodb::bson::doc! {}) + .await + .expect("count cache docs"); + assert_eq!(remaining_cache_docs, 0); + } } diff --git a/lib/src/merkle_trees/trees.rs b/lib/src/merkle_trees/trees.rs index 541921bb..78dd7870 100644 --- a/lib/src/merkle_trees/trees.rs +++ b/lib/src/merkle_trees/trees.rs @@ -203,6 +203,16 @@ where update_tree: bool, tree_id: &str, ) -> Result<(F, u64), Self::Error>; + /// appends one or more subtrees using the provided MongoDB session when present + async fn append_sub_trees_with_session( + &self, + leaves: &[F], + update_tree: bool, + tree_id: &str, + _session: Option<&mut mongodb::ClientSession>, + ) -> Result<(F, u64), Self::Error> { + self.append_sub_trees(leaves, update_tree, tree_id).await + } /// Allows on e to insert a single leaf into the tree regardless of the specified subtree size. async fn insert_leaf( &self, @@ -210,6 +220,16 @@ where update_tree: bool, tree_id: &str, ) -> Result; + /// inserts a single leaf using the provided MongoDB session when present + async fn insert_leaf_with_session( + &self, + leaf: F, + update_tree: bool, + tree_id: &str, + _session: Option<&mut mongodb::ClientSession>, + ) -> Result { + self.insert_leaf(leaf, update_tree, tree_id).await + } /// allows one to update a sub-tree async fn update_sub_tree( &self, @@ -218,6 +238,18 @@ where update_tree: bool, tree_id: &str, ) -> Result; + /// updates a subtree using the provided MongoDB session when present + async fn update_sub_tree_with_session( + &self, + sub_tree_index: u64, + leaves: &[F], + update_tree: bool, + tree_id: &str, + _session: Option<&mut mongodb::ClientSession>, + ) -> Result { + self.update_sub_tree(sub_tree_index, leaves, update_tree, tree_id) + .await + } /// get a membership proof async fn get_membership_proof( &self, @@ -227,6 +259,15 @@ where ) -> Result, Self::Error>; /// returns the node value at the given index async fn get_node(&self, index: u64, tree_id: &str) -> Result; + /// returns the node value at the given index using the provided MongoDB session when present + async fn get_node_with_session( + &self, + index: u64, + tree_id: &str, + _session: Option<&mut mongodb::ClientSession>, + ) -> Result { + self.get_node(index, tree_id).await + } /// sets the node value at the given index async fn set_node( &self, @@ -235,12 +276,39 @@ where update_tree: bool, tree_id: &str, ) -> Result<(), Self::Error>; + /// sets the node value using the provided MongoDB session when present + async fn set_node_with_session( + &self, + index: u64, + value: F, + update_tree: bool, + tree_id: &str, + _session: Option<&mut mongodb::ClientSession>, + ) -> Result<(), Self::Error> { + self.set_node(index, value, update_tree, tree_id).await + } /// determines if a leaf is in the tree async fn is_leaf(&self, leaf: &F, tree_id: &str) -> Result; /// writes the temporary node cache to the database and clears the cache. This is normally done automatically. async fn flush_cache(&self, tree_id: &str) -> Result<(), Self::Error>; + /// flushes the node cache using the provided MongoDB session when present + async fn flush_cache_with_session( + &self, + tree_id: &str, + _session: Option<&mut mongodb::ClientSession>, + ) -> Result<(), Self::Error> { + self.flush_cache(tree_id).await + } /// returns the current root of the tree async fn get_root(&self, tree_id: &str) -> Result; + /// returns the current root using the provided MongoDB session when present + async fn get_root_with_session( + &self, + tree_id: &str, + _session: Option<&mut mongodb::ClientSession>, + ) -> Result { + self.get_root(tree_id).await + } /// Inserts leaves into the tree and returns information allowing us to verify in a circuit. async fn insert_for_circuit( &self, @@ -289,6 +357,15 @@ where inner_leaf_values: &[F], tree_id: &str, ) -> Result>::Error>; + /// inserts leaves using the provided MongoDB session when present + async fn insert_leaves_with_session( + &self, + inner_leaf_values: &[F], + tree_id: &str, + _session: Option<&mut mongodb::ClientSession>, + ) -> Result>::Error> { + self.insert_leaves(inner_leaf_values, tree_id).await + } /// Inserts leaves into the tree and returns information allowing us to verify in a circuit. async fn insert_nullifiers_for_circuit( &self, @@ -321,6 +398,16 @@ pub trait IndexedLeaves { index: Option, tree_id: &str, ) -> Result, Self::Error>; + /// stores a leaf using the provided MongoDB session when present + async fn store_leaf_with_session( + &self, + leaf: F, + index: Option, + tree_id: &str, + _session: Option<&mut mongodb::ClientSession>, + ) -> Result, Self::Error> { + self.store_leaf(leaf, index, tree_id).await + } /// Searches the database for a leaf with the supplied fields. If it finds one, it returns it. async fn get_leaf( &self, @@ -328,6 +415,16 @@ pub trait IndexedLeaves { next_value: Option, tree_id: &str, ) -> Result>, Self::Error>; + /// retrieves a leaf using the provided MongoDB session when present + async fn get_leaf_with_session( + &self, + leaf_value: Option, + next_value: Option, + tree_id: &str, + _session: Option<&mut mongodb::ClientSession>, + ) -> Result>, Self::Error> { + self.get_leaf(leaf_value, next_value, tree_id).await + } /// Searches the database for the leaf that skips over the supplied value. That is finds the leaf such that /// `low_leaf.value` < `leaf_value` < `low_leaf.next_value`. If it finds one, it returns it. async fn get_low_leaf( @@ -335,6 +432,15 @@ pub trait IndexedLeaves { leaf_value: &F, tree_id: &str, ) -> Result>, Self::Error>; + /// retrieves the low leaf using the provided MongoDB session when present + async fn get_low_leaf_with_session( + &self, + leaf_value: &F, + tree_id: &str, + _session: Option<&mut mongodb::ClientSession>, + ) -> Result>, Self::Error> { + self.get_low_leaf(leaf_value, tree_id).await + } /// Updates the leaf entry stored with value `leaf` with the new `next_value`. async fn update_leaf( &self, @@ -343,6 +449,18 @@ pub trait IndexedLeaves { new_next_value: F, tree_id: &str, ) -> Result<(), Self::Error>; + /// updates a leaf using the provided MongoDB session when present + async fn update_leaf_with_session( + &self, + leaf: F, + new_next_index: u64, + new_next_value: F, + tree_id: &str, + _session: Option<&mut mongodb::ClientSession>, + ) -> Result<(), Self::Error> { + self.update_leaf(leaf, new_next_index, new_next_value, tree_id) + .await + } } pub(crate) mod helper_functions { diff --git a/lib/src/nf_client_proof.rs b/lib/src/nf_client_proof.rs index 6a2d39b9..2fac5d36 100644 --- a/lib/src/nf_client_proof.rs +++ b/lib/src/nf_client_proof.rs @@ -183,6 +183,8 @@ pub struct PrivateInputs { // so we make it private input pub nf_address: Address, pub nf_slot_id: Fr254, + /// Token ids for the two value commitments being nullified. + pub spent_nf_token_ids: [Fr254; 2], pub nullifiers_values: [Fr254; 4], pub nullifiers_salts: [Fr254; 4], pub membership_proofs: [MembershipProof; 4], @@ -227,6 +229,7 @@ impl Default for PrivateInputs { fee_token_id: Fr254::zero(), nf_address: Address::ZERO, nf_slot_id: Fr254::zero(), + spent_nf_token_ids: [Fr254::zero(); 2], nullifiers_values: [Fr254::zero(); 4], nullifiers_salts: [Fr254::zero(); 4], membership_proofs: [mproof.clone(), mproof.clone(), mproof.clone(), mproof], @@ -283,6 +286,11 @@ impl PrivateInputs { self } + pub fn spent_nf_token_ids(&mut self, token_ids: [Fr254; 2]) -> &mut Self { + self.spent_nf_token_ids = token_ids; + self + } + pub fn nullifiers_values(&mut self, nullifiers_values: &[Fr254; 4]) -> &mut Self { self.nullifiers_values = *nullifiers_values; self @@ -389,6 +397,7 @@ impl PrivateInputs { fee_token_id: self.fee_token_id, nf_address: self.nf_address, nf_slot_id: self.nf_slot_id, + spent_nf_token_ids: self.spent_nf_token_ids, nullifiers_values: self.nullifiers_values, nullifiers_salts: self.nullifiers_salts, membership_proofs: self.membership_proofs.clone(), @@ -420,6 +429,8 @@ pub struct PrivateInputsVar { pub nf_address: Variable, /// Slot Id of transaction tokens, pub nf_slot_id: Variable, + /// Token ids for the two value commitments being nullified. + pub spent_nf_token_ids: [Variable; 2], /// Nullifiers values pub nullifiers_values: [Variable; 4], /// Nullifiers salts @@ -470,6 +481,15 @@ impl PrivateInputsVar { Fr254::from(BigUint::from_bytes_be(private_inputs.nf_address.as_slice())); let nf_address = circuit.create_variable(nf_address_field)?; let nf_slot_id = circuit.create_variable(private_inputs.nf_slot_id)?; + let spent_nf_token_ids = private_inputs + .spent_nf_token_ids + .iter() + .map(|ntid| circuit.create_variable(*ntid)) + .collect::, CircuitError>>()? + .try_into() + .map_err(|_| { + CircuitError::ParameterError("Couldn't convert to fixed length array".to_string()) + })?; let nullifiers_values = private_inputs .nullifiers_values .iter() @@ -660,6 +680,7 @@ impl PrivateInputsVar { fee_token_id, nf_address, nf_slot_id, + spent_nf_token_ids, nullifiers_values, nullifiers_salts, membership_proofs, diff --git a/lib/src/nf_token_id.rs b/lib/src/nf_token_id.rs index 0079de4a..6e7b41c3 100644 --- a/lib/src/nf_token_id.rs +++ b/lib/src/nf_token_id.rs @@ -1,13 +1,26 @@ //! File contains utility functions used by the REST API, such as ones for converting from erc address and token id to //! Nightfall token id. -use crate::{error::ConversionError, hex_conversion::HexConvertible}; -use alloy::primitives::{Address, U256}; +use crate::{error::ConversionError, hex_conversion::HexConvertible, shared_entities::TokenType}; +use alloy::{ + dyn_abi::abi::encode, + primitives::{keccak256, Address, U256}, + sol_types::SolValue, +}; use ark_bn254::Fr as Fr254; use ark_ff::{BigInteger, PrimeField}; use log::debug; use num::BigUint; use sha2::{Digest, Sha256}; +const NF_TOKEN_ID_DOMAIN: u8 = 0; +const NF_SLOT_ID_DOMAIN: u8 = 1; + +fn domain_bytes(domain: u8) -> [u8; 32] { + let mut bytes = [0u8; 32]; + bytes[31] = domain; + bytes +} + #[allow(dead_code)] pub fn to_nf_token_id_from_str( erc_address: &str, @@ -29,6 +42,7 @@ pub fn to_nf_token_id_from_str( let mut input_bytes = Vec::new(); input_bytes.extend_from_slice(&erc_vec); + input_bytes.extend_from_slice(&domain_bytes(NF_TOKEN_ID_DOMAIN)); input_bytes.extend_from_slice(&token_vec); // Hash the result @@ -43,6 +57,37 @@ pub fn to_nf_token_id_from_str( Ok(Fr254::from(nf_token_id)) } +#[allow(dead_code)] +pub fn to_nf_slot_id_from_str( + erc_address: &str, + token_id: &str, + slot_id: &str, + token_type: TokenType, +) -> Result { + if token_type != TokenType::ERC3525 { + return to_nf_token_id_from_str(erc_address, token_id); + } + + let solidity_erc_address = Address::from_slice( + &Vec::::from_hex_string(erc_address).map_err(|_| ConversionError::ParseFailed)?, + ); + let mut slot_bytes = + Vec::::from_hex_string(slot_id).map_err(|_| ConversionError::ParseFailed)?; + while slot_bytes.len() < 32 { + slot_bytes.insert(0, 0); + } + let solidity_slot_id = U256::from_be_slice(&slot_bytes); + + let erc_token = solidity_erc_address.tokenize(); + let slot_id_token = solidity_slot_id.tokenize(); + let domain_token = U256::from(u64::from(NF_SLOT_ID_DOMAIN)).tokenize(); + let nf_slot_id_biguint = BigUint::from_bytes_be( + keccak256(encode(&(erc_token, domain_token, slot_id_token))).as_slice(), + ) >> 4; + + Ok(Fr254::from(nf_slot_id_biguint)) +} + pub fn to_nf_token_id_from_fr254(erc_address: Fr254, token_id: Fr254) -> Fr254 { // convert to a string and pad to 32 bytes let mut erc_address_bytes = erc_address.into_bigint().to_bytes_be(); @@ -60,6 +105,7 @@ pub fn to_nf_token_id_from_fr254(erc_address: Fr254, token_id: Fr254) -> Fr254 { let mut input_bytes = Vec::new(); input_bytes.extend_from_slice(&erc_address_bytes); + input_bytes.extend_from_slice(&domain_bytes(NF_TOKEN_ID_DOMAIN)); input_bytes.extend_from_slice(&token_id_bytes); // Hash the result @@ -91,6 +137,7 @@ pub fn to_nf_token_id_from_solidity( let mut input_bytes = Vec::new(); input_bytes.extend_from_slice(&erc_address_bytes); // 20 bytes + input_bytes.extend_from_slice(&domain_bytes(NF_TOKEN_ID_DOMAIN)); input_bytes.extend_from_slice(&token_id_bytes); // 32 bytes let mut hasher = Sha256::new(); @@ -103,6 +150,28 @@ pub fn to_nf_token_id_from_solidity( Fr254::from(hash_out) } + +pub fn to_nf_slot_id_from_solidity( + solidity_token_address: Address, + solidity_token_id: U256, + solidity_slot_id: U256, + token_type: TokenType, +) -> Fr254 { + let nf_token_id = to_nf_token_id_from_solidity(solidity_token_address, solidity_token_id); + if let TokenType::ERC3525 = token_type { + let erc_token = solidity_token_address.tokenize(); + let slot_id_token = solidity_slot_id.tokenize(); + let domain_token = U256::from(u64::from(NF_SLOT_ID_DOMAIN)).tokenize(); + let nf_slot_id_biguint = BigUint::from_bytes_be( + keccak256(encode(&(erc_token, domain_token, slot_id_token))).as_slice(), + ) >> 4; + + Fr254::from(nf_slot_id_biguint) + } else { + nf_token_id + } +} + #[cfg(test)] mod tests { use super::*; @@ -154,8 +223,14 @@ mod tests { let mut lookup_vars = Vec::<(Variable, Variable, Variable)>::new(); let erc_address_var = circuit.create_variable(erc_address_fr).unwrap(); let token_id_var = circuit.create_variable(token_id_fr).unwrap(); + let domain_var = circuit + .create_variable(Fr254::from(NF_TOKEN_ID_DOMAIN)) + .unwrap(); let (_, nf_token_id_var) = circuit - .full_shifted_sha256_hash(&[erc_address_var, token_id_var], &mut lookup_vars) + .full_shifted_sha256_hash( + &[erc_address_var, domain_var, token_id_var], + &mut lookup_vars, + ) .unwrap(); circuit.finalize_for_arithmetization().unwrap(); let nf_token_id_in_circuit = circuit.witness(nf_token_id_var).unwrap(); @@ -175,4 +250,155 @@ mod tests { ); } } + + #[test] + fn test_nf_slot_id_from_str_reuses_nf_token_id_for_non_erc3525() { + for _ in 0..10 { + let mut rng = rand::thread_rng(); + let erc_address: [u8; 20] = rng.gen(); + let erc_address_string = format!("0x{}", hex::encode(erc_address)); + let mut rng = jf_utils::test_rng(); + let raw_id = Fr254::rand(&mut rng); + let raw_id_string = Fr254::to_hex_string(&raw_id); + + let nf_token_id = to_nf_token_id_from_str(&erc_address_string, &raw_id_string).unwrap(); + let nf_slot_id = to_nf_slot_id_from_str( + &erc_address_string, + &raw_id_string, + &raw_id_string, + TokenType::ERC20, + ) + .unwrap(); + + assert_eq!( + nf_token_id, nf_slot_id, + "Non-ERC3525 slot IDs should reuse the NF token ID" + ); + } + } + + fn expected_erc3525_nf_slot_id(erc_address: Address, slot_id: U256) -> Fr254 { + let erc_token = erc_address.tokenize(); + let domain_token = U256::from(u64::from(NF_SLOT_ID_DOMAIN)).tokenize(); + let slot_id_token = slot_id.tokenize(); + Fr254::from( + BigUint::from_bytes_be( + keccak256(encode(&(erc_token, domain_token, slot_id_token))).as_slice(), + ) >> 4, + ) + } + + #[test] + fn test_nf_slot_id_from_solidity_reuses_nf_token_id_for_non_erc3525() { + let mut rng = rand::thread_rng(); + let erc_address: [u8; 20] = rng.gen(); + let solidity_erc_address = Address::from_slice(&erc_address); + + let mut field_rng = jf_utils::test_rng(); + let token_id_fr = Fr254::rand(&mut field_rng); + let token_id_string = Fr254::to_hex_string(&token_id_fr); + let mut token_id_bytes = Vec::::from_hex_string(&token_id_string).unwrap(); + while token_id_bytes.len() < 32 { + token_id_bytes.insert(0, 0); + } + let solidity_token_id = U256::from_be_slice(&token_id_bytes); + + let nf_token_id = to_nf_token_id_from_solidity(solidity_erc_address, solidity_token_id); + let nf_slot_id = to_nf_slot_id_from_solidity( + solidity_erc_address, + solidity_token_id, + U256::from(8u64), + TokenType::ERC20, + ); + + assert_eq!(nf_token_id, nf_slot_id); + } + + #[test] + fn test_nf_slot_id_from_solidity_hashes_erc3525_slot_when_slot_matches_token() { + let mut rng = rand::thread_rng(); + let erc_address: [u8; 20] = rng.gen(); + let solidity_erc_address = Address::from_slice(&erc_address); + + let token_id = U256::from(7u64); + let expected = expected_erc3525_nf_slot_id(solidity_erc_address, token_id); + + let nf_slot_id = to_nf_slot_id_from_solidity( + solidity_erc_address, + token_id, + token_id, + TokenType::ERC3525, + ); + let nf_token_id = to_nf_token_id_from_solidity(solidity_erc_address, token_id); + + assert_eq!(expected, nf_slot_id); + assert_ne!(nf_token_id, nf_slot_id); + } + + #[test] + fn test_nf_slot_id_from_solidity_hashes_erc3525_slot_when_slot_differs_from_token() { + let mut rng = rand::thread_rng(); + let erc_address: [u8; 20] = rng.gen(); + let solidity_erc_address = Address::from_slice(&erc_address); + + let token_id = U256::from(7u64); + let slot_id = U256::from(8u64); + let expected = expected_erc3525_nf_slot_id(solidity_erc_address, slot_id); + + let nf_slot_id = to_nf_slot_id_from_solidity( + solidity_erc_address, + token_id, + slot_id, + TokenType::ERC3525, + ); + + assert_eq!(expected, nf_slot_id); + } + + #[test] + fn test_nf_slot_id_from_str_hashes_erc3525_slot_when_slot_matches_token() { + let mut rng = rand::thread_rng(); + let erc_address: [u8; 20] = rng.gen(); + let erc_address_string = format!("0x{}", hex::encode(erc_address)); + let solidity_erc_address = Address::from_slice(&erc_address); + + let token_id = U256::from(7u64); + let token_id_string = Fr254::to_hex_string(&Fr254::from(7u64)); + let expected = expected_erc3525_nf_slot_id(solidity_erc_address, token_id); + + let nf_token_id = to_nf_token_id_from_str(&erc_address_string, &token_id_string).unwrap(); + let nf_slot_id = to_nf_slot_id_from_str( + &erc_address_string, + &token_id_string, + &token_id_string, + TokenType::ERC3525, + ) + .unwrap(); + + assert_eq!(expected, nf_slot_id); + assert_ne!(nf_token_id, nf_slot_id); + } + + #[test] + fn test_nf_slot_id_from_str_hashes_erc3525_slot_when_slot_differs_from_token() { + let mut rng = rand::thread_rng(); + let erc_address: [u8; 20] = rng.gen(); + let erc_address_string = format!("0x{}", hex::encode(erc_address)); + let solidity_erc_address = Address::from_slice(&erc_address); + + let slot_id = U256::from(8u64); + let token_id_string = Fr254::to_hex_string(&Fr254::from(7u64)); + let slot_id_string = Fr254::to_hex_string(&Fr254::from(8u64)); + let expected = expected_erc3525_nf_slot_id(solidity_erc_address, slot_id); + + let nf_slot_id = to_nf_slot_id_from_str( + &erc_address_string, + &token_id_string, + &slot_id_string, + TokenType::ERC3525, + ) + .unwrap(); + + assert_eq!(expected, nf_slot_id); + } } diff --git a/lib/src/plonk_prover/circuits/mod.rs b/lib/src/plonk_prover/circuits/mod.rs index f1111a89..49a8692e 100644 --- a/lib/src/plonk_prover/circuits/mod.rs +++ b/lib/src/plonk_prover/circuits/mod.rs @@ -17,3 +17,12 @@ const LABEL: &str = "Nightfall|SharedSalt"; */ pub const DOMAIN_SHARED_SALT: Fr254 = MontFp!("4832298308599927878911686715232824310149976768223104556783163253807065458"); + +// Receipt-specific domain separators for KEM-DEM, cryptographically independent +// from the on-chain protocol encryption domains (DOMAIN_KEM / DOMAIN_DEM). +// Derived as Fr254::from_le_bytes_mod_order(SHA256("Nightfall|ReceiptKEM")) and +// Fr254::from_le_bytes_mod_order(SHA256("Nightfall|ReceiptDEM")) respectively. +pub const DOMAIN_RECEIPT_KEM: Fr254 = + MontFp!("19025930337730587240715821088551072222022822490734157016697236116220883417000"); +pub const DOMAIN_RECEIPT_DEM: Fr254 = + MontFp!("1348191844062849980512103790242939271876911032825841669748694060100189132234"); diff --git a/lib/src/plonk_prover/circuits/unified_circuit.rs b/lib/src/plonk_prover/circuits/unified_circuit.rs index 7759dd65..b6ba21d2 100644 --- a/lib/src/plonk_prover/circuits/unified_circuit.rs +++ b/lib/src/plonk_prover/circuits/unified_circuit.rs @@ -72,6 +72,7 @@ impl UnifiedCircuit for PlonkCircuit { fee_token_id, nf_address, nf_slot_id, + spent_nf_token_ids, nullifiers_values, nullifiers_salts, membership_proofs, @@ -169,7 +170,7 @@ impl UnifiedCircuit for PlonkCircuit { let is_deposit = self.is_zero(nullifiers_salts[0])?; // ROLE DETECTION & DERIVED VALUES - // Determines caller's role and derives value, nf_token_id, + // Determines caller's role and derives value, output token ids, // and recipient_public_key from swap parameters. // // For swap: @@ -193,7 +194,8 @@ impl UnifiedCircuit for PlonkCircuit { // Swap-specific: derive from role let swap_value = self.conditional_select(is_party_a, value_b, value_a)?; - let swap_nf_token_id = self.conditional_select(is_party_a, nf_token_b_id, nf_token_a_id)?; + let swap_output_nf_token_id = + self.conditional_select(is_party_a, nf_token_b_id, nf_token_a_id)?; let swap_recipient_x = self.conditional_select( is_party_a, party_a_public_key.get_x(), @@ -207,7 +209,9 @@ impl UnifiedCircuit for PlonkCircuit { // Final: for transfer use value_a/party_b directly, for swap use role-based let value = self.conditional_select(is_swap, value_a, swap_value)?; - let nf_token_id = self.conditional_select(is_swap, nf_token_a_id, swap_nf_token_id)?; + let recipient_output_nf_token_id = + self.conditional_select(is_swap, nf_token_a_id, swap_output_nf_token_id)?; + let change_output_nf_token_id = spent_nf_token_ids[0]; let recipient_x = self.conditional_select(is_swap, party_b_public_key.get_x(), swap_recipient_x)?; let recipient_y = @@ -258,7 +262,7 @@ impl UnifiedCircuit for PlonkCircuit { // OWNERSHIP VERIFICATION (for all: transfer, withdraw, swap) for i in 0..4 { let is_neutral = self.is_neutral_point::(&public_keys[i])?; - let is_zero_value = self.is_zero(nullifiers_values[i])?; + let is_zero_value = self.is_zero(nullifiers_salts[i])?; let x_matches = self.is_equal(zkp_pub_key.get_x(), public_keys[i].get_x())?; let y_matches = self.is_equal(zkp_pub_key.get_y(), public_keys[i].get_y())?; @@ -361,7 +365,8 @@ impl UnifiedCircuit for PlonkCircuit { let commitments = self.verify_commitments( fee_token_id, nf_address, - nf_token_id, + recipient_output_nf_token_id, + change_output_nf_token_id, nf_slot_id, value, fee, @@ -390,7 +395,7 @@ impl UnifiedCircuit for PlonkCircuit { // Calculate nullifiers let nullifiers = self.verify_nullifiers::( fee_token_id, - nf_token_id, + spent_nf_token_ids, nf_slot_id, nullifier_key, &public_keys, @@ -406,7 +411,7 @@ impl UnifiedCircuit for PlonkCircuit { // Verify encryption of recipient's commitment preimage let public_data = self.verify_encryption( - nf_token_id, + recipient_output_nf_token_id, nf_slot_id, value, &shared_secret, diff --git a/lib/src/plonk_prover/circuits/verify/verify_commitments_gadgets.rs b/lib/src/plonk_prover/circuits/verify/verify_commitments_gadgets.rs index 10c4b429..94c8d0a4 100644 --- a/lib/src/plonk_prover/circuits/verify/verify_commitments_gadgets.rs +++ b/lib/src/plonk_prover/circuits/verify/verify_commitments_gadgets.rs @@ -14,7 +14,8 @@ where &mut self, fee_token_id: Variable, nf_address: Variable, - token_id: Variable, + recipient_nf_token_id: Variable, + change_nf_token_id: Variable, slot_id: Variable, value: Variable, fee: Variable, @@ -35,7 +36,8 @@ where &mut self, fee_token_id: Variable, nf_address: Variable, - token_id: Variable, + recipient_nf_token_id: Variable, + change_nf_token_id: Variable, slot_id: Variable, value: Variable, fee: Variable, @@ -49,7 +51,7 @@ where // new_commitments_values[1]: fee change value // Check the first commitment, Transfered to Token let first_commitment_hash = self.poseidon_hash(&[ - token_id, + recipient_nf_token_id, slot_id, value, recipient_public_key[0].get_x(), @@ -62,7 +64,7 @@ where // Check the second commitment Transfer Change Token let is_transfer_change_zero = self.is_zero(new_commitments_values[0])?; let second_commitment_hash = self.poseidon_hash(&[ - token_id, + change_nf_token_id, slot_id, new_commitments_values[0], recipient_public_key[1].get_x(), diff --git a/lib/src/plonk_prover/circuits/verify/verify_nullifiers_gadgets.rs b/lib/src/plonk_prover/circuits/verify/verify_nullifiers_gadgets.rs index 7ba02237..a47c6da1 100644 --- a/lib/src/plonk_prover/circuits/verify/verify_nullifiers_gadgets.rs +++ b/lib/src/plonk_prover/circuits/verify/verify_nullifiers_gadgets.rs @@ -25,7 +25,7 @@ where fn verify_nullifiers>( &mut self, fee_token_id: Variable, - nf_token_id: Variable, // could be nf4_token_id or fee_token_id + nf_token_ids: [Variable; 2], // could be nf4_token_id or fee_token_id nf_slot_id: Variable, nullifiers_key: Variable, public_keys: &[PointVariable; 4], @@ -45,7 +45,7 @@ where fn verify_nullifiers>( &mut self, fee_token_id: Variable, - nf_token_id: Variable, + nf_token_ids: [Variable; 2], nf_slot_id: Variable, nullifiers_key: Variable, public_keys: &[PointVariable; 4], @@ -58,7 +58,7 @@ where // Check the first nullifier, nullify Withdrawn/Transferred token let is_zero = self.is_zero(old_commitment_salts[0])?; let commitment_hash_1 = self.poseidon_hash(&[ - nf_token_id, + nf_token_ids[0], nf_slot_id, old_commitment_values[0], public_keys[0].get_x(), @@ -99,7 +99,7 @@ where let is_zero = self.is_zero(old_commitment_salts[1])?; let commitment_hash_2 = self.poseidon_hash(&[ - nf_token_id, + nf_token_ids[1], nf_slot_id, old_commitment_values[1], public_keys[1].get_x(), diff --git a/lib/src/shared_entities.rs b/lib/src/shared_entities.rs index b8b1174e..8b1db978 100644 --- a/lib/src/shared_entities.rs +++ b/lib/src/shared_entities.rs @@ -277,6 +277,13 @@ pub struct ClientTransaction

{ )] pub swap_side: Fr254, // ← SWAP_SIDE (is_party_a) pub proof: P, + /// Sender-generated random token that authorises receipt creation on every + /// proposer this transaction is sent to. Generated once by the client before + /// the fan-out and included in every copy of the transaction, so all + /// proposers store the same token and the receipt_id derived from it is + /// identical everywhere. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub receipt_token: Option, } impl ClientTransaction

{ diff --git a/lib/src/test_helpers/mocks.rs b/lib/src/test_helpers/mocks.rs index 055e043f..e362e604 100644 --- a/lib/src/test_helpers/mocks.rs +++ b/lib/src/test_helpers/mocks.rs @@ -184,6 +184,7 @@ impl Mocks { deadline: Fr254::zero(), swap_side: Fr254::zero(), proof: Self::get_mock_proof(), + receipt_token: None, } } diff --git a/lib/src/tests_utils.rs b/lib/src/tests_utils.rs index 0630387a..047f91e1 100644 --- a/lib/src/tests_utils.rs +++ b/lib/src/tests_utils.rs @@ -9,15 +9,28 @@ use testcontainers::{ use tokio::io::AsyncReadExt; use url::Host; +const TEST_MONGO_REPLICA_SET_NAME: &str = "rs0"; +const TEST_MONGO_PORT: u16 = 27017; +const TEST_MONGO_PORT_RETRY_ATTEMPTS: usize = 60; +const TEST_MONGO_PING_RETRY_ATTEMPTS: usize = 60; + pub fn get_db_connection_uri(host: Host, port: u16) -> String { - format!("mongodb://{host}:{port}") + format!( + "mongodb://{host}:{port}/?replicaSet={TEST_MONGO_REPLICA_SET_NAME}&directConnection=true" + ) +} + +fn get_direct_db_connection_uri(host: &Host, port: u16) -> String { + format!("mongodb://{host}:{port}/?directConnection=true") } pub async fn get_mongo() -> ContainerAsync { let mongo_image = GenericImage::new("mongo", "8.0") - .with_exposed_port(27017.tcp()) + .with_exposed_port(TEST_MONGO_PORT.tcp()) .with_wait_for(WaitFor::message_on_stdout("Waiting for connections")) - .with_startup_timeout(Duration::from_secs(120)); + .with_entrypoint("mongod") + .with_cmd(["--replSet", TEST_MONGO_REPLICA_SET_NAME, "--bind_ip_all"]) + .with_startup_timeout(Duration::from_secs(180)); mongo_image.start().await.unwrap() } @@ -26,17 +39,41 @@ pub async fn get_db_connection(container: &ContainerAsync) -> mong use tokio::time::{sleep, Duration}; let host = container.get_host().await.unwrap(); - let port = container.get_host_port_ipv4(27017).await.unwrap(); - let uri = get_db_connection_uri(host, port); + let mut port = None; + for _ in 0..TEST_MONGO_PORT_RETRY_ATTEMPTS { + match container.get_host_port_ipv4(TEST_MONGO_PORT.tcp()).await { + Ok(mapped) => { + port = Some(mapped); + break; + } + Err(ipv4_error) => match container.get_host_port_ipv6(TEST_MONGO_PORT.tcp()).await { + Ok(mapped) => { + port = Some(mapped); + break; + } + Err(ipv6_error) => { + warn!( + "Mongo testcontainer port mapping not ready yet (ipv4: {ipv4_error}, ipv6: {ipv6_error}), retrying..." + ); + sleep(Duration::from_secs(1)).await; + } + }, + } + } + let port = port.unwrap_or_else(|| { + panic!( + "Mongo testcontainer did not expose TCP port {TEST_MONGO_PORT} after {TEST_MONGO_PORT_RETRY_ATTEMPTS} attempts" + ) + }); + let direct_uri = get_direct_db_connection_uri(&host, port); + let rs_uri = get_db_connection_uri(host, port); let mut attempts = 0; - let client; loop { - match mongodb::Client::with_uri_str(&uri).await { + match mongodb::Client::with_uri_str(&direct_uri).await { Ok(c) => match c.database("admin").run_command(doc! {"ping": 1}).await { Ok(_) => { info!(" Mongo is ready!"); - client = c; break; } Err(e) => { @@ -49,14 +86,112 @@ pub async fn get_db_connection(container: &ContainerAsync) -> mong } attempts += 1; - if attempts >= 10 { - panic!(" MongoDB not ready after 10 attempts"); + if attempts >= TEST_MONGO_PING_RETRY_ATTEMPTS { + panic!( + "MongoDB not ready after {TEST_MONGO_PING_RETRY_ATTEMPTS} ping attempts on {direct_uri}" + ); } sleep(Duration::from_secs(1)).await; } + + let bootstrap_client = mongodb::Client::with_uri_str(&direct_uri).await.unwrap(); + initialize_replica_set(&bootstrap_client).await; + wait_for_primary(&bootstrap_client).await; + + let client = mongodb::Client::with_uri_str(&rs_uri).await.unwrap(); + client + .database("admin") + .run_command(doc! {"ping": 1}) + .await + .unwrap(); client } +async fn initialize_replica_set(client: &mongodb::Client) { + match client + .database("admin") + .run_command(doc! { + "replSetGetStatus": 1, + }) + .await + { + Ok(_) => { + info!("Mongo replica set already initialized"); + } + Err(error) => { + let should_initiate = replica_set_not_initialized(&error); + + if !should_initiate { + panic!("Failed to read replica set status: {error}"); + } + + match client + .database("admin") + .run_command(doc! { + "replSetInitiate": { + "_id": TEST_MONGO_REPLICA_SET_NAME, + "members": [{ + "_id": 0, + "host": "localhost:27017", + }], + }, + }) + .await + { + Ok(_) => info!("Initialized Mongo replica set"), + Err(error) => { + let already_initialized = replica_set_already_initialized(&error); + + if already_initialized { + info!("Mongo replica set was already initialized"); + } else { + panic!("Failed to initialize Mongo replica set: {error}"); + } + } + } + } + } +} + +fn replica_set_not_initialized(error: &mongodb::error::Error) -> bool { + let message = error.to_string(); + message.contains("NotYetInitialized") + || message.contains("no replset config has been received") + || message.contains("not yet initialized") +} + +fn replica_set_already_initialized(error: &mongodb::error::Error) -> bool { + let message = error.to_string(); + message.contains("already initialized") || message.contains("AlreadyInitialized") +} + +async fn wait_for_primary(client: &mongodb::Client) { + use tokio::time::{sleep, Duration}; + + let mut attempts = 0; + loop { + match client + .database("admin") + .run_command(doc! {"hello": 1}) + .await + { + Ok(hello) if hello.get_bool("isWritablePrimary").unwrap_or(false) => { + info!("Mongo replica set member is PRIMARY"); + return; + } + Ok(_) => warn!("Mongo replica set not PRIMARY yet, retrying..."), + Err(error) => warn!("Failed to query hello during replica set init: {error}"), + } + + attempts += 1; + if attempts >= 30 { + panic!("Mongo replica set did not reach PRIMARY state after 30 attempts"); + } + + sleep(Duration::from_secs(1)).await; + } +} + #[allow(dead_code)] /// This function is used to print the stdout of a container for test debugging pub async fn print_stdout(container: &ContainerAsync) { diff --git a/lib/src/validate_certificate.rs b/lib/src/validate_certificate.rs index ccc0b37e..db2d91fc 100644 --- a/lib/src/validate_certificate.rs +++ b/lib/src/validate_certificate.rs @@ -2,15 +2,23 @@ use super::models::CertificateReq; use crate::{ blockchain_client::BlockchainClientConnection, error::{CertificateVerificationError, NightfallContractError}, - initialisation::get_blockchain_client_connection, + initialisation::get_blockchain_client_connection_for_role, models::bad_request, verify_contract::VerifiedContracts, + wallets::validate_azure_vault_url, }; use alloy::{ primitives::{Address, U256}, providers::Provider, }; -use configuration::{addresses::get_addresses, settings::get_settings}; +use async_trait::async_trait; +use azure_identity; +use azure_security_keyvault::{prelude::*, KeyClient}; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; +use configuration::{ + addresses::get_addresses, + settings::{get_settings, WalletRole, X509SignerTypeConfig}, +}; use futures::stream::TryStreamExt; use log::{debug, error, trace, warn}; use nightfall_bindings::artifacts::X509; @@ -19,15 +27,20 @@ use openssl::{ hash::MessageDigest, pkey::{Id as PKeyId, PKey}, rsa::{Padding, Rsa}, + sha::sha256, sign::{RsaPssSaltlen, Signer as opensslSigner, Verifier}, x509::X509 as OpensslX509, }; use reqwest::StatusCode; use std::error::Error; use std::io::Read; +use std::sync::Arc; use warp::{filters::multipart::FormData, path, reply::Reply, Buf, Filter}; use x509_parser::nom::AsBytes; use zeroize::{Zeroize, ZeroizeOnDrop}; + +type CertificateSignerResult = Result>; + #[derive(Debug)] pub struct X509ValidationError; @@ -39,19 +52,97 @@ impl std::fmt::Display for X509ValidationError { impl std::error::Error for X509ValidationError {} +#[async_trait] +trait CertificateSigner: Send + Sync { + async fn sign_possession_proof( + &self, + address: &Address, + verifying_contract: &Address, + chain_id: u64, + ) -> CertificateSignerResult>; +} + +struct LocalCertificateSigner { + der_private_key: Vec, +} + +impl LocalCertificateSigner { + fn new(der_private_key: Vec) -> Self { + Self { der_private_key } + } +} + +#[async_trait] +impl CertificateSigner for LocalCertificateSigner { + async fn sign_possession_proof( + &self, + address: &Address, + verifying_contract: &Address, + chain_id: u64, + ) -> CertificateSignerResult> { + let preimage = build_certificate_possession_preimage(address, verifying_contract, chain_id); + sign_certificate_possession_preimage(&self.der_private_key, &preimage) + } +} + +#[derive(Clone)] +pub struct AzureRsaCertificateSigner { + key_client: Arc, + key_name: String, +} + +impl AzureRsaCertificateSigner { + pub fn new(vault_url: &str, key_name: &str) -> CertificateSignerResult { + validate_azure_vault_url(vault_url)?; + let credential = azure_identity::create_credential()?; + let key_client = KeyClient::new(vault_url, credential)?; + + Ok(Self { + key_client: Arc::new(key_client), + key_name: key_name.to_string(), + }) + } +} + +#[async_trait] +impl CertificateSigner for AzureRsaCertificateSigner { + async fn sign_possession_proof( + &self, + address: &Address, + verifying_contract: &Address, + chain_id: u64, + ) -> CertificateSignerResult> { + let preimage = build_certificate_possession_preimage(address, verifying_contract, chain_id); + let digest = sha256(&preimage); + let digest_base64 = URL_SAFE_NO_PAD.encode(digest); + + let sign_result = self + .key_client + .sign(&self.key_name, SignatureAlgorithm::PS256, digest_base64) + .await?; + + Ok(sign_result.signature) + } +} + pub fn certification_validation_request( + role: WalletRole, ) -> impl Filter + Clone { debug!("Received certification request"); path!("v1" / "certification") .and(warp::post()) .and(warp::multipart::form().max_length(16192)) - .and_then(handle_certificate_validation) + .and_then(move |x509_data| handle_certificate_validation(x509_data, role)) } // Middleware to validate the certificate pub async fn handle_certificate_validation( mut x509_data: FormData, + role: WalletRole, ) -> Result { + let settings = get_settings(); + let x509_signer_type = settings.x509_signer_type_for_role(role).clone(); + // Parse the certificate validation request (by FIELD NAME, not filename) let mut certificate_req = CertificateReq::default(); while let Some(part_res) = x509_data.try_next().await.transpose() { @@ -100,7 +191,9 @@ pub async fn handle_certificate_validation( if certificate_req.certificate.is_empty() { return Ok(bad_request("Missing 'certificate' field or empty file")); } - if certificate_req.certificate_private_key.is_empty() { + if x509_signer_type == X509SignerTypeConfig::Local + && certificate_req.certificate_private_key.is_empty() + { return Ok(bad_request("Missing 'priv_key' field or empty file")); } @@ -110,7 +203,7 @@ pub async fn handle_certificate_validation( let prevalidation_address = { // We do not yet have the blockchain client, but the requestor address is // exactly what will be bound, so we need it here anyway. - let conn_guard = get_blockchain_client_connection().await; + let conn_guard = get_blockchain_client_connection_for_role(role).await; let read_conn = conn_guard.read().await; read_conn.get_address() }; @@ -118,7 +211,7 @@ pub async fn handle_certificate_validation( let x509_addr = get_addresses().x509; // Resolve client - let client = get_blockchain_client_connection() + let client = get_blockchain_client_connection_for_role(role) .await .read() .await @@ -129,13 +222,18 @@ pub async fn handle_certificate_validation( warp::reject::custom(CertificateVerificationError::new("Failed to get chain ID")) })?; - if let Err(e) = prevalidate_certificate_and_key( - &certificate_req.certificate, - &certificate_req.certificate_private_key, - &prevalidation_address, - &x509_addr, - chain_id, - ) { + let prevalidation_result = match x509_signer_type { + X509SignerTypeConfig::Local => prevalidate_certificate_and_key( + &certificate_req.certificate, + &certificate_req.certificate_private_key, + &prevalidation_address, + &x509_addr, + chain_id, + ), + X509SignerTypeConfig::Azure => prevalidate_certificate(&certificate_req.certificate), + }; + + if let Err(e) = prevalidation_result { warn!("Client-side certificate prevalidation failed: {e}"); return Ok(bad_request( "Certificate / private key prevalidation failed", @@ -144,7 +242,7 @@ pub async fn handle_certificate_validation( // 2) Resolve address let blockchain_client = client.root(); - let requestor_address = get_blockchain_client_connection() + let requestor_address = get_blockchain_client_connection_for_role(role) .await .read() .await @@ -163,16 +261,41 @@ pub async fn handle_certificate_validation( let x509_instance = verified.x509; // 3) Build signature over the requester address - debug!("Signing ethereum address {requestor_address} with certificate private key"); - let ethereum_address_signature = match sign_ethereum_address( - &certificate_req.certificate_private_key, - &requestor_address, - &x509_addr, - chain_id, - ) { + let certificate_signer: Box = match x509_signer_type { + X509SignerTypeConfig::Local => { + debug!( + "Signing ethereum address {requestor_address} with local certificate private key" + ); + Box::new(LocalCertificateSigner::new( + certificate_req.certificate_private_key.clone(), + )) + } + X509SignerTypeConfig::Azure => { + let key_name = settings.x509_azure_key_name_for_role(role).map_err(|e| { + error!("Missing Azure X509 key configuration: {e}"); + warp::reject::custom(CertificateVerificationError::new( + "Failed to resolve Azure X509 signer configuration", + )) + })?; + + debug!("Signing ethereum address {requestor_address} with Azure X509 key {key_name}"); + let signer = AzureRsaCertificateSigner::new(&settings.azure_vault_url, key_name) + .map_err(|e| { + error!("Failed to create AzureRsaCertificateSigner: {e}"); + warp::reject::custom(CertificateVerificationError::new( + "Failed to initialize Azure X509 signer", + )) + })?; + Box::new(signer) + } + }; + let ethereum_address_signature = match certificate_signer + .sign_possession_proof(&requestor_address, &x509_addr, chain_id) + .await + { Ok(sig) => sig, Err(e) => { - error!("sign_ethereum_address failed: {e}"); + error!("CertificateSigner::sign_possession_proof failed: {e}"); let body = warp::reply::json(&serde_json::json!({ "status": "ok", "certified": false @@ -181,6 +304,21 @@ pub async fn handle_certificate_validation( } }; + if x509_signer_type == X509SignerTypeConfig::Azure { + if let Err(e) = verify_signature_matches_certificate( + &certificate_req.certificate, + &requestor_address, + ðereum_address_signature, + &x509_addr, + chain_id, + ) { + warn!("Certificate/signature mismatch before on-chain validation: {e}"); + return Ok(bad_request( + "Certificate does not match the configured signer", + )); + } + } + // 4) ENROLL (state-changing): write the binding on-chain and await receipt. // We want one API that validates AND enrolls, so we do the write: let check_only = false; @@ -193,6 +331,7 @@ pub async fn handle_certificate_validation( check_only, // write path 0, requestor_address, + role, ) .await { @@ -227,8 +366,12 @@ async fn validate_certificate( check_only: bool, oid_group: u32, sender_address: Address, + role: WalletRole, ) -> Result<(), Box> { - let read_connection = get_blockchain_client_connection().await.read().await; + let read_connection = get_blockchain_client_connection_for_role(role) + .await + .read() + .await; let provider = read_connection.get_client(); let blockchain_client = provider.root(); let verified = @@ -256,14 +399,11 @@ async fn validate_certificate( addr: sender_address, }; - let signer = get_blockchain_client_connection() - .await - .read() - .await - .get_signer(); + let caller = read_connection.get_address(); + let wallet = read_connection.get_wallet_type().clone(); let nonce = blockchain_client - .get_transaction_count(signer.address()) + .get_transaction_count(caller) .await .map_err(|e| NightfallContractError::X509Error(format!("Transaction unsuccesful: {e}")))?; let gas_price = blockchain_client @@ -281,7 +421,7 @@ async fn validate_certificate( .max_fee_per_gas(max_fee_per_gas) .max_priority_fee_per_gas(max_priority_fee_per_gas) .chain_id(get_settings().network.chain_id) // Linea testnet chain ID - .build_raw_transaction((*signer).clone()) + .build_raw_transaction(wallet) .await .map_err(|e| { warn!("{e}"); @@ -307,32 +447,17 @@ struct PrivateKeyMaterial { key: Vec, } -/// Sign an Ethereum address using an RSA private key -pub fn sign_ethereum_address( - der_private_key: &[u8], +fn build_certificate_possession_preimage( address: &Address, verifying_contract: &Address, chain_id: u64, -) -> Result, Box> { - // Create an RSA object from the DER-encoded private key - let mut key_material = PrivateKeyMaterial { - key: der_private_key.to_vec(), - }; - - let private_key = Rsa::private_key_from_der(&key_material.key)?; - - let pkey = PKey::from_rsa(private_key)?; - - let mut signer = opensslSigner::new(MessageDigest::sha256(), &pkey)?; - signer.set_rsa_padding(Padding::PKCS1_PSS)?; - signer.set_rsa_mgf1_md(MessageDigest::sha256())?; - signer.set_rsa_pss_saltlen(RsaPssSaltlen::DIGEST_LENGTH)?; - - // Minimal domain separation: human-readable, versioned, bound to contract + chain +) -> Vec { + // Minimal domain separation: human-readable, versioned, bound to contract + chain. // preimage = "ADDR-LINK|v1|contract:" || verifying_contract || "|chainId:" || u64_be || "|addr:" || address const PREFIX: &[u8] = b"ADDR-LINK|v1|contract:"; const SEP_CHAIN: &[u8] = b"|chainId:"; const SEP_ADDR: &[u8] = b"|addr:"; + let mut preimage = Vec::with_capacity(PREFIX.len() + 20 + SEP_CHAIN.len() + 8 + SEP_ADDR.len() + 20); preimage.extend_from_slice(PREFIX); @@ -341,14 +466,43 @@ pub fn sign_ethereum_address( preimage.extend_from_slice(&chain_id.to_be_bytes()); // 8 bytes, big-endian preimage.extend_from_slice(SEP_ADDR); preimage.extend_from_slice(address.as_bytes()); // 20 bytes + preimage +} + +fn sign_certificate_possession_preimage( + der_private_key: &[u8], + preimage: &[u8], +) -> CertificateSignerResult> { + let mut key_material = PrivateKeyMaterial { + key: der_private_key.to_vec(), + }; + + let private_key = Rsa::private_key_from_der(&key_material.key)?; + let pkey = PKey::from_rsa(private_key)?; + + let mut signer = opensslSigner::new(MessageDigest::sha256(), &pkey)?; + signer.set_rsa_padding(Padding::PKCS1_PSS)?; + signer.set_rsa_mgf1_md(MessageDigest::sha256())?; + signer.set_rsa_pss_saltlen(RsaPssSaltlen::DIGEST_LENGTH)?; + signer.update(preimage)?; - // Sign the address bytes - signer.update(&preimage)?; let signature = signer.sign_to_vec()?; - key_material.zeroize(); // Zeroize private key material + key_material.zeroize(); Ok(signature) } +/// Sign an Ethereum address using an RSA private key +pub fn sign_ethereum_address( + der_private_key: &[u8], + address: &Address, + verifying_contract: &Address, + chain_id: u64, +) -> Result, Box> { + let preimage = build_certificate_possession_preimage(address, verifying_contract, chain_id); + sign_certificate_possession_preimage(der_private_key, &preimage) + .map_err(|e| e as Box) +} + // Convenience alias so we do not keep constructing Box in the handler fn prevalidation_error(msg: &str) -> CertificateVerificationError { CertificateVerificationError::new(msg) @@ -444,6 +598,86 @@ fn prevalidate_certificate_and_key( Ok(()) } +fn prevalidate_certificate(cert_der: &[u8]) -> Result<(), CertificateVerificationError> { + let cert = OpensslX509::from_der(cert_der).map_err(|e| { + error!("X.509 parse error: {e}"); + prevalidation_error("Invalid X.509 certificate (DER parsing failed)") + })?; + + let now = Asn1Time::days_from_now(0).map_err(|e| { + error!("Asn1Time::days_from_now error: {e}"); + prevalidation_error("Internal time error while checking certificate validity") + })?; + + if cert.not_before() > now { + return Err(prevalidation_error( + "Certificate is not yet valid (not_before is in the future)", + )); + } + if cert.not_after() < now { + return Err(prevalidation_error( + "Certificate has expired (not_after is in the past)", + )); + } + + let pubkey = cert.public_key().map_err(|e| { + error!("Failed to extract public key from certificate: {e}"); + prevalidation_error("Cannot extract public key from certificate") + })?; + + if pubkey.id() == PKeyId::RSA { + let rsa_pub = pubkey.rsa().map_err(|e| { + error!("Failed to convert public key to RSA: {e}"); + prevalidation_error("Invalid RSA public key inside certificate") + })?; + if rsa_pub.size() < 2048 / 8 { + return Err(prevalidation_error( + "RSA key too short (must be at least 2048 bits)", + )); + } + } + + Ok(()) +} + +fn verify_signature_matches_certificate( + cert_der: &[u8], + address: &Address, + signature: &[u8], + verifying_contract: &Address, + chain_id: u64, +) -> Result<(), CertificateVerificationError> { + let cert = OpensslX509::from_der(cert_der).map_err(|e| { + error!("X.509 parse error while verifying certificate/signature match: {e}"); + prevalidation_error("Invalid X.509 certificate (DER parsing failed)") + })?; + + let pubkey = cert.public_key().map_err(|e| { + error!("Failed to extract public key from certificate: {e}"); + prevalidation_error("Cannot extract public key from certificate") + })?; + + let is_valid = verify_ethereum_address_signature( + &pubkey, + address, + signature, + verifying_contract, + chain_id, + ) + .map_err(|e| { + error!("Failed to verify signature with certificate public key: {e}"); + prevalidation_error("Failed to verify signature with certificate public key") + })?; + + if !is_valid { + return Err(prevalidation_error( + "Certificate public key does not match configured signer", + )); + } + + Ok(()) +} + #[allow(dead_code)] fn verify_ethereum_address_signature( pkey: &PKey, @@ -459,21 +693,8 @@ fn verify_ethereum_address_signature( verifier.set_rsa_mgf1_md(MessageDigest::sha256())?; verifier.set_rsa_pss_saltlen(RsaPssSaltlen::DIGEST_LENGTH)?; - // Minimal domain separation: human-readable, versioned, bound to contract + chain - // preimage = "ADDR-LINK|v1|contract:" || verifying_contract || "|chainId:" || u64_be || "|addr:" || address - const PREFIX: &[u8] = b"ADDR-LINK|v1|contract:"; - const SEP_CHAIN: &[u8] = b"|chainId:"; - const SEP_ADDR: &[u8] = b"|addr:"; - let mut preimage = - Vec::with_capacity(PREFIX.len() + 20 + SEP_CHAIN.len() + 8 + SEP_ADDR.len() + 20); - preimage.extend_from_slice(PREFIX); - preimage.extend_from_slice(verifying_contract.as_bytes()); // 20 bytes - preimage.extend_from_slice(SEP_CHAIN); - preimage.extend_from_slice(&chain_id.to_be_bytes()); // 8 bytes, big-endian - preimage.extend_from_slice(SEP_ADDR); - preimage.extend_from_slice(address.as_bytes()); // 20 bytes + let preimage = build_certificate_possession_preimage(address, verifying_contract, chain_id); - // Verify the signature over the structured preimage verifier.update(&preimage)?; let result = verifier.verify(signature)?; // expects same PKCS#1 v1.5 structure @@ -534,7 +755,7 @@ mod tests { "--{boundary}\r\nContent-Disposition: form-data; name=\"certificate\"; filename=\"cert.der\"\r\nContent-Type: application/octet-stream\r\n\r\nabc\r\n--{boundary}--\r\n" ); - let filter = certification_validation_request(); + let filter = certification_validation_request(WalletRole::Client); let res = warp::test::request() .method("POST") .path("/v1/certification") @@ -559,7 +780,7 @@ mod tests { "--{boundary}\r\nContent-Disposition: form-data; name=\"unexpected\"; filename=\"file.bin\"\r\nContent-Type: application/octet-stream\r\n\r\nabc\r\n--{boundary}--\r\n" ); - let filter = certification_validation_request(); + let filter = certification_validation_request(WalletRole::Client); let res = warp::test::request() .method("POST") .path("/v1/certification") diff --git a/lib/src/wallets.rs b/lib/src/wallets.rs index 72d6abad..69268319 100644 --- a/lib/src/wallets.rs +++ b/lib/src/wallets.rs @@ -3,7 +3,7 @@ use crate::{ }; use alloy::{ consensus::SignableTransaction, - network::{Ethereum, NetworkWallet, TxSigner}, + network::{Ethereum, EthereumWallet, IntoWallet, NetworkWallet, TxSigner}, primitives::{Address, Signature}, providers::{Provider, ProviderBuilder, WsConnect}, signers::{local::PrivateKeySigner, utils::public_key_to_address}, @@ -11,20 +11,67 @@ use alloy::{ use async_trait::async_trait; use azure_identity; use azure_security_keyvault::{prelude::*, KeyClient}; -use base64::prelude::*; -use configuration::settings::WalletTypeConfig; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; +use configuration::settings::{WalletRole, WalletTypeConfig}; use k256::ecdsa::{RecoveryId, Signature as K256Signature, VerifyingKey}; use k256::EncodedPoint; use log::{debug, info}; use std::sync::Arc; use url::Url; +pub(crate) fn validate_azure_vault_url( + vault_url: &str, +) -> Result<(), Box> { + let url = Url::parse(vault_url)?; + + // Refuse non-HTTPS endpoints + if url.scheme() != "https" { + return Err("Vault URL must use HTTPS".into()); + } + + // Enforce vault_url allow-list (*.vault.azure.net) + let host = url.host_str().ok_or("Invalid host")?; + if !host.ends_with(".vault.azure.net") { + return Err("Vault URL must be *.vault.azure.net".into()); + } + + Ok(()) +} + #[derive(Clone, Debug)] pub enum WalletType { Local(Box), Azure(AzureWallet), } +#[async_trait] +impl TxSigner for WalletType { + fn address(&self) -> Address { + match self { + Self::Local(signer) => signer.address(), + Self::Azure(wallet) => wallet.address(), + } + } + + async fn sign_transaction( + &self, + tx: &mut dyn SignableTransaction, + ) -> Result { + match self { + Self::Local(signer) => TxSigner::sign_transaction(&**signer, tx).await, + Self::Azure(wallet) => TxSigner::sign_transaction(wallet, tx).await, + } + } +} + +impl IntoWallet for WalletType { + type NetworkWallet = EthereumWallet; + + fn into_wallet(self) -> Self::NetworkWallet { + EthereumWallet::new(self) + } +} + /// AzureWallet /// -------------- /// This struct represents an Ethereum wallet whose private key is stored securely in Azure Key Vault. @@ -51,7 +98,7 @@ impl AzureWallet { ) -> Result> { info!(" Creating Azure Wallet"); - Self::validate_vault_url(vault_url)?; + validate_azure_vault_url(vault_url)?; // Create credential and KeyClient to communicate with Azure let credential = azure_identity::create_credential()?; let key_client = KeyClient::new(vault_url, credential)?; @@ -74,27 +121,6 @@ impl AzureWallet { }) } - /// Validates Azure Key Vault URL to prevent security vulnerabilities - /// - /// Added strict vault_url validation (HTTPS + *.vault.azure.net allow-list) - /// to prevent token exfiltration or key substitution attacks. - fn validate_vault_url(vault_url: &str) -> Result<(), Box> { - let url = Url::parse(vault_url)?; - - // Refuse non-HTTPS endpoints - if url.scheme() != "https" { - return Err("Vault URL must use HTTPS".into()); - } - - // Enforce vault_url allow-list (*.vault.azure.net) - let host = url.host_str().ok_or("Invalid host")?; - if !host.ends_with(".vault.azure.net") { - return Err("Vault URL must be *.vault.azure.net".into()); - } - - Ok(()) - } - /// Sign a message hash using the Azure Key Vault key /// --------------------------------------------------- /// The private key never leaves the HSM. The signature returned is Ethereum-compatible. @@ -103,8 +129,8 @@ impl AzureWallet { message_hash: &[u8; 32], ) -> Result> { info!(" Signing with Azure Key Vault"); - // Encode message hash in Base64 (required by Azure - let digest_base64 = BASE64_STANDARD.encode(message_hash); + // Encode the digest in JWA-style base64url before sending it to Key Vault. + let digest_base64 = URL_SAFE_NO_PAD.encode(message_hash); // Request signature from Azure Key Vault let sign_result = self @@ -319,26 +345,15 @@ impl BlockchainClientConnection for LocalWsClient { &self.wallet } - /// Get the PrivateKeySigner if using a local wallet - fn get_signer(&self) -> Arc { - match &self.wallet { - WalletType::Local(signer) => Arc::from(signer.clone()), - WalletType::Azure(_) => { - panic!( - "Cannot get PrivateKeySigner for Azure wallet - use provider methods instead" - ) - } - } - } - /// Create a new instance from configuration settings async fn try_from_settings( settings: &Self::S, + role: WalletRole, ) -> Result { - match settings.nightfall_client.wallet_type { + match settings.wallet_type_for_role(role) { // Handle different wallet types WalletTypeConfig::Local => { - info!("Creating local wallet"); + info!("Creating local wallet for role {role:?}"); // Parse the private key from settings let local_signer = settings .signing_key @@ -358,9 +373,13 @@ impl BlockchainClientConnection for LocalWsClient { }) } WalletTypeConfig::Azure => { + info!("Creating Azure wallet for role {role:?}"); // Initialize AzureWallet + let azure_key_name = settings + .azure_key_name_for_role(role) + .map_err(BlockchainClientConnectionError::InvalidWalletType)?; let azure_wallet = - AzureWallet::new(&settings.azure_vault_url, &settings.azure_key_name).await?; + AzureWallet::new(&settings.azure_vault_url, azure_key_name).await?; let ws = WsConnect::new(settings.ethereum_client_url.clone()); let provider = ProviderBuilder::new() diff --git a/nightfall.toml b/nightfall.toml index a045a652..62df7664 100644 --- a/nightfall.toml +++ b/nightfall.toml @@ -35,7 +35,8 @@ max_queue_size = 1000 url = "http://proposer:3000" log_level = "info" wallet_type = "local" -db_url = "mongodb://nf4_db_proposer:27017" +db_url = "mongodb://nf4_db_proposer:27017/?replicaSet=nf4-rs-proposer&directConnection=true" +snapshot_root_dir = "./data/proposer_snapshots/development" block_assembly_max_wait_secs = 120 block_assembly_target_fill_ratio = 0.25 block_assembly_initial_interval_secs = 15 @@ -227,7 +228,8 @@ max_queue_size = 1000 url = "http://proposer:3000" log_level = "info" wallet_type = "local" -db_url = "mongodb://nf4_db_proposer:27017" +db_url = "mongodb://nf4_db_proposer:27017/?replicaSet=nf4-rs-proposer&directConnection=true" +snapshot_root_dir = "./data/proposer_snapshots/base_sepolia" block_assembly_max_wait_secs = 120 block_assembly_target_fill_ratio = 0.25 block_assembly_initial_interval_secs = 15 @@ -275,5 +277,3 @@ oid_group = 0 [production] # This is the production environment - add an appropriate configuration here, using development as a template - - diff --git a/nightfall_client/benches/bench_unified_circuit.rs b/nightfall_client/benches/bench_unified_circuit.rs index 4ac6d2f8..0d4ff453 100644 --- a/nightfall_client/benches/bench_unified_circuit.rs +++ b/nightfall_client/benches/bench_unified_circuit.rs @@ -24,10 +24,10 @@ use lib::{ derive_key::ZKPKeys, hex_conversion::HexConvertible, nf_client_proof::{PrivateInputs, PublicInputs}, - nf_token_id::to_nf_token_id_from_str, + nf_token_id::{to_nf_slot_id_from_str, to_nf_token_id_from_str}, plonk_prover::circuits::unified_circuit::unified_circuit_builder, secret_hash::SecretHash, - shared_entities::{DepositData, DepositSecret, Preimage, Salt}, + shared_entities::{DepositData, DepositSecret, Preimage, Salt, TokenType}, }; use nf_curves::ed_on_bn254::{BabyJubjub, Fq as Fr254, Fr as BJJScalar}; use nightfall_client::driven::primitives::kemdem_functions::kemdem_encrypt; @@ -187,7 +187,13 @@ fn build_valid_transfer_inputs() -> CircuitTestInfo { let token_id_string = Fr254::to_hex_string(&token_id_fr); let nf_token_id = to_nf_token_id_from_str(&erc_address_string, &token_id_string).unwrap(); - let nf_slot_id = nf_token_id; + let nf_slot_id = to_nf_slot_id_from_str( + &erc_address_string, + &token_id_string, + &token_id_string, + TokenType::ERC20, + ) + .unwrap(); let token_id = Fr254::from_hex_string(&token_id_string).unwrap(); @@ -289,6 +295,7 @@ fn build_valid_transfer_inputs() -> CircuitTestInfo { .fee_token_id(fee_token_id) .nf_address(nf_address_h160) .value_a(value) + .spent_nf_token_ids([nf_token_id, nf_token_id]) .nf_token_a_id(nf_token_id) .nf_slot_id(nf_slot_id) .ephemeral_key(ephemeral_key) diff --git a/nightfall_client/src/domain/entities.rs b/nightfall_client/src/domain/entities.rs index 374fed36..4c0f15f9 100644 --- a/nightfall_client/src/domain/entities.rs +++ b/nightfall_client/src/domain/entities.rs @@ -23,6 +23,13 @@ pub struct Request { pub uuid: String, #[serde(skip_serializing_if = "Option::is_none")] pub child_request_args: Option, + /// The canonical proposer transaction hash (hex-encoded, 64 chars). + /// Populated after the transaction is successfully submitted to the proposer. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub tx_hash: Option, + /// Capability token returned by the proposer for authorized transfer receipt creation. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub receipt_token: Option, } /// Struct to represent the realtionship between request and commitment diff --git a/nightfall_client/src/domain/error.rs b/nightfall_client/src/domain/error.rs index f81eef39..d1687497 100644 --- a/nightfall_client/src/domain/error.rs +++ b/nightfall_client/src/domain/error.rs @@ -187,6 +187,16 @@ pub enum ClientRejection { RequestNotFound, FailedDeEscrow, SynchronisationUnavailable, + /// The receipt_token did not match any proposer's stored token for that tx_hash. + ReceiptUnauthorized, + /// A receipt already exists for this transaction with a different ciphertext. + ReceiptConflict, + /// The transaction is not yet known to any proposer (submitted too early). + ReceiptTxNotFound, + /// The request body failed validation on the proposer (e.g. malformed tx_hash). + ReceiptValidationFailed, + /// All proposers are unreachable or rejected the receipt for unexpected reasons. + ReceiptSubmissionFailed, } impl std::fmt::Display for ClientRejection { @@ -208,6 +218,24 @@ impl std::fmt::Display for ClientRejection { ClientRejection::SynchronisationUnavailable => { write!(f, "Synchronisation service unavailable") } + ClientRejection::ReceiptUnauthorized => { + write!(f, "Invalid or missing receipt_token") + } + ClientRejection::ReceiptConflict => { + write!(f, "Receipt already exists with different ciphertext") + } + ClientRejection::ReceiptTxNotFound => { + write!(f, "Transaction not found on proposer") + } + ClientRejection::ReceiptValidationFailed => { + write!(f, "Transfer receipt request failed validation") + } + ClientRejection::ReceiptSubmissionFailed => { + write!( + f, + "Receipt submission failed: no proposer accepted the receipt" + ) + } } } } diff --git a/nightfall_client/src/driven/contract_functions/nightfall_contract.rs b/nightfall_client/src/driven/contract_functions/nightfall_contract.rs index a9bfa9b0..0d4d71de 100644 --- a/nightfall_client/src/driven/contract_functions/nightfall_contract.rs +++ b/nightfall_client/src/driven/contract_functions/nightfall_contract.rs @@ -5,12 +5,7 @@ use crate::{ }; use alloy::primitives::{keccak256, Address, B256, I256}; use alloy::rpc::types::Filter; -use alloy::{ - consensus::Transaction, - dyn_abi::abi::encode, - providers::Provider, - sol_types::{SolInterface, SolValue}, -}; +use alloy::{consensus::Transaction, providers::Provider, sol_types::SolInterface}; use ark_bn254::Fr as Fr254; use ark_ff::BigInteger256; use ark_std::Zero; @@ -21,14 +16,13 @@ use lib::{ error::NightfallContractError, initialisation::get_blockchain_client_connection, log_fetcher::get_logs_paginated, - nf_token_id::to_nf_token_id_from_solidity, + nf_token_id::{to_nf_slot_id_from_solidity, to_nf_token_id_from_solidity}, secret_hash::SecretHash, shared_entities::{DepositSecret, TokenType, WithdrawData}, verify_contract::VerifiedContracts, }; use log::{debug, info}; use nightfall_bindings::artifacts::{Nightfall, IERC3525}; -use num::BigUint; impl NightfallContract for Nightfall::NightfallCalls { async fn escrow_funds( @@ -47,17 +41,11 @@ impl NightfallContract for Nightfall::NightfallCalls { let solidity_token_id = Uint256::from(token_id); let solidity_secret_hash = Uint256::from(secret_preimage.hash()?); - let blockchain_client = get_blockchain_client_connection() - .await - .read() - .await - .get_client(); - let signer = get_blockchain_client_connection() - .await - .read() - .await - .get_signer(); + let read_connection = get_blockchain_client_connection().await.read().await; + let blockchain_client = read_connection.get_client(); let client = blockchain_client.root(); + let caller = read_connection.get_address(); + let wallet = read_connection.get_wallet_type().clone(); let verified = VerifiedContracts::resolve_and_verify_contract(client.clone(), get_addresses()) .await @@ -78,12 +66,9 @@ impl NightfallContract for Nightfall::NightfallCalls { } else { fee + fee + deposit_fee }; - let nonce = client - .get_transaction_count(signer.address()) - .await - .map_err(|e| { - NightfallContractError::EscrowError(format!("Transaction unsuccesful: {e}")) - })?; + let nonce = client.get_transaction_count(caller).await.map_err(|e| { + NightfallContractError::EscrowError(format!("Transaction unsuccesful: {e}")) + })?; let gas_price = client.get_gas_price().await.map_err(|e| { NightfallContractError::EscrowError(format!("Transaction unsuccesful: {e}")) })?; @@ -105,8 +90,8 @@ impl NightfallContract for Nightfall::NightfallCalls { .gas(gas_limit) .max_fee_per_gas(max_fee_per_gas) .max_priority_fee_per_gas(max_priority_fee_per_gas) - .chain_id(get_settings().network.chain_id) // Linea testnet chain ID - .build_raw_transaction((*signer).clone()) + .chain_id(get_settings().network.chain_id) + .build_raw_transaction(wallet) .await .map_err(|e| { NightfallContractError::EscrowError(format!("Transaction unsuccesful: {e}")) @@ -125,9 +110,9 @@ impl NightfallContract for Nightfall::NightfallCalls { })?; info!("Gas used in escrow funds: {:?}", receipt.gas_used); - let slot_id = if let TokenType::ERC3525 = token_type { + let nf_slot_id = if let TokenType::ERC3525 = token_type { let erc_contract = IERC3525::new(solidity_token_address.0, client.clone()); - erc_contract + let slot_id = erc_contract .slotOf(solidity_token_id.0) .call() .await @@ -135,26 +120,26 @@ impl NightfallContract for Nightfall::NightfallCalls { NightfallContractError::EscrowError( "Could not retrieve ERC3525 slot".to_string(), ) - })? + })?; + to_nf_slot_id_from_solidity( + solidity_token_address.0, + solidity_token_id.0, + slot_id, + token_type, + ) } else { - solidity_token_id.0 + to_nf_slot_id_from_solidity( + solidity_token_address.0, + solidity_token_id.0, + solidity_token_id.0, + token_type, + ) }; // We calculate the the nf_token_id and nf_slot_id here - let erc_token = solidity_token_address.0.tokenize(); let nf_token_id = to_nf_token_id_from_solidity(solidity_token_address.0, solidity_token_id.0); - if slot_id == solidity_token_id.0 { - let nf_slot_id = nf_token_id; - Ok([nf_token_id, nf_slot_id]) - } else { - let slot_id_token = slot_id.tokenize(); - let nf_slot_id_biguint = - BigUint::from_bytes_be(keccak256(encode(&(erc_token, slot_id_token))).as_slice()) - >> 4; - let nf_slot_id = Fr254::from(nf_slot_id_biguint); - Ok([nf_token_id, nf_slot_id]) - } + Ok([nf_token_id, nf_slot_id]) } fn get_address() -> Fr254 { @@ -172,17 +157,11 @@ impl NightfallContract for Nightfall::NightfallCalls { value: data.value, withdraw_fund_salt: data.withdraw_fund_salt, }; - let blockchain_client = get_blockchain_client_connection() - .await - .read() - .await - .get_client(); - let signer = get_blockchain_client_connection() - .await - .read() - .await - .get_signer(); + let read_connection = get_blockchain_client_connection().await.read().await; + let blockchain_client = read_connection.get_client(); let client = blockchain_client.root(); + let caller = read_connection.get_address(); + let wallet = read_connection.get_wallet_type().clone(); let verified = VerifiedContracts::resolve_and_verify_contract(client.clone(), get_addresses()) @@ -194,12 +173,9 @@ impl NightfallContract for Nightfall::NightfallCalls { })?; let contract = verified.nightfall; - let nonce = client - .get_transaction_count(signer.address()) - .await - .map_err(|e| { - NightfallContractError::DeEscrowError(format!("Transaction unsuccesful: {e}")) - })?; + let nonce = client.get_transaction_count(caller).await.map_err(|e| { + NightfallContractError::DeEscrowError(format!("Transaction unsuccesful: {e}")) + })?; let gas_price = client.get_gas_price().await.map_err(|e| { NightfallContractError::DeEscrowError(format!("Transaction unsuccesful: {e}")) })?; @@ -207,13 +183,13 @@ impl NightfallContract for Nightfall::NightfallCalls { let max_priority_fee_per_gas = gas_price; let gas_limit = 5000000u64; let call = contract - .descrow_funds(decode_data, token_type.into()) + .descrow_funds(decode_data.clone(), token_type.into()) .nonce(nonce) .gas(gas_limit) .max_fee_per_gas(max_fee_per_gas) .max_priority_fee_per_gas(max_priority_fee_per_gas) - .chain_id(get_settings().network.chain_id) // Linea testnet chain ID - .build_raw_transaction((*signer).clone()) + .chain_id(get_settings().network.chain_id) + .build_raw_transaction(wallet) .await .map_err(|e| { NightfallContractError::DeEscrowError(format!("Transaction unsuccesful: {e}")) @@ -240,17 +216,10 @@ impl NightfallContract for Nightfall::NightfallCalls { async fn withdraw_available( withdraw_data: WithdrawData, ) -> Result { - let blockchain_client = get_blockchain_client_connection() - .await - .read() - .await - .get_client(); - let signer = get_blockchain_client_connection() - .await - .read() - .await - .get_signer(); + let read_connection = get_blockchain_client_connection().await.read().await; + let blockchain_client = read_connection.get_client(); let client = blockchain_client.root(); + let caller = read_connection.get_address(); let verified = VerifiedContracts::resolve_and_verify_contract(client.clone(), get_addresses()) .await @@ -271,7 +240,7 @@ impl NightfallContract for Nightfall::NightfallCalls { let result = nightfall_instance .withdraw_processed(decode_data) - .from(signer.address()) + .from(caller) .call() .await .map_err(|e| NightfallContractError::EscrowError(format!("{e}")))?; @@ -279,17 +248,10 @@ impl NightfallContract for Nightfall::NightfallCalls { } async fn get_current_layer2_blocknumber() -> Result { - let blockchain_client = get_blockchain_client_connection() - .await - .read() - .await - .get_client(); + let read_connection = get_blockchain_client_connection().await.read().await; + let blockchain_client = read_connection.get_client(); let client = blockchain_client.root(); - let signer = get_blockchain_client_connection() - .await - .read() - .await - .get_signer(); + let caller = read_connection.get_address(); let verified = VerifiedContracts::resolve_and_verify_contract(client.clone(), get_addresses()) .await @@ -302,7 +264,7 @@ impl NightfallContract for Nightfall::NightfallCalls { let l2_block = nightfall .layer2_block_number() - .from(signer.address()) + .from(caller) .call() .await .map_err(|e| NightfallContractError::EscrowError(format!("{e}")))?; @@ -310,17 +272,10 @@ impl NightfallContract for Nightfall::NightfallCalls { Ok(l2_block) } async fn get_token_info(nf_token_id: Fr254) -> Result { - let blockchain_client = get_blockchain_client_connection() - .await - .read() - .await - .get_client(); - let signer = get_blockchain_client_connection() - .await - .read() - .await - .get_signer(); + let read_connection = get_blockchain_client_connection().await.read().await; + let blockchain_client = read_connection.get_client(); let client = blockchain_client.root(); + let caller = read_connection.get_address(); let verified = VerifiedContracts::resolve_and_verify_contract(client.clone(), get_addresses()) .await @@ -333,7 +288,7 @@ impl NightfallContract for Nightfall::NightfallCalls { let token_info = nightfall .getTokenInfo(Uint256::from(nf_token_id).0) - .from(signer.address()) + .from(caller) .call() .await .map_err(|e| { @@ -348,16 +303,9 @@ impl NightfallContract for Nightfall::NightfallCalls { } async fn get_slot_info(nf_slot_id: Fr254) -> Result { - let blockchain_client = get_blockchain_client_connection() - .await - .read() - .await - .get_client(); - let signer = get_blockchain_client_connection() - .await - .read() - .await - .get_signer(); + let read_connection = get_blockchain_client_connection().await.read().await; + let blockchain_client = read_connection.get_client(); + let caller = read_connection.get_address(); let client = blockchain_client.root(); let verified = VerifiedContracts::resolve_and_verify_contract(client.clone(), get_addresses()) @@ -371,7 +319,7 @@ impl NightfallContract for Nightfall::NightfallCalls { let slot_info = nightfall .getSlotInfo(Uint256::from(nf_slot_id).0) - .from(signer.address()) + .from(caller) .call() .await .map_err(|e| { diff --git a/nightfall_client/src/driven/contract_functions/test.rs b/nightfall_client/src/driven/contract_functions/test.rs new file mode 100644 index 00000000..9e3ad587 --- /dev/null +++ b/nightfall_client/src/driven/contract_functions/test.rs @@ -0,0 +1,404 @@ +//! Implementation of the [`NightfallContract`] trait from `ports/contracts.rs`. +use crate::{domain::entities::TokenData, ports::contracts::NightfallContract}; +use alloy::primitives::{Address, B256, I256, keccak256}; +use alloy::rpc::types::Filter; +use alloy::{consensus::Transaction, providers::Provider, sol_types::SolInterface}; +use ark_bn254::Fr as Fr254; +use ark_ff::BigInteger256; +use ark_std::Zero; +use configuration::{addresses::get_addresses, settings::get_settings}; +use lib::{ + blockchain_client::BlockchainClientConnection, + contract_conversions::{Addr, FrBn254, Uint256}, + error::NightfallContractError, + initialisation::get_blockchain_client_connection, + log_fetcher::get_logs_paginated, + nf_token_id::{to_nf_slot_id_from_solidity, to_nf_token_id_from_solidity}, + secret_hash::SecretHash, + shared_entities::{DepositSecret, TokenType, WithdrawData}, + verify_contract::VerifiedContracts, +}; +use log::{debug, info}; +use nightfall_bindings::artifacts::{IERC3525, Nightfall}; + +impl NightfallContract for Nightfall::NightfallCalls { + async fn escrow_funds( + token_erc_address: Fr254, + value: Fr254, + token_id: BigInteger256, + fee: Fr254, + deposit_fee: Fr254, + secret_preimage: DepositSecret, + token_type: TokenType, + ) -> Result<[Fr254; 2], NightfallContractError> { + // Make DepositData + let solidity_fee = Uint256::from(fee); + let solidity_token_address = Addr::try_from(token_erc_address)?; + let solidity_value = Uint256::from(value); + let solidity_token_id = Uint256::from(token_id); + let solidity_secret_hash = Uint256::from(secret_preimage.hash()?); + + let blockchain_client = get_blockchain_client_connection() + .await + .read() + .await + .get_client(); + let signer = get_blockchain_client_connection() + .await + .read() + .await + .get_signer(); + let client = blockchain_client.root(); + let verified = + VerifiedContracts::resolve_and_verify_contract(client.clone(), get_addresses()) + .await + .map_err(|e| { + NightfallContractError::ContractVerificationError(format!( + "Contract verification failed during escrow_funds: {e}" + )) + })?; + let contract = verified.nightfall; + + // A deposit transaction (value_1, fee_1, deposit_fee_1), means we want to deposit value_1, with fee_1 paid to proposer, and additional deposit_fee_1 for future transactions. Two deposit data are created for a single deposit transaction: + // 1: (value: value_1, fee: fee_1), + // 2: (value: deposit_fee_1, fee: fee_1) + // therefore, the total deposit fee (msg.value) is 2 * fee_1 + deposit_fee_1 + // If a deposit transaction doesn't have a deposit fee, then the total deposit fee is equal to the fee_1 + let total_fee = if deposit_fee == Fr254::zero() { + fee + } else { + fee + fee + deposit_fee + }; + let nonce = client + .get_transaction_count(signer.address()) + .await + .map_err(|e| { + NightfallContractError::EscrowError(format!("Transaction unsuccesful: {e}")) + })?; + let gas_price = client.get_gas_price().await.map_err(|e| { + NightfallContractError::EscrowError(format!("Transaction unsuccesful: {e}")) + })?; + let max_fee_per_gas = gas_price * 2; + let max_priority_fee_per_gas = gas_price; + let gas_limit = 5000000u64; + + let call = contract + .escrow_funds( + solidity_fee.0, + solidity_token_address.0, + solidity_token_id.0, + solidity_value.0, + solidity_secret_hash.0, + token_type.into(), + ) + .value(Uint256::from(total_fee).0) + .nonce(nonce) + .gas(gas_limit) + .max_fee_per_gas(max_fee_per_gas) + .max_priority_fee_per_gas(max_priority_fee_per_gas) + .chain_id(get_settings().network.chain_id) // Linea testnet chain ID + .build_raw_transaction((*signer).clone()) + .await + .map_err(|e| { + NightfallContractError::EscrowError(format!("Transaction unsuccesful: {e}")) + })?; + + let receipt = client + .send_raw_transaction(&call) + .await + .map_err(|e| { + NightfallContractError::EscrowError(format!("Error getting receipt: {e}")) + })? + .get_receipt() + .await + .map_err(|e| { + NightfallContractError::EscrowError(format!("Transaction unsuccesful: {e}")) + })?; + + info!("Gas used in escrow funds: {:?}", receipt.gas_used); + let slot_id = if let TokenType::ERC3525 = token_type { + let erc_contract = IERC3525::new(solidity_token_address.0, client.clone()); + erc_contract + .slotOf(solidity_token_id.0) + .call() + .await + .map_err(|_| { + NightfallContractError::EscrowError( + "Could not retrieve ERC3525 slot".to_string(), + ) + })? + } else { + solidity_token_id.0 + }; + + // We calculate the the nf_token_id and nf_slot_id here + let nf_token_id = + to_nf_token_id_from_solidity(solidity_token_address.0, solidity_token_id.0); + let nf_slot_id = to_nf_slot_id_from_solidity( + solidity_token_address.0, + solidity_token_id.0, + slot_id, + token_type, + ); + Ok([nf_token_id, nf_slot_id]) + } + + fn get_address() -> Fr254 { + FrBn254::from(get_addresses().nightfall()).0 + } + + async fn de_escrow_funds( + withdraw_data: WithdrawData, + token_type: TokenType, + ) -> Result<(), NightfallContractError> { + let data = Nightfall::WithdrawData::from(withdraw_data); + let decode_data = Nightfall::WithdrawData { + nf_token_id: data.nf_token_id, + recipient_address: data.recipient_address, + value: data.value, + withdraw_fund_salt: data.withdraw_fund_salt, + }; + let blockchain_client = get_blockchain_client_connection() + .await + .read() + .await + .get_client(); + let signer = get_blockchain_client_connection() + .await + .read() + .await + .get_signer(); + let client = blockchain_client.root(); + + let verified = + VerifiedContracts::resolve_and_verify_contract(client.clone(), get_addresses()) + .await + .map_err(|e| { + NightfallContractError::ContractVerificationError(format!( + "Contract verification failed during de_escrow_funds: {e}" + )) + })?; + let contract = verified.nightfall; + + let nonce = client + .get_transaction_count(signer.address()) + .await + .map_err(|e| { + NightfallContractError::DeEscrowError(format!("Transaction unsuccesful: {e}")) + })?; + let gas_price = client.get_gas_price().await.map_err(|e| { + NightfallContractError::DeEscrowError(format!("Transaction unsuccesful: {e}")) + })?; + let max_fee_per_gas = gas_price * 2; + let max_priority_fee_per_gas = gas_price; + let gas_limit = 5000000u64; + let call = contract + .descrow_funds(decode_data, token_type.into()) + .nonce(nonce) + .gas(gas_limit) + .max_fee_per_gas(max_fee_per_gas) + .max_priority_fee_per_gas(max_priority_fee_per_gas) + .chain_id(get_settings().network.chain_id) // Linea testnet chain ID + .build_raw_transaction((*signer).clone()) + .await + .map_err(|e| { + NightfallContractError::DeEscrowError(format!("Transaction unsuccesful: {e}")) + })?; + + let receipt = client + .send_raw_transaction(&call) + .await + .map_err(|e| { + NightfallContractError::DeEscrowError(format!("Error getting receipt: {e}")) + })? + .get_receipt() + .await + .map_err(|e| { + NightfallContractError::DeEscrowError(format!("Transaction unsuccesful: {e}")) + })?; + + if !receipt.gas_used.is_zero() { + info!("Gas used in de_escrow_funds: {:?}", receipt.gas_used); + } + Ok(()) + } + + async fn withdraw_available( + withdraw_data: WithdrawData, + ) -> Result { + let blockchain_client = get_blockchain_client_connection() + .await + .read() + .await + .get_client(); + let signer = get_blockchain_client_connection() + .await + .read() + .await + .get_signer(); + let client = blockchain_client.root(); + let verified = + VerifiedContracts::resolve_and_verify_contract(client.clone(), get_addresses()) + .await + .map_err(|e| { + NightfallContractError::ContractVerificationError(format!( + "Contract verification failed during withdraw_available: {e}" + )) + })?; + let nightfall_instance = verified.nightfall; + + let data = Nightfall::WithdrawData::from(withdraw_data); + let decode_data = Nightfall::WithdrawData { + nf_token_id: data.nf_token_id, + recipient_address: data.recipient_address, + value: data.value, + withdraw_fund_salt: data.withdraw_fund_salt, + }; + + let result = nightfall_instance + .withdraw_processed(decode_data) + .from(signer.address()) + .call() + .await + .map_err(|e| NightfallContractError::EscrowError(format!("{e}")))?; + Ok(result) + } + + async fn get_current_layer2_blocknumber() -> Result { + let blockchain_client = get_blockchain_client_connection() + .await + .read() + .await + .get_client(); + let client = blockchain_client.root(); + let signer = get_blockchain_client_connection() + .await + .read() + .await + .get_signer(); + let verified = + VerifiedContracts::resolve_and_verify_contract(client.clone(), get_addresses()) + .await + .map_err(|e| { + NightfallContractError::ContractVerificationError(format!( + "Contract verification failed during get_current_layer2_blocknumber: {e}" + )) + })?; + let nightfall = verified.nightfall; + + let l2_block = nightfall + .layer2_block_number() + .from(signer.address()) + .call() + .await + .map_err(|e| NightfallContractError::EscrowError(format!("{e}")))?; + + Ok(l2_block) + } + async fn get_token_info(nf_token_id: Fr254) -> Result { + let blockchain_client = get_blockchain_client_connection() + .await + .read() + .await + .get_client(); + let signer = get_blockchain_client_connection() + .await + .read() + .await + .get_signer(); + let client = blockchain_client.root(); + let verified = + VerifiedContracts::resolve_and_verify_contract(client.clone(), get_addresses()) + .await + .map_err(|e| { + NightfallContractError::ContractVerificationError(format!( + "Contract verification failed during get_token_info: {e}" + )) + })?; + let nightfall = verified.nightfall; + + let token_info = nightfall + .getTokenInfo(Uint256::from(nf_token_id).0) + .from(signer.address()) + .call() + .await + .map_err(|e| { + NightfallContractError::EscrowError(format!("Error getting token info: {e}")) + })?; + + Ok(TokenData { + erc_address: FrBn254::from(token_info.ercAddress).into(), + token_id: BigInteger256::from(Uint256(token_info.tokenId)), + token_type: TokenType::from(token_info.tokenType), + }) + } + // given a layer 2 block number, return the layer 2 block and the sender address + async fn get_layer2_block_by_number( + block_number: I256, + ) -> Result<(Address, Nightfall::Block), NightfallContractError> { + let block_number = block_number - I256::ONE; + let blockchain_client = get_blockchain_client_connection() + .await + .read() + .await + .get_client(); + let client = blockchain_client.root(); + let nightfall_address = get_addresses().nightfall(); + let block_topic = B256::from_slice(&block_number.to_be_bytes::<32>()); + + let latest_block = client.get_block_number().await.map_err(|e| { + NightfallContractError::ProviderError(format!("get_block_number error: {e}")) + })?; + + let event_sig = B256::from(keccak256("BlockProposed(int256)")); + let filter = Filter::new() + .address(nightfall_address) + .from_block(get_settings().genesis_block as u64) + .to_block(latest_block) + .event_signature(event_sig) + .topic1(block_topic); + + let logs = get_logs_paginated( + client, + filter.clone(), + get_settings().genesis_block as u64, + latest_block, + ) + .await + .map_err(|e| NightfallContractError::ProviderError(format!("Provider error: {e}")))?; + + let log = logs + .first() + .ok_or_else(|| NightfallContractError::BlockNotFound(block_number.as_u64()))?; + + let tx_hash = log.transaction_hash.ok_or_else(|| { + NightfallContractError::MissingTransactionHash( + "Log has no transaction hash".to_string(), + ) + })?; + let tx = client + .get_transaction_by_hash(tx_hash) + .await + .map_err(|e| { + NightfallContractError::ProviderError(format!("get_transaction error: {e}")) + })? + .ok_or(NightfallContractError::TransactionNotFound(tx_hash))?; + + let sender_address = tx.inner.signer(); + debug!("Sender of transaction {tx_hash} is {sender_address}"); + + let decoded = Nightfall::NightfallCalls::abi_decode(tx.input()).map_err(|e| { + NightfallContractError::AbiDecodeError(format!("ABI decode error: {e:?}")) + })?; + + match decoded { + Nightfall::NightfallCalls::propose_block(decoded) => { + debug!("Successfully decoded block {block_number} from tx {tx_hash}"); + Ok((sender_address, decoded.blk)) + } + _ => Err(NightfallContractError::DecodedCallError( + "Decoded call was not propose_block".to_string(), + )), + } + } +} diff --git a/nightfall_client/src/driven/contract_functions/token_contracts.rs b/nightfall_client/src/driven/contract_functions/token_contracts.rs index c8f490a2..bf260ee4 100644 --- a/nightfall_client/src/driven/contract_functions/token_contracts.rs +++ b/nightfall_client/src/driven/contract_functions/token_contracts.rs @@ -39,11 +39,7 @@ impl TokenContract for IERC20::IERC20Calls { let provider = read.get_client(); let client = provider.root(); let caller = read.get_address(); - let signer = get_blockchain_client_connection() - .await - .read() - .await - .get_signer(); + let wallet = read.get_wallet_type().clone(); let nonce = client.get_transaction_count(caller).await.map_err(|e| { BlockchainClientConnectionError::ProviderError(format!( @@ -65,8 +61,8 @@ impl TokenContract for IERC20::IERC20Calls { .gas(gas_limit) .max_fee_per_gas(max_fee_per_gas) .max_priority_fee_per_gas(max_priority_fee_per_gas) - .chain_id(get_settings().network.chain_id) // Linea testnet chain ID - .build_raw_transaction((*signer).clone()) + .chain_id(get_settings().network.chain_id) + .build_raw_transaction(wallet) .await .map_err(|e| { BlockchainClientConnectionError::ProviderError(format!( @@ -127,11 +123,7 @@ impl TokenContract for IERC721::IERC721Calls { let provider = read.get_client(); let client = provider.root(); let caller = read.get_address(); - let signer = get_blockchain_client_connection() - .await - .read() - .await - .get_signer(); + let wallet = read.get_wallet_type().clone(); let nonce = client.get_transaction_count(caller).await.map_err(|e| { BlockchainClientConnectionError::ProviderError(format!( @@ -152,8 +144,8 @@ impl TokenContract for IERC721::IERC721Calls { .gas(gas_limit) .max_fee_per_gas(max_fee_per_gas) .max_priority_fee_per_gas(max_priority_fee_per_gas) - .chain_id(get_settings().network.chain_id) // Linea testnet chain ID - .build_raw_transaction((*signer).clone()) + .chain_id(get_settings().network.chain_id) + .build_raw_transaction(wallet) .await .map_err(|e| { BlockchainClientConnectionError::ProviderError(format!("Contract error: {e}")) @@ -210,11 +202,7 @@ impl TokenContract for IERC1155::IERC1155Calls { let provider = read.get_client(); let client = provider.root(); let caller = read.get_address(); - let signer = get_blockchain_client_connection() - .await - .read() - .await - .get_signer(); + let wallet = read.get_wallet_type().clone(); let erc1155 = IERC1155::new(solidity_erc_address.0, client.clone()); @@ -237,8 +225,8 @@ impl TokenContract for IERC1155::IERC1155Calls { .gas(gas_limit) .max_fee_per_gas(max_fee_per_gas) .max_priority_fee_per_gas(max_priority_fee_per_gas) - .chain_id(get_settings().network.chain_id) // Linea testnet chain ID - .build_raw_transaction((*signer).clone()) + .chain_id(get_settings().network.chain_id) + .build_raw_transaction(wallet) .await .map_err(|e| { BlockchainClientConnectionError::ProviderError(format!( @@ -292,11 +280,7 @@ impl TokenContract for IERC3525::IERC3525Calls { let provider = read.get_client(); let client = provider.root(); let caller = read.get_address(); - let signer = get_blockchain_client_connection() - .await - .read() - .await - .get_signer(); + let wallet = read.get_wallet_type().clone(); debug!("ERC3525 caller: {caller:?}"); // NOTE: IERC3525 has overloaded approve functions in many implementations. @@ -322,8 +306,8 @@ impl TokenContract for IERC3525::IERC3525Calls { .gas(gas_limit) .max_fee_per_gas(max_fee_per_gas) .max_priority_fee_per_gas(max_priority_fee_per_gas) - .chain_id(get_settings().network.chain_id) // Linea testnet chain ID - .build_raw_transaction((*signer).clone()) + .chain_id(get_settings().network.chain_id) + .build_raw_transaction(wallet) .await .map_err(|e| { BlockchainClientConnectionError::ProviderError(format!( diff --git a/nightfall_client/src/driven/db/mongo.rs b/nightfall_client/src/driven/db/mongo.rs index 803cd3a9..c4211d90 100644 --- a/nightfall_client/src/driven/db/mongo.rs +++ b/nightfall_client/src/driven/db/mongo.rs @@ -166,6 +166,8 @@ impl RequestDB for Client { uuid: request_id.to_string(), status, child_request_args: None, + tx_hash: None, + receipt_token: None, }; let result = self .database(DB) @@ -246,6 +248,36 @@ impl RequestDB for Client { debug!("{request_id} Successfully cleared child_request_args"); Some(()) } + + async fn set_request_tx_hash(&self, request_id: &str, tx_hash: &str) -> Option<()> { + let filter = doc! { "uuid": request_id }; + let update = doc! { "$set": { "tx_hash": tx_hash } }; + let result = self + .database(DB) + .collection::("requests") + .update_one(filter, update) + .await; + if let Err(e) = result { + error!("{request_id} Got an error setting tx_hash: {e}"); + return None; + } + Some(()) + } + + async fn set_request_receipt_token(&self, request_id: &str, receipt_token: &str) -> Option<()> { + let filter = doc! { "uuid": request_id }; + let update = doc! { "$set": { "receipt_token": receipt_token } }; + let result = self + .database(DB) + .collection::("requests") + .update_one(filter, update) + .await; + if let Err(e) = result { + error!("{request_id} Got an error setting receipt_token: {e}"); + return None; + } + Some(()) + } } #[derive(Clone, Debug, Deserialize, Serialize, Default)] @@ -542,9 +574,9 @@ impl CommitmentDB for Client { Ok(reserved_commitments) } - async fn get_available_commitments(&self, nf_token_id: Fr254) -> Option> { + async fn get_available_commitments(&self, nf_slot_id: Fr254) -> Option> { let filter = doc! { - "preimage.nf_token_id": nf_token_id.to_hex_string(), + "preimage.nf_slot_id": nf_slot_id.to_hex_string(), "status": "Unspent" }; let mut cursor = self diff --git a/nightfall_client/src/driven/plonk_prover_tests/unified_circuit_tests.rs b/nightfall_client/src/driven/plonk_prover_tests/unified_circuit_tests.rs index 7e16361b..0aca66cf 100644 --- a/nightfall_client/src/driven/plonk_prover_tests/unified_circuit_tests.rs +++ b/nightfall_client/src/driven/plonk_prover_tests/unified_circuit_tests.rs @@ -32,10 +32,10 @@ mod tests { derive_key::ZKPKeys, hex_conversion::HexConvertible, nf_client_proof::{PrivateInputs, PublicInputs}, - nf_token_id::to_nf_token_id_from_str, + nf_token_id::{to_nf_slot_id_from_str, to_nf_token_id_from_str}, plonk_prover::circuits::{unified_circuit::unified_circuit_builder, DOMAIN_SHARED_SALT}, secret_hash::SecretHash, - shared_entities::{DepositData, DepositSecret, Preimage, Salt}, + shared_entities::{DepositData, DepositSecret, Preimage, Salt, TokenType}, }; use nf_curves::ed_on_bn254::{BabyJubjub, Fq as Fr254, Fr as BJJScalar}; use num_bigint::BigUint; @@ -283,7 +283,13 @@ mod tests { let token_id_string = Fr254::to_hex_string(&token_id_fr); let nf_token_id = to_nf_token_id_from_str(&erc_address_string, &token_id_string).unwrap(); - let nf_slot_id = nf_token_id; + let nf_slot_id = to_nf_slot_id_from_str( + &erc_address_string, + &token_id_string, + &token_id_string, + TokenType::ERC20, + ) + .unwrap(); // make a random Nightfall address let mut bytes = rand::thread_rng(); @@ -397,6 +403,7 @@ mod tests { .fee_token_id(fee_token_id) .nf_address(nf_address_h160) .value_a(value) + .spent_nf_token_ids([nf_token_id, nf_token_id]) .nf_token_a_id(nf_token_id) .nf_slot_id(nf_slot_id) .ephemeral_key(ephemeral_key) @@ -544,7 +551,13 @@ mod tests { let token_id_string = Fr254::to_hex_string(&token_id_fr); let nf_token_id = to_nf_token_id_from_str(&erc_address_string, &token_id_string).unwrap(); - let nf_slot_id = nf_token_id; + let nf_slot_id = to_nf_slot_id_from_str( + &erc_address_string, + &token_id_string, + &token_id_string, + TokenType::ERC20, + ) + .unwrap(); let mut bytes = rand::thread_rng(); let nf_address_h160 = Address::new(bytes.gen()); @@ -606,6 +619,7 @@ mod tests { .fee_token_id(fee_token_id) .nf_address(nf_address_h160) .value_a(value) + .spent_nf_token_ids([nf_token_id, nf_token_id]) .nf_token_a_id(nf_token_id) .nf_slot_id(nf_slot_id) .ephemeral_key(ephemeral_key) @@ -721,7 +735,13 @@ mod tests { let token_id_string = Fr254::to_hex_string(&token_id_fr); let nf_token_id = to_nf_token_id_from_str(&erc_address_string, &token_id_string).unwrap(); - let nf_slot_id = nf_token_id; + let nf_slot_id = to_nf_slot_id_from_str( + &erc_address_string, + &token_id_string, + &token_id_string, + TokenType::ERC20, + ) + .unwrap(); // Generate a random withdraw address let mut withdraw_address_bytes: [u8; 20] = [0; 20]; // Initialize with zeros @@ -844,6 +864,7 @@ mod tests { .fee_token_id(fee_token_id) .nf_address(nf_address_h160) .value_a(value) + .spent_nf_token_ids([nf_token_id, nf_token_id]) .nf_token_a_id(nf_token_id) .nf_slot_id(nf_slot_id) .ephemeral_key(ephemeral_key) @@ -983,6 +1004,13 @@ mod tests { let token_id_a_string = Fr254::to_hex_string(&token_id_a_fr); let nf_token_a_id = to_nf_token_id_from_str(&erc_address_a_string, &token_id_a_string).unwrap(); + let nf_slot_a_id = to_nf_slot_id_from_str( + &erc_address_a_string, + &token_id_a_string, + &token_id_a_string, + TokenType::ERC20, + ) + .unwrap(); let erc_address_b: [u8; 20] = rand::thread_rng().gen(); let erc_address_b_string = format!("0x{}", hex::encode(erc_address_b)); @@ -1032,7 +1060,7 @@ mod tests { // I am Party A, so I send token_a with value_a let nf_token_id = nf_token_a_id; - let nf_slot_id = nf_token_id; + let nf_slot_id = nf_slot_a_id; let nullified_one = Preimage::new( nullified_value_one, @@ -1099,6 +1127,7 @@ mod tests { .fee_token_id(fee_token_id) .nf_address(nf_address_h160) .value_a(value) + .spent_nf_token_ids([nf_token_id, nf_token_id]) .nf_token_a_id(nf_token_a_id) .nf_slot_id(nf_slot_id) .ephemeral_key(ephemeral_key) @@ -1289,6 +1318,13 @@ mod tests { let token_id_b_string = Fr254::to_hex_string(&token_id_b_fr); let nf_token_b_id = to_nf_token_id_from_str(&erc_address_b_string, &token_id_b_string).unwrap(); + let nf_slot_b_id = to_nf_slot_id_from_str( + &erc_address_b_string, + &token_id_b_string, + &token_id_b_string, + TokenType::ERC20, + ) + .unwrap(); // Fee setup let nf_address_h160 = Address::new(rand::thread_rng().gen()); @@ -1320,7 +1356,7 @@ mod tests { let deadline = Fr254::from(1000u64); let nf_token_id = nf_token_b_id; - let nf_slot_id = nf_token_id; + let nf_slot_id = nf_slot_b_id; let nullified_one = Preimage::new( nullified_value_one, @@ -1385,6 +1421,7 @@ mod tests { .fee_token_id(fee_token_id) .nf_address(nf_address_h160) .value_a(value_a) + .spent_nf_token_ids([nf_token_id, nf_token_id]) .nf_token_a_id(nf_token_a_id) .nf_slot_id(nf_slot_id) .ephemeral_key(ephemeral_key) @@ -1540,7 +1577,9 @@ mod tests { keys_a: &ZKPKeys, keys_b: &ZKPKeys, nf_token_a_id: Fr254, + nf_slot_a_id: Fr254, nf_token_b_id: Fr254, + nf_slot_b_id: Fr254, value_a: Fr254, value_b: Fr254, swap_nonce: Fr254, @@ -1570,7 +1609,11 @@ mod tests { Fr254::zero(), ) }; - let nf_slot_id = nf_token_id; + let nf_slot_id = if prover_is_party_a { + nf_slot_a_id + } else { + nf_slot_b_id + }; let mut nullified_value_one = rand_96_bit(rng); let mut nullified_value_two = rand_96_bit(rng); @@ -1649,6 +1692,7 @@ mod tests { .fee_token_id(fee_token_id) .nf_address(nf_address_h160) .value_a(value_a) + .spent_nf_token_ids([nf_token_id, nf_token_id]) .nf_token_a_id(nf_token_a_id) .nf_slot_id(nf_slot_id) .ephemeral_key(ephemeral_key) @@ -1814,7 +1858,7 @@ mod tests { let secret_hash = deposit_secret.hash().expect("deposit secret hash"); let deposit_data = [ DepositData { - nf_token_id: info.private_inputs.nf_token_a_id, + nf_token_id: info.private_inputs.spent_nf_token_ids[0], nf_slot_id: info.private_inputs.nf_slot_id, value: info.private_inputs.nullifiers_values[0], secret_hash, @@ -2550,7 +2594,7 @@ mod tests { }), Box::new(|| { let mut info = build_valid_swap_inputs(); - info.private_inputs.nf_token_a_id = Fr254::from(999u64); + info.private_inputs.spent_nf_token_ids[0] = Fr254::from(999u64); info }), Box::new(|| { @@ -2570,15 +2614,18 @@ mod tests { }), ]; - for create_invalid in invalid_cases { + for (idx, create_invalid) in invalid_cases.into_iter().enumerate() { let mut info = create_invalid(); let circuit = unified_circuit_builder(&mut info.public_inputs, &mut info.private_inputs) .unwrap(); - assert!(circuit - .check_circuit_satisfiability(Vec::from(&info.public_inputs).as_slice()) - .is_err()); + assert!( + circuit + .check_circuit_satisfiability(Vec::from(&info.public_inputs).as_slice()) + .is_err(), + "invalid swap case {idx} unexpectedly satisfied the circuit" + ); } } } @@ -2627,6 +2674,13 @@ mod tests { let nf_token_a_id = to_nf_token_id_from_str(&erc_address_a_string, &Fr254::to_hex_string(&token_id_a)) .unwrap(); + let nf_slot_a_id = to_nf_slot_id_from_str( + &erc_address_a_string, + &Fr254::to_hex_string(&token_id_a), + &Fr254::to_hex_string(&token_id_a), + TokenType::ERC20, + ) + .unwrap(); let erc_address_b: [u8; 20] = rand::thread_rng().gen(); let erc_address_b_string = format!("0x{}", hex::encode(erc_address_b)); @@ -2634,6 +2688,13 @@ mod tests { let nf_token_b_id = to_nf_token_id_from_str(&erc_address_b_string, &Fr254::to_hex_string(&token_id_b)) .unwrap(); + let nf_slot_b_id = to_nf_slot_id_from_str( + &erc_address_b_string, + &Fr254::to_hex_string(&token_id_b), + &Fr254::to_hex_string(&token_id_b), + TokenType::ERC20, + ) + .unwrap(); let nf_address_h160 = Address::new(rand::thread_rng().gen()); let nf_address_token = nf_address_h160.tokenize(); @@ -2670,7 +2731,9 @@ mod tests { &keys_a, &keys_b, nf_token_a_id, + nf_slot_a_id, nf_token_b_id, + nf_slot_b_id, value_a, value_b, swap_nonce, @@ -2685,7 +2748,9 @@ mod tests { &keys_a, &keys_b, nf_token_a_id, + nf_slot_a_id, nf_token_b_id, + nf_slot_b_id, value_a, value_b, swap_nonce, diff --git a/nightfall_client/src/driven/primitives/kemdem_functions.rs b/nightfall_client/src/driven/primitives/kemdem_functions.rs index b13d5291..a25da863 100644 --- a/nightfall_client/src/driven/primitives/kemdem_functions.rs +++ b/nightfall_client/src/driven/primitives/kemdem_functions.rs @@ -2,7 +2,7 @@ use ark_ec::twisted_edwards::Affine as TEAffine; use ark_ff::{BigInteger, One, PrimeField, Zero}; use ark_serialize::{CanonicalDeserialize, CanonicalSerialize}; use jf_primitives::poseidon::{FieldHasher, Poseidon, PoseidonError}; -use lib::plonk_prover::circuits::DOMAIN_SHARED_SALT; +use lib::plonk_prover::circuits::{DOMAIN_RECEIPT_DEM, DOMAIN_RECEIPT_KEM, DOMAIN_SHARED_SALT}; use log::error; use nf_curves::ed_on_bn254::{BabyJubjub, Fq as Fr254, Fr as BJJScalar}; @@ -81,11 +81,18 @@ pub fn kemdem_decrypt( recipient_private_key: BJJScalar, cipher_text: &[Fr254], ) -> Result, PoseidonError> { + if cipher_text.len() < 5 || !is_valid_epk_sign_flag(&cipher_text[4]) { + return Err(PoseidonError::InvalidInputs); + } + // First we decompress the epk let mut point_bytes = cipher_text[3].into_bigint().to_bytes_le(); + if point_bytes.len() != 32 { + return Err(PoseidonError::InvalidInputs); + } let flag = cipher_text[4].into_bigint().to_bytes_le()[0] << 7; - // We can hard index into point_bytes because we know it is 32 bytes long. - point_bytes[31] += flag; + // The BabyJubjub compressed point encoding is exactly 32 bytes with the x-sign bit stored in the high bit. + point_bytes[31] |= flag; let epk = TEAffine::::deserialize_compressed(&*point_bytes) .map_err(|_| PoseidonError::InvalidInputs)?; @@ -114,6 +121,162 @@ pub fn kemdem_decrypt( Ok(plain_text) } +use lib::shared_entities::TokenType; + +/// Input plaintext for [`receipt_kemdem_encrypt`]. +pub struct ReceiptEncryptInput { + pub sender_public_key_x: Fr254, + pub sender_public_key_y: Fr254, + pub erc_address: Fr254, + pub token_type: TokenType, + /// `token_id` for ERC721; `value` for ERC20. + pub token_id_or_value: Fr254, + pub receiver_commitment: Fr254, +} + +/// Decrypted transfer receipt payload. +/// +/// `token_id_or_value` holds the `token_id` when `token_type` is `ERC721`, +/// and the `value` when `token_type` is `ERC20`. +#[derive(Debug, PartialEq)] +pub struct ReceiptDecryptOutput { + pub sender_public_key_x: Fr254, + pub sender_public_key_y: Fr254, + pub erc_address: Fr254, + pub token_type: TokenType, + pub token_id_or_value: Fr254, + pub receiver_commitment: Fr254, + pub shared_salt: Fr254, +} + +/// Encrypt a receipt payload using receipt-specific KEM-DEM domain separators. +/// +/// Produces a `[Fr254; 9]` ciphertext: +/// `[0..6]` = 7 encrypted plaintext fields +/// `[7]` = ephemeral public key y-coordinate +/// `[8]` = sign flag for ephemeral public key x-coordinate +/// +/// Plaintext field layout (indices 0–6): +/// 0: `sender_public_key_x` +/// 1: `sender_public_key_y` +/// 2: `erc_address` +/// 3: `token_type` (`0` = ERC20, `2` = ERC721) +/// 4: `token_id` (ERC721) or `value` (ERC20) +/// 5: `receiver_commitment` +/// 6: reserved (zero) +pub fn receipt_kemdem_encrypt( + ephemeral_private_key: BJJScalar, + recipient_public_key: TEAffine, + input: &ReceiptEncryptInput, + public_point: TEAffine, +) -> Result<[Fr254; 9], PoseidonError> { + let plain_text = [ + input.sender_public_key_x, + input.sender_public_key_y, + input.erc_address, + Fr254::from(u8::from(input.token_type) as u64), + input.token_id_or_value, + input.receiver_commitment, + Fr254::zero(), // reserved + ]; + let plain_text = &plain_text; + let shared_secret: TEAffine = (recipient_public_key * ephemeral_private_key).into(); + let poseidon = Poseidon::::new(); + + let ephemeral_public_key: TEAffine = (public_point * ephemeral_private_key).into(); + + // Derive encryption key using receipt-specific KEM domain separator. + let encryption_key = poseidon.hash(&[shared_secret.x, shared_secret.y, DOMAIN_RECEIPT_KEM])?; + + // Encrypt all 7 plaintext fields using receipt-specific DEM domain separator. + let mut cipher_text = [Fr254::zero(); 9]; + for (i, plain) in plain_text.iter().enumerate() { + let pad: Fr254 = + poseidon.hash(&[encryption_key, DOMAIN_RECEIPT_DEM, Fr254::from(i as u64)])?; + cipher_text[i] = pad + *plain; + } + + // Append compressed ephemeral public key. + let flag = if ephemeral_public_key.x > -ephemeral_public_key.x { + Fr254::one() + } else { + Fr254::zero() + }; + cipher_text[7] = ephemeral_public_key.y; + cipher_text[8] = flag; + + Ok(cipher_text) +} + +/// Decrypt a receipt ciphertext using receipt-specific KEM-DEM domain separators. +/// +/// Expects a `[Fr254; 9]` ciphertext as produced by [`receipt_kemdem_encrypt`]. +/// Returns a [`ReceiptDecryptOutput`] struct matching the encoded token type. +/// Returns `Err` if the sign flag is invalid or the token type is not supported by +/// the MVP receipt payload (`ERC20` or `ERC721`). +pub fn receipt_kemdem_decrypt( + recipient_private_key: BJJScalar, + cipher_text: &[Fr254; 9], +) -> Result { + if !is_valid_epk_sign_flag(&cipher_text[8]) { + return Err(PoseidonError::InvalidInputs); + } + + // Reconstruct the ephemeral public key from cipher_text[7] (epk.y) and [8] (flag). + let mut point_bytes = cipher_text[7].into_bigint().to_bytes_le(); + if point_bytes.len() != 32 { + return Err(PoseidonError::InvalidInputs); + } + let flag = cipher_text[8].into_bigint().to_bytes_le()[0] << 7; + // The BabyJubjub compressed point encoding is exactly 32 bytes with the x-sign bit stored in the high bit. + point_bytes[31] |= flag; + + let epk = TEAffine::::deserialize_compressed(&*point_bytes) + .map_err(|_| PoseidonError::InvalidInputs)?; + + // Compute shared secret and derive decryption key. + let shared_secret: TEAffine = (epk * recipient_private_key).into(); + let poseidon = Poseidon::::new(); + let decryption_key = poseidon.hash(&[shared_secret.x, shared_secret.y, DOMAIN_RECEIPT_KEM])?; + + // Decrypt the 7 plaintext fields. + let mut plain = [Fr254::zero(); 7]; + for (i, ct) in cipher_text.iter().take(7).enumerate() { + let pad: Fr254 = + poseidon.hash(&[decryption_key, DOMAIN_RECEIPT_DEM, Fr254::from(i as u64)])?; + plain[i] = *ct - pad; + } + + // Derive shared salt with the standard domain separator. + let shared_salt = poseidon.hash(&[shared_secret.x, shared_secret.y, DOMAIN_SHARED_SALT])?; + + // plain layout: [spk_x, spk_y, erc_address, token_type, token_id_or_value, receiver_commitment, reserved] + let token_type_byte = plain[3].into_bigint().to_bytes_le()[0]; + let token_type = decode_receipt_token_type(token_type_byte)?; + + Ok(ReceiptDecryptOutput { + sender_public_key_x: plain[0], + sender_public_key_y: plain[1], + erc_address: plain[2], + token_type, + token_id_or_value: plain[4], + receiver_commitment: plain[5], + shared_salt, + }) +} + +fn is_valid_epk_sign_flag(flag: &Fr254) -> bool { + *flag == Fr254::zero() || *flag == Fr254::one() +} + +fn decode_receipt_token_type(token_type_byte: u8) -> Result { + match token_type_byte { + 0 => Ok(TokenType::ERC20), + 2 => Ok(TokenType::ERC721), + _ => Err(PoseidonError::InvalidInputs), + } +} + #[cfg(test)] mod tests { use super::*; @@ -152,8 +315,9 @@ mod tests { &plain_text, public_point, ) - .unwrap(); - let decrypted = kemdem_decrypt(recipient_private_key, &cipher_text).unwrap(); + .expect("kemdem_encrypt should succeed for transfer"); + let decrypted = kemdem_decrypt(recipient_private_key, &cipher_text) + .expect("kemdem_decrypt should succeed for transfer"); for (plain_text, decrypted_text) in plain_text.iter().zip(decrypted.iter()) { assert_eq!(*plain_text, *decrypted_text); } @@ -165,7 +329,7 @@ mod tests { // Derive a shared salt from the shared secret using domain-separated Poseidon hash. let shared_salt = poseidon .hash(&[shared_secret.x, shared_secret.y, DOMAIN_SHARED_SALT]) - .unwrap(); + .expect("Poseidon hash should succeed for shared salt"); assert_eq!(shared_salt, decrypted[3]); } @@ -184,8 +348,310 @@ mod tests { &plain_text, public_point, ) - .unwrap(); + .expect("kemdem_encrypt should succeed for withdraw"); assert_eq!(&plain_text, &cipher_text[..3]); } + + #[test] + fn compute_receipt_domain_constants() { + use sha2::{Digest, Sha256}; + + let kem_digest = Sha256::digest("Nightfall|ReceiptKEM".as_bytes()); + let dem_digest = Sha256::digest("Nightfall|ReceiptDEM".as_bytes()); + + let expected_kem = Fr254::from_le_bytes_mod_order(&kem_digest); + let expected_dem = Fr254::from_le_bytes_mod_order(&dem_digest); + + assert_eq!(expected_kem, DOMAIN_RECEIPT_KEM); + assert_eq!(expected_dem, DOMAIN_RECEIPT_DEM); + } + + #[test] + fn test_receipt_kemdem_round_trip_erc721() { + let rng = &mut test_rng(); + let recipient_private_key = BJJScalar::rand(rng); + let ephemeral_private_key = BJJScalar::rand(rng); + let public_point = TEAffine::::new(GENERATOR_X, GENERATOR_Y); + let recipient_public_key: TEAffine = + (public_point * recipient_private_key).into(); + + let input = ReceiptEncryptInput { + sender_public_key_x: Fr254::rand(rng), + sender_public_key_y: Fr254::rand(rng), + erc_address: Fr254::rand(rng), + token_type: TokenType::ERC721, + token_id_or_value: Fr254::rand(rng), + receiver_commitment: Fr254::rand(rng), + }; + + let cipher_text = receipt_kemdem_encrypt( + ephemeral_private_key, + recipient_public_key, + &input, + public_point, + ) + .expect("receipt_kemdem_encrypt should succeed for ERC721"); + + assert_eq!(cipher_text.len(), 9); + + let out = receipt_kemdem_decrypt(recipient_private_key, &cipher_text) + .expect("receipt_kemdem_decrypt should succeed for ERC721"); + + assert_eq!(out.sender_public_key_x, input.sender_public_key_x); + assert_eq!(out.sender_public_key_y, input.sender_public_key_y); + assert_eq!(out.erc_address, input.erc_address); + assert_eq!(out.token_type, TokenType::ERC721); + assert_eq!(out.token_id_or_value, input.token_id_or_value); + assert_eq!(out.receiver_commitment, input.receiver_commitment); + } + + #[test] + fn test_receipt_kemdem_round_trip_erc20() { + let rng = &mut test_rng(); + let recipient_private_key = BJJScalar::rand(rng); + let ephemeral_private_key = BJJScalar::rand(rng); + let public_point = TEAffine::::new(GENERATOR_X, GENERATOR_Y); + let recipient_public_key: TEAffine = + (public_point * recipient_private_key).into(); + + let input = ReceiptEncryptInput { + sender_public_key_x: Fr254::rand(rng), + sender_public_key_y: Fr254::rand(rng), + erc_address: Fr254::rand(rng), + token_type: TokenType::ERC20, + token_id_or_value: Fr254::rand(rng), + receiver_commitment: Fr254::rand(rng), + }; + + let cipher_text = receipt_kemdem_encrypt( + ephemeral_private_key, + recipient_public_key, + &input, + public_point, + ) + .expect("receipt_kemdem_encrypt should succeed for ERC20"); + + assert_eq!(cipher_text.len(), 9); + + let out = receipt_kemdem_decrypt(recipient_private_key, &cipher_text) + .expect("receipt_kemdem_decrypt should succeed for ERC20"); + + assert_eq!(out.sender_public_key_x, input.sender_public_key_x); + assert_eq!(out.sender_public_key_y, input.sender_public_key_y); + assert_eq!(out.erc_address, input.erc_address); + assert_eq!(out.token_type, TokenType::ERC20); + assert_eq!(out.token_id_or_value, input.token_id_or_value); + assert_eq!(out.receiver_commitment, input.receiver_commitment); + } + + #[test] + fn test_receipt_kemdem_ecdh_consistency() { + let rng = &mut test_rng(); + let recipient_private_key = BJJScalar::rand(rng); + let ephemeral_private_key = BJJScalar::rand(rng); + let public_point = TEAffine::::new(GENERATOR_X, GENERATOR_Y); + let recipient_public_key: TEAffine = + (public_point * recipient_private_key).into(); + let ephemeral_public_key: TEAffine = + (public_point * ephemeral_private_key).into(); + + let shared_secret_sender: TEAffine = + (recipient_public_key * ephemeral_private_key).into(); + let shared_secret_recipient: TEAffine = + (ephemeral_public_key * recipient_private_key).into(); + + assert_eq!(shared_secret_sender, shared_secret_recipient); + } + + #[test] + fn test_receipt_kemdem_domain_separation() { + let rng = &mut test_rng(); + let recipient_private_key = BJJScalar::rand(rng); + let ephemeral_private_key = BJJScalar::rand(rng); + let public_point = TEAffine::::new(GENERATOR_X, GENERATOR_Y); + let recipient_public_key: TEAffine = + (public_point * recipient_private_key).into(); + + // Encrypt 3 fields using the regular protocol KEM-DEM + let proto_plain = vec![Fr254::rand(rng), Fr254::rand(rng), Fr254::rand(rng)]; + let proto_ct = kemdem_encrypt::( + ephemeral_private_key, + recipient_public_key, + &proto_plain, + public_point, + ) + .expect("kemdem_encrypt should succeed for proto plaintext"); + + // Encrypt using receipt KEM-DEM — first 3 plaintext fields share values with proto_plain + let receipt_input = ReceiptEncryptInput { + sender_public_key_x: proto_plain[0], + sender_public_key_y: proto_plain[1], + erc_address: proto_plain[2], + token_type: TokenType::ERC721, + token_id_or_value: Fr254::rand(rng), + receiver_commitment: Fr254::rand(rng), + }; + let receipt_ct = receipt_kemdem_encrypt( + ephemeral_private_key, + recipient_public_key, + &receipt_input, + public_point, + ) + .expect("receipt_kemdem_encrypt should succeed for domain separation test"); + + // Domain separation: the first 3 ciphertext elements must differ + // even though the plaintext and ephemeral key are the same. + assert_ne!(proto_ct[0], receipt_ct[0]); + assert_ne!(proto_ct[1], receipt_ct[1]); + assert_ne!(proto_ct[2], receipt_ct[2]); + } + + #[test] + fn test_receipt_kemdem_wrong_key_fails() { + let rng = &mut test_rng(); + let recipient_private_key = BJJScalar::rand(rng); + let wrong_private_key = BJJScalar::rand(rng); + let ephemeral_private_key = BJJScalar::rand(rng); + let public_point = TEAffine::::new(GENERATOR_X, GENERATOR_Y); + let recipient_public_key: TEAffine = + (public_point * recipient_private_key).into(); + + let input = ReceiptEncryptInput { + sender_public_key_x: Fr254::rand(rng), + sender_public_key_y: Fr254::rand(rng), + erc_address: Fr254::rand(rng), + token_type: TokenType::ERC721, + token_id_or_value: Fr254::rand(rng), + receiver_commitment: Fr254::rand(rng), + }; + + let cipher_text = receipt_kemdem_encrypt( + ephemeral_private_key, + recipient_public_key, + &input, + public_point, + ) + .expect("receipt_kemdem_encrypt should succeed before testing wrong-key decryption"); + + // Decrypt with wrong key. In the overwhelming majority of cases the token-type field + // will be invalid for the MVP receipt format and decryption should fail. + match receipt_kemdem_decrypt(wrong_private_key, &cipher_text) { + Err(PoseidonError::InvalidInputs) => {} + Ok(out) => assert_ne!(out.sender_public_key_x, input.sender_public_key_x), + } + } + + #[test] + fn test_receipt_kemdem_rejects_unsupported_token_type() { + let rng = &mut test_rng(); + let recipient_private_key = BJJScalar::rand(rng); + let ephemeral_private_key = BJJScalar::rand(rng); + let public_point = TEAffine::::new(GENERATOR_X, GENERATOR_Y); + let recipient_public_key: TEAffine = + (public_point * recipient_private_key).into(); + + let input = ReceiptEncryptInput { + sender_public_key_x: Fr254::rand(rng), + sender_public_key_y: Fr254::rand(rng), + erc_address: Fr254::rand(rng), + token_type: TokenType::ERC20, + token_id_or_value: Fr254::rand(rng), + receiver_commitment: Fr254::rand(rng), + }; + + let mut cipher_text = receipt_kemdem_encrypt( + ephemeral_private_key, + recipient_public_key, + &input, + public_point, + ) + .expect("receipt_kemdem_encrypt should succeed before tampering with token type"); + + let shared_secret: TEAffine = + (recipient_public_key * ephemeral_private_key).into(); + let poseidon = Poseidon::::new(); + let encryption_key = poseidon + .hash(&[shared_secret.x, shared_secret.y, DOMAIN_RECEIPT_KEM]) + .expect("Poseidon hash should succeed for encryption key"); + let token_type_index = 3u64; + let token_type_pad = poseidon + .hash(&[ + encryption_key, + DOMAIN_RECEIPT_DEM, + Fr254::from(token_type_index), + ]) + .expect("Poseidon hash should succeed for token type pad"); + cipher_text[3] = token_type_pad + Fr254::from(1u64); + + assert!(matches!( + receipt_kemdem_decrypt(recipient_private_key, &cipher_text), + Err(PoseidonError::InvalidInputs) + )); + } + + #[test] + fn test_receipt_kemdem_rejects_invalid_sign_flag() { + let rng = &mut test_rng(); + let recipient_private_key = BJJScalar::rand(rng); + let ephemeral_private_key = BJJScalar::rand(rng); + let public_point = TEAffine::::new(GENERATOR_X, GENERATOR_Y); + let recipient_public_key: TEAffine = + (public_point * recipient_private_key).into(); + + let input = ReceiptEncryptInput { + sender_public_key_x: Fr254::rand(rng), + sender_public_key_y: Fr254::rand(rng), + erc_address: Fr254::rand(rng), + token_type: TokenType::ERC20, + token_id_or_value: Fr254::rand(rng), + receiver_commitment: Fr254::rand(rng), + }; + + let mut cipher_text = receipt_kemdem_encrypt( + ephemeral_private_key, + recipient_public_key, + &input, + public_point, + ) + .expect("receipt_kemdem_encrypt should succeed before tampering with sign flag"); + cipher_text[8] = Fr254::from(2u64); + + assert!(receipt_kemdem_decrypt(recipient_private_key, &cipher_text).is_err()); + } + + #[test] + fn test_receipt_ciphertext_size() { + let rng = &mut test_rng(); + let recipient_private_key = BJJScalar::rand(rng); + let ephemeral_private_key = BJJScalar::rand(rng); + let public_point = TEAffine::::new(GENERATOR_X, GENERATOR_Y); + let recipient_public_key: TEAffine = + (public_point * recipient_private_key).into(); + + let input = ReceiptEncryptInput { + sender_public_key_x: Fr254::rand(rng), + sender_public_key_y: Fr254::rand(rng), + erc_address: Fr254::rand(rng), + token_type: TokenType::ERC20, + token_id_or_value: Fr254::rand(rng), + receiver_commitment: Fr254::rand(rng), + }; + + let cipher_text = receipt_kemdem_encrypt( + ephemeral_private_key, + recipient_public_key, + &input, + public_point, + ) + .expect("receipt_kemdem_encrypt should succeed for size check"); + + let mut serialized_cipher_text = Vec::new(); + cipher_text + .serialize_uncompressed(&mut serialized_cipher_text) + .expect("receipt ciphertext should serialize successfully"); + + assert_eq!(cipher_text.len(), 9); + assert_eq!(serialized_cipher_text.len(), 288); + } } diff --git a/nightfall_client/src/driven/queue.rs b/nightfall_client/src/driven/queue.rs index 59eba83d..a13d0048 100644 --- a/nightfall_client/src/driven/queue.rs +++ b/nightfall_client/src/driven/queue.rs @@ -130,6 +130,8 @@ mod tests { status, uuid: "test-request".to_string(), child_request_args: None, + tx_hash: None, + receipt_token: None, } } diff --git a/nightfall_client/src/drivers/rest/client_nf_3.rs b/nightfall_client/src/drivers/rest/client_nf_3.rs index b7b344d0..7ab4267c 100644 --- a/nightfall_client/src/drivers/rest/client_nf_3.rs +++ b/nightfall_client/src/drivers/rest/client_nf_3.rs @@ -246,6 +246,25 @@ fn validate_withdraw_request_payload(req: &NF3WithdrawRequest) -> Result<(), Str validate_asset_constraints(token_type, value, token_id) } +async fn resolve_target_slot_id_from_token_id( + db: &mongodb::Client, + nf_token_id: Fr254, +) -> Result { + db.get_all_commitments() + .await + .ok() + .and_then(|entries| { + entries.into_iter().map(|(_, entry)| entry).find(|entry| { + entry.get_status() == CommitmentStatus::Unspent + && entry.get_nf_token_id() == nf_token_id + }) + }) + .map(|entry| entry.get_nf_slot_id()) + .ok_or_else(|| { + TransactionHandlerError::CustomError("Requested token is not yet deposited".to_string()) + }) +} + pub fn cancel_swap_request( ) -> impl Filter + Clone { path!("v1" / "swap" / "cancel-request") @@ -1364,10 +1383,13 @@ where { let db = get_db_connection().await; let fee_token_id = get_fee_token_id(); - let spend_value_commitments = find_usable_commitments(nf_token_id, value,db) - .await.map_err(|e|{ - error!("{id} Could not find enough usable value commitments to complete this transfer, suggest depositing more tokens: {e}"); - TransactionHandlerError::CustomError(e.to_string())})?; + let nf_slot_id = resolve_target_slot_id_from_token_id(db, nf_token_id).await?; + let spend_value_commitments = find_usable_commitments(nf_slot_id, value, db) + .await + .map_err(|e| { + error!("{id} Could not find enough usable value commitments to complete this transfer, suggest depositing more tokens: {e}"); + TransactionHandlerError::CustomError(e.to_string()) + })?; let spend_fee_commitments = if fee.is_zero() { [Preimage::default(), Preimage::default()] } else { @@ -1429,7 +1451,7 @@ where let new_commitment_two = if !token_change.is_zero() { Preimage::new( token_change, - nf_token_id, + spend_commitments[0].get_nf_token_id(), spend_commitments[0].get_nf_slot_id(), keys.zkp_public_key, Salt::new_transfer_salt(), @@ -1579,10 +1601,14 @@ where { let fee_token_id = get_fee_token_id(); - let spend_value_commitments = find_usable_commitments(nf_token_id, value,db) - .await.map_err(|e|{ - error!("{id} Could not find enough usable value commitments to complete this withdraw, suggest depositing more tokens: {e}"); - TransactionHandlerError::CustomError(e.to_string())})?; + //getting the slot_id for the token_id + let nf_slot_id = resolve_target_slot_id_from_token_id(db, nf_token_id).await?; + let spend_value_commitments = find_usable_commitments(nf_slot_id, value, db) + .await + .map_err(|e| { + error!("{id} Could not find enough usable value commitments to complete this withdraw, suggest depositing more tokens: {e}"); + TransactionHandlerError::CustomError(e.to_string()) + })?; let spend_fee_commitments = if fee.is_zero() { [Preimage::default(), Preimage::default()] } else { @@ -1638,7 +1664,7 @@ where let new_commitment_two = if !token_change.is_zero() { Preimage::new( token_change, - nf_token_id, + spend_commitments[0].get_nf_token_id(), spend_commitments[0].get_nf_slot_id(), keys.zkp_public_key, Salt::new_transfer_salt(), @@ -1899,21 +1925,21 @@ where let db = get_db_connection().await; let fee_token_id = get_fee_token_id(); - let spend_value_commitments = find_usable_commitments(nf_token_id, value, db) + let nf_slot_id = resolve_target_slot_id_from_token_id(db, nf_token_id).await?; + let spend_value_commitments = find_usable_commitments(nf_slot_id, value, db) .await .map_err(|e| { - error!("{id} Could not find enough usable value commitments for swap: {e}"); + error!("{id} Could not find enough usable value commitments to complete this withdraw, suggest depositing more tokens: {e}"); TransactionHandlerError::CustomError(e.to_string()) })?; - let spend_fee_commitments = if fee.is_zero() { [Preimage::default(), Preimage::default()] } else { match find_usable_commitments(fee_token_id, fee, db).await { Ok(commitments) => commitments, Err(e) => { - debug!("{id} Could not find enough usable fee commitments: {e}"); - // Rollback value commitments + error!("{id} Could not find enough usable fee commitments to complete this withdraw, suggest depositing more fee: {e}"); + // rollback the value commitments to unspent if fails to find fee commitments let value_commitment_ids = spend_value_commitments .iter() .filter_map(|c| c.hash().ok()) @@ -1924,7 +1950,6 @@ where } } }; - spend_commitments = [ spend_value_commitments[0], spend_value_commitments[1], @@ -1977,7 +2002,7 @@ where let new_commitment_two = if !token_change.is_zero() { Preimage::new( token_change, - nf_token_id, + spend_commitments[0].get_nf_token_id(), spend_commitments[0].get_nf_slot_id(), keys.zkp_public_key, Salt::new_transfer_salt(), @@ -2328,6 +2353,8 @@ mod tests { status: RequestStatus::Submitted, uuid: request_id.to_string(), child_request_args, + tx_hash: None, + receipt_token: None, } } @@ -3773,6 +3800,8 @@ mod tests { status: RequestStatus::Processing, uuid: request_id.to_string(), child_request_args: Some(child_args), + tx_hash: None, + receipt_token: None, }) .await; @@ -3800,6 +3829,8 @@ mod tests { status: RequestStatus::Expired, uuid: request_id.to_string(), child_request_args: None, + tx_hash: None, + receipt_token: None, }) .await; @@ -3838,6 +3869,8 @@ mod tests { status: RequestStatus::Expired, uuid: request_id.to_string(), child_request_args: Some(child_args), + tx_hash: None, + receipt_token: None, }) .await; db.push_commitment(mock_commitment( @@ -3960,6 +3993,8 @@ mod tests { status: RequestStatus::Expired, uuid: request_id.to_string(), child_request_args: Some(child_args), + tx_hash: None, + receipt_token: None, }) .await; @@ -4009,6 +4044,8 @@ mod tests { status: RequestStatus::Expired, uuid: request_id.to_string(), child_request_args: Some(child_args), + tx_hash: None, + receipt_token: None, }) .await; db.push_commitment(mock_commitment(commitment_id, CommitmentStatus::Unspent)) diff --git a/nightfall_client/src/drivers/rest/client_operation.rs b/nightfall_client/src/drivers/rest/client_operation.rs index e4597303..b6e29936 100644 --- a/nightfall_client/src/drivers/rest/client_operation.rs +++ b/nightfall_client/src/drivers/rest/client_operation.rs @@ -1,6 +1,6 @@ use crate::{ domain::{ - entities::{CommitmentStatus, Operation, RequestStatus}, + entities::{CommitmentStatus, Operation, OperationType, RequestStatus}, error::TransactionHandlerError, notifications::NotificationPayload, }, @@ -34,7 +34,7 @@ use nf_curves::ed_on_bn254::BabyJubjub; use nf_curves::ed_on_bn254::Fr as BJJScalar; use nightfall_bindings::artifacts::ProposerManager; use reqwest::{Client, Error as ReqwestError}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use std::{error::Error, fmt::Debug, time::Duration}; use tokio::time::sleep; use url::Url; @@ -61,6 +61,12 @@ pub struct SwapParams { pub struct SubmittedOperation

{ pub payload: NotificationPayload, pub transaction: ClientTransaction

, + pub receipt_token: Option, +} + +#[derive(Debug, Deserialize)] +struct ProposerTransactionAcceptedResponse { + receipt_token: Option, } #[allow(clippy::too_many_arguments)] @@ -130,9 +136,7 @@ where commitment_entry.native_slot_id = Some(slot_info.slot_id.to_hex_string()); } Err(e) => { - warn!( - "{id} Could not enrich commitment with native slot metadata: {e}" - ); + warn!("{id} Could not enrich commitment with native slot metadata: {e}"); } } commitment_entries.push(commitment_entry); @@ -192,9 +196,21 @@ where error!("{id} {e}"); TransactionHandlerError::CustomError(e.to_string()) })?; + + // Generate a single receipt_token for Transfer transactions before fan-out + // so every proposer receives and stores the same token. This ensures that + // the receipt_id derived from it (HMAC of receipt_token + tx_hash) is + // identical on every proposer regardless of which one the receiver queries. + if operation.operation_type == OperationType::Transfer { + use rand::{rngs::OsRng, RngCore}; + let mut bytes = [0u8; 32]; + OsRng.fill_bytes(&mut bytes); + operation_result.receipt_token = Some(hex::encode(bytes)); + } + // having done that, we can submit the nighfall transaction to proposer offchain - let tx_receipt = process_transaction_offchain(&operation_result, id) + let (tx_receipt, _receipt_token) = process_transaction_offchain(&operation_result, id) .await .map_err(|e| TransactionHandlerError::CustomError(e.to_string()))?; info!("{id} {} transaction submitted", operation.operation_type); @@ -234,9 +250,15 @@ where let uuid = serde_json::to_string(id).map_err(TransactionHandlerError::JsonConversionError)?; + // Use the locally-generated receipt_token as the authoritative value + // rather than the proposer-echoed one. The proposer stores and echoes the + // token unchanged, so the values are always identical — but sourcing it + // locally removes any implicit dependency on the proposer round-trip. + let authoritative_receipt_token = operation_result.receipt_token.clone(); Ok(SubmittedOperation { payload: NotificationPayload::TransactionEvent { response, uuid }, transaction: operation_result, + receipt_token: authoritative_receipt_token, }) } @@ -256,7 +278,7 @@ where E: ProvingEngine

+ Send + Sync, N: NightfallContract, { - Ok(submit_client_operation::( + let submitted = submit_client_operation::( operation, spend_commitments, new_commitments, @@ -266,8 +288,20 @@ where swap_params, id, ) - .await? - .payload) + .await?; + + // Persist the canonical proposer tx_hash on the request record so the + // wallet can retrieve it via GET /v1/request/{uuid} without needing a webhook. + if let Ok(hash_bytes) = submitted.transaction.hash() { + let tx_hash_hex: String = hash_bytes.iter().map(|b| format!("{b:02x}")).collect(); + let db = crate::initialisation::get_db_connection().await; + let _ = db.set_request_tx_hash(id, &tx_hash_hex).await; + if let Some(receipt_token) = submitted.receipt_token.as_deref() { + let _ = db.set_request_receipt_token(id, receipt_token).await; + } + } + + Ok(submitted.payload) } /// Only retry on network issues or timeouts @@ -282,7 +316,7 @@ async fn send_to_proposer_with_retry( id: &str, max_retries: u32, initial_backoff: Duration, -) -> Result<(), (String, bool)> { +) -> Result, (String, bool)> { let url = match Url::parse(&proposer.url).and_then(|base| base.join("/v1/transaction")) { Ok(u) => u, Err(e) => { @@ -304,8 +338,17 @@ async fn send_to_proposer_with_retry( Ok(response) => { let status = response.status(); if status.is_success() { + let response_body = response.text().await.unwrap_or_default(); + let parsed = + serde_json::from_str::(&response_body) + .map_err(|e| { + ( + format!("Failed to parse proposer transaction response: {e}"), + false, + ) + })?; debug!("{id} Successfully sent transaction to proposer at {url}"); - return Ok(()); + return Ok(parsed.receipt_token); } else { let body = response.text().await.unwrap_or_default(); error!("{id} Error from proposer: HTTP {status} — Body: {body}"); @@ -564,7 +607,7 @@ pub async fn request_swap_cancel( pub async fn process_transaction_offchain( l2_transaction: &ClientTransaction

, id: &str, -) -> Result, Box> { +) -> Result<(Option, Option), Box> { info!("{id} Sending client transaction to all proposers concurrently."); const MAX_RETRIES: u32 = 3; const INITIAL_BACKOFF: Duration = Duration::from_millis(500); @@ -600,11 +643,15 @@ pub async fn process_transaction_offchain( let mut any_success = false; let mut any_retriable_failures = false; + let mut receipt_token = None; for result in results { match result { - Ok(_) => { + Ok(token) => { any_success = true; + if receipt_token.is_none() { + receipt_token = token; + } } Err((msg, retriable)) => { warn!("{id} Proposer error: {msg}"); @@ -617,7 +664,7 @@ pub async fn process_transaction_offchain( if any_success { db.update_request(id, RequestStatus::Submitted).await; - Ok(None) + Ok((None, receipt_token)) } else if any_retriable_failures { db.update_request(id, RequestStatus::ProposerUnreachable) .await; diff --git a/nightfall_client/src/drivers/rest/mod.rs b/nightfall_client/src/drivers/rest/mod.rs index 1e8fae20..afe623a6 100644 --- a/nightfall_client/src/drivers/rest/mod.rs +++ b/nightfall_client/src/drivers/rest/mod.rs @@ -1,5 +1,6 @@ use crate::ports::contracts::NightfallContract; use balance::{get_balance, get_fee_balance, get_l1_balance}; +use configuration::settings::WalletRole; use lib::{ health_check::health_route, nf_client_proof::Proof, validate_certificate::certification_validation_request, validate_keys::keys_validation_request, @@ -27,6 +28,7 @@ use self::{ request_status::{get_queue_length, get_request_status}, synchronisation::synchronisation, token_info::get_token_info, + transfer_receipts::submit_transfer_receipt, }; pub mod balance; @@ -38,6 +40,7 @@ pub mod proposers; pub(crate) mod request_status; mod synchronisation; mod token_info; +pub mod transfer_receipts; pub mod withdraw; pub fn routes() -> impl Filter + Clone @@ -52,13 +55,14 @@ where .or(swap_request::

()) .or(cancel_swap_request()) .or(settle_expired_swap_request()) + .or(submit_transfer_receipt()) .or(get_commitment()) .or(get_all_commitments()) .or(get_commitments_by_token_type()) .or(get_max_transferable_amount_by_token_type()) .or(derive_key_mnemonic()) .or(get_proposers()) - .or(certification_validation_request()) + .or(certification_validation_request(WalletRole::Client)) .or(keys_validation_request()) .or(get_balance()) .or(get_fee_balance()) @@ -119,6 +123,26 @@ async fn handle_rejection(err: Rejection) -> Result Ok(reply::with_status( + "Invalid or missing receipt_token", + StatusCode::UNAUTHORIZED, + )), + ReceiptConflict => Ok(reply::with_status( + "Receipt already exists with different ciphertext", + StatusCode::CONFLICT, + )), + ReceiptTxNotFound => Ok(reply::with_status( + "Transaction not found on proposer", + StatusCode::BAD_REQUEST, + )), + ReceiptValidationFailed => Ok(reply::with_status( + "Transfer receipt request failed validation", + StatusCode::BAD_REQUEST, + )), + ReceiptSubmissionFailed => Ok(reply::with_status( + "Receipt submission failed: no proposer accepted the receipt", + StatusCode::SERVICE_UNAVAILABLE, + )), InvalidTokenType => Ok(reply::with_status( "Invalid Token Type", StatusCode::BAD_REQUEST, diff --git a/nightfall_client/src/drivers/rest/request_status.rs b/nightfall_client/src/drivers/rest/request_status.rs index 418fe8ff..ac3f72bf 100644 --- a/nightfall_client/src/drivers/rest/request_status.rs +++ b/nightfall_client/src/drivers/rest/request_status.rs @@ -124,6 +124,8 @@ mod tests { status, uuid: "test-request".to_string(), child_request_args, + tx_hash: None, + receipt_token: None, } } diff --git a/nightfall_client/src/drivers/rest/transfer_receipts.rs b/nightfall_client/src/drivers/rest/transfer_receipts.rs new file mode 100644 index 00000000..5fb11dd3 --- /dev/null +++ b/nightfall_client/src/drivers/rest/transfer_receipts.rs @@ -0,0 +1,264 @@ +use crate::domain::error::ClientRejection; +use configuration::addresses::get_addresses; +use futures::future::join_all; +use lib::{ + blockchain_client::BlockchainClientConnection, initialisation::get_blockchain_client_connection, +}; +use log::{debug, error, warn}; +use nightfall_bindings::artifacts::ProposerManager; +use reqwest::Client; +use serde::{Deserialize, Serialize}; +use std::time::Duration; +use tokio::time::sleep; +use url::Url; +use warp::{path, Filter, Rejection, Reply}; + +/// Request body accepted by `POST /v1/transfer-receipts` on the **client**. +/// It is forwarded verbatim to every proposer's matching endpoint. +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct SubmitReceiptRequest { + pub tx_hash: String, + pub ciphertext: String, + pub version: Option, + pub receipt_token: String, +} + +/// The subset of the proposer's `CreateTransferReceiptResponse` that the +/// client proxies back to the caller. +#[derive(Debug, Serialize, Deserialize)] +struct ProposerReceiptResponse { + receipt_id: String, + status: serde_json::Value, +} + +#[derive(Debug)] +enum ProposerReceiptError { + /// 401 from proposer — token mismatch. + Unauthorized, + /// 409 from proposer — different ciphertext already stored. + Conflict, + /// 400 from proposer — tx not yet in proposer mempool. + TxNotFound, + /// 400 from proposer — request body failed validation (e.g. malformed tx_hash). + ValidationFailed, + /// Network or unexpected HTTP error. + Other(String), +} + +pub fn submit_transfer_receipt() -> impl Filter + Clone +{ + path!("v1" / "transfer-receipts") + .and(warp::post()) + .and(warp::body::content_length_limit(1024 * 64)) + .and(warp::body::json()) + .and_then(handle_submit_transfer_receipt) +} + +async fn handle_submit_transfer_receipt( + request: SubmitReceiptRequest, +) -> Result { + const MAX_RETRIES: u32 = 3; + const INITIAL_BACKOFF: Duration = Duration::from_millis(500); + + let client = Client::new(); + + let blockchain_client = get_blockchain_client_connection() + .await + .read() + .await + .get_client(); + let round_robin_instance = + ProposerManager::new(get_addresses().round_robin, blockchain_client.root()); + + let proposers = round_robin_instance + .get_proposers() + .call() + .await + .map_err(|_| warp::reject::custom(ClientRejection::ProposerError))?; + + let futures: Vec<_> = proposers + .into_iter() + .map(|proposer| { + send_receipt_to_proposer_with_retry( + client.clone(), + proposer, + request.clone(), + MAX_RETRIES, + INITIAL_BACKOFF, + ) + }) + .collect(); + + let results = join_all(futures).await; + + // Aggregate: the first success wins. Track error categories so we can + // return the most informative failure if every proposer rejects. + let mut first_success: Option<(ProposerReceiptResponse, warp::http::StatusCode)> = None; + let mut saw_conflict = false; + let mut saw_unauthorized = false; + let mut saw_tx_not_found = false; + let mut saw_validation_failed = false; + + for result in results { + match result { + Ok((resp, http_status)) => { + if first_success.is_none() { + first_success = Some((resp, http_status)); + } + } + Err(ProposerReceiptError::Conflict) => saw_conflict = true, + Err(ProposerReceiptError::Unauthorized) => saw_unauthorized = true, + Err(ProposerReceiptError::TxNotFound) => saw_tx_not_found = true, + Err(ProposerReceiptError::ValidationFailed) => saw_validation_failed = true, + Err(ProposerReceiptError::Other(msg)) => { + warn!("Proposer receipt error: {msg}"); + } + } + } + + if let Some((resp, http_status)) = first_success { + return Ok(warp::reply::with_status( + warp::reply::json(&resp), + http_status, + )); + } + + // No proposer accepted — return the most specific error. + if saw_conflict { + return Err(warp::reject::custom(ClientRejection::ReceiptConflict)); + } + if saw_unauthorized { + return Err(warp::reject::custom(ClientRejection::ReceiptUnauthorized)); + } + if saw_validation_failed { + return Err(warp::reject::custom( + ClientRejection::ReceiptValidationFailed, + )); + } + if saw_tx_not_found { + return Err(warp::reject::custom(ClientRejection::ReceiptTxNotFound)); + } + + Err(warp::reject::custom( + ClientRejection::ReceiptSubmissionFailed, + )) +} + +async fn send_receipt_to_proposer_with_retry( + client: Client, + proposer: ProposerManager::Proposer, + request: SubmitReceiptRequest, + max_retries: u32, + initial_backoff: Duration, +) -> Result<(ProposerReceiptResponse, warp::http::StatusCode), ProposerReceiptError> { + let url = Url::parse(&proposer.url) + .and_then(|base| base.join("/v1/transfer-receipts")) + .map_err(|e| ProposerReceiptError::Other(format!("Invalid URL: {e}")))?; + + for attempt in 1..=max_retries { + let body = serde_json::to_string(&request) + .expect("SubmitReceiptRequest serialization cannot fail"); + + let resp = client + .post(url.clone()) + .header("Content-Type", "application/json") + .header("Content-Length", body.len().to_string()) + .body(body) + .send() + .await; + + match resp { + Ok(response) => { + let status = response.status(); + match status { + s if s == reqwest::StatusCode::CREATED => { + let body = response.text().await.unwrap_or_default(); + let parsed = serde_json::from_str::(&body) + .map_err(|e| { + ProposerReceiptError::Other(format!("Parse error: {e}")) + })?; + debug!("Receipt stored (201 Created) at proposer {}", proposer.url); + return Ok((parsed, warp::http::StatusCode::CREATED)); + } + s if s == reqwest::StatusCode::OK => { + let body = response.text().await.unwrap_or_default(); + let parsed = serde_json::from_str::(&body) + .map_err(|e| { + ProposerReceiptError::Other(format!("Parse error: {e}")) + })?; + debug!( + "Receipt idempotent replay (200 OK) at proposer {}", + proposer.url + ); + return Ok((parsed, warp::http::StatusCode::OK)); + } + s if s == reqwest::StatusCode::UNAUTHORIZED => { + error!("Receipt unauthorized (401) at proposer {}", proposer.url); + return Err(ProposerReceiptError::Unauthorized); + } + s if s == reqwest::StatusCode::CONFLICT => { + error!("Receipt conflict (409) at proposer {}", proposer.url); + return Err(ProposerReceiptError::Conflict); + } + s if s == reqwest::StatusCode::BAD_REQUEST => { + let body = response.text().await.unwrap_or_default(); + warn!( + "Receipt bad request (400) at proposer {}: {}", + proposer.url, body + ); + // Proposer uses 400 for two distinct cases: + // "Transfer receipt transaction not found" → tx not yet in mempool (retriable) + // "Transfer receipt creation failed" → validation/input error (fatal) + if body.contains("not found") { + return Err(ProposerReceiptError::TxNotFound); + } else { + return Err(ProposerReceiptError::ValidationFailed); + } + } + s if matches!( + s, + reqwest::StatusCode::BAD_GATEWAY + | reqwest::StatusCode::SERVICE_UNAVAILABLE + | reqwest::StatusCode::GATEWAY_TIMEOUT + ) && attempt < max_retries => + { + let backoff = initial_backoff * 2u32.pow(attempt - 1); + warn!( + "Proposer {} returned {s}, retrying in {backoff:?}", + proposer.url + ); + sleep(backoff).await; + continue; + } + s => { + let body = response.text().await.unwrap_or_default(); + return Err(ProposerReceiptError::Other(format!( + "Proposer {} returned HTTP {s}: {body}", + proposer.url + ))); + } + } + } + Err(err) => { + if attempt < max_retries { + let backoff = initial_backoff * 2u32.pow(attempt - 1); + warn!( + "Network error sending receipt to proposer {}, retrying in {backoff:?}: {err}", + proposer.url + ); + sleep(backoff).await; + continue; + } + return Err(ProposerReceiptError::Other(format!( + "Network error for proposer {}: {err}", + proposer.url + ))); + } + } + } + + Err(ProposerReceiptError::Other(format!( + "Max retries exhausted for proposer {}", + proposer.url + ))) +} diff --git a/nightfall_client/src/drivers/rest/withdraw.rs b/nightfall_client/src/drivers/rest/withdraw.rs index d6f6be4e..a11c51fa 100644 --- a/nightfall_client/src/drivers/rest/withdraw.rs +++ b/nightfall_client/src/drivers/rest/withdraw.rs @@ -2,7 +2,7 @@ use super::client_nf_3::parse_token_type; use crate::ports::contracts::NightfallContract; use ::nightfall_bindings::artifacts::Nightfall; use lib::{client_models::DeEscrowDataReq, shared_entities::WithdrawData as NFWithdrawData}; -use log::{debug, error}; +use log::{debug, info, error}; use reqwest::StatusCode; use warp::{reject, Reply}; @@ -19,7 +19,7 @@ pub async fn handle_de_escrow(data: DeEscrowDataReq) -> Result { if b { - debug!("Withdraw is on chain, attempting to de-escrow funds"); + info!("Withdraw is on chain, attempting to de-escrow funds"); Nightfall::NightfallCalls::de_escrow_funds(withdraw_data, token_type) .await .map_err(|e| { @@ -36,7 +36,7 @@ pub async fn handle_de_escrow(data: DeEscrowDataReq) -> Result { - debug!("Nightfall contract error: {e}"); + info!("Nightfall contract error: {e}"); Err(reject::custom( crate::domain::error::ClientRejection::FailedDeEscrow, )) diff --git a/nightfall_client/src/ports/db.rs b/nightfall_client/src/ports/db.rs index d113d828..299bd3ea 100644 --- a/nightfall_client/src/ports/db.rs +++ b/nightfall_client/src/ports/db.rs @@ -17,6 +17,8 @@ pub trait RequestDB { async fn update_request(&self, id: &str, status: RequestStatus) -> Option<()>; async fn update_request_child_args(&self, id: &str, child_args: &str) -> Option<()>; async fn clear_request_child_args(&self, id: &str) -> Option<()>; + async fn set_request_tx_hash(&self, id: &str, tx_hash: &str) -> Option<()>; + async fn set_request_receipt_token(&self, id: &str, receipt_token: &str) -> Option<()>; } #[async_trait] @@ -43,7 +45,7 @@ where &self, commitment_ids: Vec, ) -> Result, &'static str>; - async fn get_available_commitments(&self, nf_token_id: Fr254) -> Option>; + async fn get_available_commitments(&self, nf_slot_id: Fr254) -> Option>; async fn mark_commitments_pending_creation(&self, commitments: Vec) -> Option<()>; async fn mark_commitments_unspent( &self, diff --git a/nightfall_client/src/services/client_operation.rs b/nightfall_client/src/services/client_operation.rs index 07587b03..4887b59f 100644 --- a/nightfall_client/src/services/client_operation.rs +++ b/nightfall_client/src/services/client_operation.rs @@ -161,7 +161,11 @@ where .map_err(|_| "Transaction could not be completed due to an invalid root key.")?; // Construct Private Inputs [ Commitment value, salt, recipient public_key]; let nf_address = get_addresses().nightfall(); - let nf_token_id = spend_commitments[0].get_nf_token_id(); + let spent_nf_token_ids = [ + spend_commitments[0].get_nf_token_id(), + spend_commitments[1].get_nf_token_id(), + ]; + let nf_token_a_id = new_commitments[0].get_nf_token_id(); let fee_token_id = get_fee_token_id(); let (mut public_inputs, mut private_inputs) = ( PublicInputs::new() @@ -171,7 +175,8 @@ where PrivateInputs::new() .nf_address(nf_address) .value_a(new_commitments[0].get_value()) - .nf_token_a_id(nf_token_id) + .spent_nf_token_ids(spent_nf_token_ids) + .nf_token_a_id(nf_token_a_id) .nf_slot_id(nf_slot_id) .fee_token_id(fee_token_id) .nullifiers_values(&spend_commitments.map(|c| c.get_value())) @@ -219,6 +224,9 @@ where deadline: public_inputs.deadline, swap_side: public_inputs.swap_side, proof, + // receipt_token is set by the caller (drivers layer) after + // proof generation and before the transaction is sent to proposers. + receipt_token: None, }), Err(e) => { error!("{id} Proving error {e:?}"); @@ -276,6 +284,10 @@ where warn!("{id} Value or fee not conserved in this transaction: rejecting"); return Err("Value or fee not conserved in this transaction: rejecting"); } + let spent_nf_token_ids = [ + spend_commitments[0].get_nf_token_id(), + spend_commitments[1].get_nf_token_id(), + ]; // Collect the public keys from the nullified commitments let public_keys: [TEAffine; 4] = spend_commitments @@ -304,6 +316,7 @@ where .party_a_public_key(party_a_public_key) .party_b_public_key(party_b_public_key) .value_a(value_a) + .spent_nf_token_ids(spent_nf_token_ids) .nf_token_a_id(nf_token_a_id) .nf_token_b_id(nf_token_b_id) .value_b(value_b) @@ -353,6 +366,7 @@ where deadline: public_inputs.deadline, swap_side: public_inputs.swap_side, proof, + receipt_token: None, }), Err(e) => { error!("{id} Swap proving error {e:?}"); diff --git a/nightfall_client/src/services/commitment_selection.rs b/nightfall_client/src/services/commitment_selection.rs index a0eb3fec..4f71ffc8 100644 --- a/nightfall_client/src/services/commitment_selection.rs +++ b/nightfall_client/src/services/commitment_selection.rs @@ -33,13 +33,13 @@ const MAX_POSSIBLE_COMMITMENTS: usize = 2; // The function returns exactly MAX_POSSIBLE_COMMITMENTS preimages, with unused slots // filled with Preimage::default(). pub async fn find_usable_commitments( - target_token_id: Fr254, + target_slot_id: Fr254, target_value: Fr254, db: &Client, ) -> Result<[Preimage; MAX_POSSIBLE_COMMITMENTS], &'static str> { // Verify enough commitments and get sorted available commitments let (avaliable_sorted_commitments, min_num_c) = - verify_enough_commitments(target_token_id, target_value, db).await?; + verify_enough_commitments(target_slot_id, target_value, db).await?; // Determine max number of commitments to use let max_num_c = avaliable_sorted_commitments @@ -247,23 +247,22 @@ fn calculate_minimum_commitments( // Fetch and filter on-chain commitments async fn fetch_on_chain_commitments( db: &Client, - token_id: Fr254, + slot_id: Fr254, ) -> Result, &'static str> { let commitments = db - .get_available_commitments(token_id) + .get_available_commitments(slot_id) .await .ok_or("No commmitments found in the db")?; Ok(commitments.into_iter().map(|c| c.get_preimage()).collect()) } async fn verify_enough_commitments( - target_token_id: Fr254, + target_slot_id: Fr254, target_value: Fr254, db: &Client, ) -> Result<(std::vec::Vec, usize), &'static str> { // Fetch on-chain commitments for the non-fee component - let mut on_chain_old_value_commitments = - fetch_on_chain_commitments(db, target_token_id).await?; + let mut on_chain_old_value_commitments = fetch_on_chain_commitments(db, target_slot_id).await?; on_chain_old_value_commitments.sort_by_key(|a| a.get_value()); trace!("On-chain commitments for value: {on_chain_old_value_commitments:?}"); @@ -290,6 +289,15 @@ mod test { use mongodb::bson::doc; use url::Host; + fn test_preimage(value: u64, id: u64) -> Preimage { + Preimage { + value: Fr254::from(value), + nf_token_id: Fr254::from(id), + nf_slot_id: Fr254::from(id), + ..Default::default() + } + } + #[tokio::test] async fn test_find_usable_commitments_success() { // 1. Setup: start Mongo test container and get DB connection @@ -308,11 +316,7 @@ mod test { // Insert commitments for token_id = 1 (value commitments) let value_commitments = vec![ CommitmentEntry::new( - Preimage { - value: Fr254::from(5u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(5, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -320,11 +324,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(6u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(6, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -332,11 +332,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(7u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(7, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -383,11 +379,7 @@ mod test { // Insert commitments for token_id = 2 (fee commitments) let fee_commitments = vec![ CommitmentEntry::new( - Preimage { - value: Fr254::from(2u64), - nf_token_id: Fr254::from(2u64), - ..Default::default() - }, + test_preimage(2, 2), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -395,11 +387,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(12u64), - nf_token_id: Fr254::from(2u64), - ..Default::default() - }, + test_preimage(12, 2), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -407,11 +395,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(13u64), - nf_token_id: Fr254::from(2u64), - ..Default::default() - }, + test_preimage(13, 2), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -460,11 +444,7 @@ mod test { let commitments = vec![ // Value commitments for nf_token_id: 1 CommitmentEntry::new( - Preimage { - value: Fr254::from(1u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(1, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -472,11 +452,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(2u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(2, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -484,11 +460,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(3u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(3, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -496,11 +468,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(4u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(4, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -509,11 +477,7 @@ mod test { ), // Fee commitments for nf_token_id: 2 CommitmentEntry::new( - Preimage { - value: Fr254::from(1u64), - nf_token_id: Fr254::from(2u64), - ..Default::default() - }, + test_preimage(1, 2), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -521,11 +485,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(2u64), - nf_token_id: Fr254::from(2u64), - ..Default::default() - }, + test_preimage(2, 2), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -533,11 +493,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(5u64), - nf_token_id: Fr254::from(2u64), - ..Default::default() - }, + test_preimage(5, 2), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -545,11 +501,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(3u64), - nf_token_id: Fr254::from(2u64), - ..Default::default() - }, + test_preimage(3, 2), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -557,11 +509,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(6u64), - nf_token_id: Fr254::from(2u64), - ..Default::default() - }, + test_preimage(6, 2), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -648,11 +596,7 @@ mod test { let commitments = vec![ CommitmentEntry::new( - Preimage { - value: Fr254::from(5u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(5, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -660,11 +604,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(6u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(6, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -672,11 +612,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(7u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(7, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -739,11 +675,7 @@ mod test { let value_commitments = vec![ // Only insert commitments for nf_token_id: 1 CommitmentEntry::new( - Preimage { - value: Fr254::from(5u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(5, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -751,11 +683,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(6u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(6, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -763,11 +691,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(7u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(7, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -824,11 +748,7 @@ mod test { let fee_commitments = vec![ // Only insert commitments for nf_token_id: 2 CommitmentEntry::new( - Preimage { - value: Fr254::from(5u64), - nf_token_id: Fr254::from(2u64), - ..Default::default() - }, + test_preimage(5, 2), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -836,11 +756,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(6u64), - nf_token_id: Fr254::from(2u64), - ..Default::default() - }, + test_preimage(6, 2), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -848,11 +764,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(12u64), - nf_token_id: Fr254::from(2u64), - ..Default::default() - }, + test_preimage(12, 2), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -860,11 +772,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(2u64), - nf_token_id: Fr254::from(2u64), - ..Default::default() - }, + test_preimage(2, 2), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -872,11 +780,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(13u64), - nf_token_id: Fr254::from(2u64), - ..Default::default() - }, + test_preimage(13, 2), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -925,11 +829,7 @@ mod test { let commitments = vec![ CommitmentEntry::new( - Preimage { - value: Fr254::from(5u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(5, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -937,11 +837,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(6u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(6, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -949,11 +845,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(7u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(7, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -1002,11 +894,7 @@ mod test { let commitments = vec![ CommitmentEntry::new( - Preimage { - value: Fr254::from(5u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(5, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -1014,11 +902,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(6u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(6, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -1026,11 +910,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(7u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(7, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -1081,11 +961,7 @@ mod test { let commitments = vec![ CommitmentEntry::new( - Preimage { - value: Fr254::from(10u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(10, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, @@ -1093,11 +969,7 @@ mod test { None, ), CommitmentEntry::new( - Preimage { - value: Fr254::from(20u64), - nf_token_id: Fr254::from(1u64), - ..Default::default() - }, + test_preimage(20, 1), Fr254::default(), CommitmentStatus::Unspent, TokenType::ERC1155, diff --git a/nightfall_client/tests/api_stateful.rs b/nightfall_client/tests/api_stateful.rs index 0cdcaabb..83a06c4f 100644 --- a/nightfall_client/tests/api_stateful.rs +++ b/nightfall_client/tests/api_stateful.rs @@ -7,7 +7,7 @@ use lib::{ nf_token_id::to_nf_token_id_from_str, plonk_prover::plonk_proof::PlonkProof, shared_entities::{Preimage, TokenType, WithdrawData}, - tests_utils::{get_db_connection_uri, get_mongo}, + tests_utils::{get_db_connection as get_test_db_connection, get_db_connection_uri, get_mongo}, }; use mongodb::bson::doc; use nightfall_bindings::artifacts::Nightfall; @@ -77,6 +77,7 @@ impl NightfallContract for MockNightfall { async fn setup_test_db() -> ContainerAsync { let container = get_mongo().await; + let _ = get_test_db_connection(&container).await; let host = container.get_host().await.expect("mongo host"); let port = container .get_host_port_ipv4(27017) diff --git a/nightfall_proposer/Cargo.toml b/nightfall_proposer/Cargo.toml index f6457b2d..1ee1aa99 100644 --- a/nightfall_proposer/Cargo.toml +++ b/nightfall_proposer/Cargo.toml @@ -50,6 +50,8 @@ jf-utils = { git = "https://git@github.com/EYBlockchain/nightfish_CE.git", featu nf-curves = { git = "https://git@github.com/EYBlockchain/nightfish_CE.git" } itertools = { version = "0.10.5", default-features = false } sha2 = { version = "0.10.9", default-features = false } +hmac = "0.12" +rand = "0.8.5" [dev-dependencies] criterion = { version = "0.4", features = ["html_reports"] } diff --git a/nightfall_proposer/src/domain/entities.rs b/nightfall_proposer/src/domain/entities.rs index 96c324d6..c146072b 100644 --- a/nightfall_proposer/src/domain/entities.rs +++ b/nightfall_proposer/src/domain/entities.rs @@ -1,3 +1,4 @@ +use alloy::primitives::TxHash; use ark_bn254::Fr as Fr254; use ark_serialize::SerializationError; use lib::{ @@ -6,9 +7,10 @@ use lib::{ shared_entities::{ClientTransaction, OnChainTransaction}, }; use log::error; -use serde::{Deserialize, Deserializer, Serialize}; +use mongodb::bson::DateTime; +use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; use sha3::{Digest, Keccak256}; -use std::fmt::Debug; +use std::{fmt, fmt::Debug}; /// A Block struct representing NF block /// NOTE: This is not finalised yet, we may need to change fields to this struct @@ -31,6 +33,31 @@ pub struct Block { pub block_number: u64, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub enum PendingBlockState { + Reserved, + ReadyToPropose, + BroadcastPending, +} + +fn default_pending_block_state() -> PendingBlockState { + PendingBlockState::ReadyToPropose +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct PendingBlock { + pub layer2_block_number: u64, + #[serde(default = "default_pending_block_state")] + pub state: PendingBlockState, + #[serde(default)] + pub broadcast_tx_hash: Option, + #[serde(default)] + pub broadcast_receipt_checks: u32, + pub block: Option, + pub selected_deposits: Vec>, + pub selected_client_transaction_hashes: Vec>, +} + /// Struct used to represent deposit data, used in making deposit proofs by the proposer. #[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq)] pub struct DepositDatawithFee { @@ -39,6 +66,210 @@ pub struct DepositDatawithFee { pub fee: Fr254, /// deposit data pub deposit_data: DepositData, + #[serde(default)] + pub reserved: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct L1Ref { + pub block_number: u64, + pub tx_hash: TxHash, + pub log_index: u64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SyncState { + #[serde(rename = "_id")] + pub id: String, + pub last_applied_l2_block: u64, + pub fingerprint: String, + pub l1_ref: L1Ref, + pub schema_version: u32, + pub updated_at: DateTime, +} + +impl SyncState { + pub const DOCUMENT_ID: &'static str = "proposer"; + pub const SCHEMA_VERSION: u32 = 1; + + pub fn new( + last_applied_l2_block: u64, + fingerprint: String, + l1_ref: L1Ref, + updated_at: DateTime, + ) -> Self { + Self { + id: Self::DOCUMENT_ID.to_string(), + last_applied_l2_block, + fingerprint, + l1_ref, + schema_version: Self::SCHEMA_VERSION, + updated_at, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SnapshotCollectionManifest { + pub collection_name: String, + pub file_name: String, + pub document_count: u64, + pub sha256: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ProposerSnapshotManifest { + pub snapshot_id: String, + pub schema_version: u32, + pub created_at: DateTime, + pub storage_format: String, + pub database: String, + pub last_applied_l2_block: u64, + pub fingerprint: String, + pub l1_ref: L1Ref, + pub collections: Vec, + pub overall_sha256: String, +} + +impl ProposerSnapshotManifest { + pub const SCHEMA_VERSION: u32 = 1; + pub const STORAGE_FORMAT: &'static str = "mongo-relaxed-extjsonl-v1"; + + pub fn new( + snapshot_id: String, + created_at: DateTime, + database: String, + sync_state: &SyncState, + collections: Vec, + overall_sha256: String, + ) -> Self { + Self { + snapshot_id, + schema_version: Self::SCHEMA_VERSION, + created_at, + storage_format: Self::STORAGE_FORMAT.to_string(), + database, + last_applied_l2_block: sync_state.last_applied_l2_block, + fingerprint: sync_state.fingerprint.clone(), + l1_ref: sync_state.l1_ref.clone(), + collections, + overall_sha256, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum RestoreJournalPhase { + LoadingShadow, + SwapInProgress, + RollbackInProgress, + SwapComplete, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum RestoreJournalStep { + BackupPending, + BackupCreated, + NonSnapshotCleanupPending, + NonSnapshotCleanupApplied, + RollbackPending, + RollbackStarted, + RollbackApplied, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct RestoreJournalCollection { + pub live: String, + pub shadow: String, + pub backup: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct RestoreJournal { + #[serde(rename = "_id")] + pub id: String, + pub schema_version: u32, + pub snapshot_id: String, + pub snapshot_dir: String, + pub manifest_overall_sha256: String, + pub phase: RestoreJournalPhase, + pub current_index: Option, + pub current_step: Option, + pub collections: Vec, + pub started_at: DateTime, + pub updated_at: DateTime, +} + +impl RestoreJournal { + pub const DOCUMENT_ID: &'static str = "proposer"; + pub const SCHEMA_VERSION: u32 = 1; + + pub fn new_loading_shadow( + snapshot_id: String, + snapshot_dir: String, + manifest_overall_sha256: String, + collections: Vec, + now: DateTime, + ) -> Self { + Self { + id: Self::DOCUMENT_ID.to_string(), + schema_version: Self::SCHEMA_VERSION, + snapshot_id, + snapshot_dir, + manifest_overall_sha256, + phase: RestoreJournalPhase::LoadingShadow, + current_index: None, + current_step: None, + collections, + started_at: now, + updated_at: now, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum StartupReplayResetPhase { + DbCleanupPending, + DbCleanupApplied, + AuxiliaryCleanupPending, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct StartupReplayResetMarker { + #[serde(rename = "_id")] + pub id: String, + pub schema_version: u32, + pub phase: StartupReplayResetPhase, + pub started_at: DateTime, + pub updated_at: DateTime, +} + +impl StartupReplayResetMarker { + pub const DOCUMENT_ID: &'static str = "proposer"; + pub const SCHEMA_VERSION: u32 = 1; + + pub fn new(phase: StartupReplayResetPhase, now: DateTime) -> Self { + Self { + id: Self::DOCUMENT_ID.to_string(), + schema_version: Self::SCHEMA_VERSION, + phase, + started_at: now, + updated_at: now, + } + } + + pub fn with_phase(&self, phase: StartupReplayResetPhase, now: DateTime) -> Self { + Self { + id: self.id.clone(), + schema_version: self.schema_version, + phase, + started_at: self.started_at, + updated_at: now, + } + } } impl DepositDatawithFee { @@ -64,6 +295,7 @@ impl DepositDatawithFee { pub enum TxLifecycle { Mempool, Selected { block_l2: u64 }, + Included { block_l2: u64 }, Cancelled, Dropped, } @@ -78,6 +310,7 @@ impl TxLifecycle { pub fn block_l2(&self) -> Option { match self { Self::Selected { block_l2 } => Some(*block_l2), + Self::Included { block_l2 } => Some(*block_l2), _ => None, } } @@ -90,6 +323,10 @@ impl TxLifecycle { matches!(self, Self::Selected { .. }) } + pub fn is_included(&self) -> bool { + matches!(self, Self::Included { .. }) + } + pub fn is_cancelled(&self) -> bool { matches!(self, Self::Cancelled) } @@ -102,6 +339,10 @@ pub struct ClientTransactionWithMetaData

{ pub hash: Vec, #[serde(serialize_with = "ark_se_hex", deserialize_with = "ark_de_hex")] pub historic_roots: Vec, + /// Capability token required to create a transfer receipt for this transaction. + /// Generated at submission time and returned to the submitter. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub receipt_token: Option, } #[derive(Deserialize)] @@ -113,6 +354,8 @@ struct ClientTransactionWithMetaDataSerde

{ pub in_mempool: Option, #[serde(default)] pub cancelled_explicitly: bool, + #[serde(default)] + pub receipt_token: Option, pub hash: Vec, #[serde(deserialize_with = "ark_de_hex")] pub historic_roots: Vec, @@ -127,14 +370,20 @@ where D: Deserializer<'de>, { let helper = ClientTransactionWithMetaDataSerde::

::deserialize(deserializer)?; - // Legacy migration: a document with cancelled_explicitly=false, - // in_mempool=false, block_l2=None corresponds to a stale/expired - // removal in the pre-lifecycle codebase. All persisted transactions - // historically entered the mempool with in_mempool=true first - // (see nightfall_client_transaction.rs), so this triplet - // unambiguously represents a Dropped state. If a future code - // path persists transactions in this triplet for a different - // reason, this assumption must be revisited. + // Legacy migration support: startup backfills explicit lifecycle + // fields before runtime queries rely on them. This fallback keeps + // raw legacy documents deserializable during that migration window. + // + // In the pre-lifecycle codebase, all persisted client transactions + // first entered the mempool with `in_mempool=true` (see + // nightfall_client_transaction.rs). Therefore the legacy triplet + // `cancelled_explicitly=false`, `in_mempool=false`, `block_l2=None` + // cannot mean "never seen" or "still pending"; it unambiguously + // represents a stale/expired removal, so we map it to + // `TxLifecycle::Dropped`. + // + // If a future code path persists transactions in this triplet for a + // different reason, this assumption must be revisited. let lifecycle = helper.lifecycle.unwrap_or_else(|| { if helper.cancelled_explicitly { TxLifecycle::Cancelled @@ -152,10 +401,154 @@ where lifecycle, hash: helper.hash, historic_roots: helper.historic_roots, + receipt_token: helper.receipt_token, }) } } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TransferReceiptStatus { + Pending, + IncludedL2, + Failed, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct TxHashBytes([u8; 32]); + +impl TxHashBytes { + pub fn from_u32_slice(bytes: &[u32]) -> Option { + if bytes.len() != 32 || bytes.iter().any(|&byte| byte > u8::MAX as u32) { + return None; + } + + let mut tx_hash = [0u8; 32]; + for (target, source) in tx_hash.iter_mut().zip(bytes.iter()) { + *target = *source as u8; + } + Some(Self(tx_hash)) + } + + pub fn from_hex(hex_str: &str) -> Option { + if hex_str.len() != 64 || !hex_str.chars().all(|c| c.is_ascii_hexdigit()) { + return None; + } + let bytes = hex::decode(hex_str).ok()?; + let arr: [u8; 32] = bytes.try_into().ok()?; + Some(Self(arr)) + } + + pub fn as_hex(&self) -> String { + hex::encode(self.0) + } + + pub fn as_u32_vec(&self) -> Vec { + self.0.iter().map(|&byte| byte as u32).collect() + } +} + +impl Serialize for TxHashBytes { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&self.as_hex()) + } +} + +impl<'de> Deserialize<'de> for TxHashBytes { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct TxHashBytesVisitor; + + impl<'de> de::Visitor<'de> for TxHashBytesVisitor { + type Value = TxHashBytes; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("a 64-character hex string or 32 byte values") + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + if value.len() != 64 || value.chars().any(|c| !c.is_ascii_hexdigit()) { + return Err(E::custom("tx_hash must be a 64-character hex string")); + } + + let decoded = hex::decode(value).map_err(E::custom)?; + let bytes: [u8; 32] = decoded + .try_into() + .map_err(|_| E::custom("tx_hash must decode to 32 bytes"))?; + Ok(TxHashBytes(bytes)) + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: de::SeqAccess<'de>, + { + let mut bytes = [0u8; 32]; + for (index, target) in bytes.iter_mut().enumerate() { + let Some(byte) = seq.next_element::()? else { + return Err(de::Error::invalid_length(index, &self)); + }; + if byte > u8::MAX as u32 { + return Err(de::Error::custom("tx_hash byte value exceeds 255")); + } + *target = byte as u8; + } + + if seq.next_element::()?.is_some() { + return Err(de::Error::custom("tx_hash must contain exactly 32 bytes")); + } + + Ok(TxHashBytes(bytes)) + } + } + + deserializer.deserialize_any(TxHashBytesVisitor) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TransferReceipt { + /// Deterministic unique identifier for this receipt, derived as + /// `HMAC-SHA256(receipt_token, tx_hash)`, hex-encoded (64 lowercase hex chars). + /// Identical across all proposers for the same transfer, allowing the receiver + /// to query any available proposer using the same `receipt_id`. + pub receipt_id: String, + /// The canonical proposer-side transaction hash of the transfer this receipt is + /// associated with, stored as 32 raw bytes. Used as the primary lookup key when + /// checking for duplicate receipts on the same transaction. + pub tx_hash: TxHashBytes, + /// The opaque KEM-DEM encrypted receipt payload supplied by the sender wallet, + /// hex-encoded. For `version = 1` this is exactly 576 hex characters + /// (9 × 32-byte BN254 field elements: 7 encrypted plaintext fields followed by + /// the ephemeral public key y-coordinate and x-sign flag). The proposer stores + /// this without interpreting or decrypting it. + pub ciphertext: String, + /// Identifies the ciphertext format and encoding rules. Currently only `1` is + /// accepted, corresponding to the fixed `[Fr254; 9]` layout. Persisting the + /// version with each receipt allows safe format evolution in future versions + /// without breaking existing stored receipts, and is also used during idempotent + /// replay to detect conflicting re-submissions. + pub version: u8, + /// Lifecycle status of the associated transfer transaction. In the current + /// transfer-receipt flow this is typically refreshed from live transaction state + /// on read and returned as `Pending` or `IncludedL2`; the enum also retains + /// `Failed` for compatibility with the broader receipt status model. + pub status: TransferReceiptStatus, + /// Unix timestamp (seconds) when this receipt was first stored by the proposer. + pub created_at_unix: i64, + /// Unix timestamp (seconds) when this receipt was last updated, for example + /// when `status` was refreshed after the referenced transaction was included + /// in a Layer 2 block. + pub updated_at_unix: i64, +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct HistoricRoot( #[serde(serialize_with = "ark_se_hex", deserialize_with = "ark_de_hex")] pub Fr254, @@ -192,6 +585,7 @@ mod tests { lifecycle: TxLifecycle::Selected { block_l2: 7 }, hash: vec![1, 2, 3], historic_roots: vec![], + receipt_token: None, }; let mut value = to_value(&tx).expect("serialize tx"); let map = value @@ -219,6 +613,7 @@ mod tests { lifecycle: TxLifecycle::Cancelled, hash: vec![4, 5, 6], historic_roots: vec![Fr254::from(9u64)], + receipt_token: None, }; let value = to_value(&tx).expect("serialize tx"); diff --git a/nightfall_proposer/src/domain/error.rs b/nightfall_proposer/src/domain/error.rs index 20b7528a..871cd6e4 100644 --- a/nightfall_proposer/src/domain/error.rs +++ b/nightfall_proposer/src/domain/error.rs @@ -46,6 +46,11 @@ pub enum ProposerRejection { FailedToRemoveProposer, FailedToWithdrawStake, ProviderError, + TransferReceiptCreationFailed, + TransferReceiptNotFound, + TransferReceiptTxNotFound, + TransferReceiptConflict, + TransferReceiptUnauthorized, } impl std::fmt::Display for ProposerRejection { @@ -63,6 +68,19 @@ impl std::fmt::Display for ProposerRejection { ProposerRejection::FailedToRemoveProposer => write!(f, "Failed to remove proposer"), ProposerRejection::FailedToWithdrawStake => write!(f, "Failed to withdraw stake"), ProposerRejection::ProviderError => write!(f, "Provider error"), + ProposerRejection::TransferReceiptCreationFailed => { + write!(f, "Transfer receipt creation failed") + } + ProposerRejection::TransferReceiptNotFound => write!(f, "Transfer receipt not found"), + ProposerRejection::TransferReceiptTxNotFound => { + write!(f, "Transfer receipt transaction not found") + } + ProposerRejection::TransferReceiptConflict => { + write!(f, "Transfer receipt already exists for this transaction") + } + ProposerRejection::TransferReceiptUnauthorized => { + write!(f, "Invalid or missing receipt token for this transaction") + } } } } diff --git a/nightfall_proposer/src/driven/db/client_transaction_state.rs b/nightfall_proposer/src/driven/db/client_transaction_state.rs new file mode 100644 index 00000000..999eaff2 --- /dev/null +++ b/nightfall_proposer/src/driven/db/client_transaction_state.rs @@ -0,0 +1,787 @@ +use crate::{ + domain::entities::{ClientTransactionWithMetaData, TxLifecycle}, + driven::db::mongo_db::{StoredBlock, DB}, + ports::db::BlockStorageDB, +}; +use ark_ff::Zero; +use lib::hex_conversion::HexConvertible; +use mongodb::{ + bson::{doc, Bson, Document}, + Client, +}; +use std::collections::{HashMap, HashSet}; + +pub const CLIENT_TRANSACTIONS_COLLECTION: &str = "ClientTransactions"; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct LegacyClientTransactionLifecycleMigrationStats { + pub mempool_backfilled: u64, + pub selected_backfilled: u64, + pub included_backfilled: u64, + pub cancelled_backfilled: u64, + pub dropped_backfilled: u64, +} + +impl LegacyClientTransactionLifecycleMigrationStats { + pub fn total_backfilled(&self) -> u64 { + self.mempool_backfilled + + self.selected_backfilled + + self.included_backfilled + + self.cancelled_backfilled + + self.dropped_backfilled + } +} + +pub(crate) fn lifecycle_bson(lifecycle: &TxLifecycle) -> Bson { + mongodb::bson::to_bson(lifecycle).expect("TxLifecycle should serialize to BSON") +} + +pub(crate) fn mempool_transactions_filter() -> Document { + doc! { "lifecycle.state": "mempool" } +} + +pub(crate) fn selected_transactions_filter() -> Document { + doc! { "lifecycle.state": "selected" } +} + +pub(crate) fn selected_or_included_transactions_filter() -> Document { + doc! { "lifecycle.state": { "$in": ["selected", "included"] } } +} + +pub(crate) fn selected_transactions_filter_for_block(block_l2: i64) -> Document { + doc! { + "lifecycle.state": "selected", + "lifecycle.block_l2": Bson::Int64(block_l2), + } +} + +pub(crate) fn selected_transactions_filter_after_block(block_l2: i64) -> Document { + doc! { + "lifecycle.state": "selected", + "lifecycle.block_l2": { "$gt": Bson::Int64(block_l2) }, + } +} + +pub(crate) fn included_transactions_filter_after_block(block_l2: i64) -> Document { + doc! { + "lifecycle.state": "included", + "lifecycle.block_l2": { "$gt": Bson::Int64(block_l2) }, + } +} + +fn included_transactions_filter() -> Document { + doc! { "lifecycle.state": "included" } +} + +#[cfg(test)] +fn legacy_transactions_filter() -> Document { + doc! { "lifecycle": { "$exists": false } } +} + +fn legacy_mempool_transactions_filter() -> Document { + doc! { + "lifecycle": { "$exists": false }, + "in_mempool": true, + "cancelled_explicitly": { "$ne": true } + } +} + +fn legacy_cancelled_transactions_filter() -> Document { + doc! { + "lifecycle": { "$exists": false }, + "cancelled_explicitly": true + } +} + +fn legacy_selected_or_included_transactions_filter() -> Document { + doc! { + "lifecycle": { "$exists": false }, + "in_mempool": { "$ne": true }, + "block_l2": { "$exists": true, "$ne": Bson::Null }, + "cancelled_explicitly": { "$ne": true } + } +} + +fn legacy_dropped_transactions_filter() -> Document { + doc! { + "lifecycle": { "$exists": false }, + "in_mempool": { "$ne": true }, + "cancelled_explicitly": { "$ne": true }, + "$or": [ + doc! { "block_l2": { "$exists": false } }, + doc! { "block_l2": Bson::Null } + ] + } +} + +fn stored_block_commitments_map(stored_blocks: Vec) -> HashMap> { + stored_blocks + .into_iter() + .map(|block| { + ( + block.layer2_block_number, + block.commitments.into_iter().collect::>(), + ) + }) + .collect() +} + +fn transaction_commitments_hex

(transaction: &ClientTransactionWithMetaData

) -> Vec { + transaction + .client_transaction + .commitments + .iter() + .filter(|commitment| !commitment.is_zero()) + .map(|commitment| commitment.to_hex_string()) + .collect() +} + +fn classify_legacy_active_transaction

( + transaction: &ClientTransactionWithMetaData

, + stored_blocks: &HashMap>, +) -> Result { + let block_l2 = transaction.lifecycle.block_l2().ok_or_else(|| { + format!( + "Legacy client transaction {:?} is missing block_l2 during lifecycle backfill", + transaction.hash + ) + })?; + let transaction_commitments = transaction_commitments_hex(transaction); + + let lifecycle = match stored_blocks.get(&block_l2) { + Some(block_commitments) + if !transaction_commitments.is_empty() + && transaction_commitments + .iter() + .all(|commitment| block_commitments.contains(commitment)) => + { + TxLifecycle::Included { block_l2 } + } + _ => TxLifecycle::Selected { block_l2 }, + }; + + Ok(lifecycle) +} + +async fn backfill_legacy_transactions_by_filter( + client: &Client, + filter: Document, + lifecycle: TxLifecycle, +) -> Result { + client + .database(DB) + .collection::(CLIENT_TRANSACTIONS_COLLECTION) + .update_many( + filter, + doc! { + "$set": { "lifecycle": lifecycle_bson(&lifecycle) } + }, + ) + .await + .map(|result| result.modified_count) + .map_err(|error| { + format!("Could not backfill proposer client transaction lifecycle: {error}") + }) +} + +pub(crate) async fn backfill_legacy_client_transaction_lifecycle( + client: &Client, +) -> Result { + let mempool_backfilled = backfill_legacy_transactions_by_filter( + client, + legacy_mempool_transactions_filter(), + TxLifecycle::Mempool, + ) + .await?; + let cancelled_backfilled = backfill_legacy_transactions_by_filter( + client, + legacy_cancelled_transactions_filter(), + TxLifecycle::Cancelled, + ) + .await?; + let dropped_backfilled = backfill_legacy_transactions_by_filter( + client, + legacy_dropped_transactions_filter(), + TxLifecycle::Dropped, + ) + .await?; + + let active_collection = client + .database(DB) + .collection::>(CLIENT_TRANSACTIONS_COLLECTION); + let mut cursor = active_collection + .find(legacy_selected_or_included_transactions_filter()) + .await + .map_err(|error| { + format!( + "Could not query legacy proposer client transactions for lifecycle backfill: {error}" + ) + })?; + + let stored_blocks = stored_block_commitments_map( + ::get_all_blocks(client) + .await + .ok_or_else(|| { + "Could not read proposer StoredBlocks during lifecycle backfill".to_string() + })?, + ); + let raw_collection = client + .database(DB) + .collection::(CLIENT_TRANSACTIONS_COLLECTION); + + let mut selected_backfilled = 0_u64; + let mut included_backfilled = 0_u64; + + while cursor.advance().await.map_err(|error| { + format!( + "Could not iterate legacy proposer client transactions for lifecycle backfill: {error}" + ) + })? { + let transaction = cursor.deserialize_current().map_err(|error| { + format!( + "Could not deserialize legacy proposer client transaction for lifecycle backfill: {error}" + ) + })?; + + let lifecycle = classify_legacy_active_transaction(&transaction, &stored_blocks)?; + let result = raw_collection + .update_one( + doc! { + "hash": &transaction.hash, + "lifecycle": { "$exists": false } + }, + doc! { "$set": { "lifecycle": lifecycle_bson(&lifecycle) } }, + ) + .await + .map_err(|error| { + format!( + "Could not persist lifecycle backfill for proposer client transaction {:?}: {error}", + transaction.hash + ) + })?; + + if result.modified_count != 1 { + return Err(format!( + "Lifecycle backfill expected to update exactly one proposer client transaction for hash {:?}, updated {} instead", + transaction.hash, result.modified_count + )); + } + + match lifecycle { + TxLifecycle::Selected { .. } => selected_backfilled += 1, + TxLifecycle::Included { .. } => included_backfilled += 1, + _ => {} + } + } + + Ok(LegacyClientTransactionLifecycleMigrationStats { + mempool_backfilled, + selected_backfilled, + included_backfilled, + cancelled_backfilled, + dropped_backfilled, + }) +} + +pub(crate) async fn selected_client_transaction_count(client: &Client) -> Result { + client + .database(DB) + .collection::(CLIENT_TRANSACTIONS_COLLECTION) + .count_documents(selected_transactions_filter()) + .await + .map_err(|error| { + format!("Could not inspect proposer selected client transactions: {error}") + }) +} + +pub(crate) async fn selected_client_transaction_count_after_block( + client: &Client, + last_applied_l2_block: u64, +) -> Result { + let last_applied_l2_block = i64::try_from(last_applied_l2_block).map_err(|_| { + format!("Could not convert proposer last applied L2 block {last_applied_l2_block} into i64") + })?; + + client + .database(DB) + .collection::(CLIENT_TRANSACTIONS_COLLECTION) + .count_documents(selected_transactions_filter_after_block(last_applied_l2_block)) + .await + .map_err(|error| { + format!( + "Could not inspect proposer selected client transactions beyond L2 block {last_applied_l2_block}: {error}" + ) + }) +} + +pub(crate) async fn restore_all_selected_transactions_to_mempool( + client: &Client, +) -> Result { + client + .database(DB) + .collection::(CLIENT_TRANSACTIONS_COLLECTION) + .update_many( + selected_transactions_filter(), + doc! { + "$set": { + "lifecycle": lifecycle_bson(&TxLifecycle::Mempool) + } + }, + ) + .await + .map(|result| result.modified_count) + .map_err(|error| { + format!( + "Could not restore proposer selected client transactions to the mempool: {error}" + ) + }) +} + +pub(crate) async fn restore_all_included_transactions_to_mempool( + client: &Client, +) -> Result { + client + .database(DB) + .collection::(CLIENT_TRANSACTIONS_COLLECTION) + .update_many( + included_transactions_filter(), + doc! { + "$set": { + "lifecycle": lifecycle_bson(&TxLifecycle::Mempool) + } + }, + ) + .await + .map(|result| result.modified_count) + .map_err(|error| { + format!( + "Could not restore proposer included client transactions to the mempool: {error}" + ) + }) +} + +pub(crate) async fn restore_selected_transactions_to_mempool_after_block( + client: &Client, + last_applied_l2_block: u64, +) -> Result { + let last_applied_l2_block = i64::try_from(last_applied_l2_block).map_err(|_| { + format!("Could not convert proposer last applied L2 block {last_applied_l2_block} into i64") + })?; + + client + .database(DB) + .collection::(CLIENT_TRANSACTIONS_COLLECTION) + .update_many( + selected_transactions_filter_after_block(last_applied_l2_block), + doc! { + "$set": { + "lifecycle": lifecycle_bson(&TxLifecycle::Mempool) + } + }, + ) + .await + .map(|result| result.modified_count) + .map_err(|error| { + format!( + "Could not restore proposer selected client transactions beyond L2 block {last_applied_l2_block} to the mempool: {error}" + ) + }) +} + +pub(crate) async fn restore_selected_transactions_to_mempool_after_block_with_session( + client: &Client, + last_applied_l2_block: u64, + session: &mut mongodb::ClientSession, +) -> Result { + let last_applied_l2_block = i64::try_from(last_applied_l2_block).map_err(|_| { + format!("Could not convert proposer last applied L2 block {last_applied_l2_block} into i64") + })?; + + client + .database(DB) + .collection::(CLIENT_TRANSACTIONS_COLLECTION) + .update_many( + selected_transactions_filter_after_block(last_applied_l2_block), + doc! { + "$set": { + "lifecycle": lifecycle_bson(&TxLifecycle::Mempool) + } + }, + ) + .session(&mut *session) + .await + .map(|result| result.modified_count) + .map_err(|error| { + format!( + "Could not restore proposer selected client transactions beyond L2 block {last_applied_l2_block} to the mempool: {error}" + ) + }) +} + +pub(crate) async fn restore_included_transactions_to_mempool_after_block_with_session( + client: &Client, + last_applied_l2_block: u64, + session: &mut mongodb::ClientSession, +) -> Result { + let last_applied_l2_block = i64::try_from(last_applied_l2_block).map_err(|_| { + format!("Could not convert proposer last applied L2 block {last_applied_l2_block} into i64") + })?; + + client + .database(DB) + .collection::(CLIENT_TRANSACTIONS_COLLECTION) + .update_many( + included_transactions_filter_after_block(last_applied_l2_block), + doc! { + "$set": { + "lifecycle": lifecycle_bson(&TxLifecycle::Mempool) + } + }, + ) + .session(&mut *session) + .await + .map(|result| result.modified_count) + .map_err(|error| { + format!( + "Could not restore proposer included client transactions beyond L2 block {last_applied_l2_block} to the mempool: {error}" + ) + }) +} + +pub(crate) async fn remove_all_mempool_client_transactions_with_session( + client: &Client, + session: &mut mongodb::ClientSession, +) -> Result { + client + .database(DB) + .collection::(CLIENT_TRANSACTIONS_COLLECTION) + .delete_many(mempool_transactions_filter()) + .session(&mut *session) + .await + .map(|result| result.deleted_count) + .map_err(|error| format!("Could not discard proposer mempool client transactions: {error}")) +} + +#[cfg(test)] +pub(crate) fn count_legacy_client_transactions_filter() -> Document { + legacy_transactions_filter() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + domain::entities::TxLifecycle, + ports::db::{BlockStorageDB, TransactionsDB}, + }; + use alloy::primitives::Address; + use ark_bn254::Fr as Fr254; + use ark_serialize::SerializationError; + use lib::{ + nf_client_proof::Proof, + shared_entities::{ClientTransaction, CompressedSecrets}, + tests_utils::{get_db_connection, get_mongo}, + }; + use mongodb::bson::doc; + use serde::{Deserialize, Serialize}; + use tokio::sync::{Mutex, OnceCell}; + + async fn client_transaction_state_test_lock() -> tokio::sync::MutexGuard<'static, ()> { + static LOCK: OnceCell> = OnceCell::const_new(); + LOCK.get_or_init(|| async { Mutex::new(()) }) + .await + .lock() + .await + } + + #[derive(Clone, Debug, Default, Deserialize, Serialize)] + struct MockProof; + + impl Proof for MockProof { + fn compress_proof(&self) -> Result { + Ok(alloy::primitives::Bytes::new()) + } + + fn from_compressed( + _compressed: alloy::primitives::Bytes, + ) -> Result { + Ok(Self) + } + } + + fn test_transaction( + seed: u32, + lifecycle: TxLifecycle, + commitment: Fr254, + ) -> ClientTransactionWithMetaData { + ClientTransactionWithMetaData { + client_transaction: ClientTransaction { + commitments: [commitment, Fr254::zero(), Fr254::zero(), Fr254::zero()], + compressed_secrets: CompressedSecrets::default(), + proof: MockProof, + ..Default::default() + }, + lifecycle, + hash: vec![seed, seed + 1, seed + 2], + historic_roots: vec![], + receipt_token: None, + } + } + + fn legacy_document_from_transaction( + transaction: &ClientTransactionWithMetaData, + legacy_kind: &str, + ) -> Document { + let mut document = + mongodb::bson::to_document(transaction).expect("serialize client transaction"); + document.remove("lifecycle"); + + match legacy_kind { + "mempool" => { + document.insert("in_mempool", true); + document.insert("cancelled_explicitly", false); + } + "cancelled" => { + document.insert("in_mempool", false); + document.insert("cancelled_explicitly", true); + } + "selected_or_included" => { + document.insert("in_mempool", false); + document.insert("cancelled_explicitly", false); + document.insert( + "block_l2", + Bson::Int64( + i64::try_from(transaction.lifecycle.block_l2().expect("block_l2")) + .expect("block_l2 fits i64"), + ), + ); + } + "dropped" => { + document.insert("in_mempool", false); + document.insert("cancelled_explicitly", false); + } + other => panic!("unexpected legacy kind {other}"), + } + + document + } + + #[tokio::test] + async fn selected_filter_excludes_included_new_and_legacy_shapes() { + let _lock = client_transaction_state_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + let collection = client + .database(DB) + .collection::(CLIENT_TRANSACTIONS_COLLECTION); + collection + .insert_many(vec![ + doc! { + "hash": [1_i32, 2_i32, 3_i32], + "lifecycle": { "state": "selected", "block_l2": 7_i64 } + }, + doc! { + "hash": [4_i32, 5_i32, 6_i32], + "lifecycle": { "state": "included", "block_l2": 7_i64 } + }, + doc! { + "hash": [7_i32, 8_i32, 9_i32], + "in_mempool": false, + "block_l2": 7_i64, + "cancelled_explicitly": false + }, + ]) + .await + .expect("insert lifecycle filter fixtures"); + + let matched = collection + .count_documents(selected_transactions_filter()) + .await + .expect("count selected filter matches"); + assert_eq!( + matched, 1, + "selected filter should only match the explicit Selected lifecycle state" + ); + } + + #[tokio::test] + async fn legacy_client_transaction_lifecycle_backfill_is_idempotent() { + let _lock = client_transaction_state_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + let selected_transaction = test_transaction( + 10, + TxLifecycle::Selected { block_l2: 30 }, + Fr254::from(310_u64), + ); + let included_transaction = test_transaction( + 20, + TxLifecycle::Selected { block_l2: 31 }, + Fr254::from(311_u64), + ); + let mempool_transaction = test_transaction(30, TxLifecycle::Mempool, Fr254::from(312_u64)); + let cancelled_transaction = + test_transaction(40, TxLifecycle::Cancelled, Fr254::from(313_u64)); + let dropped_transaction = test_transaction(50, TxLifecycle::Dropped, Fr254::from(314_u64)); + + let collection = client + .database(DB) + .collection::(CLIENT_TRANSACTIONS_COLLECTION); + collection + .insert_many(vec![ + legacy_document_from_transaction(&selected_transaction, "selected_or_included"), + legacy_document_from_transaction(&included_transaction, "selected_or_included"), + legacy_document_from_transaction(&mempool_transaction, "mempool"), + legacy_document_from_transaction(&cancelled_transaction, "cancelled"), + legacy_document_from_transaction(&dropped_transaction, "dropped"), + ]) + .await + .expect("insert legacy lifecycle fixtures"); + + client + .store_block(&StoredBlock { + layer2_block_number: 31, + commitments: transaction_commitments_hex(&included_transaction), + proposer_address: Address::from([31_u8; 20]), + }) + .await + .expect("store included block fixture"); + + let first_stats = backfill_legacy_client_transaction_lifecycle(&client) + .await + .expect("first lifecycle backfill should succeed"); + assert_eq!( + first_stats, + LegacyClientTransactionLifecycleMigrationStats { + mempool_backfilled: 1, + selected_backfilled: 1, + included_backfilled: 1, + cancelled_backfilled: 1, + dropped_backfilled: 1, + } + ); + assert_eq!( + collection + .count_documents(count_legacy_client_transactions_filter()) + .await + .expect("count legacy rows after first migration"), + 0 + ); + + assert_eq!( + >::get_transaction( + &client, + &selected_transaction.hash + ) + .await + .expect("read selected transaction after first migration") + .lifecycle, + TxLifecycle::Selected { block_l2: 30 } + ); + assert_eq!( + >::get_transaction( + &client, + &included_transaction.hash + ) + .await + .expect("read included transaction after first migration") + .lifecycle, + TxLifecycle::Included { block_l2: 31 } + ); + assert!(>::get_transaction( + &client, + &mempool_transaction.hash + ) + .await + .expect("read mempool transaction after first migration") + .lifecycle + .is_mempool()); + assert!(>::get_transaction( + &client, + &cancelled_transaction.hash + ) + .await + .expect("read cancelled transaction after first migration") + .lifecycle + .is_cancelled()); + assert_eq!( + >::get_transaction( + &client, + &dropped_transaction.hash + ) + .await + .expect("read dropped transaction after first migration") + .lifecycle, + TxLifecycle::Dropped + ); + + let second_stats = backfill_legacy_client_transaction_lifecycle(&client) + .await + .expect("second lifecycle backfill should be a no-op"); + assert_eq!( + second_stats, + LegacyClientTransactionLifecycleMigrationStats::default() + ); + assert_eq!( + collection + .count_documents(count_legacy_client_transactions_filter()) + .await + .expect("count legacy rows after second migration"), + 0 + ); + } + + #[tokio::test] + async fn legacy_withdrawal_without_commitments_is_not_backfilled_as_included() { + let _lock = client_transaction_state_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + let withdrawal_transaction = + test_transaction(60, TxLifecycle::Selected { block_l2: 32 }, Fr254::zero()); + + let collection = client + .database(DB) + .collection::(CLIENT_TRANSACTIONS_COLLECTION); + collection + .insert_one(legacy_document_from_transaction( + &withdrawal_transaction, + "selected_or_included", + )) + .await + .expect("insert legacy withdrawal fixture"); + + client + .store_block(&StoredBlock { + layer2_block_number: 32, + commitments: vec![Fr254::from(320_u64).to_hex_string()], + proposer_address: Address::from([32_u8; 20]), + }) + .await + .expect("store block fixture for legacy withdrawal"); + + let stats = backfill_legacy_client_transaction_lifecycle(&client) + .await + .expect("legacy withdrawal lifecycle backfill should succeed"); + assert_eq!( + stats, + LegacyClientTransactionLifecycleMigrationStats { + selected_backfilled: 1, + ..Default::default() + } + ); + + assert_eq!( + >::get_transaction( + &client, + &withdrawal_transaction.hash + ) + .await + .expect("read withdrawal transaction after migration") + .lifecycle, + TxLifecycle::Selected { block_l2: 32 } + ); + } +} diff --git a/nightfall_proposer/src/driven/db/mod.rs b/nightfall_proposer/src/driven/db/mod.rs index 9664826c..c99e79e8 100644 --- a/nightfall_proposer/src/driven/db/mod.rs +++ b/nightfall_proposer/src/driven/db/mod.rs @@ -1,4 +1,6 @@ +pub mod client_transaction_state; pub mod commitment_tree; pub mod historic_root_tree; pub mod mongo_db; pub mod nullifier_tree; +pub mod snapshot; diff --git a/nightfall_proposer/src/driven/db/mongo_db.rs b/nightfall_proposer/src/driven/db/mongo_db.rs index e6a44a30..4f8c62c8 100644 --- a/nightfall_proposer/src/driven/db/mongo_db.rs +++ b/nightfall_proposer/src/driven/db/mongo_db.rs @@ -1,8 +1,18 @@ use crate::{ domain::entities::{ - ClientTransactionWithMetaData, DepositDatawithFee, HistoricRoot, TxLifecycle, + ClientTransactionWithMetaData, DepositDatawithFee, HistoricRoot, PendingBlock, + RestoreJournal, StartupReplayResetMarker, SyncState, TransferReceipt, + TransferReceiptStatus, TxHashBytes, TxLifecycle, + }, + driven::db::client_transaction_state::{ + selected_or_included_transactions_filter, selected_transactions_filter, + selected_transactions_filter_for_block, + }, + ports::db::{ + BlockStorageDB, HistoricRootsDB, PendingBlockDB, RestoreJournalDB, + StartupReplayResetMarkerDB, SyncStateDB, TransactionsDB, TransferReceiptDB, + TransferReceiptStoreError, }, - ports::db::{BlockStorageDB, HistoricRootsDB, TransactionsDB}, }; use alloy::primitives::Address; use ark_bn254::Fr as Fr254; @@ -12,6 +22,7 @@ use lib::{ error::ConversionError, hex_conversion::HexConvertible, nf_client_proof::Proof, shared_entities::ClientTransaction, }; +use log::warn; use mongodb::bson::{doc, Bson, Document}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -20,9 +31,9 @@ fn lifecycle_bson(lifecycle: &TxLifecycle) -> Bson { mongodb::bson::to_bson(lifecycle).expect("TxLifecycle should serialize to BSON") } -// Temporary migration support: queries must match both the new explicit -// lifecycle document and the legacy triplet {in_mempool, block_l2, -// cancelled_explicitly} until existing proposer data has been backfilled. +// Startup migration backfills legacy client transaction lifecycle fields. The +// remaining legacy query branches below are only retained for mempool and +// cancelled-state compatibility while older data drains out of the system. fn legacy_mempool_filter() -> Document { doc! { "lifecycle": { "$exists": false }, @@ -31,15 +42,6 @@ fn legacy_mempool_filter() -> Document { } } -fn legacy_selected_filter() -> Document { - doc! { - "lifecycle": { "$exists": false }, - "in_mempool": { "$ne": true }, - "block_l2": { "$exists": true, "$ne": Bson::Null }, - "cancelled_explicitly": { "$ne": true } - } -} - fn mempool_state_filter() -> Document { doc! { "$or": [ @@ -49,35 +51,16 @@ fn mempool_state_filter() -> Document { } } -fn selected_state_filter() -> Document { - doc! { - "$or": [ - doc! { "lifecycle.state": "selected" }, - legacy_selected_filter() - ] - } -} - -fn selected_state_filter_for_block(block_l2: i64) -> Document { +fn swap_link_filter(swap_link: &Fr254) -> Document { doc! { - "$or": [ - doc! { - "lifecycle.state": "selected", - "lifecycle.block_l2": Bson::Int64(block_l2) - }, - doc! { - "lifecycle": { "$exists": false }, - "in_mempool": { "$ne": true }, - "block_l2": Bson::Int64(block_l2), - "cancelled_explicitly": { "$ne": true } - } - ] + "client_transaction.swap_link": swap_link.to_hex_string() } } -fn swap_link_filter(swap_link: &Fr254) -> Document { +fn deposit_identity_filter(deposit: &DepositDatawithFee) -> Document { doc! { - "client_transaction.swap_link": swap_link.to_hex_string() + "deposit_data.secret_hash": deposit.deposit_data.secret_hash.to_hex_string(), + "deposit_data.nf_slot_id": deposit.deposit_data.nf_slot_id.to_hex_string(), } } @@ -95,8 +78,26 @@ fn cancelled_state_filter() -> Document { pub const DB: &str = "nightfall"; const COLLECTION: &str = "ClientTransactions"; -const DEPOSIT_COLLECTION: &str = "Deposits"; +pub const DEPOSIT_COLLECTION: &str = "Deposits"; pub const PROPOSED_BLOCKS_COLLECTION: &str = "ProposedBlocks"; +pub const SYNC_STATE_COLLECTION: &str = "sync_state"; +pub const RESTORE_JOURNAL_COLLECTION: &str = "restore_journal"; +pub const STARTUP_REPLAY_RESET_MARKER_COLLECTION: &str = "startup_replay_reset_marker"; +const TRANSFER_RECEIPTS_COLLECTION: &str = "TransferReceipts"; +const PENDING_BLOCKS_COLLECTION: &str = "PendingBlocks"; + +fn deposit_filters(deposits: &[DepositDatawithFee]) -> Vec { + deposits.iter().map(deposit_identity_filter).collect() +} + +fn available_deposits_filter() -> Document { + doc! { + "$or": [ + { "reserved": false }, + { "reserved": { "$exists": false } } + ] + } +} #[async_trait::async_trait] impl<'a, P> TransactionsDB<'a, P> for mongodb::Client @@ -160,7 +161,25 @@ where async fn get_all_selected_client_transactions( &self, ) -> Option, ClientTransactionWithMetaData

)>> { - let filter = selected_state_filter(); + let filter = selected_transactions_filter(); + let mut cursor: mongodb::Cursor> = self + .database(DB) + .collection::>(COLLECTION) + .find(filter) + .await + .ok()?; + let mut result: Vec<(Vec, ClientTransactionWithMetaData

)> = Vec::new(); + while cursor.advance().await.ok()? { + let v: ClientTransactionWithMetaData

= cursor.deserialize_current().ok()?; + result.push((v.hash.clone(), v)); + } + Some(result) + } + + async fn get_all_selected_or_included_client_transactions( + &self, + ) -> Option, ClientTransactionWithMetaData

)>> { + let filter = selected_or_included_transactions_filter(); let mut cursor: mongodb::Cursor> = self .database(DB) .collection::>(COLLECTION) @@ -202,7 +221,7 @@ where swap_link: &Fr254, ) -> Result { let mut filter = swap_link_filter(swap_link); - filter.extend(selected_state_filter()); + filter.extend(selected_or_included_transactions_filter()); self.database(DB) .collection::>(COLLECTION) .count_documents(filter) @@ -236,6 +255,33 @@ where Some(result.modified_count) } + async fn set_client_transactions_in_mempool_by_hashes( + &self, + transaction_hashes: &[Vec], + in_mempool: bool, + ) -> Option { + if transaction_hashes.is_empty() { + return Some(0); + } + + let lifecycle = if in_mempool { + TxLifecycle::Mempool + } else { + TxLifecycle::Dropped + }; + let filter = doc! {"hash": { "$in": transaction_hashes }}; + let update = doc! {"$set": { + "lifecycle": lifecycle_bson(&lifecycle) + }}; + let result = self + .database(DB) + .collection::>(COLLECTION) + .update_many(filter, update) + .await + .ok()?; + Some(result.modified_count) + } + async fn mark_transactions_selected_for_block( &self, txs: &[ClientTransactionWithMetaData

], @@ -275,7 +321,7 @@ where let mut filter = doc! { "hash": { "$in": k } }; - filter.extend(selected_state_filter_for_block(block_l2)); + filter.extend(selected_transactions_filter_for_block(block_l2)); let update = doc! {"$set": { "lifecycle": lifecycle_bson(&TxLifecycle::Mempool) }}; @@ -288,6 +334,54 @@ where Some(result.modified_count) } + async fn mark_transactions_included_by_hashes( + &self, + transaction_hashes: &[Vec], + ) -> Option { + if transaction_hashes.is_empty() { + return Some(0); + } + + let collection = self + .database(DB) + .collection::>(COLLECTION); + let mut modified = 0u64; + + for hash in transaction_hashes { + let Some(transaction) = + >::get_transaction(self, hash).await + else { + continue; + }; + let Some(block_l2) = transaction.lifecycle.block_l2() else { + continue; + }; + let Ok(block_l2_i64) = i64::try_from(block_l2) else { + continue; + }; + + let mut filter = doc! { "hash": hash }; + filter.extend(selected_transactions_filter_for_block(block_l2_i64)); + let update = doc! {"$set": { + "lifecycle": lifecycle_bson(&TxLifecycle::Included { block_l2 }) + }}; + + let result = match collection.update_one(filter, update).await { + Ok(result) => result, + Err(error) => { + warn!( + "Failed lifecycle promotion to Included for proposer client transaction \ + {hash:?} at L2 block {block_l2}: {error}" + ); + return None; + } + }; + modified += result.modified_count; + } + + Some(modified) + } + async fn drop_transactions(&self, txs: &[ClientTransactionWithMetaData

]) -> Option { if txs.is_empty() { return Some(0); @@ -328,12 +422,9 @@ where } async fn find_deposit(&self, v: &DepositDatawithFee) -> Option { - // we'll compute the hash of the transaction and then look it up in the database - let hash = v.hash().ok()?; - let filter = doc! {"hash": hash}; self.database(DB) - .collection::(COLLECTION) - .find_one(filter) + .collection::(DEPOSIT_COLLECTION) + .find_one(deposit_identity_filter(v)) .await .expect("Database error") // we can't really proceed at this point } @@ -347,11 +438,22 @@ where let collection = self .database(DB) .collection::(DEPOSIT_COLLECTION); + let mut inserted_count = 0_u64; - // Directly insert Vec instead of converting to Document - let result = collection.insert_many(deposits).await.ok()?; + for deposit in deposits { + let filter = deposit_identity_filter(&deposit); + let update = doc! { + "$setOnInsert": mongodb::bson::to_document(&deposit).ok()? + }; + let result = collection + .update_one(filter, update) + .upsert(true) + .await + .ok()?; + inserted_count += result.upserted_id.is_some() as u64; + } - Some(result.inserted_ids.len() as u64) + Some(inserted_count) } // Retrieve deposits from the mempool @@ -359,7 +461,7 @@ where let collection = self .database(DB) .collection::(DEPOSIT_COLLECTION); - let mut cursor = collection.find(doc! {}).await.ok()?; + let mut cursor = collection.find(available_deposits_filter()).await.ok()?; let mut result: Vec = Vec::new(); while cursor.advance().await.ok()? { @@ -377,7 +479,7 @@ where async fn count_mempool_deposits(&self) -> Result { self.database(DB) .collection::(DEPOSIT_COLLECTION) - .count_documents(doc! {}) + .count_documents(available_deposits_filter()) .await } @@ -395,16 +497,7 @@ where .database(DB) .collection::(DEPOSIT_COLLECTION); - // Fetch all documents in the collection - let delete_conditions: Vec<_> = used_deposits - .iter() - .map(|d| { - doc! { - "deposit_data.secret_hash": d.deposit_data.secret_hash.to_hex_string(), - "deposit_data.nf_slot_id": d.deposit_data.nf_slot_id.to_hex_string(), - } - }) - .collect(); + let delete_conditions = deposit_filters(&used_deposits); let filter = doc! { "$or": delete_conditions }; @@ -413,6 +506,42 @@ where Some(result.deleted_count) } + async fn set_mempool_deposits_reserved( + &self, + deposits: Vec>, + reserved: bool, + ) -> Option { + let deposits: Vec = deposits.into_iter().flatten().collect(); + if deposits.is_empty() { + return Some(0); + } + + let filter = doc! { + "$or": deposit_filters(&deposits) + }; + let update = doc! {"$set": { "reserved": reserved }}; + let result = self + .database(DB) + .collection::(DEPOSIT_COLLECTION) + .update_many(filter, update) + .await + .ok()?; + Some(result.modified_count) + } + + async fn clear_all_mempool_deposit_reservations(&self) -> Option { + let result = self + .database(DB) + .collection::(DEPOSIT_COLLECTION) + .update_many( + doc! { "reserved": true }, + doc! { "$set": { "reserved": false } }, + ) + .await + .ok()?; + Some(result.modified_count) + } + // Remove all deposits from the mempool async fn remove_all_mempool_deposits(&self) -> Option { let collection = self @@ -521,33 +650,43 @@ impl StoredBlock { #[async_trait::async_trait] impl BlockStorageDB for mongodb::Client { async fn store_block(&self, block: &StoredBlock) -> Option<()> { - // check if the block already exists let filter = doc! { "layer2_block_number": block.layer2_block_number as i64 }; - let existing_block = self + let collection = self .database(DB) - .collection::(PROPOSED_BLOCKS_COLLECTION) - .find_one(filter.clone()) + .collection::(PROPOSED_BLOCKS_COLLECTION); + let result = collection + .replace_one(filter, block) + .upsert(true) .await .ok()?; - if existing_block.is_some() { - // if the block already exists, we need to update it - let update = doc! { "$set": { "commitments": block.commitments.clone() } }; - self.database(DB) - .collection::(PROPOSED_BLOCKS_COLLECTION) - .update_one(filter, update) - .await - .ok()?; - return Some(()); + if result.matched_count == 0 && result.upserted_id.is_none() { + return None; } - // if the block doesn't exist, we need to insert it - self.database(DB) - .collection::(PROPOSED_BLOCKS_COLLECTION) - .insert_one(block) - .await - .ok()?; Some(()) } + async fn store_block_with_session( + &self, + block: &StoredBlock, + session: &mut mongodb::ClientSession, + ) -> Result<(), mongodb::error::Error> { + let filter = doc! { "layer2_block_number": block.layer2_block_number as i64 }; + let collection = self + .database(DB) + .collection::(PROPOSED_BLOCKS_COLLECTION); + let result = collection + .replace_one(filter, block) + .upsert(true) + .session(&mut *session) + .await?; + if result.matched_count == 0 && result.upserted_id.is_none() { + return Err(mongodb::error::Error::custom( + "Failed to upsert proposed block", + )); + } + Ok(()) + } + async fn get_block_by_number(&self, block_number: u64) -> Option { let filter = doc! { "layer2_block_number": block_number as i64 }; self.database(DB) @@ -575,12 +714,427 @@ impl BlockStorageDB for mongodb::Client { .ok()?; Some(()) } + + async fn delete_all_blocks(&self) -> Option { + let result = self + .database(DB) + .collection::(PROPOSED_BLOCKS_COLLECTION) + .delete_many(doc! {}) + .await + .ok()?; + Some(result.deleted_count) + } + + async fn delete_block_by_number_with_session( + &self, + block_number: u64, + session: &mut mongodb::ClientSession, + ) -> Result<(), mongodb::error::Error> { + let filter = doc! { "layer2_block_number": block_number as i64 }; + self.database(DB) + .collection::(PROPOSED_BLOCKS_COLLECTION) + .delete_one(filter) + .session(&mut *session) + .await?; + Ok(()) + } + + async fn delete_all_blocks_with_session( + &self, + session: &mut mongodb::ClientSession, + ) -> Result { + let result = self + .database(DB) + .collection::(PROPOSED_BLOCKS_COLLECTION) + .delete_many(doc! {}) + .session(&mut *session) + .await?; + Ok(result.deleted_count) + } +} + +#[async_trait::async_trait] +impl SyncStateDB for mongodb::Client { + async fn update_sync_state_with_session( + &self, + state: &SyncState, + session: &mut mongodb::ClientSession, + ) -> Result<(), mongodb::error::Error> { + let filter = doc! { "_id": SyncState::DOCUMENT_ID }; + let collection = self + .database(DB) + .collection::(SYNC_STATE_COLLECTION); + let result = collection + .replace_one(filter, state) + .upsert(true) + .session(&mut *session) + .await?; + + if result.matched_count == 0 && result.upserted_id.is_none() { + return Err(mongodb::error::Error::custom( + "Failed to upsert proposer sync_state", + )); + } + + Ok(()) + } + + async fn get_sync_state(&self) -> Option { + self.database(DB) + .collection::(SYNC_STATE_COLLECTION) + .find_one(doc! { "_id": SyncState::DOCUMENT_ID }) + .await + .ok()? + } + + async fn delete_sync_state_with_session( + &self, + session: &mut mongodb::ClientSession, + ) -> Result<(), mongodb::error::Error> { + self.database(DB) + .collection::(SYNC_STATE_COLLECTION) + .delete_one(doc! { "_id": SyncState::DOCUMENT_ID }) + .session(&mut *session) + .await?; + Ok(()) + } +} + +#[async_trait::async_trait] +impl RestoreJournalDB for mongodb::Client { + async fn upsert_restore_journal( + &self, + journal: &RestoreJournal, + ) -> Result<(), mongodb::error::Error> { + let result = self + .database(DB) + .collection::(RESTORE_JOURNAL_COLLECTION) + .replace_one(doc! { "_id": RestoreJournal::DOCUMENT_ID }, journal) + .upsert(true) + .await?; + + if result.matched_count == 0 && result.upserted_id.is_none() { + return Err(mongodb::error::Error::custom( + "Failed to upsert proposer restore_journal", + )); + } + + Ok(()) + } + + async fn get_restore_journal(&self) -> Option { + self.database(DB) + .collection::(RESTORE_JOURNAL_COLLECTION) + .find_one(doc! { "_id": RestoreJournal::DOCUMENT_ID }) + .await + .ok()? + } + + async fn delete_restore_journal(&self) -> Result<(), mongodb::error::Error> { + self.database(DB) + .collection::(RESTORE_JOURNAL_COLLECTION) + .delete_one(doc! { "_id": RestoreJournal::DOCUMENT_ID }) + .await?; + Ok(()) + } +} + +#[async_trait::async_trait] +impl StartupReplayResetMarkerDB for mongodb::Client { + async fn upsert_startup_replay_reset_marker( + &self, + marker: &StartupReplayResetMarker, + ) -> Result<(), mongodb::error::Error> { + let result = self + .database(DB) + .collection::(STARTUP_REPLAY_RESET_MARKER_COLLECTION) + .replace_one( + doc! { "_id": StartupReplayResetMarker::DOCUMENT_ID }, + marker, + ) + .upsert(true) + .await?; + + if result.matched_count == 0 && result.upserted_id.is_none() { + return Err(mongodb::error::Error::custom( + "Failed to upsert proposer startup_replay_reset_marker", + )); + } + + Ok(()) + } + + async fn upsert_startup_replay_reset_marker_with_session( + &self, + marker: &StartupReplayResetMarker, + session: &mut mongodb::ClientSession, + ) -> Result<(), mongodb::error::Error> { + let result = self + .database(DB) + .collection::(STARTUP_REPLAY_RESET_MARKER_COLLECTION) + .replace_one( + doc! { "_id": StartupReplayResetMarker::DOCUMENT_ID }, + marker, + ) + .upsert(true) + .session(&mut *session) + .await?; + + if result.matched_count == 0 && result.upserted_id.is_none() { + return Err(mongodb::error::Error::custom( + "Failed to upsert proposer startup_replay_reset_marker", + )); + } + + Ok(()) + } + + async fn get_startup_replay_reset_marker(&self) -> Option { + self.database(DB) + .collection::(STARTUP_REPLAY_RESET_MARKER_COLLECTION) + .find_one(doc! { "_id": StartupReplayResetMarker::DOCUMENT_ID }) + .await + .ok()? + } + + async fn delete_startup_replay_reset_marker(&self) -> Result<(), mongodb::error::Error> { + self.database(DB) + .collection::(STARTUP_REPLAY_RESET_MARKER_COLLECTION) + .delete_one(doc! { "_id": StartupReplayResetMarker::DOCUMENT_ID }) + .await?; + Ok(()) + } +} + +/// Creates a unique index for proposer deposit replay identity. +/// Must be called once at proposer startup. +pub async fn ensure_deposit_indexes(client: &mongodb::Client) -> Result<(), mongodb::error::Error> { + use mongodb::options::IndexOptions; + use mongodb::IndexModel; + + let database = client.database(DB); + let existing_collections = database.list_collection_names().await?; + if !existing_collections + .iter() + .any(|name| name == DEPOSIT_COLLECTION) + { + database.create_collection(DEPOSIT_COLLECTION).await?; + } + + let collection = database.collection::(DEPOSIT_COLLECTION); + + let deposit_identity_index = IndexModel::builder() + .keys(doc! { + "deposit_data.secret_hash": 1, + "deposit_data.nf_slot_id": 1, + }) + .options(IndexOptions::builder().unique(true).build()) + .build(); + + collection.create_index(deposit_identity_index).await?; + Ok(()) +} + +/// Creates unique indexes on `receipt_id` and `tx_hash` in the TransferReceipts collection. +/// Must be called once at proposer startup. +pub async fn ensure_transfer_receipt_indexes( + client: &mongodb::Client, +) -> Result<(), mongodb::error::Error> { + use mongodb::options::IndexOptions; + use mongodb::IndexModel; + + let collection = client + .database(DB) + .collection::(TRANSFER_RECEIPTS_COLLECTION); + + let receipt_id_index = IndexModel::builder() + .keys(doc! { "receipt_id": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(); + + let tx_hash_hex_index = IndexModel::builder() + .keys(doc! { "tx_hash": 1 }) + .options(IndexOptions::builder().unique(true).build()) + .build(); + + collection.create_index(receipt_id_index).await?; + collection.create_index(tx_hash_hex_index).await?; + Ok(()) +} + +#[async_trait::async_trait] +impl TransferReceiptDB for mongodb::Client { + async fn store_transfer_receipt( + &self, + receipt: TransferReceipt, + ) -> Result<(), TransferReceiptStoreError> { + let result = self + .database(DB) + .collection::(TRANSFER_RECEIPTS_COLLECTION) + .insert_one(receipt) + .await; + + match result { + Ok(_) => Ok(()), + Err(e) => { + // MongoDB duplicate key error code is 11000. + let is_dup = matches!( + *e.kind, + mongodb::error::ErrorKind::Write( + mongodb::error::WriteFailure::WriteError(ref we) + ) if we.code == 11000 + ); + if is_dup { + Err(TransferReceiptStoreError::DuplicateKey) + } else { + Err(TransferReceiptStoreError::Other(e.to_string())) + } + } + } + } + + async fn get_transfer_receipt(&self, receipt_id: &str) -> Option { + let filter = doc! { "receipt_id": receipt_id }; + match self + .database(DB) + .collection::(TRANSFER_RECEIPTS_COLLECTION) + .find_one(filter) + .await + { + Ok(result) => result, + Err(e) => { + log::warn!("Failed to query transfer receipt by id={receipt_id}: {e}"); + None + } + } + } + + async fn get_transfer_receipt_by_tx_hash( + &self, + tx_hash: &TxHashBytes, + ) -> Option { + let filter = doc! { "tx_hash": tx_hash.as_hex() }; + match self + .database(DB) + .collection::(TRANSFER_RECEIPTS_COLLECTION) + .find_one(filter) + .await + { + Ok(result) => result, + Err(e) => { + log::warn!( + "Failed to query transfer receipt by tx_hash={}: {e}", + tx_hash.as_hex() + ); + None + } + } + } + + async fn set_transfer_receipt_status( + &self, + receipt_id: &str, + status: TransferReceiptStatus, + updated_at_unix: i64, + ) -> Option<()> { + let filter = doc! { "receipt_id": receipt_id }; + let status_bson = match mongodb::bson::to_bson(&status) { + Ok(b) => b, + Err(e) => { + log::warn!("Failed to serialize receipt status for id={receipt_id}: {e}"); + return None; + } + }; + let update = doc! { + "$set": { + "status": status_bson, + "updated_at_unix": updated_at_unix, + } + }; + match self + .database(DB) + .collection::(TRANSFER_RECEIPTS_COLLECTION) + .update_one(filter, update) + .await + { + Ok(_) => Some(()), + Err(e) => { + log::warn!("Failed to update transfer receipt status for id={receipt_id}: {e}"); + None + } + } + } +} + +#[async_trait::async_trait] +impl PendingBlockDB for mongodb::Client { + async fn store_pending_block(&self, pending_block: &PendingBlock) -> Option<()> { + let filter = doc! { "layer2_block_number": pending_block.layer2_block_number as i64 }; + self.database(DB) + .collection::(PENDING_BLOCKS_COLLECTION) + .replace_one(filter, pending_block) + .upsert(true) + .await + .ok()?; + Some(()) + } + + async fn get_pending_block(&self, block_number: u64) -> Option { + let filter = doc! { "layer2_block_number": block_number as i64 }; + self.database(DB) + .collection::(PENDING_BLOCKS_COLLECTION) + .find_one(filter) + .await + .ok()? + } + + async fn get_all_pending_blocks(&self) -> Option> { + let cursor = self + .database(DB) + .collection::(PENDING_BLOCKS_COLLECTION) + .find(doc! {}) + .await + .ok()?; + cursor.try_collect().await.ok() + } + + async fn delete_pending_block(&self, block_number: u64) -> Option<()> { + let filter = doc! { "layer2_block_number": block_number as i64 }; + self.database(DB) + .collection::(PENDING_BLOCKS_COLLECTION) + .delete_one(filter) + .await + .ok()?; + Some(()) + } + + async fn delete_all_pending_blocks(&self) -> Option { + let result = self + .database(DB) + .collection::(PENDING_BLOCKS_COLLECTION) + .delete_many(doc! {}) + .await + .ok()?; + Some(result.deleted_count) + } + + async fn delete_all_pending_blocks_with_session( + &self, + session: &mut mongodb::ClientSession, + ) -> Result { + let result = self + .database(DB) + .collection::(PENDING_BLOCKS_COLLECTION) + .delete_many(doc! {}) + .session(&mut *session) + .await?; + Ok(result.deleted_count) + } } #[cfg(test)] mod test { use super::*; - use ark_bn254::Fr as Fr254; use ark_std::UniformRand; #[test] @@ -610,29 +1164,26 @@ mod test { } #[test] - fn selected_filter_matches_new_and_legacy_shapes() { - let filter = selected_state_filter(); - let branches = filter - .get_array("$or") - .expect("selected filter should contain transitional branches"); + fn selected_filter_matches_only_explicit_selected_lifecycle_state() { + let filter = selected_transactions_filter(); - assert_eq!(branches.len(), 2); assert_eq!( - branches[0] - .as_document() - .and_then(|doc| doc.get_str("lifecycle.state").ok()), - Some("selected") + filter.get_str("lifecycle.state"), + Ok("selected"), + "selected filter should only match the explicit Selected lifecycle state" ); - let legacy = branches[1] - .as_document() - .expect("legacy selected branch should be a document"); + } + + #[test] + fn selected_or_included_filter_matches_explicit_lifecycle_states_only() { + let filter = selected_or_included_transactions_filter(); + let lifecycle_state = filter + .get_document("lifecycle.state") + .expect("selected-or-included filter should match multiple explicit lifecycle states"); assert_eq!( - legacy - .get_document("lifecycle") - .and_then(|doc| doc.get_bool("$exists")), - Ok(false) + lifecycle_state.get_array("$in").expect("$in states").len(), + 2 ); - assert!(legacy.get("block_l2").is_some()); } #[test] diff --git a/nightfall_proposer/src/driven/db/snapshot.rs b/nightfall_proposer/src/driven/db/snapshot.rs new file mode 100644 index 00000000..1820e27a --- /dev/null +++ b/nightfall_proposer/src/driven/db/snapshot.rs @@ -0,0 +1,6381 @@ +use crate::{ + domain::entities::{ + ProposerSnapshotManifest, RestoreJournal, RestoreJournalCollection, RestoreJournalPhase, + RestoreJournalStep, SnapshotCollectionManifest, SyncState, + }, + driven::db::client_transaction_state::{ + remove_all_mempool_client_transactions_with_session, + restore_included_transactions_to_mempool_after_block_with_session, + restore_selected_transactions_to_mempool_after_block_with_session, + }, + driven::db::mongo_db::{ + ensure_deposit_indexes, StoredBlock, DB, DEPOSIT_COLLECTION, PROPOSED_BLOCKS_COLLECTION, + RESTORE_JOURNAL_COLLECTION, SYNC_STATE_COLLECTION, + }, + initialisation::{ + validate_live_proposer_state_consistency, + validate_snapshotted_live_proposer_state_consistency, + }, + ports::db::{PendingBlockDB, RestoreJournalDB, StartupReplayResetMarkerDB, SyncStateDB}, + ports::trees::{CommitmentTree, HistoricRootTree, NullifierTree}, +}; +use ark_bn254::Fr as Fr254; +use lib::hex_conversion::HexConvertible; +use lib::merkle_trees::trees::TreeMetadata; +use log::{debug, error, warn}; +use mongodb::bson::{Bson, Document}; +use mongodb::options::ReadConcern; +use sha2::{Digest, Sha256}; +use std::{ + error::Error, + fmt::{Display, Formatter}, + path::{Path, PathBuf}, + process, + sync::OnceLock, +}; +use tokio::{ + fs::{self, File}, + io::{AsyncBufReadExt, AsyncWriteExt, BufReader}, +}; + +const TEMP_SNAPSHOT_DIR_PREFIX: &str = ".tmp-proposer-snapshot-"; +static PROPOSER_STATE_MAINTENANCE_LOCK: OnceLock> = OnceLock::new(); + +#[cfg(test)] +use std::{collections::HashSet, sync::Mutex}; + +#[cfg(test)] +static ENABLED_FAILPOINTS: OnceLock>> = OnceLock::new(); +#[cfg(test)] +static SNAPSHOT_TEST_LOCK: OnceLock> = OnceLock::new(); + +#[cfg(test)] +fn enabled_failpoints() -> &'static Mutex> { + ENABLED_FAILPOINTS.get_or_init(|| Mutex::new(HashSet::new())) +} + +#[cfg(test)] +pub(crate) fn clear_all_test_failpoints() { + enabled_failpoints() + .lock() + .expect("failpoint lock poisoned") + .clear(); +} + +fn maybe_crash_at_failpoint(_name: &str) { + #[cfg(test)] + { + if enabled_failpoints() + .lock() + .expect("failpoint lock poisoned") + .contains(_name) + { + panic!("simulated crash at failpoint {_name}"); + } + } +} + +fn expected_historic_root_sub_tree_count(last_applied_l2_block: u64) -> Result { + last_applied_l2_block.checked_add(2).ok_or_else(|| { + SnapshotError::InvalidSnapshotSyncState( + "sync_state last_applied_l2_block overflowed while validating historic roots" + .to_string(), + ) + }) +} + +fn proposer_state_maintenance_lock() -> &'static tokio::sync::Mutex<()> { + PROPOSER_STATE_MAINTENANCE_LOCK.get_or_init(|| tokio::sync::Mutex::new(())) +} + +#[cfg(test)] +pub(crate) async fn snapshot_test_lock() -> tokio::sync::MutexGuard<'static, ()> { + SNAPSHOT_TEST_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await +} + +pub(crate) fn try_acquire_proposer_state_maintenance_guard( +) -> Option> { + proposer_state_maintenance_lock().try_lock().ok() +} + +pub(crate) async fn acquire_proposer_state_maintenance_guard( +) -> tokio::sync::MutexGuard<'static, ()> { + proposer_state_maintenance_lock().lock().await +} + +#[cfg(test)] +struct TestFailpointGuard { + name: String, +} + +#[cfg(test)] +impl TestFailpointGuard { + fn enable(name: &str) -> Self { + enabled_failpoints() + .lock() + .expect("failpoint lock poisoned") + .insert(name.to_string()); + Self { + name: name.to_string(), + } + } +} + +#[cfg(test)] +impl Drop for TestFailpointGuard { + fn drop(&mut self) { + enabled_failpoints() + .lock() + .expect("failpoint lock poisoned") + .remove(&self.name); + } +} + +#[derive(Debug)] +pub enum SnapshotError { + Io(std::io::Error), + Mongo(mongodb::error::Error), + SerdeJson(serde_json::Error), + MissingSyncState, + MissingSnapshotCollection(String), + InvalidSnapshotSyncState(String), + MissingRestoreJournal, + UnsupportedManifestSchemaVersion(u32), + UnsupportedManifestStorageFormat(String), + UnexpectedSnapshotCollection(String), + DuplicateSnapshotCollection(String), + InvalidSnapshotFileName(String), + MissingSnapshotFile(String), + UnexpectedRestoreJournalPhase { + expected: String, + actual: String, + }, + RestoreInvariantViolation(String), + CollectionChecksumMismatch { + collection_name: String, + expected: String, + actual: String, + }, + OverallChecksumMismatch { + expected: String, + actual: String, + }, + DocumentCountMismatch { + collection_name: String, + expected: u64, + actual: u64, + }, + SnapshotManifestSyncStateMismatch { + field: &'static str, + manifest_value: String, + snapshotted_value: String, + }, +} + +impl Display for SnapshotError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::Io(error) => write!(f, "I/O error while creating snapshot: {error}"), + Self::Mongo(error) => write!(f, "MongoDB error while creating snapshot: {error}"), + Self::SerdeJson(error) => { + write!(f, "Serialization error while creating snapshot: {error}") + } + Self::MissingSyncState => write!( + f, + "Cannot create proposer snapshot without a persisted sync_state" + ), + Self::MissingSnapshotCollection(collection_name) => write!( + f, + "Cannot create proposer snapshot because canonical collection {collection_name} is missing" + ), + Self::InvalidSnapshotSyncState(message) => { + write!(f, "Snapshot sync_state is invalid: {message}") + } + Self::MissingRestoreJournal => { + write!(f, "Cannot continue restore flow without a persisted restore_journal") + } + Self::UnsupportedManifestSchemaVersion(schema_version) => write!( + f, + "Unsupported proposer snapshot manifest schema version: {schema_version}" + ), + Self::UnsupportedManifestStorageFormat(storage_format) => write!( + f, + "Unsupported proposer snapshot storage format: {storage_format}" + ), + Self::UnexpectedSnapshotCollection(collection_name) => write!( + f, + "Snapshot manifest contains unsupported proposer collection: {collection_name}" + ), + Self::DuplicateSnapshotCollection(collection_name) => write!( + f, + "Snapshot manifest contains duplicate proposer collection entry: {collection_name}" + ), + Self::InvalidSnapshotFileName(file_name) => write!( + f, + "Snapshot manifest contains invalid collection file name: {file_name}" + ), + Self::MissingSnapshotFile(file_name) => { + write!(f, "Snapshot file is missing from snapshot directory: {file_name}") + } + Self::UnexpectedRestoreJournalPhase { expected, actual } => write!( + f, + "Unexpected restore_journal phase: expected {expected}, got {actual}" + ), + Self::RestoreInvariantViolation(message) => { + write!(f, "Restore invariant violated: {message}") + } + Self::CollectionChecksumMismatch { + collection_name, + expected, + actual, + } => write!( + f, + "Snapshot collection checksum mismatch for {collection_name}: expected {expected}, got {actual}" + ), + Self::OverallChecksumMismatch { expected, actual } => write!( + f, + "Snapshot overall checksum mismatch: expected {expected}, got {actual}" + ), + Self::DocumentCountMismatch { + collection_name, + expected, + actual, + } => write!( + f, + "Snapshot document count mismatch for {collection_name}: expected {expected}, got {actual}" + ), + Self::SnapshotManifestSyncStateMismatch { + field, + manifest_value, + snapshotted_value, + } => write!( + f, + "Snapshot manifest field {field} does not match snapshotted sync_state: manifest={manifest_value}, snapshot={snapshotted_value}" + ), + } + } +} + +impl Error for SnapshotError {} + +impl From for SnapshotError { + fn from(error: std::io::Error) -> Self { + Self::Io(error) + } +} + +impl From for SnapshotError { + fn from(error: mongodb::error::Error) -> Self { + Self::Mongo(error) + } +} + +impl From for SnapshotError { + fn from(error: serde_json::Error) -> Self { + Self::SerdeJson(error) + } +} + +fn snapshot_error_to_mongo(error: SnapshotError) -> mongodb::error::Error { + match error { + SnapshotError::Mongo(error) => error, + other => mongodb::error::Error::custom(other.to_string()), + } +} + +fn mutable_tree_collection_names(tree_name: &str) -> [String; 3] { + [ + format!("{tree_name}_metadata"), + format!("{tree_name}_nodes"), + format!("{tree_name}_cache"), + ] +} + +fn proposer_snapshot_collection_names() -> Vec { + let mut names = Vec::new(); + names.extend(mutable_tree_collection_names( + >::TREE_NAME, + )); + names.extend(mutable_tree_collection_names( + >::TREE_NAME, + )); + names.extend(mutable_tree_collection_names( + >::TREE_NAME, + )); + names.push(format!( + "{}_indexed_leaves", + >::TREE_NAME + )); + names.push(DEPOSIT_COLLECTION.to_string()); + names.push(PROPOSED_BLOCKS_COLLECTION.to_string()); + names.push(SYNC_STATE_COLLECTION.to_string()); + names +} + +fn restore_journal_collections_for_canonical_snapshot() -> Vec { + proposer_snapshot_collection_names() + .into_iter() + .map(|collection_name| RestoreJournalCollection { + shadow: shadow_collection_name(&collection_name), + backup: backup_collection_name(&collection_name), + live: collection_name, + }) + .collect() +} + +pub(crate) fn required_proposer_snapshot_collection_names() -> Vec { + vec![ + format!( + "{}_metadata", + >::TREE_NAME + ), + format!( + "{}_metadata", + >::TREE_NAME + ), + format!( + "{}_metadata", + >::TREE_NAME + ), + format!( + "{}_indexed_leaves", + >::TREE_NAME + ), + DEPOSIT_COLLECTION.to_string(), + PROPOSED_BLOCKS_COLLECTION.to_string(), + SYNC_STATE_COLLECTION.to_string(), + ] +} + +fn validate_snapshot_manifest_collection_set( + manifest: &ProposerSnapshotManifest, +) -> Result<(), SnapshotError> { + let canonical = proposer_snapshot_collection_names(); + let required = required_proposer_snapshot_collection_names(); + let mut seen = std::collections::HashSet::new(); + let manifest_names: std::collections::HashSet<&str> = manifest + .collections + .iter() + .map(|collection| collection.collection_name.as_str()) + .collect(); + + for collection in &manifest.collections { + if !seen.insert(collection.collection_name.as_str()) { + return Err(SnapshotError::DuplicateSnapshotCollection( + collection.collection_name.clone(), + )); + } + + if !canonical + .iter() + .any(|name| name == &collection.collection_name) + { + return Err(SnapshotError::UnexpectedSnapshotCollection( + collection.collection_name.clone(), + )); + } + } + + for required_collection in required { + if !manifest_names.contains(required_collection.as_str()) { + return Err(SnapshotError::MissingSnapshotCollection( + required_collection, + )); + } + } + + Ok(()) +} + +fn is_optional_snapshot_collection(collection_name: &str) -> bool { + proposer_snapshot_collection_names() + .into_iter() + .any(|name| name == collection_name) + && !required_proposer_snapshot_collection_names() + .into_iter() + .any(|name| name == collection_name) +} + +fn validate_snapshot_file_name(file_name: &str) -> Result<(), SnapshotError> { + let mut components = Path::new(file_name).components(); + match (components.next(), components.next()) { + (Some(std::path::Component::Normal(component)), None) + if !component.to_string_lossy().is_empty() => + { + Ok(()) + } + _ => Err(SnapshotError::InvalidSnapshotFileName( + file_name.to_string(), + )), + } +} + +fn shadow_collection_name(live_collection_name: &str) -> String { + format!("restore_shadow__{live_collection_name}") +} + +fn backup_collection_name(live_collection_name: &str) -> String { + format!("restore_backup__{live_collection_name}") +} + +fn restore_collections_in_swap_order( + collections: &[RestoreJournalCollection], +) -> Vec { + let mut ordered = collections.to_vec(); + ordered.sort_by(|left, right| { + match ( + left.live == SYNC_STATE_COLLECTION, + right.live == SYNC_STATE_COLLECTION, + ) { + (false, true) => std::cmp::Ordering::Less, + (true, false) => std::cmp::Ordering::Greater, + _ => left.live.cmp(&right.live), + } + }); + ordered +} + +fn restore_phase_name(phase: &RestoreJournalPhase) -> &'static str { + match phase { + RestoreJournalPhase::LoadingShadow => "loading_shadow", + RestoreJournalPhase::SwapInProgress => "swap_in_progress", + RestoreJournalPhase::RollbackInProgress => "rollback_in_progress", + RestoreJournalPhase::SwapComplete => "swap_complete", + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct RestoreCollectionLayout { + live_exists: bool, + shadow_exists: bool, + backup_exists: bool, +} + +impl RestoreCollectionLayout { + fn is_pending_swap(self) -> bool { + !self.backup_exists && self.shadow_exists + } + + fn is_backup_created(self) -> bool { + self.backup_exists && !self.live_exists + } + + fn is_live_restored(self) -> bool { + self.live_exists && !self.shadow_exists + } + + fn needs_current_rollback(self) -> bool { + self.backup_exists || self.is_live_restored() + } +} + +async fn inspect_restore_collection_layout( + database: &mongodb::Database, + collection: &RestoreJournalCollection, +) -> Result { + Ok(RestoreCollectionLayout { + live_exists: collection_exists(database, &collection.live).await?, + shadow_exists: collection_exists(database, &collection.shadow).await?, + backup_exists: collection_exists(database, &collection.backup).await?, + }) +} + +async fn rollback_indices_from_swap_state( + database: &mongodb::Database, + journal: &RestoreJournal, + ordered: &[RestoreJournalCollection], +) -> Result, SnapshotError> { + let collection_count = ordered.len(); + let current_index = journal.current_index.unwrap_or(0) as usize; + if current_index >= collection_count && collection_count > 0 { + return Err(SnapshotError::RestoreInvariantViolation(format!( + "current_index {current_index} is out of bounds for {collection_count} collections" + ))); + } + + let include_current = if let Some(collection) = ordered.get(current_index) { + let layout = inspect_restore_collection_layout(database, collection).await?; + let include_current = layout.needs_current_rollback(); + if matches!( + journal.current_step, + Some(RestoreJournalStep::BackupCreated) + ) && !include_current + { + return Err(SnapshotError::RestoreInvariantViolation(format!( + "rollback expected current collection {} to retain restorable state after BackupCreated, but its namespace layout no longer does", + collection.live + ))); + } + include_current + } else { + false + }; + + let mut rollback_indices: Vec = if include_current { + (0..=current_index).collect() + } else { + (0..current_index).collect() + }; + rollback_indices.reverse(); + Ok(rollback_indices) +} + +fn compute_overall_checksum(collections: &[SnapshotCollectionManifest]) -> String { + let mut overall_checksum = Sha256::new(); + let mut collections_for_checksum: Vec<&SnapshotCollectionManifest> = + collections.iter().collect(); + collections_for_checksum + .sort_by(|left, right| left.collection_name.cmp(&right.collection_name)); + for collection in collections_for_checksum { + overall_checksum.update(collection.collection_name.as_bytes()); + overall_checksum.update(b"|"); + overall_checksum.update(collection.file_name.as_bytes()); + overall_checksum.update(b"|"); + overall_checksum.update(collection.document_count.to_le_bytes()); + overall_checksum.update(b"|"); + overall_checksum.update(collection.sha256.as_bytes()); + overall_checksum.update(b"\n"); + } + + hex::encode(overall_checksum.finalize()) +} + +pub async fn cleanup_orphaned_proposer_snapshot_temp_dirs( + snapshot_root_dir: &Path, +) -> Result<(), SnapshotError> { + if !fs::try_exists(snapshot_root_dir).await? { + return Ok(()); + } + + let mut entries = fs::read_dir(snapshot_root_dir).await?; + while let Some(entry) = entries.next_entry().await? { + if !entry.file_type().await?.is_dir() { + continue; + } + + let file_name = entry.file_name(); + if file_name + .to_string_lossy() + .starts_with(TEMP_SNAPSHOT_DIR_PREFIX) + { + fs::remove_dir_all(entry.path()).await?; + } + } + + Ok(()) +} + +async fn export_collection( + database: &mongodb::Database, + snapshot_dir: &Path, + collection_name: &str, + session: &mut mongodb::ClientSession, +) -> Result { + let file_name = format!("{collection_name}.jsonl"); + let file_path = snapshot_dir.join(&file_name); + let mut file = File::create(file_path).await?; + let collection = database.collection::(collection_name); + let mut cursor = collection + .find(mongodb::bson::doc! {}) + .session(&mut *session) + .await?; + let mut document_count = 0_u64; + let mut checksum = Sha256::new(); + + while cursor.advance(&mut *session).await? { + let document = cursor.deserialize_current()?; + let json_value = Bson::Document(document).into_relaxed_extjson(); + let line = serde_json::to_string(&json_value)?; + file.write_all(line.as_bytes()).await?; + file.write_all(b"\n").await?; + checksum.update(line.as_bytes()); + checksum.update(b"\n"); + document_count += 1; + } + + file.flush().await?; + file.sync_all().await?; + + Ok(SnapshotCollectionManifest { + collection_name: collection_name.to_string(), + file_name, + document_count, + sha256: hex::encode(checksum.finalize()), + }) +} + +async fn sync_path(path: &Path) -> Result<(), SnapshotError> { + let path = path.to_path_buf(); + tokio::task::spawn_blocking(move || -> Result<(), std::io::Error> { + std::fs::File::open(path)?.sync_all()?; + Ok(()) + }) + .await + .map_err(|join_error| { + SnapshotError::Io(std::io::Error::other(format!( + "Could not wait for fsync task to finish: {join_error}" + ))) + })??; + Ok(()) +} + +pub async fn create_proposer_snapshot( + client: &mongodb::Client, + snapshot_root_dir: &Path, +) -> Result { + let _maintenance_guard = acquire_proposer_state_maintenance_guard().await; + create_proposer_snapshot_unlocked(client, snapshot_root_dir).await +} + +pub(crate) async fn create_proposer_snapshot_unlocked( + client: &mongodb::Client, + snapshot_root_dir: &Path, +) -> Result { + let database = client.database(DB); + // listCollections is not supported inside multi-document transactions. + // We therefore discover the stable proposer collection set before opening + // the snapshot read transaction. This is acceptable because these + // collection names are created during proposer initialization and are not + // expected to change while a snapshot is taken. + let existing_collections = database.list_collection_names().await?; + for collection_name in required_proposer_snapshot_collection_names() { + if !existing_collections + .iter() + .any(|name| name == &collection_name) + { + return Err(SnapshotError::MissingSnapshotCollection(collection_name)); + } + } + let snapshot_root_dir = snapshot_root_dir.to_path_buf(); + fs::create_dir_all(&snapshot_root_dir).await?; + cleanup_orphaned_proposer_snapshot_temp_dirs(&snapshot_root_dir).await?; + let mut session = client.start_session().await?; + // temp_snapshot_dir must stay a sibling of the final snapshot_dir under + // snapshot_root_dir so the final rename remains atomic and intra-filesystem. + // Do not move temp snapshot staging to a different filesystem (for example + // /tmp) without adding an explicit copy fallback for EXDEV. + let temp_snapshot_dir = snapshot_root_dir.join(format!( + "{TEMP_SNAPSHOT_DIR_PREFIX}{}-{}", + mongodb::bson::DateTime::now().timestamp_millis(), + process::id() + )); + fs::create_dir_all(&temp_snapshot_dir).await?; + let temp_snapshot_dir_for_export = temp_snapshot_dir.clone(); + let snapshot_result = session + .start_transaction() + .read_concern(ReadConcern::snapshot()) + .and_run2(async move |session| { + let sync_state = database + .collection::(SYNC_STATE_COLLECTION) + .find_one(mongodb::bson::doc! { "_id": SyncState::DOCUMENT_ID }) + .session(&mut *session) + .await? + .ok_or_else(|| snapshot_error_to_mongo(SnapshotError::MissingSyncState))?; + + validate_snapshot_state_with_session(&database, &sync_state, session) + .await + .map_err(snapshot_error_to_mongo)?; + + let created_at = mongodb::bson::DateTime::now(); + let snapshot_id = format!( + "proposer-l2-{}-{}", + sync_state.last_applied_l2_block, + created_at.timestamp_millis() + ); + + let mut collections = Vec::new(); + for collection_name in proposer_snapshot_collection_names() { + if existing_collections.iter().any(|name| name == &collection_name) { + collections.push( + export_collection( + &database, + &temp_snapshot_dir_for_export, + &collection_name, + session, + ) + .await + .map_err(snapshot_error_to_mongo)?, + ); + if collections.len() == 1 { + maybe_crash_at_failpoint("snapshot_after_first_collection_exported"); + } + } else { + debug!( + "Skipping optional proposer snapshot collection {collection_name}: collection not found in database {DB}" + ); + } + } + + Ok::< + ( + SyncState, + Vec, + String, + mongodb::bson::DateTime, + ), + mongodb::error::Error, + >((sync_state, collections, snapshot_id, created_at)) + }) + .await; + + let (sync_state, collections, snapshot_id, created_at) = match snapshot_result { + Ok(result) => result, + Err(error) => { + let _ = fs::remove_dir_all(&temp_snapshot_dir).await; + return Err(SnapshotError::from(error)); + } + }; + let snapshot_dir = snapshot_root_dir.join(&snapshot_id); + + let overall_checksum = compute_overall_checksum(&collections); + let manifest = ProposerSnapshotManifest::new( + snapshot_id, + created_at, + DB.to_string(), + &sync_state, + collections, + overall_checksum, + ); + let result = async { + let manifest_path: PathBuf = temp_snapshot_dir.join("manifest.json"); + let manifest_bytes = serde_json::to_vec_pretty(&manifest)?; + let mut manifest_file = File::create(&manifest_path).await?; + manifest_file.write_all(&manifest_bytes).await?; + manifest_file.flush().await?; + manifest_file.sync_all().await?; + fs::rename(&temp_snapshot_dir, &snapshot_dir).await?; + sync_path(&snapshot_root_dir).await?; + Ok::<(), SnapshotError>(()) + } + .await; + + if let Err(error) = result { + let _ = fs::remove_dir_all(&temp_snapshot_dir).await; + return Err(error); + } + + Ok(manifest) +} + +async fn load_snapshot_manifest( + snapshot_dir: &Path, +) -> Result { + let manifest_path = snapshot_dir.join("manifest.json"); + let manifest_bytes = fs::read(manifest_path).await?; + let manifest: ProposerSnapshotManifest = serde_json::from_slice(&manifest_bytes)?; + + if manifest.schema_version != ProposerSnapshotManifest::SCHEMA_VERSION { + return Err(SnapshotError::UnsupportedManifestSchemaVersion( + manifest.schema_version, + )); + } + + if manifest.storage_format != ProposerSnapshotManifest::STORAGE_FORMAT { + return Err(SnapshotError::UnsupportedManifestStorageFormat( + manifest.storage_format, + )); + } + + validate_snapshot_manifest_collection_set(&manifest)?; + + Ok(manifest) +} + +pub(crate) async fn load_and_validate_snapshot_manifest( + snapshot_dir: &Path, +) -> Result { + let manifest = load_snapshot_manifest(snapshot_dir).await?; + validate_snapshot_files(snapshot_dir, &manifest).await?; + let snapshotted_sync_state = load_snapshotted_sync_state(snapshot_dir, &manifest).await?; + validate_manifest_against_snapshotted_sync_state(&manifest, &snapshotted_sync_state)?; + Ok(manifest) +} + +async fn tree_sub_tree_count_with_session( + database: &mongodb::Database, + tree_name: &str, + session: &mut mongodb::ClientSession, +) -> Result { + let metadata_collection_name = format!("{tree_name}_metadata"); + let metadata = database + .collection::>(&metadata_collection_name) + .find_one(mongodb::bson::doc! { "_id": 0 }) + .session(&mut *session) + .await? + .ok_or_else(|| { + SnapshotError::InvalidSnapshotSyncState(format!( + "tree metadata for {tree_name} is missing while creating a snapshot" + )) + })?; + Ok(metadata.sub_tree_count) +} + +async fn highest_stored_block_with_session( + database: &mongodb::Database, + session: &mut mongodb::ClientSession, +) -> Result, SnapshotError> { + database + .collection::(PROPOSED_BLOCKS_COLLECTION) + .find_one(mongodb::bson::doc! {}) + .sort(mongodb::bson::doc! { "layer2_block_number": -1_i32 }) + .session(&mut *session) + .await + .map_err(SnapshotError::from) +} + +async fn reserved_deposit_count_with_session( + database: &mongodb::Database, + session: &mut mongodb::ClientSession, +) -> Result { + database + .collection::(DEPOSIT_COLLECTION) + .count_documents(mongodb::bson::doc! { "reserved": true }) + .session(&mut *session) + .await + .map_err(SnapshotError::from) +} + +async fn validate_snapshot_state_with_session( + database: &mongodb::Database, + sync_state: &SyncState, + session: &mut mongodb::ClientSession, +) -> Result<(), SnapshotError> { + if sync_state.schema_version != SyncState::SCHEMA_VERSION { + return Err(SnapshotError::InvalidSnapshotSyncState(format!( + "sync_state uses unsupported schema version {}", + sync_state.schema_version + ))); + } + + let stored_block = database + .collection::(PROPOSED_BLOCKS_COLLECTION) + .find_one(mongodb::bson::doc! { + "layer2_block_number": sync_state.last_applied_l2_block as i64 + }) + .session(&mut *session) + .await? + .ok_or_else(|| { + SnapshotError::InvalidSnapshotSyncState(format!( + "sync_state references L2 block {}, but StoredBlock at that height is missing", + sync_state.last_applied_l2_block + )) + })?; + + let stored_fingerprint = stored_block.hash().to_hex_string(); + if stored_fingerprint != sync_state.fingerprint { + return Err(SnapshotError::InvalidSnapshotSyncState(format!( + "sync_state fingerprint {} does not match StoredBlock fingerprint {} at L2 block {}", + sync_state.fingerprint, stored_fingerprint, sync_state.last_applied_l2_block + ))); + } + + if let Some(highest_stored_block) = highest_stored_block_with_session(database, session).await? + { + if highest_stored_block.layer2_block_number > sync_state.last_applied_l2_block { + return Err(SnapshotError::InvalidSnapshotSyncState(format!( + "highest StoredBlock {} is ahead of sync_state-applied block {}", + highest_stored_block.layer2_block_number, sync_state.last_applied_l2_block + ))); + } + } + + let reserved_deposit_count = reserved_deposit_count_with_session(database, session).await?; + if reserved_deposit_count > 0 { + return Err(SnapshotError::InvalidSnapshotSyncState(format!( + "Deposits contains {reserved_deposit_count} reserved selection(s) ahead of sync_state-applied block {}", + sync_state.last_applied_l2_block + ))); + } + + let commitment_sub_tree_count = tree_sub_tree_count_with_session( + database, + >::TREE_NAME, + session, + ) + .await?; + let historic_root_sub_tree_count = tree_sub_tree_count_with_session( + database, + >::TREE_NAME, + session, + ) + .await?; + let expected_historic_root_sub_tree_count = + expected_historic_root_sub_tree_count(sync_state.last_applied_l2_block)?; + + if historic_root_sub_tree_count <= 1 { + return Err(SnapshotError::InvalidSnapshotSyncState( + "historic root tree only contains the zero leaf".to_string(), + )); + } + + if historic_root_sub_tree_count > expected_historic_root_sub_tree_count { + return Err(SnapshotError::InvalidSnapshotSyncState(format!( + "historic root tree sub_tree_count {} is ahead of sync_state-applied block {} (maximum coherent value {})", + historic_root_sub_tree_count, + sync_state.last_applied_l2_block, + expected_historic_root_sub_tree_count + ))); + } + + if historic_root_sub_tree_count < expected_historic_root_sub_tree_count { + return Err(SnapshotError::InvalidSnapshotSyncState(format!( + "historic root tree sub_tree_count {} is behind sync_state-applied block {} (required coherent value {})", + historic_root_sub_tree_count, + sync_state.last_applied_l2_block, + expected_historic_root_sub_tree_count + ))); + } + + if commitment_sub_tree_count == 0 && !stored_block.commitments.is_empty() { + return Err(SnapshotError::InvalidSnapshotSyncState(format!( + "commitment tree is empty while StoredBlock {} contains commitments", + sync_state.last_applied_l2_block + ))); + } + + Ok(()) +} + +async fn ensure_empty_shadow_collection( + database: &mongodb::Database, + shadow_collection_name: &str, +) -> Result<(), SnapshotError> { + database.create_collection(shadow_collection_name).await?; + Ok(()) +} + +async fn ensure_restored_snapshot_collection_indexes( + client: &mongodb::Client, +) -> Result<(), SnapshotError> { + // Deposits is currently the only snapshot-restored collection with required + // secondary indexes beyond MongoDB's default _id index. + ensure_deposit_indexes(client).await?; + Ok(()) +} + +pub async fn find_latest_valid_proposer_snapshot( + snapshot_root_dir: &Path, + max_last_applied_l2_block: u64, +) -> Result, SnapshotError> { + if !fs::try_exists(snapshot_root_dir).await? { + return Ok(None); + } + + let mut entries = fs::read_dir(snapshot_root_dir).await?; + let mut candidates = Vec::new(); + + while let Some(entry) = entries.next_entry().await? { + let file_type = entry.file_type().await?; + if !file_type.is_dir() { + continue; + } + + let snapshot_dir = entry.path(); + let Some(snapshot_dir_name) = snapshot_dir.file_name().and_then(|name| name.to_str()) + else { + continue; + }; + if snapshot_dir_name.starts_with(TEMP_SNAPSHOT_DIR_PREFIX) { + continue; + } + let manifest = match load_and_validate_snapshot_manifest(&snapshot_dir).await { + Ok(manifest) => manifest, + Err(error) => { + warn!( + "Skipping proposer snapshot at {}: validation failed: {error}", + snapshot_dir.display() + ); + continue; + } + }; + + if manifest.last_applied_l2_block > max_last_applied_l2_block { + continue; + } + + candidates.push((snapshot_dir, manifest)); + } + + candidates.sort_by(|left, right| { + right + .1 + .last_applied_l2_block + .cmp(&left.1.last_applied_l2_block) + .then_with(|| right.1.snapshot_id.cmp(&left.1.snapshot_id)) + }); + + Ok(candidates.into_iter().next()) +} + +async fn validate_snapshot_files( + snapshot_dir: &Path, + manifest: &ProposerSnapshotManifest, +) -> Result<(), SnapshotError> { + for collection in &manifest.collections { + validate_snapshot_file_name(&collection.file_name)?; + let file_path = snapshot_dir.join(&collection.file_name); + if !fs::try_exists(&file_path).await? { + return Err(SnapshotError::MissingSnapshotFile( + collection.file_name.clone(), + )); + } + + let file = File::open(file_path).await?; + let mut reader = BufReader::new(file).lines(); + let mut checksum = Sha256::new(); + let mut document_count = 0_u64; + + while let Some(line) = reader.next_line().await? { + checksum.update(line.as_bytes()); + checksum.update(b"\n"); + document_count += 1; + } + + let actual_checksum = hex::encode(checksum.finalize()); + if actual_checksum != collection.sha256 { + return Err(SnapshotError::CollectionChecksumMismatch { + collection_name: collection.collection_name.clone(), + expected: collection.sha256.clone(), + actual: actual_checksum, + }); + } + + if document_count != collection.document_count { + return Err(SnapshotError::DocumentCountMismatch { + collection_name: collection.collection_name.clone(), + expected: collection.document_count, + actual: document_count, + }); + } + } + + let overall_checksum = compute_overall_checksum(&manifest.collections); + if overall_checksum != manifest.overall_sha256 { + return Err(SnapshotError::OverallChecksumMismatch { + expected: manifest.overall_sha256.clone(), + actual: overall_checksum, + }); + } + + Ok(()) +} + +fn snapshot_collection_manifest<'a>( + manifest: &'a ProposerSnapshotManifest, + collection_name: &str, +) -> Result<&'a SnapshotCollectionManifest, SnapshotError> { + manifest + .collections + .iter() + .find(|collection| collection.collection_name == collection_name) + .ok_or_else(|| SnapshotError::MissingSnapshotCollection(collection_name.to_string())) +} + +async fn load_snapshotted_sync_state( + snapshot_dir: &Path, + manifest: &ProposerSnapshotManifest, +) -> Result { + let sync_state_manifest = snapshot_collection_manifest(manifest, SYNC_STATE_COLLECTION)?; + validate_snapshot_file_name(&sync_state_manifest.file_name)?; + + let file = File::open(snapshot_dir.join(&sync_state_manifest.file_name)).await?; + let mut reader = BufReader::new(file).lines(); + let Some(line) = reader.next_line().await? else { + return Err(SnapshotError::InvalidSnapshotSyncState( + "sync_state snapshot file is empty".to_string(), + )); + }; + let sync_state_json: serde_json::Value = serde_json::from_str(&line)?; + let sync_state_bson = Bson::try_from(sync_state_json).map_err(|error| { + SnapshotError::InvalidSnapshotSyncState(format!( + "could not parse sync_state snapshot extended JSON: {error}" + )) + })?; + let Bson::Document(sync_state_document) = sync_state_bson else { + return Err(SnapshotError::InvalidSnapshotSyncState( + "sync_state snapshot line is not a BSON document".to_string(), + )); + }; + let sync_state: SyncState = + mongodb::bson::from_document(sync_state_document).map_err(|error| { + SnapshotError::InvalidSnapshotSyncState(format!( + "could not deserialize sync_state snapshot document: {error}" + )) + })?; + + if reader.next_line().await?.is_some() { + return Err(SnapshotError::InvalidSnapshotSyncState( + "sync_state snapshot file contains multiple documents".to_string(), + )); + } + + if sync_state.id != SyncState::DOCUMENT_ID { + return Err(SnapshotError::InvalidSnapshotSyncState(format!( + "unexpected sync_state document id {}", + sync_state.id + ))); + } + + Ok(sync_state) +} + +fn validate_manifest_against_snapshotted_sync_state( + manifest: &ProposerSnapshotManifest, + snapshotted_sync_state: &SyncState, +) -> Result<(), SnapshotError> { + if manifest.last_applied_l2_block != snapshotted_sync_state.last_applied_l2_block { + return Err(SnapshotError::SnapshotManifestSyncStateMismatch { + field: "last_applied_l2_block", + manifest_value: manifest.last_applied_l2_block.to_string(), + snapshotted_value: snapshotted_sync_state.last_applied_l2_block.to_string(), + }); + } + + if manifest.fingerprint != snapshotted_sync_state.fingerprint { + return Err(SnapshotError::SnapshotManifestSyncStateMismatch { + field: "fingerprint", + manifest_value: manifest.fingerprint.clone(), + snapshotted_value: snapshotted_sync_state.fingerprint.clone(), + }); + } + + if manifest.l1_ref.block_number != snapshotted_sync_state.l1_ref.block_number { + return Err(SnapshotError::SnapshotManifestSyncStateMismatch { + field: "l1_ref.block_number", + manifest_value: manifest.l1_ref.block_number.to_string(), + snapshotted_value: snapshotted_sync_state.l1_ref.block_number.to_string(), + }); + } + + if manifest.l1_ref.tx_hash != snapshotted_sync_state.l1_ref.tx_hash { + return Err(SnapshotError::SnapshotManifestSyncStateMismatch { + field: "l1_ref.tx_hash", + manifest_value: manifest.l1_ref.tx_hash.to_string(), + snapshotted_value: snapshotted_sync_state.l1_ref.tx_hash.to_string(), + }); + } + + if manifest.l1_ref.log_index != snapshotted_sync_state.l1_ref.log_index { + return Err(SnapshotError::SnapshotManifestSyncStateMismatch { + field: "l1_ref.log_index", + manifest_value: manifest.l1_ref.log_index.to_string(), + snapshotted_value: snapshotted_sync_state.l1_ref.log_index.to_string(), + }); + } + + Ok(()) +} + +async fn import_collection_into_shadow( + database: &mongodb::Database, + snapshot_dir: &Path, + manifest: &SnapshotCollectionManifest, + shadow_collection_name: &str, +) -> Result<(), SnapshotError> { + validate_snapshot_file_name(&manifest.file_name)?; + database.create_collection(shadow_collection_name).await?; + let file = File::open(snapshot_dir.join(&manifest.file_name)).await?; + let mut reader = BufReader::new(file).lines(); + let shadow_collection = database.collection::(shadow_collection_name); + + while let Some(line) = reader.next_line().await? { + let json_value: serde_json::Value = serde_json::from_str(&line)?; + let bson_value = Bson::try_from(json_value).map_err(|error| { + SnapshotError::SerdeJson(serde_json::Error::io(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "could not parse snapshot line for collection {} as extended JSON: {error}", + manifest.collection_name + ), + ))) + })?; + let Bson::Document(document) = bson_value else { + return Err(SnapshotError::SerdeJson(serde_json::Error::io( + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "snapshot line for collection {} is not a BSON document", + manifest.collection_name + ), + ), + ))); + }; + shadow_collection.insert_one(document).await?; + } + + let actual_count = shadow_collection + .count_documents(mongodb::bson::doc! {}) + .await?; + if actual_count != manifest.document_count { + return Err(SnapshotError::DocumentCountMismatch { + collection_name: manifest.collection_name.clone(), + expected: manifest.document_count, + actual: actual_count, + }); + } + + Ok(()) +} + +async fn collection_exists( + database: &mongodb::Database, + collection_name: &str, +) -> Result { + let names = database.list_collection_names().await?; + Ok(names.iter().any(|name| name == collection_name)) +} + +async fn snapshot_checksum_for_collection( + database: &mongodb::Database, + collection_name: &str, +) -> Result<(u64, String), SnapshotError> { + let collection = database.collection::(collection_name); + let mut cursor = collection.find(mongodb::bson::doc! {}).await?; + let mut document_count = 0_u64; + let mut checksum = Sha256::new(); + + while cursor.advance().await? { + let document = cursor.deserialize_current()?; + let json_value = Bson::Document(document).into_relaxed_extjson(); + let line = serde_json::to_string(&json_value)?; + checksum.update(line.as_bytes()); + checksum.update(b"\n"); + document_count += 1; + } + + Ok((document_count, hex::encode(checksum.finalize()))) +} + +async fn optional_live_matches_expected_snapshot_state( + database: &mongodb::Database, + journal: &RestoreJournal, + collection: &RestoreJournalCollection, +) -> Result { + let manifest = load_and_validate_snapshot_manifest(Path::new(&journal.snapshot_dir)).await?; + if manifest.overall_sha256 != journal.manifest_overall_sha256 { + return Err(SnapshotError::RestoreInvariantViolation(format!( + "snapshot manifest checksum {} no longer matches restore journal checksum {} while verifying optional collection {}", + manifest.overall_sha256, + journal.manifest_overall_sha256, + collection.live, + ))); + } + + let (document_count, checksum) = + snapshot_checksum_for_collection(database, &collection.live).await?; + + let expected = manifest + .collections + .iter() + .find(|manifest_collection| manifest_collection.collection_name == collection.live); + + Ok(match expected { + Some(expected) => document_count == expected.document_count && checksum == expected.sha256, + None => document_count == 0, + }) +} + +async fn rename_collection( + client: &mongodb::Client, + from: &str, + to: &str, + drop_target: bool, +) -> Result<(), SnapshotError> { + client + .database("admin") + .run_command(mongodb::bson::doc! { + "renameCollection": format!("{DB}.{from}"), + "to": format!("{DB}.{to}"), + "dropTarget": drop_target, + }) + .await?; + Ok(()) +} + +async fn drop_collection_if_exists( + database: &mongodb::Database, + collection_name: &str, +) -> Result<(), SnapshotError> { + if collection_exists(database, collection_name).await? { + database + .collection::(collection_name) + .drop() + .await?; + } + Ok(()) +} + +async fn begin_rollback_from_swap_journal( + client: &mongodb::Client, + journal: &mut RestoreJournal, +) -> Result<(), SnapshotError> { + let ordered = restore_collections_in_swap_order(&journal.collections); + let database = client.database(DB); + let rollback_indices = rollback_indices_from_swap_state(&database, journal, &ordered).await?; + + journal.collections = ordered; + journal.phase = RestoreJournalPhase::RollbackInProgress; + journal.current_index = rollback_indices.first().map(|index| *index as u32); + journal.current_step = journal + .current_index + .map(|_| RestoreJournalStep::RollbackPending); + journal.updated_at = mongodb::bson::DateTime::now(); + client.upsert_restore_journal(journal).await?; + Ok(()) +} + +async fn begin_rollback_from_swap_complete_journal( + client: &mongodb::Client, + journal: &mut RestoreJournal, +) -> Result<(), SnapshotError> { + let ordered = restore_collections_in_swap_order(&journal.collections); + + journal.collections = ordered.clone(); + journal.phase = RestoreJournalPhase::RollbackInProgress; + journal.current_index = ordered.len().checked_sub(1).map(|index| index as u32); + journal.current_step = journal + .current_index + .map(|_| RestoreJournalStep::RollbackPending); + journal.updated_at = mongodb::bson::DateTime::now(); + client.upsert_restore_journal(journal).await?; + Ok(()) +} + +async fn complete_rollback_from_journal( + client: &mongodb::Client, + journal: &mut RestoreJournal, +) -> Result<(), SnapshotError> { + if journal.phase != RestoreJournalPhase::RollbackInProgress { + return Err(SnapshotError::UnexpectedRestoreJournalPhase { + expected: "rollback_in_progress".to_string(), + actual: restore_phase_name(&journal.phase).to_string(), + }); + } + + let database = client.database(DB); + let ordered = restore_collections_in_swap_order(&journal.collections); + journal.collections = ordered.clone(); + + let mut first_iteration = true; + while let Some(current_index) = journal.current_index { + let index = current_index as usize; + if index >= ordered.len() { + return Err(SnapshotError::RestoreInvariantViolation(format!( + "rollback current_index {index} is out of bounds for {} collections", + ordered.len() + ))); + } + + let collection = &ordered[index]; + let current_step = journal + .current_step + .clone() + .unwrap_or(RestoreJournalStep::RollbackPending); + match current_step { + RestoreJournalStep::RollbackPending | RestoreJournalStep::RollbackStarted => { + let resuming_started_step = + matches!(current_step, RestoreJournalStep::RollbackStarted); + if matches!(current_step, RestoreJournalStep::RollbackPending) { + journal.current_step = Some(RestoreJournalStep::RollbackStarted); + journal.updated_at = mongodb::bson::DateTime::now(); + client.upsert_restore_journal(journal).await?; + if first_iteration { + maybe_crash_at_failpoint("restore_after_first_rollback_progress_persist"); + } + } + + let layout = inspect_restore_collection_layout(&database, collection).await?; + + if layout.backup_exists { + rename_collection(client, &collection.backup, &collection.live, true).await?; + if first_iteration { + maybe_crash_at_failpoint("restore_after_first_rollback_live_restore"); + } + + journal.current_step = Some(RestoreJournalStep::RollbackApplied); + journal.updated_at = mongodb::bson::DateTime::now(); + client.upsert_restore_journal(journal).await?; + } else if is_optional_snapshot_collection(&collection.live) && layout.live_exists { + if resuming_started_step + && !optional_live_matches_expected_snapshot_state( + &database, journal, collection, + ) + .await? + { + return Err(SnapshotError::RestoreInvariantViolation(format!( + "rollback cannot prove optional live collection {} still matches the restored snapshot state after crash; manual intervention required", + collection.live + ))); + } + + drop_collection_if_exists(&database, &collection.live).await?; + + journal.current_step = Some(RestoreJournalStep::RollbackApplied); + journal.updated_at = mongodb::bson::DateTime::now(); + client.upsert_restore_journal(journal).await?; + } else if (resuming_started_step + && layout.live_exists + && !is_optional_snapshot_collection(&collection.live)) + || (is_optional_snapshot_collection(&collection.live) + && (layout.shadow_exists || resuming_started_step)) + { + journal.current_step = Some(RestoreJournalStep::RollbackApplied); + journal.updated_at = mongodb::bson::DateTime::now(); + client.upsert_restore_journal(journal).await?; + } else { + return Err(SnapshotError::RestoreInvariantViolation(format!( + "rollback cannot restore live collection {} because backup {} is missing; manual intervention required", + collection.live, collection.backup + ))); + } + } + RestoreJournalStep::RollbackApplied => {} + step => { + return Err(SnapshotError::RestoreInvariantViolation(format!( + "rollback expected rollback step at index {index}, found {step:?}" + ))); + } + } + + journal.current_index = index.checked_sub(1).map(|next| next as u32); + journal.current_step = journal + .current_index + .map(|_| RestoreJournalStep::RollbackPending); + journal.updated_at = mongodb::bson::DateTime::now(); + client.upsert_restore_journal(journal).await?; + first_iteration = false; + } + + maybe_crash_at_failpoint("restore_after_rollback_renames_complete"); + + for collection in &ordered { + drop_collection_if_exists(&database, &collection.shadow).await?; + drop_collection_if_exists(&database, &collection.backup).await?; + } + + client.delete_restore_journal().await?; + Ok(()) +} + +async fn rollback_shadow_swap( + client: &mongodb::Client, + journal: &mut RestoreJournal, +) -> Result<(), SnapshotError> { + if journal.phase == RestoreJournalPhase::SwapInProgress { + begin_rollback_from_swap_journal(client, journal).await?; + } + + complete_rollback_from_journal(client, journal).await +} + +async fn rollback_completed_swap_from_backups( + client: &mongodb::Client, + journal: &mut RestoreJournal, +) -> Result<(), SnapshotError> { + if journal.phase == RestoreJournalPhase::SwapComplete { + begin_rollback_from_swap_complete_journal(client, journal).await?; + } + + complete_rollback_from_journal(client, journal).await +} + +async fn cleanup_loading_shadow_restore( + client: &mongodb::Client, + journal: &RestoreJournal, +) -> Result<(), SnapshotError> { + let database = client.database(DB); + for collection in &journal.collections { + drop_collection_if_exists(&database, &collection.shadow).await?; + drop_collection_if_exists(&database, &collection.backup).await?; + } + client.delete_restore_journal().await?; + Ok(()) +} + +async fn complete_shadow_swap_from_journal( + client: &mongodb::Client, + journal: &mut RestoreJournal, +) -> Result<(), SnapshotError> { + if journal.phase != RestoreJournalPhase::SwapInProgress { + return Err(SnapshotError::UnexpectedRestoreJournalPhase { + expected: "swap_in_progress".to_string(), + actual: restore_phase_name(&journal.phase).to_string(), + }); + } + + let database = client.database(DB); + let ordered = restore_collections_in_swap_order(&journal.collections); + journal.collections = ordered.clone(); + + let start_index = journal.current_index.unwrap_or(0) as usize; + if start_index >= ordered.len() { + return Err(SnapshotError::RestoreInvariantViolation(format!( + "current_index {start_index} is out of bounds for {} collections", + ordered.len() + ))); + } + + for (index, collection) in ordered.iter().enumerate().skip(start_index) { + let step = if index == start_index { + journal + .current_step + .clone() + .unwrap_or(RestoreJournalStep::BackupPending) + } else { + RestoreJournalStep::BackupPending + }; + + match step { + RestoreJournalStep::BackupPending => { + let layout = inspect_restore_collection_layout(&database, collection).await?; + + if layout.is_live_restored() { + if !layout.backup_exists { + if !is_optional_snapshot_collection(&collection.live) { + return Err(SnapshotError::RestoreInvariantViolation(format!( + "live collection {} is already restored but backup {} is missing; resume cannot prove the pre-restore state still exists", + collection.live, collection.backup + ))); + } + + let matches_snapshot = optional_live_matches_expected_snapshot_state( + &database, + journal, + collection, + ) + .await + .map_err(|error| { + SnapshotError::RestoreInvariantViolation(format!( + "optional live collection {} is present while both shadow {} and backup {} are missing, and recovery could not verify it matches the snapshot state: {}", + collection.live, collection.shadow, collection.backup, error + )) + })?; + + if !matches_snapshot { + return Err(SnapshotError::RestoreInvariantViolation(format!( + "optional live collection {} is present while both shadow {} and backup {} are missing; resume cannot prove snapshot state replaced the pre-restore live collection", + collection.live, collection.shadow, collection.backup + ))); + } + } + } else if layout.is_backup_created() { + journal.current_index = Some(index as u32); + journal.current_step = Some(RestoreJournalStep::BackupCreated); + journal.updated_at = mongodb::bson::DateTime::now(); + client.upsert_restore_journal(journal).await?; + + if !layout.shadow_exists { + return Err(SnapshotError::RestoreInvariantViolation(format!( + "shadow collection {} is missing while resume expects BackupCreated", + collection.shadow + ))); + } + + rename_collection(client, &collection.shadow, &collection.live, false).await?; + } else if layout.is_pending_swap() && layout.live_exists { + rename_collection(client, &collection.live, &collection.backup, false).await?; + maybe_crash_at_failpoint("restore_after_live_rename_before_backup_created"); + + journal.current_index = Some(index as u32); + journal.current_step = Some(RestoreJournalStep::BackupCreated); + journal.updated_at = mongodb::bson::DateTime::now(); + client.upsert_restore_journal(journal).await?; + maybe_crash_at_failpoint("restore_after_backup_created"); + rename_collection(client, &collection.shadow, &collection.live, false).await?; + } else if layout.is_pending_swap() + && is_optional_snapshot_collection(&collection.live) + { + rename_collection(client, &collection.shadow, &collection.live, false).await?; + } else { + return Err(SnapshotError::RestoreInvariantViolation(format!( + "collection {} cannot resume swap from BackupPending with namespace layout live_exists={}, shadow_exists={}, backup_exists={}", + collection.live, + layout.live_exists, + layout.shadow_exists, + layout.backup_exists + ))); + } + } + RestoreJournalStep::BackupCreated => { + let layout = inspect_restore_collection_layout(&database, collection).await?; + + if layout.is_live_restored() { + if !layout.backup_exists && !is_optional_snapshot_collection(&collection.live) { + return Err(SnapshotError::RestoreInvariantViolation(format!( + "live collection {} is already restored but backup {} is missing while resume expects BackupCreated", + collection.live, collection.backup + ))); + } + } else if !layout.backup_exists { + return Err(SnapshotError::RestoreInvariantViolation(format!( + "backup collection {} is missing while resume expects BackupCreated", + collection.backup + ))); + } else if !layout.shadow_exists { + return Err(SnapshotError::RestoreInvariantViolation(format!( + "shadow collection {} is missing while resume expects BackupCreated", + collection.shadow + ))); + } else { + rename_collection(client, &collection.shadow, &collection.live, false).await?; + } + } + step => { + return Err(SnapshotError::RestoreInvariantViolation(format!( + "swap expected backup step at index {index}, found {step:?}" + ))); + } + } + + if index + 1 < ordered.len() { + journal.current_index = Some((index + 1) as u32); + journal.current_step = Some(RestoreJournalStep::BackupPending); + journal.updated_at = mongodb::bson::DateTime::now(); + client.upsert_restore_journal(journal).await?; + } + } + + journal.phase = RestoreJournalPhase::SwapComplete; + journal.current_index = None; + journal.current_step = Some(RestoreJournalStep::NonSnapshotCleanupPending); + journal.updated_at = mongodb::bson::DateTime::now(); + client.upsert_restore_journal(journal).await?; + maybe_crash_at_failpoint("restore_after_swap_complete"); + + Ok(()) +} + +pub async fn load_proposer_snapshot_into_shadow( + client: &mongodb::Client, + snapshot_dir: &Path, +) -> Result { + let snapshot_dir = snapshot_dir.to_path_buf(); + let manifest = load_and_validate_snapshot_manifest(&snapshot_dir).await?; + let manifest_collections_by_name: std::collections::HashMap<_, _> = manifest + .collections + .iter() + .map(|collection| (collection.collection_name.as_str(), collection)) + .collect(); + + let database = client.database(DB); + let existing_collections = database.list_collection_names().await?; + let restore_collections = restore_journal_collections_for_canonical_snapshot(); + + let now = mongodb::bson::DateTime::now(); + let mut journal = RestoreJournal::new_loading_shadow( + manifest.snapshot_id.clone(), + snapshot_dir.to_string_lossy().to_string(), + manifest.overall_sha256.clone(), + restore_collections.clone(), + now, + ); + client.upsert_restore_journal(&journal).await?; + + for restore_collection in &restore_collections { + if existing_collections + .iter() + .any(|name| name == &restore_collection.shadow) + { + database + .collection::(&restore_collection.shadow) + .drop() + .await?; + } + + if existing_collections + .iter() + .any(|name| name == &restore_collection.backup) + { + database + .collection::(&restore_collection.backup) + .drop() + .await?; + } + } + + for restore_collection in &restore_collections { + if let Some(manifest_collection) = + manifest_collections_by_name.get(restore_collection.live.as_str()) + { + import_collection_into_shadow( + &database, + &snapshot_dir, + manifest_collection, + &restore_collection.shadow, + ) + .await?; + } else { + ensure_empty_shadow_collection(&database, &restore_collection.shadow).await?; + } + maybe_crash_at_failpoint("restore_after_first_shadow_collection_imported"); + } + + journal.updated_at = mongodb::bson::DateTime::now(); + client.upsert_restore_journal(&journal).await?; + + Ok(journal) +} + +pub async fn restore_proposer_snapshot( + client: &mongodb::Client, + snapshot_dir: &Path, +) -> Result { + let _maintenance_guard = acquire_proposer_state_maintenance_guard().await; + load_proposer_snapshot_into_shadow(client, snapshot_dir).await?; + swap_proposer_shadow_into_live(client).await?; + cleanup_after_proposer_shadow_swap(client).await?; + + client + .get_sync_state() + .await + .ok_or(SnapshotError::MissingSyncState) +} + +pub async fn swap_proposer_shadow_into_live( + client: &mongodb::Client, +) -> Result { + let mut journal = client + .get_restore_journal() + .await + .ok_or(SnapshotError::MissingRestoreJournal)?; + + if journal.phase != RestoreJournalPhase::LoadingShadow { + return Err(SnapshotError::UnexpectedRestoreJournalPhase { + expected: "loading_shadow".to_string(), + actual: format!("{:?}", journal.phase), + }); + } + + let ordered = restore_collections_in_swap_order(&journal.collections); + journal.collections = ordered.clone(); + journal.phase = RestoreJournalPhase::SwapInProgress; + journal.current_index = Some(0); + journal.current_step = Some(RestoreJournalStep::BackupPending); + journal.updated_at = mongodb::bson::DateTime::now(); + client.upsert_restore_journal(&journal).await?; + + if let Err(error) = complete_shadow_swap_from_journal(client, &mut journal).await { + if matches!(error, SnapshotError::RestoreInvariantViolation(_)) { + rollback_shadow_swap(client, &mut journal).await?; + } + return Err(error); + } + + Ok(journal) +} + +pub async fn cleanup_after_proposer_shadow_swap( + client: &mongodb::Client, +) -> Result<(), SnapshotError> { + let journal = client + .get_restore_journal() + .await + .ok_or(SnapshotError::MissingRestoreJournal)?; + + if journal.phase != RestoreJournalPhase::SwapComplete { + return Err(SnapshotError::UnexpectedRestoreJournalPhase { + expected: "swap_complete".to_string(), + actual: format!("{:?}", journal.phase), + }); + } + + validate_restored_snapshot_state_before_cleanup(client).await?; + complete_cleanup_after_proposer_shadow_swap(client, &journal).await +} + +async fn validate_restored_snapshot_state_before_cleanup( + client: &mongodb::Client, +) -> Result<(), SnapshotError> { + ensure_restored_snapshot_collection_indexes(client).await?; + validate_snapshotted_live_proposer_state_consistency(client) + .await + .map_err(SnapshotError::RestoreInvariantViolation) +} + +async fn complete_cleanup_after_proposer_shadow_swap( + client: &mongodb::Client, + journal: &RestoreJournal, +) -> Result<(), SnapshotError> { + let cleanup_stats = apply_non_snapshot_cleanup_if_needed(client, journal).await?; + maybe_crash_at_failpoint("restore_after_non_snapshot_cleanup_applied"); + log_non_snapshot_cleanup_stats(&cleanup_stats); + validate_live_proposer_state_consistency(client) + .await + .map_err(SnapshotError::RestoreInvariantViolation)?; + + let database = client.database(DB); + let ordered = restore_collections_in_swap_order(&journal.collections); + + for collection in &ordered { + drop_collection_if_exists(&database, &collection.backup).await?; + maybe_crash_at_failpoint("restore_after_first_backup_cleanup"); + } + + for collection in &ordered { + drop_collection_if_exists(&database, &collection.shadow).await?; + } + + clear_startup_replay_reset_marker_if_present(client).await?; + client.delete_restore_journal().await?; + Ok(()) +} + +async fn clear_startup_replay_reset_marker_if_present( + client: &mongodb::Client, +) -> Result<(), SnapshotError> { + if client.get_startup_replay_reset_marker().await.is_none() { + return Ok(()); + } + + client + .delete_startup_replay_reset_marker() + .await + .map_err(SnapshotError::Mongo) +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +struct NonSnapshotCleanupStats { + last_applied_l2_block: u64, + deleted_pending_blocks: u64, + cleared_reserved_deposits: u64, + removed_mempool_transactions: u64, + restored_selected_transactions: u64, + restored_included_transactions: u64, +} + +async fn apply_non_snapshot_cleanup_if_needed( + client: &mongodb::Client, + journal: &RestoreJournal, +) -> Result { + match journal.current_step.clone() { + Some(RestoreJournalStep::NonSnapshotCleanupApplied) => { + Ok(NonSnapshotCleanupStats::default()) + } + Some(RestoreJournalStep::NonSnapshotCleanupPending) | None => { + cleanup_non_snapshot_restore_state(client, journal).await + } + Some(step) => Err(SnapshotError::RestoreInvariantViolation(format!( + "swap_complete cleanup expected non_snapshot_cleanup step, found {step:?}" + ))), + } +} + +async fn get_sync_state_with_session( + client: &mongodb::Client, + session: &mut mongodb::ClientSession, +) -> Result { + client + .database(DB) + .collection::(SYNC_STATE_COLLECTION) + .find_one(mongodb::bson::doc! { "_id": SyncState::DOCUMENT_ID }) + .session(&mut *session) + .await? + .ok_or_else(|| { + mongodb::error::Error::custom( + "Could not load restored sync_state during snapshot restore finalization", + ) + }) +} + +async fn clear_all_reserved_deposits_with_session( + client: &mongodb::Client, + session: &mut mongodb::ClientSession, +) -> Result { + client + .database(DB) + .collection::(DEPOSIT_COLLECTION) + .update_many( + mongodb::bson::doc! { "reserved": true }, + mongodb::bson::doc! { "$set": { "reserved": false } }, + ) + .session(&mut *session) + .await + .map(|result| result.modified_count) +} + +async fn upsert_restore_journal_with_session( + client: &mongodb::Client, + journal: &RestoreJournal, + session: &mut mongodb::ClientSession, +) -> Result<(), mongodb::error::Error> { + let result = client + .database(DB) + .collection::(RESTORE_JOURNAL_COLLECTION) + .replace_one( + mongodb::bson::doc! { "_id": RestoreJournal::DOCUMENT_ID }, + journal, + ) + .upsert(true) + .session(&mut *session) + .await?; + + if result.matched_count == 0 && result.upserted_id.is_none() { + return Err(mongodb::error::Error::custom( + "Failed to upsert proposer restore_journal", + )); + } + + Ok(()) +} + +async fn cleanup_non_snapshot_restore_state( + client: &mongodb::Client, + journal: &RestoreJournal, +) -> Result { + let journal_for_update = journal.clone(); + let client_for_transaction = client.clone(); + let mut session = client.start_session().await?; + session + .start_transaction() + .and_run2(async move |session| { + let sync_state = get_sync_state_with_session(&client_for_transaction, session).await?; + let deleted_pending_blocks = client_for_transaction + .delete_all_pending_blocks_with_session(session) + .await + .map_err(|error| { + mongodb::error::Error::custom(format!( + "Could not delete persisted PendingBlocks during snapshot restore finalization: {error}" + )) + })?; + let cleared_reserved_deposits = + clear_all_reserved_deposits_with_session(&client_for_transaction, session) + .await + .map_err(|error| { + mongodb::error::Error::custom(format!( + "Could not clear proposer Deposits reservations during snapshot restore finalization: {error}" + )) + })?; + let removed_mempool_transactions = + remove_all_mempool_client_transactions_with_session( + &client_for_transaction, + session, + ) + .await + .map_err(mongodb::error::Error::custom)?; + let restored_selected_transactions = + restore_selected_transactions_to_mempool_after_block_with_session( + &client_for_transaction, + sync_state.last_applied_l2_block, + session, + ) + .await + .map_err(mongodb::error::Error::custom)?; + let restored_included_transactions = + restore_included_transactions_to_mempool_after_block_with_session( + &client_for_transaction, + sync_state.last_applied_l2_block, + session, + ) + .await + .map_err(mongodb::error::Error::custom)?; + + let mut updated_journal = journal_for_update.clone(); + updated_journal.current_step = Some(RestoreJournalStep::NonSnapshotCleanupApplied); + updated_journal.updated_at = mongodb::bson::DateTime::now(); + upsert_restore_journal_with_session( + &client_for_transaction, + &updated_journal, + session, + ) + .await?; + + Ok::(NonSnapshotCleanupStats { + last_applied_l2_block: sync_state.last_applied_l2_block, + deleted_pending_blocks, + cleared_reserved_deposits, + removed_mempool_transactions, + restored_selected_transactions, + restored_included_transactions, + }) + }) + .await + .map_err(SnapshotError::Mongo) +} + +fn log_non_snapshot_cleanup_stats(stats: &NonSnapshotCleanupStats) { + if stats.deleted_pending_blocks > 0 + || stats.cleared_reserved_deposits > 0 + || stats.removed_mempool_transactions > 0 + || stats.restored_selected_transactions > 0 + || stats.restored_included_transactions > 0 + { + warn!( + "Snapshot restore discarded {} persisted PendingBlock(s), cleared {} reserved deposit selection(s), dropped {} mempool client transaction(s), restored {} selected client transaction(s) beyond restored sync_state L2 block {} to the mempool, and restored {} included client transaction(s) beyond that restored sync_state to the mempool before final live-state validation", + stats.deleted_pending_blocks, + stats.cleared_reserved_deposits, + stats.removed_mempool_transactions, + stats.restored_selected_transactions, + stats.last_applied_l2_block, + stats.restored_included_transactions, + ); + } +} + +pub async fn recover_from_restore_journal(client: &mongodb::Client) -> Result<(), SnapshotError> { + let _maintenance_guard = acquire_proposer_state_maintenance_guard().await; + recover_from_restore_journal_unlocked(client).await +} + +async fn recover_from_restore_journal_unlocked( + client: &mongodb::Client, +) -> Result<(), SnapshotError> { + let journal = match client.get_restore_journal().await { + Some(journal) => journal, + None => return Ok(()), + }; + + match journal.phase { + RestoreJournalPhase::LoadingShadow => { + cleanup_loading_shadow_restore(client, &journal).await?; + Ok(()) + } + RestoreJournalPhase::SwapInProgress => { + let mut journal = journal; + match complete_shadow_swap_from_journal(client, &mut journal).await { + Ok(()) => Ok(()), + Err(error) => { + if matches!(error, SnapshotError::RestoreInvariantViolation(_)) { + if let Err(rollback_error) = + rollback_shadow_swap(client, &mut journal).await + { + error!( + "Proposer restore rollback cannot complete cleanly: {rollback_error}. Manual intervention required." + ); + return Err(rollback_error); + } + } + Err(error) + } + } + } + RestoreJournalPhase::RollbackInProgress => { + let mut journal = journal; + if let Err(error) = complete_rollback_from_journal(client, &mut journal).await { + error!( + "Proposer restore rollback remains incomplete: {error}. Manual intervention required." + ); + return Err(error); + } + Err(SnapshotError::RestoreInvariantViolation( + "restore was rolled back after a crash mid-rollback; original restore did not complete".to_string(), + )) + } + RestoreJournalPhase::SwapComplete => { + let mut journal = journal; + match validate_restored_snapshot_state_before_cleanup(client).await { + Ok(()) => complete_cleanup_after_proposer_shadow_swap(client, &journal).await, + Err(error) => { + if matches!(error, SnapshotError::RestoreInvariantViolation(_)) { + if let Err(rollback_error) = + rollback_completed_swap_from_backups(client, &mut journal).await + { + error!( + "Proposer restore rollback from swap_complete cannot complete cleanly: {rollback_error}. Manual intervention required." + ); + return Err(rollback_error); + } + } + Err(error) + } + } + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::{ + domain::entities::{ + Block, ClientTransactionWithMetaData, DepositDatawithFee, L1Ref, PendingBlock, + PendingBlockState, RestoreJournal, RestoreJournalCollection, RestoreJournalPhase, + RestoreJournalStep, StartupReplayResetMarker, StartupReplayResetPhase, SyncState, + TxLifecycle, + }, + driven::db::mongo_db::{ + ensure_deposit_indexes, StoredBlock, DEPOSIT_COLLECTION, RESTORE_JOURNAL_COLLECTION, + }, + ports::{ + db::{ + BlockStorageDB, PendingBlockDB, RestoreJournalDB, StartupReplayResetMarkerDB, + SyncStateDB, TransactionsDB, + }, + trees::{CommitmentTree, HistoricRootTree, NullifierTree}, + }, + }; + use alloy::primitives::{Address, Bytes, TxHash}; + use ark_ff::Zero; + use ark_serialize::SerializationError; + use lib::merkle_trees::trees::{MutableTree, TreeMetadata}; + use lib::nf_client_proof::Proof; + use lib::shared_entities::{ClientTransaction, CompressedSecrets, DepositData}; + use lib::tests_utils::{get_db_connection, get_mongo}; + use serde::{Deserialize, Serialize}; + use std::{ + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Duration, + }; + pub(crate) async fn snapshot_test_lock() -> tokio::sync::MutexGuard<'static, ()> { + super::snapshot_test_lock().await + } + + #[derive(Clone, Debug, Default, Deserialize, Serialize)] + struct MockProof; + + impl Proof for MockProof { + fn compress_proof(&self) -> Result { + Ok(Bytes::new()) + } + + fn from_compressed(_compressed: Bytes) -> Result { + Ok(Self) + } + } + + fn test_selected_client_transaction( + seed: u32, + block_l2: u64, + commitment: Fr254, + ) -> ClientTransactionWithMetaData { + ClientTransactionWithMetaData { + client_transaction: ClientTransaction { + commitments: [commitment, Fr254::zero(), Fr254::zero(), Fr254::zero()], + compressed_secrets: CompressedSecrets::default(), + proof: MockProof, + ..Default::default() + }, + lifecycle: TxLifecycle::Selected { block_l2 }, + hash: vec![seed, seed + 1, seed + 2], + historic_roots: vec![], + receipt_token: None, + } + } + + async fn persist_sync_state(client: &mongodb::Client, sync_state: &SyncState) { + let mut session = client.start_session().await.expect("start session"); + let client_for_write = client.clone(); + let sync_state_for_write = sync_state.clone(); + session + .start_transaction() + .and_run2(async move |session| { + client_for_write + .update_sync_state_with_session(&sync_state_for_write, session) + .await?; + Ok::<(), mongodb::error::Error>(()) + }) + .await + .expect("write sync_state"); + + let stored_block = client + .get_block_by_number(sync_state.last_applied_l2_block) + .await + .expect("stored block should exist for test sync_state"); + + let commitment_metadata = client + .database(DB) + .collection::>(&format!( + "{}_metadata", + >::TREE_NAME + )) + .find_one(mongodb::bson::doc! { "_id": 0 }) + .await + .expect("read commitment metadata"); + if commitment_metadata + .as_ref() + .is_some_and(|metadata| metadata.sub_tree_count == 0) + && !stored_block.commitments.is_empty() + { + >::insert_leaf( + client, + Fr254::from(sync_state.last_applied_l2_block + 1), + true, + >::TREE_NAME, + ) + .await + .expect("materialize commitment tree state for snapshot test"); + } + + let target_historic_root_sub_tree_count = sync_state + .last_applied_l2_block + .checked_add(2) + .expect("historic root count should not overflow in snapshot tests"); + let mut historic_root_sub_tree_count = client + .database(DB) + .collection::>(&format!( + "{}_metadata", + >::TREE_NAME + )) + .find_one(mongodb::bson::doc! { "_id": 0 }) + .await + .expect("read historic root metadata") + .expect("historic root metadata should exist for snapshot tests") + .sub_tree_count; + while historic_root_sub_tree_count < target_historic_root_sub_tree_count { + let commitment_root = >::get_root(client) + .await + .expect("read commitment root"); + >::append_historic_commitment_root( + client, + &commitment_root, + true, + ) + .await + .expect("materialize historic root state for snapshot test"); + historic_root_sub_tree_count += 1; + } + } + + async fn initialize_snapshot_test_trees(client: &mongodb::Client) { + >::new_commitment_tree(client, 29, 3) + .await + .expect("create commitment tree"); + >::new_historic_root_tree(client, 32) + .await + .expect("create historic root tree"); + >::append_historic_commitment_root( + client, + &Fr254::zero(), + true, + ) + .await + .expect("append zero historic root"); + >::new_nullifier_tree(client, 29, 3) + .await + .expect("create nullifier tree"); + ensure_deposit_indexes(client) + .await + .expect("create deposit indexes"); + } + + async fn create_snapshot_fixture( + client: &mongodb::Client, + layer2_block_number: u64, + commitment: &str, + proposer_byte: u8, + l1_block_number: u64, + snapshot_root_prefix: &str, + ) -> (PathBuf, ProposerSnapshotManifest, StoredBlock, SyncState) { + let stored_block = StoredBlock { + layer2_block_number, + commitments: vec![commitment.to_string()], + proposer_address: Address::from([proposer_byte; 20]), + }; + client + .store_block(&stored_block) + .await + .expect("store block for snapshot fixture"); + + let sync_state = SyncState::new( + stored_block.layer2_block_number, + stored_block.hash().to_hex_string(), + L1Ref { + block_number: l1_block_number, + tx_hash: TxHash::from([proposer_byte; 32]), + log_index: proposer_byte as u64, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(client, &sync_state).await; + + let snapshot_root = std::env::temp_dir().join(format!( + "{snapshot_root_prefix}-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(client, &snapshot_root) + .await + .expect("create snapshot fixture"); + + (snapshot_root, manifest, stored_block, sync_state) + } + + async fn first_existing_live_restore_collection( + client: &mongodb::Client, + journal: &[RestoreJournalCollection], + ) -> (usize, RestoreJournalCollection) { + let existing_collections = client + .database(DB) + .list_collection_names() + .await + .expect("list collections"); + restore_collections_in_swap_order(journal) + .into_iter() + .enumerate() + .find(|(_, collection)| { + existing_collections + .iter() + .any(|name| name == &collection.live) + }) + .expect("at least one live restore collection should exist") + } + + async fn next_required_live_restore_collection( + client: &mongodb::Client, + journal: &[RestoreJournalCollection], + after_index: usize, + ) -> (usize, RestoreJournalCollection) { + let existing_collections = client + .database(DB) + .list_collection_names() + .await + .expect("list collections"); + restore_collections_in_swap_order(journal) + .into_iter() + .enumerate() + .skip(after_index + 1) + .find(|(_, collection)| { + existing_collections + .iter() + .any(|name| name == &collection.live) + && required_proposer_snapshot_collection_names() + .iter() + .any(|name| name == &collection.live) + }) + .expect( + "expected another required live restore collection after the first processed one", + ) + } + + async fn write_manifest(snapshot_dir: &Path, manifest: &ProposerSnapshotManifest) { + fs::write( + snapshot_dir.join("manifest.json"), + serde_json::to_vec_pretty(manifest).expect("serialize manifest"), + ) + .await + .expect("write manifest"); + } + + async fn read_manifest(snapshot_dir: &Path) -> ProposerSnapshotManifest { + serde_json::from_slice( + &fs::read(snapshot_dir.join("manifest.json")) + .await + .expect("read manifest"), + ) + .expect("deserialize manifest") + } + + async fn prepare_mid_swap_pending_state( + client: &mongodb::Client, + snapshot_root_prefix: &str, + snapshot_block_number: u64, + newer_live_block_number: u64, + ) -> (PathBuf, RestoreJournal, SyncState) { + initialize_snapshot_test_trees(client).await; + + let (snapshot_root, manifest, _, _) = create_snapshot_fixture( + client, + snapshot_block_number, + "0xsnapshot", + snapshot_block_number as u8, + snapshot_block_number * 100, + snapshot_root_prefix, + ) + .await; + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let newer_live_block = StoredBlock { + layer2_block_number: newer_live_block_number, + commitments: vec!["0xnewer-live".to_string()], + proposer_address: Address::from([newer_live_block_number as u8; 20]), + }; + client + .store_block(&newer_live_block) + .await + .expect("store newer live block"); + let newer_live_sync_state = SyncState::new( + newer_live_block.layer2_block_number, + newer_live_block.hash().to_hex_string(), + L1Ref { + block_number: newer_live_block_number * 100, + tx_hash: TxHash::from([newer_live_block_number as u8; 32]), + log_index: newer_live_block_number, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(client, &newer_live_sync_state).await; + + let mut journal = load_proposer_snapshot_into_shadow(client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + let ordered = restore_collections_in_swap_order(&journal.collections); + assert!( + ordered.len() > 1, + "expected multiple collections in restore order" + ); + let (first_index, first) = + first_existing_live_restore_collection(client, &journal.collections).await; + + rename_collection(client, &first.live, &first.backup, false) + .await + .expect("rename first live to backup"); + rename_collection(client, &first.shadow, &first.live, false) + .await + .expect("rename first shadow to live"); + + let (next_index, _) = + next_required_live_restore_collection(client, &journal.collections, first_index).await; + + journal.collections = ordered; + journal.phase = RestoreJournalPhase::SwapInProgress; + journal.current_index = Some(next_index as u32); + journal.current_step = Some(RestoreJournalStep::BackupPending); + journal.updated_at = mongodb::bson::DateTime::now(); + client + .upsert_restore_journal(&journal) + .await + .expect("persist mid-swap journal"); + + (snapshot_root, journal, newer_live_sync_state) + } + + #[tokio::test] + async fn recover_from_restore_journal_is_noop_when_idle() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + recover_from_restore_journal(&client) + .await + .expect("idle recover should be a no-op"); + + assert_eq!(client.get_restore_journal().await, None); + } + + #[tokio::test] + async fn proposer_state_maintenance_try_lock_skips_while_held() { + let _snapshot_test_lock = snapshot_test_lock().await; + + let guard = acquire_proposer_state_maintenance_guard().await; + assert!( + try_acquire_proposer_state_maintenance_guard().is_none(), + "best-effort snapshot work should skip while maintenance is in progress" + ); + drop(guard); + + assert!( + try_acquire_proposer_state_maintenance_guard().is_some(), + "maintenance lock should become available again after release" + ); + } + + #[tokio::test] + async fn proposer_state_maintenance_critical_path_waits_for_release() { + let _snapshot_test_lock = snapshot_test_lock().await; + + let guard = acquire_proposer_state_maintenance_guard().await; + let acquired = Arc::new(AtomicBool::new(false)); + let acquired_for_task = Arc::clone(&acquired); + tokio::spawn(async move { + let guard = acquire_proposer_state_maintenance_guard().await; + acquired_for_task.store(true, Ordering::Release); + drop(guard); + }); + + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + !acquired.load(Ordering::Acquire), + "critical restore path should wait while a snapshot-style holder still owns the lock" + ); + drop(guard); + } + + #[tokio::test] + async fn create_proposer_snapshot_waits_for_maintenance_lock_release() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let stored_block = StoredBlock { + layer2_block_number: 1, + commitments: vec!["0xlock-test".to_string()], + proposer_address: Address::from([7; 20]), + }; + client + .store_block(&stored_block) + .await + .expect("store block"); + persist_sync_state( + &client, + &SyncState::new( + stored_block.layer2_block_number, + stored_block.hash().to_hex_string(), + L1Ref { + block_number: 100, + tx_hash: TxHash::from([7; 32]), + log_index: 0, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-snapshot-lock-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + + let guard = acquire_proposer_state_maintenance_guard().await; + let snapshot_client = client.clone(); + let snapshot_root_for_task = snapshot_root.clone(); + let handle = tokio::spawn(async move { + create_proposer_snapshot(&snapshot_client, &snapshot_root_for_task).await + }); + + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + !handle.is_finished(), + "public snapshot API should wait for the maintenance lock rather than racing restore work" + ); + + drop(guard); + let manifest = handle + .await + .expect("snapshot task should join") + .expect("snapshot should succeed after lock release"); + assert!( + fs::try_exists(snapshot_root.join(manifest.snapshot_id)) + .await + .expect("check snapshot output"), + "snapshot directory should exist after successful snapshot creation" + ); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot lock root"); + } + + #[tokio::test] + async fn recover_from_restore_journal_cleans_loading_shadow_leftovers() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + let database = client.database(DB); + + let collections = vec![ + RestoreJournalCollection { + live: PROPOSED_BLOCKS_COLLECTION.to_string(), + shadow: shadow_collection_name(PROPOSED_BLOCKS_COLLECTION), + backup: backup_collection_name(PROPOSED_BLOCKS_COLLECTION), + }, + RestoreJournalCollection { + live: SYNC_STATE_COLLECTION.to_string(), + shadow: shadow_collection_name(SYNC_STATE_COLLECTION), + backup: backup_collection_name(SYNC_STATE_COLLECTION), + }, + ]; + + for collection in &collections { + database + .collection::(&collection.shadow) + .insert_one(mongodb::bson::doc! { "_id": 1, "value": "shadow" }) + .await + .expect("seed shadow"); + database + .collection::(&collection.backup) + .insert_one(mongodb::bson::doc! { "_id": 1, "value": "backup" }) + .await + .expect("seed backup"); + } + + let journal = RestoreJournal::new_loading_shadow( + "snapshot-loading-shadow".to_string(), + "/tmp/snapshot-loading-shadow".to_string(), + "deadbeef".to_string(), + collections.clone(), + mongodb::bson::DateTime::now(), + ); + client.upsert_restore_journal(&journal).await.unwrap(); + + recover_from_restore_journal(&client) + .await + .expect("loading_shadow recover should clean leftovers"); + + assert_eq!(client.get_restore_journal().await, None); + let names = database + .list_collection_names() + .await + .expect("list collections"); + for collection in collections { + assert!(!names.iter().any(|name| name == &collection.shadow)); + assert!(!names.iter().any(|name| name == &collection.backup)); + } + } + + #[tokio::test] + async fn create_proposer_snapshot_crash_leaves_temp_dir_that_can_be_cleaned() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_snapshot_test_trees(&client).await; + + let stored_block = StoredBlock { + layer2_block_number: 21, + commitments: vec!["0xcrash".to_string()], + proposer_address: Address::from([21u8; 20]), + }; + client + .store_block(&stored_block) + .await + .expect("store block"); + + persist_sync_state( + &client, + &SyncState::new( + stored_block.layer2_block_number, + stored_block.hash().to_hex_string(), + L1Ref { + block_number: 2100, + tx_hash: TxHash::from([21u8; 32]), + log_index: 0, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-snapshot-crash-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + + let failpoint = TestFailpointGuard::enable("snapshot_after_first_collection_exported"); + let crash_client = client.clone(); + let crash_root = snapshot_root.clone(); + let join_error = + tokio::spawn(async move { create_proposer_snapshot(&crash_client, &crash_root).await }) + .await + .expect_err("snapshot creation should panic at failpoint"); + assert!(join_error.is_panic()); + drop(failpoint); + + let mut temp_dirs = Vec::new(); + let mut entries = fs::read_dir(&snapshot_root) + .await + .expect("read snapshot root"); + while let Some(entry) = entries + .next_entry() + .await + .expect("read snapshot root entry") + { + if entry + .file_name() + .to_string_lossy() + .starts_with(TEMP_SNAPSHOT_DIR_PREFIX) + { + temp_dirs.push(entry.path()); + } + } + assert!( + !temp_dirs.is_empty(), + "crash should leave at least one temp dir" + ); + + cleanup_orphaned_proposer_snapshot_temp_dirs(&snapshot_root) + .await + .expect("cleanup orphaned temp dirs"); + + let mut entries = fs::read_dir(&snapshot_root) + .await + .expect("read cleaned snapshot root"); + while let Some(entry) = entries.next_entry().await.expect("read cleaned root entry") { + assert!( + !entry + .file_name() + .to_string_lossy() + .starts_with(TEMP_SNAPSHOT_DIR_PREFIX), + "orphaned temp dir should have been removed" + ); + } + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn create_proposer_snapshot_writes_manifest_and_collections() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_snapshot_test_trees(&client).await; + + let stored_block = StoredBlock { + layer2_block_number: 7, + commitments: vec!["0xabc".to_string()], + proposer_address: Address::from([3u8; 20]), + }; + client + .store_block(&stored_block) + .await + .expect("store block"); + + let sync_state = SyncState::new( + stored_block.layer2_block_number, + stored_block.hash().to_hex_string(), + L1Ref { + block_number: 1234, + tx_hash: TxHash::from([9u8; 32]), + log_index: 2, + }, + mongodb::bson::DateTime::now(), + ); + + persist_sync_state(&client, &sync_state).await; + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-snapshot-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + assert!(snapshot_dir.join("manifest.json").exists()); + assert!(manifest + .collections + .iter() + .any(|collection| collection.collection_name == DEPOSIT_COLLECTION)); + assert!(manifest + .collections + .iter() + .any(|collection| collection.collection_name == PROPOSED_BLOCKS_COLLECTION)); + assert!(manifest + .collections + .iter() + .any(|collection| collection.collection_name == SYNC_STATE_COLLECTION)); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn load_proposer_snapshot_into_shadow_rejects_wrong_schema_version() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let (snapshot_root, manifest, _, _) = create_snapshot_fixture( + &client, + 31, + "0xschema", + 31, + 3100, + "nf4-proposer-schema-version-test", + ) + .await; + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + let mut broken_manifest = read_manifest(&snapshot_dir).await; + broken_manifest.schema_version = 999; + write_manifest(&snapshot_dir, &broken_manifest).await; + + let error = load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect_err("loading snapshot should reject unsupported schema version"); + assert!(matches!( + error, + SnapshotError::UnsupportedManifestSchemaVersion(999) + )); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn load_proposer_snapshot_into_shadow_rejects_wrong_storage_format() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let (snapshot_root, manifest, _, _) = create_snapshot_fixture( + &client, + 32, + "0xstorage", + 32, + 3200, + "nf4-proposer-storage-format-test", + ) + .await; + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + let mut broken_manifest = read_manifest(&snapshot_dir).await; + broken_manifest.storage_format = "broken-format".to_string(); + write_manifest(&snapshot_dir, &broken_manifest).await; + + let error = load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect_err("loading snapshot should reject unsupported storage format"); + assert!(matches!( + error, + SnapshotError::UnsupportedManifestStorageFormat(ref actual) + if actual == "broken-format" + )); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn load_proposer_snapshot_into_shadow_rejects_unexpected_manifest_collection() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let (snapshot_root, manifest, _, _) = create_snapshot_fixture( + &client, + 33, + "0xunexpected-collection", + 33, + 3300, + "nf4-proposer-unexpected-collection-test", + ) + .await; + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + let mut broken_manifest = read_manifest(&snapshot_dir).await; + broken_manifest + .collections + .push(SnapshotCollectionManifest { + collection_name: "TotallyUnexpectedCollection".to_string(), + file_name: "TotallyUnexpectedCollection.jsonl".to_string(), + document_count: 0, + sha256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + .to_string(), + }); + write_manifest(&snapshot_dir, &broken_manifest).await; + fs::write(snapshot_dir.join("TotallyUnexpectedCollection.jsonl"), b"") + .await + .expect("write unexpected snapshot file"); + + let error = load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect_err("loading snapshot should reject unexpected manifest collection"); + assert!(matches!( + error, + SnapshotError::UnexpectedSnapshotCollection(ref actual) + if actual == "TotallyUnexpectedCollection" + )); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn load_proposer_snapshot_into_shadow_rejects_invalid_manifest_file_name() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let (snapshot_root, manifest, _, _) = create_snapshot_fixture( + &client, + 34, + "0xinvalid-file-name", + 34, + 3400, + "nf4-proposer-invalid-file-name-test", + ) + .await; + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + let mut broken_manifest = read_manifest(&snapshot_dir).await; + broken_manifest + .collections + .first_mut() + .expect("manifest should contain at least one collection") + .file_name = "../../outside.jsonl".to_string(); + write_manifest(&snapshot_dir, &broken_manifest).await; + + let error = load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect_err("loading snapshot should reject invalid manifest file names"); + assert!(matches!( + error, + SnapshotError::InvalidSnapshotFileName(ref actual) if actual == "../../outside.jsonl" + )); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn load_proposer_snapshot_into_shadow_rejects_missing_snapshot_file() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let (snapshot_root, manifest, _, _) = create_snapshot_fixture( + &client, + 33, + "0xmissing-file", + 33, + 3300, + "nf4-proposer-missing-file-test", + ) + .await; + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + let removed_file = manifest + .collections + .first() + .expect("manifest should contain at least one collection") + .file_name + .clone(); + fs::remove_file(snapshot_dir.join(&removed_file)) + .await + .expect("remove snapshot file"); + + let error = load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect_err("loading snapshot should reject missing snapshot file"); + assert!(matches!( + error, + SnapshotError::MissingSnapshotFile(ref actual) if actual == &removed_file + )); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn load_proposer_snapshot_into_shadow_rejects_bad_collection_checksum() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let (snapshot_root, manifest, _, _) = create_snapshot_fixture( + &client, + 34, + "0xbad-sha", + 34, + 3400, + "nf4-proposer-bad-collection-sha-test", + ) + .await; + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + let target_collection = manifest + .collections + .iter() + .find(|collection| collection.document_count > 0) + .unwrap_or_else(|| { + manifest + .collections + .first() + .expect("manifest should contain at least one collection") + }); + fs::write( + snapshot_dir.join(&target_collection.file_name), + b"{\"corrupted\":true}\n", + ) + .await + .expect("corrupt collection file"); + + let error = load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect_err("loading snapshot should reject bad collection checksum"); + assert!(matches!( + error, + SnapshotError::CollectionChecksumMismatch { ref collection_name, .. } + if collection_name == &target_collection.collection_name + )); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn load_proposer_snapshot_into_shadow_rejects_bad_overall_checksum() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let (snapshot_root, manifest, _, _) = create_snapshot_fixture( + &client, + 35, + "0xbad-overall", + 35, + 3500, + "nf4-proposer-bad-overall-sha-test", + ) + .await; + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + let mut broken_manifest = read_manifest(&snapshot_dir).await; + broken_manifest.overall_sha256 = "definitely-wrong".to_string(); + write_manifest(&snapshot_dir, &broken_manifest).await; + + let error = load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect_err("loading snapshot should reject bad overall checksum"); + assert!(matches!( + error, + SnapshotError::OverallChecksumMismatch { ref expected, .. } + if expected == "definitely-wrong" + )); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn load_proposer_snapshot_into_shadow_rejects_manifest_sync_state_mismatch() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let (snapshot_root, manifest, _, _) = create_snapshot_fixture( + &client, + 36, + "0xmanifest-mismatch", + 36, + 3600, + "nf4-proposer-manifest-mismatch-test", + ) + .await; + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + let mut broken_manifest = read_manifest(&snapshot_dir).await; + broken_manifest.last_applied_l2_block += 1; + write_manifest(&snapshot_dir, &broken_manifest).await; + + let error = load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect_err("loading snapshot should reject manifest metadata drift"); + assert!(matches!( + error, + SnapshotError::SnapshotManifestSyncStateMismatch { + field: "last_applied_l2_block", + .. + } + )); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn find_latest_valid_proposer_snapshot_prefers_highest_restorable_block() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let first_block = StoredBlock { + layer2_block_number: 4, + commitments: vec!["0xaaa".to_string()], + proposer_address: Address::from([4u8; 20]), + }; + client + .store_block(&first_block) + .await + .expect("store first block"); + persist_sync_state( + &client, + &SyncState::new( + first_block.layer2_block_number, + first_block.hash().to_hex_string(), + L1Ref { + block_number: 100, + tx_hash: TxHash::from([4u8; 32]), + log_index: 0, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-discovery-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let first_manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create first snapshot"); + + let second_block = StoredBlock { + layer2_block_number: 7, + commitments: vec!["0xbbb".to_string()], + proposer_address: Address::from([7u8; 20]), + }; + client + .store_block(&second_block) + .await + .expect("store second block"); + persist_sync_state( + &client, + &SyncState::new( + second_block.layer2_block_number, + second_block.hash().to_hex_string(), + L1Ref { + block_number: 200, + tx_hash: TxHash::from([7u8; 32]), + log_index: 0, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + let second_manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create second snapshot"); + + let (selected_dir, selected_manifest) = + find_latest_valid_proposer_snapshot(&snapshot_root, 4) + .await + .expect("discover snapshot") + .expect("snapshot should exist"); + assert_eq!(selected_manifest.last_applied_l2_block, 4); + assert_eq!(selected_manifest.snapshot_id, first_manifest.snapshot_id); + assert_eq!( + selected_dir, + snapshot_root.join(&first_manifest.snapshot_id) + ); + + let (_, selected_manifest) = find_latest_valid_proposer_snapshot(&snapshot_root, 7) + .await + .expect("discover snapshot") + .expect("snapshot should exist"); + assert_eq!(selected_manifest.last_applied_l2_block, 7); + assert_eq!(selected_manifest.snapshot_id, second_manifest.snapshot_id); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn find_latest_valid_proposer_snapshot_ignores_temp_snapshot_dirs() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let canonical_block = StoredBlock { + layer2_block_number: 5, + commitments: vec!["0xaaa".to_string()], + proposer_address: Address::from([5u8; 20]), + }; + client + .store_block(&canonical_block) + .await + .expect("store canonical block"); + persist_sync_state( + &client, + &SyncState::new( + canonical_block.layer2_block_number, + canonical_block.hash().to_hex_string(), + L1Ref { + block_number: 150, + tx_hash: TxHash::from([5u8; 32]), + log_index: 0, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-temp-discovery-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create canonical snapshot"); + let canonical_snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let temp_snapshot_dir = snapshot_root.join(format!( + "{TEMP_SNAPSHOT_DIR_PREFIX}{}-{}", + manifest.last_applied_l2_block + 1, + manifest.snapshot_id + )); + fs::create_dir_all(&temp_snapshot_dir) + .await + .expect("create temp snapshot dir"); + for collection in &manifest.collections { + fs::copy( + canonical_snapshot_dir.join(&collection.file_name), + temp_snapshot_dir.join(&collection.file_name), + ) + .await + .expect("copy collection snapshot into temp dir"); + } + let mut temp_manifest = manifest.clone(); + temp_manifest.snapshot_id = temp_snapshot_dir + .file_name() + .and_then(|name| name.to_str()) + .expect("temp snapshot dir name should be valid utf-8") + .to_string(); + temp_manifest.last_applied_l2_block += 1; + write_manifest(&temp_snapshot_dir, &temp_manifest).await; + + let (selected_dir, selected_manifest) = find_latest_valid_proposer_snapshot( + &snapshot_root, + temp_manifest.last_applied_l2_block, + ) + .await + .expect("discover snapshot") + .expect("snapshot should exist"); + + assert_eq!(selected_dir, canonical_snapshot_dir); + assert_eq!(selected_manifest.snapshot_id, manifest.snapshot_id); + assert_eq!( + selected_manifest.last_applied_l2_block, + manifest.last_applied_l2_block + ); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn load_proposer_snapshot_into_shadow_populates_shadow_collections_and_journal() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + >::new_commitment_tree(&client, 29, 3) + .await + .expect("create commitment tree"); + >::new_historic_root_tree(&client, 32) + .await + .expect("create historic root tree"); + >::append_historic_commitment_root( + &client, + &Fr254::zero(), + true, + ) + .await + .expect("append zero historic root"); + >::new_nullifier_tree(&client, 29, 3) + .await + .expect("create nullifier tree"); + ensure_deposit_indexes(&client) + .await + .expect("create deposit indexes"); + + let stored_block = StoredBlock { + layer2_block_number: 8, + commitments: vec!["0xdef".to_string()], + proposer_address: Address::from([4u8; 20]), + }; + client + .store_block(&stored_block) + .await + .expect("store block"); + + let sync_state = SyncState::new( + stored_block.layer2_block_number, + stored_block.hash().to_hex_string(), + L1Ref { + block_number: 5678, + tx_hash: TxHash::from([8u8; 32]), + log_index: 4, + }, + mongodb::bson::DateTime::now(), + ); + + persist_sync_state(&client, &sync_state).await; + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-shadow-load-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let journal = load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + + let stored_journal = client + .get_restore_journal() + .await + .expect("restore_journal should exist"); + assert_eq!(stored_journal, journal); + assert_eq!( + stored_journal.phase, + crate::domain::entities::RestoreJournalPhase::LoadingShadow + ); + + let restore_journal_docs = client + .database(DB) + .collection::(RESTORE_JOURNAL_COLLECTION) + .count_documents(mongodb::bson::doc! {}) + .await + .expect("count restore_journal docs"); + assert_eq!(restore_journal_docs, 1); + + for collection in &manifest.collections { + let shadow_name = format!("restore_shadow__{}", collection.collection_name); + let shadow_count = client + .database(DB) + .collection::(&shadow_name) + .count_documents(mongodb::bson::doc! {}) + .await + .expect("count shadow documents"); + assert_eq!( + shadow_count, collection.document_count, + "shadow collection {shadow_name} should contain the manifest document count" + ); + } + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn recover_from_restore_journal_cleans_real_loading_shadow_crash() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_snapshot_test_trees(&client).await; + + let stored_block = StoredBlock { + layer2_block_number: 22, + commitments: vec!["0xshadow".to_string()], + proposer_address: Address::from([22u8; 20]), + }; + client + .store_block(&stored_block) + .await + .expect("store block"); + + persist_sync_state( + &client, + &SyncState::new( + stored_block.layer2_block_number, + stored_block.hash().to_hex_string(), + L1Ref { + block_number: 2200, + tx_hash: TxHash::from([22u8; 32]), + log_index: 1, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-loading-shadow-crash-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let failpoint = + TestFailpointGuard::enable("restore_after_first_shadow_collection_imported"); + let crash_client = client.clone(); + let crash_snapshot_dir = snapshot_dir.clone(); + let join_error = tokio::spawn(async move { + load_proposer_snapshot_into_shadow(&crash_client, &crash_snapshot_dir).await + }) + .await + .expect_err("loading shadow should panic at failpoint"); + assert!(join_error.is_panic()); + drop(failpoint); + + let journal = client + .get_restore_journal() + .await + .expect("journal should exist after loading_shadow crash"); + assert_eq!(journal.phase, RestoreJournalPhase::LoadingShadow); + + recover_from_restore_journal(&client) + .await + .expect("recover should clean loading_shadow leftovers"); + + assert_eq!(client.get_restore_journal().await, None); + let names = client + .database(DB) + .list_collection_names() + .await + .expect("list collection names"); + for collection in journal.collections { + assert!(!names.iter().any(|name| name == &collection.shadow)); + assert!(!names.iter().any(|name| name == &collection.backup)); + } + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn swap_proposer_shadow_into_live_marks_swap_complete_and_keeps_backups() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + >::new_commitment_tree(&client, 29, 3) + .await + .expect("create commitment tree"); + >::new_historic_root_tree(&client, 32) + .await + .expect("create historic root tree"); + >::append_historic_commitment_root( + &client, + &Fr254::zero(), + true, + ) + .await + .expect("append zero historic root"); + >::new_nullifier_tree(&client, 29, 3) + .await + .expect("create nullifier tree"); + ensure_deposit_indexes(&client) + .await + .expect("create deposit indexes"); + + let snapshot_block = StoredBlock { + layer2_block_number: 8, + commitments: vec!["0xdef".to_string()], + proposer_address: Address::from([4u8; 20]), + }; + client + .store_block(&snapshot_block) + .await + .expect("store snapshot block"); + + let snapshot_sync_state = SyncState::new( + snapshot_block.layer2_block_number, + snapshot_block.hash().to_hex_string(), + L1Ref { + block_number: 5678, + tx_hash: TxHash::from([8u8; 32]), + log_index: 4, + }, + mongodb::bson::DateTime::now(), + ); + + persist_sync_state(&client, &snapshot_sync_state).await; + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-shadow-swap-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let new_live_block = StoredBlock { + layer2_block_number: 9, + commitments: vec!["0x123".to_string()], + proposer_address: Address::from([5u8; 20]), + }; + client + .store_block(&new_live_block) + .await + .expect("store newer live block"); + + let new_live_sync_state = SyncState::new( + new_live_block.layer2_block_number, + new_live_block.hash().to_hex_string(), + L1Ref { + block_number: 91011, + tx_hash: TxHash::from([7u8; 32]), + log_index: 5, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &new_live_sync_state).await; + + load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + let journal = swap_proposer_shadow_into_live(&client) + .await + .expect("swap shadow into live"); + + assert_eq!(journal.phase, RestoreJournalPhase::SwapComplete); + assert_eq!(journal.current_index, None); + assert_eq!( + journal.current_step, + Some(RestoreJournalStep::NonSnapshotCleanupPending) + ); + + let live_sync_state = client + .get_sync_state() + .await + .expect("live sync_state should exist"); + assert_eq!(live_sync_state.last_applied_l2_block, 8); + assert_eq!( + live_sync_state.fingerprint, + snapshot_block.hash().to_hex_string() + ); + + let live_blocks_count = client + .database(DB) + .collection::(PROPOSED_BLOCKS_COLLECTION) + .count_documents(mongodb::bson::doc! {}) + .await + .expect("count live proposed blocks"); + assert_eq!(live_blocks_count, 1); + + let backup_blocks_count = client + .database(DB) + .collection::(&backup_collection_name(PROPOSED_BLOCKS_COLLECTION)) + .count_documents(mongodb::bson::doc! {}) + .await + .expect("count backup proposed blocks"); + assert_eq!(backup_blocks_count, 2); + + let shadow_sync_state_exists = client + .database(DB) + .list_collection_names() + .await + .expect("list collections") + .iter() + .any(|name| name == &shadow_collection_name(SYNC_STATE_COLLECTION)); + assert!( + !shadow_sync_state_exists, + "shadow sync_state should have been renamed away" + ); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn recover_from_restore_journal_resumes_real_mid_swap_crash() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_snapshot_test_trees(&client).await; + + let snapshot_block = StoredBlock { + layer2_block_number: 23, + commitments: vec!["0xresume".to_string()], + proposer_address: Address::from([23u8; 20]), + }; + client + .store_block(&snapshot_block) + .await + .expect("store snapshot block"); + + persist_sync_state( + &client, + &SyncState::new( + snapshot_block.layer2_block_number, + snapshot_block.hash().to_hex_string(), + L1Ref { + block_number: 2300, + tx_hash: TxHash::from([23u8; 32]), + log_index: 2, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-mid-swap-crash-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let newer_live_block = StoredBlock { + layer2_block_number: 24, + commitments: vec!["0xlive".to_string()], + proposer_address: Address::from([24u8; 20]), + }; + client + .store_block(&newer_live_block) + .await + .expect("store newer live block"); + persist_sync_state( + &client, + &SyncState::new( + newer_live_block.layer2_block_number, + newer_live_block.hash().to_hex_string(), + L1Ref { + block_number: 2400, + tx_hash: TxHash::from([24u8; 32]), + log_index: 3, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + + load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + + let failpoint = TestFailpointGuard::enable("restore_after_backup_created"); + let crash_client = client.clone(); + let join_error = + tokio::spawn(async move { swap_proposer_shadow_into_live(&crash_client).await }) + .await + .expect_err("swap should panic at failpoint"); + assert!(join_error.is_panic()); + drop(failpoint); + + let journal = client + .get_restore_journal() + .await + .expect("journal should exist after mid-swap crash"); + assert_eq!(journal.phase, RestoreJournalPhase::SwapInProgress); + assert_eq!( + journal.current_step, + Some(RestoreJournalStep::BackupCreated) + ); + + recover_from_restore_journal(&client) + .await + .expect("recover should resume swap"); + + let resumed = client + .get_restore_journal() + .await + .expect("journal should remain at swap_complete after resume"); + assert_eq!(resumed.phase, RestoreJournalPhase::SwapComplete); + + let live_sync_state = client + .get_sync_state() + .await + .expect("restored live sync_state should exist"); + assert_eq!( + live_sync_state.last_applied_l2_block, + snapshot_block.layer2_block_number + ); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn recover_from_restore_journal_rolls_back_mid_swap_crash_before_backup_created_journal_persist( + ) { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_snapshot_test_trees(&client).await; + + let snapshot_block = StoredBlock { + layer2_block_number: 27, + commitments: vec!["0xpre-journal-crash".to_string()], + proposer_address: Address::from([27u8; 20]), + }; + client + .store_block(&snapshot_block) + .await + .expect("store snapshot block"); + persist_sync_state( + &client, + &SyncState::new( + snapshot_block.layer2_block_number, + snapshot_block.hash().to_hex_string(), + L1Ref { + block_number: 2700, + tx_hash: TxHash::from([27u8; 32]), + log_index: 2, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-pre-backup-created-crash-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let newer_live_block = StoredBlock { + layer2_block_number: 28, + commitments: vec!["0xnewer-live".to_string()], + proposer_address: Address::from([28u8; 20]), + }; + client + .store_block(&newer_live_block) + .await + .expect("store newer live block"); + let newer_live_sync_state = SyncState::new( + newer_live_block.layer2_block_number, + newer_live_block.hash().to_hex_string(), + L1Ref { + block_number: 2800, + tx_hash: TxHash::from([28u8; 32]), + log_index: 3, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &newer_live_sync_state).await; + + load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + + let failpoint = + TestFailpointGuard::enable("restore_after_live_rename_before_backup_created"); + let crash_client = client.clone(); + let join_error = + tokio::spawn(async move { swap_proposer_shadow_into_live(&crash_client).await }) + .await + .expect_err("swap should panic after live rename and before journal persist"); + assert!(join_error.is_panic()); + drop(failpoint); + + let journal = client + .get_restore_journal() + .await + .expect("journal should exist after pre-journal mid-swap crash"); + assert_eq!(journal.phase, RestoreJournalPhase::SwapInProgress); + assert_eq!( + journal.current_step, + Some(RestoreJournalStep::BackupPending) + ); + + let ordered = restore_collections_in_swap_order(&journal.collections); + let current_collection = &ordered[journal.current_index.expect("current_index") as usize]; + client + .database(DB) + .collection::(¤t_collection.shadow) + .drop() + .await + .expect("drop current shadow collection to force rollback"); + + let error = recover_from_restore_journal(&client) + .await + .expect_err("recovery should roll back after resume detects broken current shadow"); + assert!(matches!(error, SnapshotError::RestoreInvariantViolation(_))); + assert_eq!(client.get_restore_journal().await, None); + + let live_sync_state = client + .get_sync_state() + .await + .expect("pre-restore live sync_state should be restored"); + assert_eq!(live_sync_state, newer_live_sync_state); + + let collection_names = client + .database(DB) + .list_collection_names() + .await + .expect("list collections"); + assert!(!collection_names + .iter() + .any(|name| name.starts_with("restore_shadow__"))); + assert!(!collection_names + .iter() + .any(|name| name.starts_with("restore_backup__"))); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn recover_from_restore_journal_rolls_back_broken_invariant_and_restores_live_state() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + let (snapshot_root, journal, newer_live_sync_state) = + prepare_mid_swap_pending_state(&client, "nf4-proposer-rollback-success-test", 41, 42) + .await; + + let ordered = restore_collections_in_swap_order(&journal.collections); + let broken_collection = &ordered[journal.current_index.expect("current_index") as usize]; + client + .database(DB) + .collection::(&broken_collection.live) + .drop() + .await + .expect("drop live collection to force invariant violation"); + + let error = recover_from_restore_journal(&client) + .await + .expect_err("recovery should surface invariant violation after rollback"); + assert!(matches!(error, SnapshotError::RestoreInvariantViolation(_))); + + assert_eq!( + client.get_restore_journal().await, + None, + "successful rollback should clear restore_journal" + ); + + let live_sync_state = client + .get_sync_state() + .await + .expect("pre-swap live sync_state should be restored"); + assert_eq!(live_sync_state, newer_live_sync_state); + + let collection_names = client + .database(DB) + .list_collection_names() + .await + .expect("list collections"); + assert!(!collection_names + .iter() + .any(|name| name.starts_with("restore_shadow__"))); + assert!(!collection_names + .iter() + .any(|name| name.starts_with("restore_backup__"))); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn recover_from_restore_journal_resumes_real_mid_rollback_crash() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + let (snapshot_root, journal, newer_live_sync_state) = + prepare_mid_swap_pending_state(&client, "nf4-proposer-mid-rollback-crash-test", 51, 52) + .await; + + let ordered = restore_collections_in_swap_order(&journal.collections); + let broken_collection = &ordered[journal.current_index.expect("current_index") as usize]; + client + .database(DB) + .collection::(&broken_collection.live) + .drop() + .await + .expect("drop live collection to force rollback path"); + + let failpoint = TestFailpointGuard::enable("restore_after_first_rollback_progress_persist"); + let crash_client = client.clone(); + let join_error = + tokio::spawn(async move { recover_from_restore_journal(&crash_client).await }) + .await + .expect_err("rollback recovery should panic at failpoint"); + assert!(join_error.is_panic()); + drop(failpoint); + + let persisted_journal = client + .get_restore_journal() + .await + .expect("journal should remain after rollback crash"); + assert_eq!( + persisted_journal.phase, + RestoreJournalPhase::RollbackInProgress + ); + let expected_rollback_index = journal + .current_index + .and_then(|index| index.checked_sub(1)) + .expect("mid-rollback crash should start after at least one processed collection"); + assert_eq!( + persisted_journal.current_index, + Some(expected_rollback_index) + ); + assert_eq!( + persisted_journal.current_step, + Some(RestoreJournalStep::RollbackStarted) + ); + + let error = recover_from_restore_journal(&client) + .await + .expect_err("resume should surface original invariant violation after rollback"); + assert!(matches!(error, SnapshotError::RestoreInvariantViolation(_))); + assert_eq!(client.get_restore_journal().await, None); + + let live_sync_state = client + .get_sync_state() + .await + .expect("pre-swap live sync_state should be restored after resumed rollback"); + assert_eq!(live_sync_state, newer_live_sync_state); + + let collection_names = client + .database(DB) + .list_collection_names() + .await + .expect("list collections"); + assert!(!collection_names + .iter() + .any(|name| name.starts_with("restore_shadow__"))); + assert!(!collection_names + .iter() + .any(|name| name.starts_with("restore_backup__"))); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn recover_from_restore_journal_resumes_after_crash_post_backup_restore_pre_rollback_applied( + ) { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + let (snapshot_root, mut journal, newer_live_sync_state) = prepare_mid_swap_pending_state( + &client, + "nf4-proposer-mid-rollback-live-restore-crash-test", + 61, + 62, + ) + .await; + + let ordered = restore_collections_in_swap_order(&journal.collections); + let (first_processed_index, _) = + first_existing_live_restore_collection(&client, &journal.collections).await; + let rollback_trigger_index = first_processed_index + 1; + assert!( + rollback_trigger_index < ordered.len(), + "expected another collection after the first processed live collection" + ); + let broken_collection = &ordered[rollback_trigger_index]; + journal.collections = ordered.clone(); + journal.phase = RestoreJournalPhase::SwapInProgress; + journal.current_index = Some(rollback_trigger_index as u32); + journal.current_step = Some(RestoreJournalStep::BackupPending); + journal.updated_at = mongodb::bson::DateTime::now(); + client + .upsert_restore_journal(&journal) + .await + .expect("persist rollback live-restore crash fixture journal"); + + drop_collection_if_exists(&client.database(DB), &broken_collection.live) + .await + .expect("drop current live collection when present"); + client + .database(DB) + .collection::(&broken_collection.shadow) + .drop() + .await + .expect("drop current shadow collection to force rollback path"); + + let failpoint = TestFailpointGuard::enable("restore_after_first_rollback_live_restore"); + let crash_client = client.clone(); + let join_error = + tokio::spawn(async move { recover_from_restore_journal(&crash_client).await }) + .await + .expect_err( + "rollback recovery should panic after restoring the first live collection", + ); + assert!(join_error.is_panic()); + drop(failpoint); + + let persisted_journal = client + .get_restore_journal() + .await + .expect("journal should remain after rollback crash"); + assert_eq!( + persisted_journal.phase, + RestoreJournalPhase::RollbackInProgress + ); + let expected_rollback_index = rollback_trigger_index + .checked_sub(1) + .expect("rollback should start at the first processed collection") + as u32; + assert_eq!( + persisted_journal.current_index, + Some(expected_rollback_index) + ); + assert_eq!( + persisted_journal.current_step, + Some(RestoreJournalStep::RollbackStarted) + ); + + let error = recover_from_restore_journal(&client) + .await + .expect_err("resume should complete rollback and surface the original restore failure"); + assert!(matches!(error, SnapshotError::RestoreInvariantViolation(_))); + assert_eq!(client.get_restore_journal().await, None); + + let live_sync_state = client + .get_sync_state() + .await + .expect("pre-swap live sync_state should be restored after resumed rollback"); + assert_eq!(live_sync_state, newer_live_sync_state); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn recover_from_restore_journal_resumes_optional_drop_after_crash_at_rollback_started() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let snapshot_block = StoredBlock { + layer2_block_number: 63, + commitments: Vec::new(), + proposer_address: Address::from([63u8; 20]), + }; + client + .store_block(&snapshot_block) + .await + .expect("store snapshot block"); + let snapshot_sync_state = SyncState::new( + snapshot_block.layer2_block_number, + snapshot_block.hash().to_hex_string(), + L1Ref { + block_number: 6300, + tx_hash: TxHash::from([63u8; 32]), + log_index: 63, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &snapshot_sync_state).await; + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-optional-rollback-drop-resume-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let commitment_nodes_collection = format!( + "{}_nodes", + >::TREE_NAME + ); + assert!( + !manifest + .collections + .iter() + .any(|collection| collection.collection_name == commitment_nodes_collection), + "snapshot fixture should omit empty optional commitment nodes collection" + ); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let newer_live_block = StoredBlock { + layer2_block_number: 64, + commitments: Vec::new(), + proposer_address: Address::from([64u8; 20]), + }; + client + .store_block(&newer_live_block) + .await + .expect("store newer live block"); + let newer_live_sync_state = SyncState::new( + newer_live_block.layer2_block_number, + newer_live_block.hash().to_hex_string(), + L1Ref { + block_number: 6400, + tx_hash: TxHash::from([64u8; 32]), + log_index: 64, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &newer_live_sync_state).await; + let mut journal = load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + let ordered = restore_collections_in_swap_order(&journal.collections); + let target_index = ordered + .iter() + .position(|collection| collection.live == commitment_nodes_collection) + .expect("target optional collection should exist in restore order"); + let rollback_trigger_index = target_index + 1; + assert!( + rollback_trigger_index < ordered.len(), + "expected a collection after the optional rollback target" + ); + + let database = client.database(DB); + for collection in ordered.iter().take(target_index + 1) { + let layout = inspect_restore_collection_layout(&database, collection) + .await + .expect("inspect processed prefix layout"); + if layout.live_exists { + rename_collection(&client, &collection.live, &collection.backup, false) + .await + .expect("move processed live collection to backup"); + } + if layout.shadow_exists { + rename_collection(&client, &collection.shadow, &collection.live, false) + .await + .expect("move processed shadow collection into live"); + } + } + + let target = &ordered[target_index]; + assert!( + collection_exists(&database, &target.live) + .await + .expect("inspect optional live collection after forward swap"), + "optional target should be live before rollback begins" + ); + assert!( + !collection_exists(&database, &target.backup) + .await + .expect("inspect optional backup collection"), + "optional target should have no backup when rollback action is drop" + ); + + let broken_collection = &ordered[rollback_trigger_index]; + database + .collection::(&broken_collection.live) + .drop() + .await + .expect("drop next live collection to force rollback path"); + + journal.collections = ordered.clone(); + journal.phase = RestoreJournalPhase::SwapInProgress; + journal.current_index = Some(rollback_trigger_index as u32); + journal.current_step = Some(RestoreJournalStep::BackupPending); + journal.updated_at = mongodb::bson::DateTime::now(); + client + .upsert_restore_journal(&journal) + .await + .expect("persist rollback crash fixture journal"); + + let failpoint = TestFailpointGuard::enable("restore_after_first_rollback_progress_persist"); + let crash_client = client.clone(); + let join_error = + tokio::spawn(async move { recover_from_restore_journal(&crash_client).await }) + .await + .expect_err("rollback recovery should panic after persisting RollbackStarted"); + assert!(join_error.is_panic()); + drop(failpoint); + + let persisted_journal = client + .get_restore_journal() + .await + .expect("journal should remain after rollback crash"); + assert_eq!( + persisted_journal.phase, + RestoreJournalPhase::RollbackInProgress + ); + assert_eq!(persisted_journal.current_index, Some(target_index as u32)); + assert_eq!( + persisted_journal.current_step, + Some(RestoreJournalStep::RollbackStarted) + ); + assert!( + collection_exists(&database, &target.live) + .await + .expect("inspect optional live collection before resumed rollback"), + "optional live collection should still exist immediately after the crash" + ); + + let error = recover_from_restore_journal(&client) + .await + .expect_err("resume should complete rollback and surface the original restore failure"); + assert!(matches!(error, SnapshotError::RestoreInvariantViolation(_))); + assert_eq!(client.get_restore_journal().await, None); + assert_eq!(client.get_sync_state().await, Some(newer_live_sync_state)); + assert!( + !collection_exists(&database, &target.live) + .await + .expect("inspect optional live collection after resumed rollback"), + "resumed rollback should drop the optional live collection when no backup existed" + ); + + let collection_names = client + .database(DB) + .list_collection_names() + .await + .expect("list collections"); + assert!(!collection_names + .iter() + .any(|name| name.starts_with("restore_shadow__"))); + assert!(!collection_names + .iter() + .any(|name| name.starts_with("restore_backup__"))); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn recover_from_restore_journal_fails_closed_when_optional_live_state_changes_after_rollback_started( + ) { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let snapshot_block = StoredBlock { + layer2_block_number: 65, + commitments: Vec::new(), + proposer_address: Address::from([65u8; 20]), + }; + client + .store_block(&snapshot_block) + .await + .expect("store snapshot block"); + let snapshot_sync_state = SyncState::new( + snapshot_block.layer2_block_number, + snapshot_block.hash().to_hex_string(), + L1Ref { + block_number: 6500, + tx_hash: TxHash::from([65u8; 32]), + log_index: 65, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &snapshot_sync_state).await; + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-optional-rollback-stale-live-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let commitment_nodes_collection = format!( + "{}_nodes", + >::TREE_NAME + ); + assert!( + !manifest + .collections + .iter() + .any(|collection| collection.collection_name == commitment_nodes_collection), + "snapshot fixture should omit empty optional commitment nodes collection" + ); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let newer_live_block = StoredBlock { + layer2_block_number: 66, + commitments: Vec::new(), + proposer_address: Address::from([66u8; 20]), + }; + client + .store_block(&newer_live_block) + .await + .expect("store newer live block"); + let newer_live_sync_state = SyncState::new( + newer_live_block.layer2_block_number, + newer_live_block.hash().to_hex_string(), + L1Ref { + block_number: 6600, + tx_hash: TxHash::from([66u8; 32]), + log_index: 66, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &newer_live_sync_state).await; + let mut journal = load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + let ordered = restore_collections_in_swap_order(&journal.collections); + let target_index = ordered + .iter() + .position(|collection| collection.live == commitment_nodes_collection) + .expect("target optional collection should exist in restore order"); + let rollback_trigger_index = target_index + 1; + assert!( + rollback_trigger_index < ordered.len(), + "expected a collection after the optional rollback target" + ); + + let database = client.database(DB); + for collection in ordered.iter().take(target_index + 1) { + let layout = inspect_restore_collection_layout(&database, collection) + .await + .expect("inspect processed prefix layout"); + if layout.live_exists { + rename_collection(&client, &collection.live, &collection.backup, false) + .await + .expect("move processed live collection to backup"); + } + if layout.shadow_exists { + rename_collection(&client, &collection.shadow, &collection.live, false) + .await + .expect("move processed shadow collection into live"); + } + } + + let target = &ordered[target_index]; + let broken_collection = &ordered[rollback_trigger_index]; + database + .collection::(&broken_collection.live) + .drop() + .await + .expect("drop next live collection to force rollback path"); + + journal.collections = ordered.clone(); + journal.phase = RestoreJournalPhase::SwapInProgress; + journal.current_index = Some(rollback_trigger_index as u32); + journal.current_step = Some(RestoreJournalStep::BackupPending); + journal.updated_at = mongodb::bson::DateTime::now(); + client + .upsert_restore_journal(&journal) + .await + .expect("persist rollback crash fixture journal"); + + let failpoint = TestFailpointGuard::enable("restore_after_first_rollback_progress_persist"); + let crash_client = client.clone(); + let join_error = + tokio::spawn(async move { recover_from_restore_journal(&crash_client).await }) + .await + .expect_err("rollback recovery should panic after persisting RollbackStarted"); + assert!(join_error.is_panic()); + drop(failpoint); + + let persisted_journal = client + .get_restore_journal() + .await + .expect("journal should remain after rollback crash"); + assert_eq!( + persisted_journal.phase, + RestoreJournalPhase::RollbackInProgress + ); + assert_eq!(persisted_journal.current_index, Some(target_index as u32)); + assert_eq!( + persisted_journal.current_step, + Some(RestoreJournalStep::RollbackStarted) + ); + + client + .database(DB) + .collection::(&target.live) + .insert_one(mongodb::bson::doc! { "_id": 4242_i64, "value": "stale-after-crash" }) + .await + .expect("mutate optional live collection after crash to simulate stale state"); + + let error = recover_from_restore_journal(&client) + .await + .expect_err("resume should fail closed when optional live state no longer matches the restored snapshot"); + assert!(matches!(error, SnapshotError::RestoreInvariantViolation(_))); + + let persisted_journal = client + .get_restore_journal() + .await + .expect("journal should remain after fail-closed optional rollback resume"); + assert_eq!( + persisted_journal.phase, + RestoreJournalPhase::RollbackInProgress + ); + assert_eq!(persisted_journal.current_index, Some(target_index as u32)); + assert_eq!( + persisted_journal.current_step, + Some(RestoreJournalStep::RollbackStarted) + ); + assert!( + collection_exists(&database, &target.live) + .await + .expect("inspect optional live collection after fail-closed recovery"), + "fail-closed recovery should leave the optional live collection in place for manual inspection" + ); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn recover_from_restore_journal_keeps_journal_when_rollback_fails() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + let (snapshot_root, journal, _) = + prepare_mid_swap_pending_state(&client, "nf4-proposer-rollback-failure-test", 43, 44) + .await; + + let (first_processed_index, first_processed) = + first_existing_live_restore_collection(&client, &journal.collections).await; + let ordered = restore_collections_in_swap_order(&journal.collections); + let broken_collection = &ordered[journal.current_index.expect("current_index") as usize]; + + client + .database(DB) + .collection::(&broken_collection.live) + .drop() + .await + .expect("drop live collection to trigger invariant violation"); + client + .database(DB) + .collection::(&first_processed.backup) + .drop() + .await + .expect("drop backup collection to make rollback impossible"); + + let error = recover_from_restore_journal(&client) + .await + .expect_err("recovery should fail when rollback itself cannot complete"); + assert!(matches!(error, SnapshotError::RestoreInvariantViolation(_))); + + let persisted_journal = client + .get_restore_journal() + .await + .expect("journal should remain when rollback fails"); + assert_eq!( + persisted_journal.phase, + RestoreJournalPhase::RollbackInProgress + ); + assert_eq!( + persisted_journal.current_index, + Some(first_processed_index as u32) + ); + assert_eq!( + persisted_journal.current_step, + Some(RestoreJournalStep::RollbackStarted) + ); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn cleanup_after_shadow_swap_returns_to_idle_and_keeps_live_restored_state() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + >::new_commitment_tree(&client, 29, 3) + .await + .expect("create commitment tree"); + >::new_historic_root_tree(&client, 32) + .await + .expect("create historic root tree"); + >::append_historic_commitment_root( + &client, + &Fr254::zero(), + true, + ) + .await + .expect("append zero historic root"); + >::new_nullifier_tree(&client, 29, 3) + .await + .expect("create nullifier tree"); + ensure_deposit_indexes(&client) + .await + .expect("create deposit indexes"); + + let snapshot_pending_deposit = DepositDatawithFee { + fee: Fr254::from(3u64), + deposit_data: DepositData { + nf_token_id: Fr254::from(101u64), + nf_slot_id: Fr254::from(102u64), + value: Fr254::from(103u64), + secret_hash: Fr254::from(104u64), + }, + reserved: false, + }; + >::set_mempool_deposits( + &client, + vec![snapshot_pending_deposit], + ) + .await + .expect("store snapshot pending deposit"); + + let snapshot_block = StoredBlock { + layer2_block_number: 10, + commitments: vec!["0xbeef".to_string()], + proposer_address: Address::from([6u8; 20]), + }; + client + .store_block(&snapshot_block) + .await + .expect("store snapshot block"); + + let snapshot_sync_state = SyncState::new( + snapshot_block.layer2_block_number, + snapshot_block.hash().to_hex_string(), + L1Ref { + block_number: 1112, + tx_hash: TxHash::from([6u8; 32]), + log_index: 6, + }, + mongodb::bson::DateTime::now(), + ); + + persist_sync_state(&client, &snapshot_sync_state).await; + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-shadow-cleanup-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let newer_live_block = StoredBlock { + layer2_block_number: 11, + commitments: vec!["0xcafe".to_string()], + proposer_address: Address::from([7u8; 20]), + }; + client + .store_block(&newer_live_block) + .await + .expect("store newer live block"); + + let newer_live_sync_state = SyncState::new( + newer_live_block.layer2_block_number, + newer_live_block.hash().to_hex_string(), + L1Ref { + block_number: 1314, + tx_hash: TxHash::from([5u8; 32]), + log_index: 7, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &newer_live_sync_state).await; + let stale_live_only_deposit = DepositDatawithFee { + fee: Fr254::from(8u64), + deposit_data: DepositData { + nf_token_id: Fr254::from(201u64), + nf_slot_id: Fr254::from(202u64), + value: Fr254::from(203u64), + secret_hash: Fr254::from(204u64), + }, + reserved: false, + }; + >::set_mempool_deposits( + &client, + vec![stale_live_only_deposit], + ) + .await + .expect("store stale live-only deposit"); + + load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + swap_proposer_shadow_into_live(&client) + .await + .expect("swap shadow into live"); + cleanup_after_proposer_shadow_swap(&client) + .await + .expect("cleanup after swap"); + + assert_eq!( + client.get_restore_journal().await, + None, + "cleanup should return the restore flow to idle" + ); + + let collection_names = client + .database(DB) + .list_collection_names() + .await + .expect("list collections"); + for collection in manifest.collections { + assert!( + !collection_names + .iter() + .any(|name| name == &backup_collection_name(&collection.collection_name)), + "backup collection should be removed for {}", + collection.collection_name + ); + assert!( + !collection_names + .iter() + .any(|name| name == &shadow_collection_name(&collection.collection_name)), + "shadow collection should be removed for {}", + collection.collection_name + ); + } + + let live_sync_state = client + .get_sync_state() + .await + .expect("live sync_state should still exist"); + assert_eq!(live_sync_state.last_applied_l2_block, 10); + assert_eq!( + live_sync_state.fingerprint, + snapshot_block.hash().to_hex_string() + ); + + let live_blocks_count = client + .database(DB) + .collection::(PROPOSED_BLOCKS_COLLECTION) + .count_documents(mongodb::bson::doc! {}) + .await + .expect("count live proposed blocks"); + assert_eq!(live_blocks_count, 1); + + let restored_deposits = >::get_mempool_deposits(&client) + .await + .expect("restored deposits should exist"); + assert_eq!(restored_deposits, vec![snapshot_pending_deposit]); + + let duplicate_insert_error = client + .database(DB) + .collection::(DEPOSIT_COLLECTION) + .insert_one(snapshot_pending_deposit) + .await + .expect_err("restored deposit collection should keep its unique index"); + assert!(duplicate_insert_error.to_string().contains("E11000")); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn restore_proposer_snapshot_cleans_non_snapshotted_recovery_state_before_validation() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_snapshot_test_trees(&client).await; + + let selected_commitment = Fr254::from(777u64); + let snapshot_block = StoredBlock { + layer2_block_number: 10, + commitments: vec![selected_commitment.to_hex_string()], + proposer_address: Address::from([8u8; 20]), + }; + client + .store_block(&snapshot_block) + .await + .expect("store snapshot block"); + let snapshot_sync_state = SyncState::new( + snapshot_block.layer2_block_number, + snapshot_block.hash().to_hex_string(), + L1Ref { + block_number: 2000, + tx_hash: TxHash::from([8u8; 32]), + log_index: 8, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &snapshot_sync_state).await; + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-restore-cleans-nonsnapshotted-state-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let reserved_deposit = DepositDatawithFee { + fee: Fr254::from(9u64), + deposit_data: DepositData { + nf_token_id: Fr254::from(301u64), + nf_slot_id: Fr254::from(302u64), + value: Fr254::from(303u64), + secret_hash: Fr254::from(304u64), + }, + reserved: true, + }; + >::set_mempool_deposits( + &client, + vec![reserved_deposit], + ) + .await + .expect("store reserved deposit outside the snapshot"); + + let selected_transaction = test_selected_client_transaction( + 41, + snapshot_sync_state.last_applied_l2_block, + selected_commitment, + ); + client + .store_transaction(selected_transaction.clone()) + .await + .expect("store lingering selected transaction outside the snapshot"); + + let speculative_selected_transaction = test_selected_client_transaction( + 42, + snapshot_sync_state.last_applied_l2_block + 1, + Fr254::from(902u64), + ); + client + .store_transaction(speculative_selected_transaction.clone()) + .await + .expect("store speculative selected transaction outside the snapshot"); + + let speculative_included_transaction = ClientTransactionWithMetaData { + lifecycle: TxLifecycle::Included { + block_l2: snapshot_sync_state.last_applied_l2_block + 2, + }, + ..test_selected_client_transaction( + 43, + snapshot_sync_state.last_applied_l2_block + 2, + Fr254::from(903u64), + ) + }; + client + .store_transaction(speculative_included_transaction.clone()) + .await + .expect("store speculative included transaction outside the snapshot"); + + client + .store_pending_block(&PendingBlock { + layer2_block_number: snapshot_sync_state.last_applied_l2_block + 1, + state: PendingBlockState::ReadyToPropose, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: Some(Block::default()), + selected_deposits: vec![vec![reserved_deposit]], + selected_client_transaction_hashes: vec![selected_transaction.hash.clone()], + }) + .await + .expect("store stale pending block outside the snapshot"); + + let removed_mempool_transaction = + test_selected_client_transaction(51, 0, Fr254::from(901u64)); + let removed_mempool_transaction = ClientTransactionWithMetaData { + lifecycle: TxLifecycle::Mempool, + ..removed_mempool_transaction + }; + client + .store_transaction(removed_mempool_transaction.clone()) + .await + .expect("store stale mempool transaction outside the snapshot"); + + let restored_sync_state = restore_proposer_snapshot(&client, &snapshot_dir) + .await + .expect("restore should clean non-snapshotted recovery state before validation"); + assert_eq!(restored_sync_state, snapshot_sync_state); + + assert!( + client + .get_all_pending_blocks() + .await + .expect("read pending blocks after restore") + .is_empty(), + "restore should discard lingering PendingBlocks that are not part of the snapshot" + ); + assert_eq!( + client + .database(DB) + .collection::(DEPOSIT_COLLECTION) + .count_documents(mongodb::bson::doc! { "reserved": true }) + .await + .expect("count reserved deposits after restore"), + 0, + "restore should clear lingering reserved deposits before validation" + ); + assert!( + >::get_transaction( + &client, + &selected_transaction.hash, + ) + .await + .expect("selected transaction should still exist after restore cleanup") + .lifecycle + .is_selected(), + "restore should preserve selected transactions already canonical at the restored sync_state height" + ); + assert!( + >::get_transaction( + &client, + &speculative_selected_transaction.hash, + ) + .await + .expect("speculative selected transaction should still exist after restore cleanup") + .lifecycle + .is_mempool(), + "restore should normalize speculative Selected transactions beyond the restored sync_state back to the mempool before validation" + ); + assert!( + >::get_transaction( + &client, + &speculative_included_transaction.hash, + ) + .await + .expect("speculative included transaction should still exist after restore cleanup") + .lifecycle + .is_mempool(), + "restore should restore speculative Included transactions beyond the restored sync_state to the mempool before validation" + ); + assert!( + >::get_transaction( + &client, + &removed_mempool_transaction.hash, + ) + .await + .is_none(), + "restore should still drop non-selected mempool transactions during recovery" + ); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn restore_replaces_absent_optional_live_collections_with_empty_state() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let snapshot_block = StoredBlock { + layer2_block_number: 3, + commitments: Vec::new(), + proposer_address: Address::from([3u8; 20]), + }; + client + .store_block(&snapshot_block) + .await + .expect("store snapshot block"); + let snapshot_sync_state = SyncState::new( + snapshot_block.layer2_block_number, + snapshot_block.hash().to_hex_string(), + L1Ref { + block_number: 303, + tx_hash: TxHash::from([3u8; 32]), + log_index: 3, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &snapshot_sync_state).await; + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-optional-live-reset-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let commitment_nodes_collection = format!( + "{}_nodes", + >::TREE_NAME + ); + assert!( + !manifest + .collections + .iter() + .any(|collection| collection.collection_name == commitment_nodes_collection), + "snapshot fixture should omit optional empty commitment nodes collection" + ); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + client + .database(DB) + .collection::(&commitment_nodes_collection) + .insert_one(mongodb::bson::doc! { "_id": 999_i64, "value": "stale" }) + .await + .expect("seed stale live commitment nodes"); + + let newer_live_block = StoredBlock { + layer2_block_number: 4, + commitments: Vec::new(), + proposer_address: Address::from([4u8; 20]), + }; + client + .store_block(&newer_live_block) + .await + .expect("store newer live block"); + let newer_live_sync_state = SyncState::new( + newer_live_block.layer2_block_number, + newer_live_block.hash().to_hex_string(), + L1Ref { + block_number: 404, + tx_hash: TxHash::from([4u8; 32]), + log_index: 4, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &newer_live_sync_state).await; + restore_proposer_snapshot(&client, &snapshot_dir) + .await + .expect("restore snapshot"); + + let live_commitment_nodes_count = client + .database(DB) + .collection::(&commitment_nodes_collection) + .count_documents(mongodb::bson::doc! {}) + .await + .expect("count restored commitment nodes"); + assert_eq!(live_commitment_nodes_count, 0); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn recover_from_restore_journal_rolls_back_when_optional_shadow_disappears_before_swap() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let snapshot_block = StoredBlock { + layer2_block_number: 5, + commitments: Vec::new(), + proposer_address: Address::from([5u8; 20]), + }; + client + .store_block(&snapshot_block) + .await + .expect("store snapshot block"); + let snapshot_sync_state = SyncState::new( + snapshot_block.layer2_block_number, + snapshot_block.hash().to_hex_string(), + L1Ref { + block_number: 505, + tx_hash: TxHash::from([5u8; 32]), + log_index: 5, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &snapshot_sync_state).await; + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-optional-shadow-missing-recovery-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let commitment_nodes_collection = format!( + "{}_nodes", + >::TREE_NAME + ); + assert!( + !manifest + .collections + .iter() + .any(|collection| collection.collection_name == commitment_nodes_collection), + "snapshot fixture should omit empty optional commitment nodes collection" + ); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + client + .database(DB) + .collection::(&commitment_nodes_collection) + .insert_one(mongodb::bson::doc! { "_id": 111_i64, "value": "stale-live-node" }) + .await + .expect("seed stale optional live collection"); + + let newer_live_block = StoredBlock { + layer2_block_number: 6, + commitments: Vec::new(), + proposer_address: Address::from([6u8; 20]), + }; + client + .store_block(&newer_live_block) + .await + .expect("store newer live block"); + let newer_live_sync_state = SyncState::new( + newer_live_block.layer2_block_number, + newer_live_block.hash().to_hex_string(), + L1Ref { + block_number: 606, + tx_hash: TxHash::from([6u8; 32]), + log_index: 6, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &newer_live_sync_state).await; + let mut journal = load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + let ordered = restore_collections_in_swap_order(&journal.collections); + let target_index = ordered + .iter() + .position(|collection| collection.live == commitment_nodes_collection) + .expect("target optional collection should be part of restore ordering"); + + let database = client.database(DB); + for collection in ordered.iter().take(target_index) { + let layout = inspect_restore_collection_layout(&database, collection) + .await + .expect("inspect processed prefix layout"); + if layout.live_exists { + rename_collection(&client, &collection.live, &collection.backup, false) + .await + .expect("move processed live collection to backup"); + } + if layout.shadow_exists { + rename_collection(&client, &collection.shadow, &collection.live, false) + .await + .expect("move processed shadow collection into live"); + } + } + + let target = &ordered[target_index]; + database + .collection::(&target.shadow) + .drop() + .await + .expect("drop optional shadow collection before it is swapped"); + + journal.collections = ordered; + journal.phase = RestoreJournalPhase::SwapInProgress; + journal.current_index = Some(target_index as u32); + journal.current_step = Some(RestoreJournalStep::BackupPending); + journal.updated_at = mongodb::bson::DateTime::now(); + client + .upsert_restore_journal(&journal) + .await + .expect("persist mid-swap journal at optional collection"); + + let error = recover_from_restore_journal(&client) + .await + .expect_err("recovery should roll back when an optional shadow disappears"); + assert!(matches!(error, SnapshotError::RestoreInvariantViolation(_))); + assert_eq!(client.get_restore_journal().await, None); + assert_eq!( + client.get_sync_state().await, + Some(newer_live_sync_state.clone()) + ); + + let live_commitment_nodes_count = client + .database(DB) + .collection::(&commitment_nodes_collection) + .count_documents(mongodb::bson::doc! {}) + .await + .expect("count optional live collection after rollback"); + assert_eq!( + live_commitment_nodes_count, 0, + "rollback should not keep the stale optional live collection when its shadow vanished before swap" + ); + + let collection_names = client + .database(DB) + .list_collection_names() + .await + .expect("list collections"); + assert!(!collection_names + .iter() + .any(|name| name.starts_with("restore_shadow__"))); + assert!(!collection_names + .iter() + .any(|name| name.starts_with("restore_backup__"))); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn recover_from_restore_journal_accepts_optional_collection_already_restored_without_backup( + ) { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let snapshot_block = StoredBlock { + layer2_block_number: 7, + commitments: Vec::new(), + proposer_address: Address::from([7u8; 20]), + }; + client + .store_block(&snapshot_block) + .await + .expect("store snapshot block"); + let snapshot_sync_state = SyncState::new( + snapshot_block.layer2_block_number, + snapshot_block.hash().to_hex_string(), + L1Ref { + block_number: 707, + tx_hash: TxHash::from([7u8; 32]), + log_index: 7, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &snapshot_sync_state).await; + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-optional-live-restored-recovery-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let commitment_nodes_collection = format!( + "{}_nodes", + >::TREE_NAME + ); + assert!( + !manifest + .collections + .iter() + .any(|collection| collection.collection_name == commitment_nodes_collection), + "snapshot fixture should omit empty optional commitment nodes collection" + ); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let newer_live_block = StoredBlock { + layer2_block_number: 8, + commitments: Vec::new(), + proposer_address: Address::from([8u8; 20]), + }; + client + .store_block(&newer_live_block) + .await + .expect("store newer live block"); + let newer_live_sync_state = SyncState::new( + newer_live_block.layer2_block_number, + newer_live_block.hash().to_hex_string(), + L1Ref { + block_number: 808, + tx_hash: TxHash::from([8u8; 32]), + log_index: 8, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &newer_live_sync_state).await; + let mut journal = load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + let ordered = restore_collections_in_swap_order(&journal.collections); + let target_index = ordered + .iter() + .position(|collection| collection.live == commitment_nodes_collection) + .expect("target optional collection should be part of restore ordering"); + + let database = client.database(DB); + for collection in ordered.iter().take(target_index) { + let layout = inspect_restore_collection_layout(&database, collection) + .await + .expect("inspect processed prefix layout"); + if layout.live_exists { + rename_collection(&client, &collection.live, &collection.backup, false) + .await + .expect("move processed live collection to backup"); + } + if layout.shadow_exists { + rename_collection(&client, &collection.shadow, &collection.live, false) + .await + .expect("move processed shadow collection into live"); + } + } + + let target = &ordered[target_index]; + rename_collection(&client, &target.shadow, &target.live, false) + .await + .expect("simulate optional shadow already renamed into live"); + + journal.collections = ordered; + journal.phase = RestoreJournalPhase::SwapInProgress; + journal.current_index = Some(target_index as u32); + journal.current_step = Some(RestoreJournalStep::BackupPending); + journal.updated_at = mongodb::bson::DateTime::now(); + client + .upsert_restore_journal(&journal) + .await + .expect("persist mid-swap journal at already-restored optional collection"); + + recover_from_restore_journal(&client).await.expect( + "recovery should accept optional collection already restored from the snapshot", + ); + + let resumed = client + .get_restore_journal() + .await + .expect("journal should remain at swap_complete after resume"); + assert_eq!(resumed.phase, RestoreJournalPhase::SwapComplete); + assert_eq!(client.get_sync_state().await, Some(snapshot_sync_state)); + + let live_commitment_nodes_count = client + .database(DB) + .collection::(&commitment_nodes_collection) + .count_documents(mongodb::bson::doc! {}) + .await + .expect("count optional live collection after resume"); + assert_eq!(live_commitment_nodes_count, 0); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn recover_from_restore_journal_resumes_swap_in_progress() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + >::new_commitment_tree(&client, 29, 3) + .await + .expect("create commitment tree"); + >::new_historic_root_tree(&client, 32) + .await + .expect("create historic root tree"); + >::append_historic_commitment_root( + &client, + &Fr254::zero(), + true, + ) + .await + .expect("append zero historic root"); + >::new_nullifier_tree(&client, 29, 3) + .await + .expect("create nullifier tree"); + ensure_deposit_indexes(&client) + .await + .expect("create deposit indexes"); + + let snapshot_block = StoredBlock { + layer2_block_number: 12, + commitments: vec!["0xaaa".to_string()], + proposer_address: Address::from([8u8; 20]), + }; + client + .store_block(&snapshot_block) + .await + .expect("store snapshot block"); + + let snapshot_sync_state = SyncState::new( + snapshot_block.layer2_block_number, + snapshot_block.hash().to_hex_string(), + L1Ref { + block_number: 2122, + tx_hash: TxHash::from([4u8; 32]), + log_index: 8, + }, + mongodb::bson::DateTime::now(), + ); + + persist_sync_state(&client, &snapshot_sync_state).await; + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-recover-swap-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let newer_live_block = StoredBlock { + layer2_block_number: 13, + commitments: vec!["0xbbb".to_string()], + proposer_address: Address::from([9u8; 20]), + }; + client + .store_block(&newer_live_block) + .await + .expect("store newer live block"); + + let newer_live_sync_state = SyncState::new( + newer_live_block.layer2_block_number, + newer_live_block.hash().to_hex_string(), + L1Ref { + block_number: 2324, + tx_hash: TxHash::from([3u8; 32]), + log_index: 9, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &newer_live_sync_state).await; + + let mut journal = load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + let ordered = restore_collections_in_swap_order(&journal.collections); + let (first_index, first) = + first_existing_live_restore_collection(&client, &journal.collections).await; + + rename_collection(&client, &first.live, &first.backup, false) + .await + .expect("simulate backup rename"); + + journal.collections = ordered; + journal.phase = RestoreJournalPhase::SwapInProgress; + journal.current_index = Some(first_index as u32); + journal.current_step = Some(RestoreJournalStep::BackupCreated); + journal.updated_at = mongodb::bson::DateTime::now(); + client.upsert_restore_journal(&journal).await.unwrap(); + + recover_from_restore_journal(&client) + .await + .expect("recover should resume swap"); + + let resumed = client + .get_restore_journal() + .await + .expect("journal should remain at swap_complete after resume"); + assert_eq!(resumed.phase, RestoreJournalPhase::SwapComplete); + + let live_sync_state = client + .get_sync_state() + .await + .expect("live sync_state should exist"); + assert_eq!(live_sync_state.last_applied_l2_block, 12); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn cleanup_after_shadow_swap_keeps_backups_when_restored_state_is_incoherent() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let snapshot_block = StoredBlock { + layer2_block_number: 6, + commitments: Vec::new(), + proposer_address: Address::from([6u8; 20]), + }; + client + .store_block(&snapshot_block) + .await + .expect("store snapshot block"); + let snapshot_sync_state = SyncState::new( + snapshot_block.layer2_block_number, + snapshot_block.hash().to_hex_string(), + L1Ref { + block_number: 606, + tx_hash: TxHash::from([6u8; 32]), + log_index: 6, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &snapshot_sync_state).await; + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-cleanup-validation-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let newer_live_block = StoredBlock { + layer2_block_number: 7, + commitments: Vec::new(), + proposer_address: Address::from([7u8; 20]), + }; + client + .store_block(&newer_live_block) + .await + .expect("store newer live block"); + let newer_live_sync_state = SyncState::new( + newer_live_block.layer2_block_number, + newer_live_block.hash().to_hex_string(), + L1Ref { + block_number: 707, + tx_hash: TxHash::from([7u8; 32]), + log_index: 7, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &newer_live_sync_state).await; + load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + swap_proposer_shadow_into_live(&client) + .await + .expect("swap shadow into live"); + + let reserved_deposit = DepositDatawithFee { + fee: Fr254::from(17u64), + deposit_data: DepositData { + nf_token_id: Fr254::from(701u64), + nf_slot_id: Fr254::from(702u64), + value: Fr254::from(703u64), + secret_hash: Fr254::from(704u64), + }, + reserved: true, + }; + >::set_mempool_deposits( + &client, + vec![reserved_deposit], + ) + .await + .expect("store reserved deposit outside the snapshot"); + + let selected_transaction = test_selected_client_transaction( + 61, + newer_live_sync_state.last_applied_l2_block, + Fr254::from(761u64), + ); + client + .store_transaction(selected_transaction.clone()) + .await + .expect("store lingering selected transaction outside the snapshot"); + + client + .store_pending_block(&PendingBlock { + layer2_block_number: newer_live_sync_state.last_applied_l2_block + 1, + state: PendingBlockState::ReadyToPropose, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: Some(Block::default()), + selected_deposits: vec![vec![reserved_deposit]], + selected_client_transaction_hashes: vec![selected_transaction.hash.clone()], + }) + .await + .expect("store stale pending block outside the snapshot"); + + let mempool_transaction = ClientTransactionWithMetaData { + lifecycle: TxLifecycle::Mempool, + ..test_selected_client_transaction(71, 0, Fr254::from(771u64)) + }; + client + .store_transaction(mempool_transaction.clone()) + .await + .expect("store mempool transaction outside the snapshot"); + + client + .delete_block_by_number(snapshot_block.layer2_block_number) + .await + .expect("delete restored stored block"); + + let error = cleanup_after_proposer_shadow_swap(&client) + .await + .expect_err("cleanup should refuse to discard backups for incoherent restore"); + assert!(matches!(error, SnapshotError::RestoreInvariantViolation(_))); + assert!( + client.get_restore_journal().await.is_some(), + "restore journal should be preserved for manual recovery" + ); + assert!( + client + .database(DB) + .list_collection_names() + .await + .expect("list collections") + .iter() + .any(|name| name == &backup_collection_name(PROPOSED_BLOCKS_COLLECTION)), + "backup collections should remain available for rollback" + ); + assert_eq!( + client + .get_all_pending_blocks() + .await + .expect("read pending blocks after failed cleanup") + .len(), + 1, + "failed cleanup should not discard non-snapshotted PendingBlocks before validation succeeds" + ); + assert_eq!( + client + .database(DB) + .collection::(DEPOSIT_COLLECTION) + .count_documents(mongodb::bson::doc! { "reserved": true }) + .await + .expect("count reserved deposits after failed cleanup"), + 1, + "failed cleanup should not clear reserved deposits before snapshot validation succeeds" + ); + assert!(matches!( + >::get_transaction( + &client, + &selected_transaction.hash, + ) + .await + .expect("selected transaction should still exist after failed cleanup") + .lifecycle, + TxLifecycle::Selected { .. } + )); + assert!(matches!( + >::get_transaction( + &client, + &mempool_transaction.hash, + ) + .await + .expect("mempool transaction should still exist after failed cleanup") + .lifecycle, + TxLifecycle::Mempool + )); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn create_proposer_snapshot_rejects_tree_state_ahead_of_sync_state() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let stored_block = StoredBlock { + layer2_block_number: 0, + commitments: Vec::new(), + proposer_address: Address::from([1u8; 20]), + }; + client + .store_block(&stored_block) + .await + .expect("store block"); + let sync_state = SyncState::new( + stored_block.layer2_block_number, + stored_block.hash().to_hex_string(), + L1Ref { + block_number: 100, + tx_hash: TxHash::from([1u8; 32]), + log_index: 1, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &sync_state).await; + >::append_historic_commitment_root( + &client, + &Fr254::from(1u64), + true, + ) + .await + .expect("append applied historic root"); + >::append_historic_commitment_root( + &client, + &Fr254::from(2u64), + true, + ) + .await + .expect("append extra historic root"); + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-snapshot-inconsistent-state-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let error = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect_err("snapshot should reject tree state ahead of sync_state"); + assert!(error.to_string().contains("creating snapshot")); + } + + #[tokio::test] + async fn create_proposer_snapshot_rejects_stored_block_ahead_of_sync_state() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let applied_block = StoredBlock { + layer2_block_number: 0, + commitments: Vec::new(), + proposer_address: Address::from([2u8; 20]), + }; + client + .store_block(&applied_block) + .await + .expect("store applied block"); + persist_sync_state( + &client, + &SyncState::new( + applied_block.layer2_block_number, + applied_block.hash().to_hex_string(), + L1Ref { + block_number: 200, + tx_hash: TxHash::from([2u8; 32]), + log_index: 2, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + + let speculative_block = StoredBlock { + layer2_block_number: 1, + commitments: vec!["0xspeculative".to_string()], + proposer_address: Address::from([3u8; 20]), + }; + client + .store_block(&speculative_block) + .await + .expect("store speculative block ahead of sync_state"); + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-snapshot-ahead-stored-block-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let error = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect_err("snapshot should reject speculative StoredBlocks ahead of sync_state"); + assert!(error + .to_string() + .contains("highest StoredBlock 1 is ahead of sync_state-applied block 0")); + } + + #[tokio::test] + async fn create_proposer_snapshot_rejects_reserved_deposits_ahead_of_sync_state() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let applied_block = StoredBlock { + layer2_block_number: 0, + commitments: Vec::new(), + proposer_address: Address::from([4u8; 20]), + }; + client + .store_block(&applied_block) + .await + .expect("store applied block"); + persist_sync_state( + &client, + &SyncState::new( + applied_block.layer2_block_number, + applied_block.hash().to_hex_string(), + L1Ref { + block_number: 400, + tx_hash: TxHash::from([4u8; 32]), + log_index: 4, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + >::set_mempool_deposits( + &client, + vec![DepositDatawithFee { + fee: Fr254::from(4u64), + deposit_data: DepositData { + nf_token_id: Fr254::from(41u64), + nf_slot_id: Fr254::from(42u64), + value: Fr254::from(43u64), + secret_hash: Fr254::from(44u64), + }, + reserved: true, + }], + ) + .await + .expect("store reserved deposit"); + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-snapshot-reserved-deposits-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let error = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect_err("snapshot should reject reserved deposits ahead of sync_state"); + assert!(error.to_string().contains( + "Deposits contains 1 reserved selection(s) ahead of sync_state-applied block 0" + )); + } + + #[tokio::test] + async fn recover_from_restore_journal_completes_swap_complete_cleanup() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + >::new_commitment_tree(&client, 29, 3) + .await + .expect("create commitment tree"); + >::new_historic_root_tree(&client, 32) + .await + .expect("create historic root tree"); + >::append_historic_commitment_root( + &client, + &Fr254::zero(), + true, + ) + .await + .expect("append zero historic root"); + >::new_nullifier_tree(&client, 29, 3) + .await + .expect("create nullifier tree"); + ensure_deposit_indexes(&client) + .await + .expect("create deposit indexes"); + + let snapshot_block = StoredBlock { + layer2_block_number: 14, + commitments: vec!["0xccc".to_string()], + proposer_address: Address::from([10u8; 20]), + }; + client + .store_block(&snapshot_block) + .await + .expect("store snapshot block"); + + let snapshot_sync_state = SyncState::new( + snapshot_block.layer2_block_number, + snapshot_block.hash().to_hex_string(), + L1Ref { + block_number: 2526, + tx_hash: TxHash::from([2u8; 32]), + log_index: 10, + }, + mongodb::bson::DateTime::now(), + ); + + persist_sync_state(&client, &snapshot_sync_state).await; + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-recover-complete-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let newer_live_block = StoredBlock { + layer2_block_number: 15, + commitments: vec!["0xddd".to_string()], + proposer_address: Address::from([11u8; 20]), + }; + client + .store_block(&newer_live_block) + .await + .expect("store newer live block"); + + let newer_live_sync_state = SyncState::new( + newer_live_block.layer2_block_number, + newer_live_block.hash().to_hex_string(), + L1Ref { + block_number: 2728, + tx_hash: TxHash::from([1u8; 32]), + log_index: 11, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &newer_live_sync_state).await; + + load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + swap_proposer_shadow_into_live(&client) + .await + .expect("swap shadow into live"); + + recover_from_restore_journal(&client) + .await + .expect("recover should clean swap_complete state"); + + assert_eq!(client.get_restore_journal().await, None); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn recover_from_restore_journal_rolls_back_incoherent_swap_complete_state() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let snapshot_block = StoredBlock { + layer2_block_number: 29, + commitments: Vec::new(), + proposer_address: Address::from([29u8; 20]), + }; + client + .store_block(&snapshot_block) + .await + .expect("store snapshot block"); + let snapshot_sync_state = SyncState::new( + snapshot_block.layer2_block_number, + snapshot_block.hash().to_hex_string(), + L1Ref { + block_number: 2900, + tx_hash: TxHash::from([29u8; 32]), + log_index: 29, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &snapshot_sync_state).await; + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-recover-swap-complete-rollback-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let newer_live_block = StoredBlock { + layer2_block_number: 30, + commitments: Vec::new(), + proposer_address: Address::from([30u8; 20]), + }; + client + .store_block(&newer_live_block) + .await + .expect("store newer live block"); + let newer_live_sync_state = SyncState::new( + newer_live_block.layer2_block_number, + newer_live_block.hash().to_hex_string(), + L1Ref { + block_number: 3000, + tx_hash: TxHash::from([30u8; 32]), + log_index: 30, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &newer_live_sync_state).await; + load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + swap_proposer_shadow_into_live(&client) + .await + .expect("swap shadow into live"); + + let reserved_deposit = DepositDatawithFee { + fee: Fr254::from(27u64), + deposit_data: DepositData { + nf_token_id: Fr254::from(2901u64), + nf_slot_id: Fr254::from(2902u64), + value: Fr254::from(2903u64), + secret_hash: Fr254::from(2904u64), + }, + reserved: true, + }; + >::set_mempool_deposits( + &client, + vec![reserved_deposit], + ) + .await + .expect("store reserved deposit outside the snapshot"); + + let selected_transaction = test_selected_client_transaction( + 81, + newer_live_sync_state.last_applied_l2_block, + Fr254::from(2981u64), + ); + client + .store_transaction(selected_transaction.clone()) + .await + .expect("store lingering selected transaction outside the snapshot"); + + client + .store_pending_block(&PendingBlock { + layer2_block_number: newer_live_sync_state.last_applied_l2_block + 1, + state: PendingBlockState::ReadyToPropose, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: Some(Block::default()), + selected_deposits: vec![vec![reserved_deposit]], + selected_client_transaction_hashes: vec![selected_transaction.hash.clone()], + }) + .await + .expect("store stale pending block outside the snapshot"); + + let mempool_transaction = ClientTransactionWithMetaData { + lifecycle: TxLifecycle::Mempool, + ..test_selected_client_transaction(91, 0, Fr254::from(2991u64)) + }; + client + .store_transaction(mempool_transaction.clone()) + .await + .expect("store mempool transaction outside the snapshot"); + + client + .delete_block_by_number(snapshot_block.layer2_block_number) + .await + .expect("delete restored stored block"); + + let error = recover_from_restore_journal(&client) + .await + .expect_err("recovery should roll back incoherent swap_complete state"); + assert!(matches!(error, SnapshotError::RestoreInvariantViolation(_))); + assert_eq!(client.get_restore_journal().await, None); + assert_eq!( + client.get_sync_state().await, + Some(newer_live_sync_state.clone()) + ); + assert_eq!( + client + .get_all_pending_blocks() + .await + .expect("read pending blocks after rollback") + .len(), + 1, + "swap_complete rollback should leave non-snapshotted PendingBlocks untouched" + ); + assert!(matches!( + >::get_transaction( + &client, + &selected_transaction.hash, + ) + .await + .expect("selected transaction should still exist after rollback") + .lifecycle, + TxLifecycle::Selected { .. } + )); + assert!(matches!( + >::get_transaction( + &client, + &mempool_transaction.hash, + ) + .await + .expect("mempool transaction should still exist after rollback") + .lifecycle, + TxLifecycle::Mempool + )); + + let collection_names = client + .database(DB) + .list_collection_names() + .await + .expect("list collections"); + assert!(!collection_names + .iter() + .any(|name| name.starts_with("restore_shadow__"))); + assert!(!collection_names + .iter() + .any(|name| name.starts_with("restore_backup__"))); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn recover_from_restore_journal_keeps_journal_when_swap_complete_rollback_fails() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + initialize_snapshot_test_trees(&client).await; + + let snapshot_block = StoredBlock { + layer2_block_number: 31, + commitments: Vec::new(), + proposer_address: Address::from([31u8; 20]), + }; + client + .store_block(&snapshot_block) + .await + .expect("store snapshot block"); + let snapshot_sync_state = SyncState::new( + snapshot_block.layer2_block_number, + snapshot_block.hash().to_hex_string(), + L1Ref { + block_number: 3100, + tx_hash: TxHash::from([31u8; 32]), + log_index: 31, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &snapshot_sync_state).await; + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-recover-swap-complete-rollback-failure-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let newer_live_block = StoredBlock { + layer2_block_number: 32, + commitments: Vec::new(), + proposer_address: Address::from([32u8; 20]), + }; + client + .store_block(&newer_live_block) + .await + .expect("store newer live block"); + let _newer_live_sync_state = SyncState::new( + newer_live_block.layer2_block_number, + newer_live_block.hash().to_hex_string(), + L1Ref { + block_number: 3200, + tx_hash: TxHash::from([32u8; 32]), + log_index: 32, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &_newer_live_sync_state).await; + load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + swap_proposer_shadow_into_live(&client) + .await + .expect("swap shadow into live"); + + client + .delete_block_by_number(snapshot_block.layer2_block_number) + .await + .expect("delete restored stored block"); + client + .database(DB) + .collection::(&backup_collection_name(PROPOSED_BLOCKS_COLLECTION)) + .drop() + .await + .expect("drop required backup collection"); + + let error = recover_from_restore_journal(&client).await.expect_err( + "recovery should fail closed when swap_complete rollback loses a required backup", + ); + assert!(matches!(error, SnapshotError::RestoreInvariantViolation(_))); + + let persisted_journal = client + .get_restore_journal() + .await + .expect("journal should remain when swap_complete rollback fails"); + assert_eq!( + persisted_journal.phase, + RestoreJournalPhase::RollbackInProgress + ); + assert_eq!( + persisted_journal.current_step, + Some(RestoreJournalStep::RollbackStarted) + ); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn recover_from_restore_journal_cleans_real_swap_complete_crash() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_snapshot_test_trees(&client).await; + + let snapshot_block = StoredBlock { + layer2_block_number: 25, + commitments: vec!["0xcleanup".to_string()], + proposer_address: Address::from([25u8; 20]), + }; + client + .store_block(&snapshot_block) + .await + .expect("store snapshot block"); + + persist_sync_state( + &client, + &SyncState::new( + snapshot_block.layer2_block_number, + snapshot_block.hash().to_hex_string(), + L1Ref { + block_number: 2500, + tx_hash: TxHash::from([25u8; 32]), + log_index: 4, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-cleanup-crash-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let newer_live_block = StoredBlock { + layer2_block_number: 26, + commitments: vec!["0xnewer".to_string()], + proposer_address: Address::from([26u8; 20]), + }; + client + .store_block(&newer_live_block) + .await + .expect("store newer live block"); + persist_sync_state( + &client, + &SyncState::new( + newer_live_block.layer2_block_number, + newer_live_block.hash().to_hex_string(), + L1Ref { + block_number: 2600, + tx_hash: TxHash::from([26u8; 32]), + log_index: 5, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + + load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + swap_proposer_shadow_into_live(&client) + .await + .expect("swap shadow into live"); + + let failpoint = TestFailpointGuard::enable("restore_after_first_backup_cleanup"); + let crash_client = client.clone(); + let join_error = + tokio::spawn(async move { cleanup_after_proposer_shadow_swap(&crash_client).await }) + .await + .expect_err("cleanup should panic at failpoint"); + assert!(join_error.is_panic()); + drop(failpoint); + + recover_from_restore_journal(&client) + .await + .expect("recover should finish swap_complete cleanup"); + + assert_eq!(client.get_restore_journal().await, None); + let live_sync_state = client + .get_sync_state() + .await + .expect("restored live sync_state should still exist"); + assert_eq!( + live_sync_state.last_applied_l2_block, + snapshot_block.layer2_block_number + ); + + let names = client + .database(DB) + .list_collection_names() + .await + .expect("list collection names"); + assert!(!names + .iter() + .any(|name| name.starts_with("restore_shadow__"))); + assert!(!names + .iter() + .any(|name| name.starts_with("restore_backup__"))); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn cleanup_after_proposer_shadow_swap_clears_stale_startup_replay_reset_marker() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_snapshot_test_trees(&client).await; + + let snapshot_block = StoredBlock { + layer2_block_number: 27, + commitments: vec!["0xrestore-marker".to_string()], + proposer_address: Address::from([27u8; 20]), + }; + client + .store_block(&snapshot_block) + .await + .expect("store snapshot block"); + persist_sync_state( + &client, + &SyncState::new( + snapshot_block.layer2_block_number, + snapshot_block.hash().to_hex_string(), + L1Ref { + block_number: 2700, + tx_hash: TxHash::from([27u8; 32]), + log_index: 6, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-restore-marker-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let newer_live_block = StoredBlock { + layer2_block_number: 28, + commitments: vec!["0xnewer-live".to_string()], + proposer_address: Address::from([28u8; 20]), + }; + client + .store_block(&newer_live_block) + .await + .expect("store newer live block"); + persist_sync_state( + &client, + &SyncState::new( + newer_live_block.layer2_block_number, + newer_live_block.hash().to_hex_string(), + L1Ref { + block_number: 2800, + tx_hash: TxHash::from([28u8; 32]), + log_index: 7, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + + client + .upsert_startup_replay_reset_marker(&StartupReplayResetMarker::new( + StartupReplayResetPhase::DbCleanupPending, + mongodb::bson::DateTime::now(), + )) + .await + .expect("persist stale startup replay reset marker"); + + load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + swap_proposer_shadow_into_live(&client) + .await + .expect("swap shadow into live"); + + cleanup_after_proposer_shadow_swap(&client) + .await + .expect("cleanup should clear stale startup replay reset marker"); + + assert_eq!(client.get_restore_journal().await, None); + assert_eq!(client.get_startup_replay_reset_marker().await, None); + let live_sync_state = client + .get_sync_state() + .await + .expect("restored live sync_state should remain"); + assert_eq!( + live_sync_state.last_applied_l2_block, + snapshot_block.layer2_block_number + ); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn recover_from_restore_journal_skips_non_snapshot_cleanup_once_applied() { + let _snapshot_test_lock = snapshot_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_snapshot_test_trees(&client).await; + + let snapshot_block = StoredBlock { + layer2_block_number: 25, + commitments: vec!["0xcleanup-selected".to_string()], + proposer_address: Address::from([31u8; 20]), + }; + client + .store_block(&snapshot_block) + .await + .expect("store snapshot block"); + + let snapshot_sync_state = SyncState::new( + snapshot_block.layer2_block_number, + snapshot_block.hash().to_hex_string(), + L1Ref { + block_number: 2500, + tx_hash: TxHash::from([31u8; 32]), + log_index: 4, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &snapshot_sync_state).await; + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-proposer-cleanup-applied-crash-test-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let newer_live_block = StoredBlock { + layer2_block_number: 26, + commitments: vec!["0xnewer".to_string()], + proposer_address: Address::from([32u8; 20]), + }; + client + .store_block(&newer_live_block) + .await + .expect("store newer live block"); + persist_sync_state( + &client, + &SyncState::new( + newer_live_block.layer2_block_number, + newer_live_block.hash().to_hex_string(), + L1Ref { + block_number: 2600, + tx_hash: TxHash::from([32u8; 32]), + log_index: 5, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + + let selected_transaction = test_selected_client_transaction( + 401, + snapshot_block.layer2_block_number + 1, + Fr254::from(444_u64), + ); + client + .store_transaction(selected_transaction.clone()) + .await + .expect("store selected transaction ahead of snapshot state"); + + load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + swap_proposer_shadow_into_live(&client) + .await + .expect("swap shadow into live"); + + let failpoint = TestFailpointGuard::enable("restore_after_non_snapshot_cleanup_applied"); + let crash_client = client.clone(); + let join_error = + tokio::spawn(async move { cleanup_after_proposer_shadow_swap(&crash_client).await }) + .await + .expect_err("cleanup should panic after non-snapshot cleanup commit"); + assert!(join_error.is_panic()); + drop(failpoint); + + let journal = client + .get_restore_journal() + .await + .expect("journal should remain after cleanup-applied crash"); + assert_eq!(journal.phase, RestoreJournalPhase::SwapComplete); + assert_eq!( + journal.current_step, + Some(RestoreJournalStep::NonSnapshotCleanupApplied) + ); + assert!( + >::get_transaction( + &client, + &selected_transaction.hash, + ) + .await + .expect("selected transaction should still exist after cleanup-applied crash") + .lifecycle + .is_mempool(), + "cleanup should have already restored the selected transaction to the mempool before the crash" + ); + + recover_from_restore_journal(&client) + .await + .expect("recover should skip already-applied non-snapshot cleanup"); + + assert_eq!(client.get_restore_journal().await, None); + assert!( + >::get_transaction( + &client, + &selected_transaction.hash, + ) + .await + .expect("transaction should still exist after swap_complete recovery") + .lifecycle + .is_mempool(), + "recovery should not delete transactions preserved by the first cleanup pass" + ); + + fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } +} diff --git a/nightfall_proposer/src/driven/mock_prover.rs b/nightfall_proposer/src/driven/mock_prover.rs index 0b2ac6f4..5b8abd4e 100644 --- a/nightfall_proposer/src/driven/mock_prover.rs +++ b/nightfall_proposer/src/driven/mock_prover.rs @@ -248,6 +248,14 @@ impl RecursiveProvingEngine for MockProver { deposit_data: &[DepositData; 4], public_inputs: &mut PublicInputs, ) -> Result { + if deposit_data + .iter() + .all(|deposit| *deposit == DepositData::default()) + { + *public_inputs = PublicInputs::for_deposit(); + return Ok(PlonkProof::default()); + } + create_unified_deposit_proof(deposit_data, public_inputs).map_err(Self::Error::from) } } diff --git a/nightfall_proposer/src/driven/nightfall_client_transaction.rs b/nightfall_proposer/src/driven/nightfall_client_transaction.rs index 69afe10b..fd5c79cf 100644 --- a/nightfall_proposer/src/driven/nightfall_client_transaction.rs +++ b/nightfall_proposer/src/driven/nightfall_client_transaction.rs @@ -81,7 +81,7 @@ impl, P: Proof> reject::Reject for ClientTransactionError( client_transaction: ClientTransaction

, -) -> Result<(), ClientTransactionError> +) -> Result, ClientTransactionError> where E: ProvingEngine

, P: Proof, @@ -140,11 +140,18 @@ where // 5) Validate that we can convert the transaction into a form suitable for the nightfall contract bindings let _: OnChainTransaction = (&client_transaction).into(); + + // Pass the sender-supplied receipt_token through unchanged. A token is + // only meaningful for transfer transactions; for deposits and withdrawals + // the client never sets this field and we store None. + let receipt_token = client_transaction.receipt_token.clone(); + let client_transaction_with_metadata = ClientTransactionWithMetaData::

{ client_transaction: client_transaction.clone(), lifecycle: TxLifecycle::Mempool, hash: hash.to_vec(), historic_roots: vec![client_transaction.historic_commitment_root], + receipt_token: receipt_token.clone(), }; // 6) Validate that the first nullifier is not zero (we must nullify the first spent commitment) @@ -156,7 +163,7 @@ where info!("Client Transaction is valid, storing in database"); let key = db.store_transaction(client_transaction_with_metadata).await; match key { - Some(_key) => Ok(()), + Some(_key) => Ok(receipt_token), None => Err(ClientTransactionError::CouldNotStoreTransaction), } } @@ -175,7 +182,8 @@ where let ctx = >::find_deposit(db, &deposit_data).await; // if it is, we should return an error if ctx.is_some() { - return Err(ClientTransactionError::TransactionAlreadyExists); + info!("Deposit transaction already exists in mempool, treating replay as a no-op"); + return Ok(()); } info!("Deposit Transaction is valid, storing in database"); let key = diff --git a/nightfall_proposer/src/driven/nightfall_contract.rs b/nightfall_proposer/src/driven/nightfall_contract.rs index b3c70369..7dc7fe60 100644 --- a/nightfall_proposer/src/driven/nightfall_contract.rs +++ b/nightfall_proposer/src/driven/nightfall_contract.rs @@ -1,10 +1,16 @@ //! Implementation of the [`NightfallContract`] trait from `nightfall_proposer/src/ports/contracts.rs`. use crate::{ - domain::entities::Block, initialisation::get_blockchain_client_connection, - ports::contracts::NightfallContract, + domain::entities::Block, + initialisation::get_blockchain_client_connection, + ports::contracts::{ + BroadcastUnknownReason, NightfallContract, NotBroadcastReason, ProposeBlockOutcome, + }, +}; +use alloy::{ + primitives::{TxHash, I256}, + providers::Provider, }; -use alloy::primitives::I256; use configuration::{addresses::get_addresses, settings::get_settings}; use lib::{ blockchain_client::BlockchainClientConnection, error::NightfallContractError, @@ -15,73 +21,97 @@ use nightfall_bindings::artifacts::Nightfall; #[async_trait::async_trait] impl NightfallContract for Nightfall::NightfallCalls { - async fn propose_block(block: Block) -> Result<(), NightfallContractError> { - let blockchain_client = get_blockchain_client_connection() - .await - .read() - .await - .get_client(); + async fn propose_block(block: Block) -> Result { + let read_connection = get_blockchain_client_connection().await.read().await; + let blockchain_client = read_connection.get_client(); let client = blockchain_client.root(); - let signer = get_blockchain_client_connection() - .await - .read() - .await - .get_signer(); + let caller = read_connection.get_address(); + let wallet = read_connection.get_wallet_type().clone(); let verified = - VerifiedContracts::resolve_and_verify_contract(client.clone(), get_addresses()) + match VerifiedContracts::resolve_and_verify_contract(client.clone(), get_addresses()) .await - .map_err(|e| { - NightfallContractError::ContractVerificationError(format!( - "Contract verification failed during get_token_info: {e}" - )) - })?; + { + Ok(verified) => verified, + Err(_) => { + return Ok(ProposeBlockOutcome::NotBroadcast { + reason: NotBroadcastReason::TransactionPreparationFailed, + }); + } + }; let nightfall = verified.nightfall; // Convert the block transactions to the Nightfall format let blk: Nightfall::Block = block.into(); - let nonce = blockchain_client - .get_transaction_count(signer.address()) - .await - .map_err(|_| NightfallContractError::TransactionError)?; - let gas_price = blockchain_client - .get_gas_price() - .await - .map_err(|_| NightfallContractError::TransactionError)?; + let nonce = match blockchain_client.get_transaction_count(caller).await { + Ok(nonce) => nonce, + Err(_) => { + return Ok(ProposeBlockOutcome::NotBroadcast { + reason: NotBroadcastReason::TransactionPreparationFailed, + }); + } + }; + let gas_price = match blockchain_client.get_gas_price().await { + Ok(gas_price) => gas_price, + Err(_) => { + return Ok(ProposeBlockOutcome::NotBroadcast { + reason: NotBroadcastReason::TransactionPreparationFailed, + }); + } + }; let max_fee_per_gas = gas_price * 2; let max_priority_fee_per_gas = gas_price; let gas_limit = 5000000u64; - let raw_tx = nightfall + let raw_tx = match nightfall .propose_block(blk) .nonce(nonce) .gas(gas_limit) .max_fee_per_gas(max_fee_per_gas) .max_priority_fee_per_gas(max_priority_fee_per_gas) - .chain_id(get_settings().network.chain_id) // Linea testnet chain ID - .build_raw_transaction((*signer).clone()) + .chain_id(get_settings().network.chain_id) + .build_raw_transaction(wallet) .await - .map_err(|_| NightfallContractError::TransactionError)?; + { + Ok(raw_tx) => raw_tx, + Err(_) => { + return Ok(ProposeBlockOutcome::NotBroadcast { + reason: NotBroadcastReason::TransactionPreparationFailed, + }); + } + }; - let receipt = blockchain_client - .send_raw_transaction(&raw_tx) - .await - .map_err(|_| { - NightfallContractError::BlockProposalError( - "Failed to send raw transaction in propose_block".to_string(), - ) - })? - .get_receipt() - .await - .map_err(|_| { - NightfallContractError::BlockProposalError( - "Failed to get transaction receipt in propose_block".to_string(), - ) - })?; + let pending_tx = match blockchain_client.send_raw_transaction(&raw_tx).await { + Ok(pending_tx) => pending_tx, + Err(_) => { + return Ok(ProposeBlockOutcome::NotBroadcast { + reason: NotBroadcastReason::SendRawTransactionFailed, + }); + } + }; + let tx_hash = *pending_tx.tx_hash(); + + let receipt = match pending_tx.get_receipt().await { + Ok(receipt) => receipt, + Err(_) => { + return Ok(ProposeBlockOutcome::BroadcastUnknown { + tx_hash, + reason: BroadcastUnknownReason::ReceiptUnavailable, + }); + } + }; info!( "The L2 block was sent to L1. Received receipt for submitted block with hash: {}, gas used was: {}", receipt.transaction_hash, receipt.gas_used ); - Ok(()) + if receipt.status() { + Ok(ProposeBlockOutcome::Submitted { + tx_hash: receipt.transaction_hash, + }) + } else { + Ok(ProposeBlockOutcome::Reverted { + tx_hash: receipt.transaction_hash, + }) + } } async fn get_current_layer2_blocknumber() -> Result { @@ -106,4 +136,21 @@ impl NightfallContract for Nightfall::NightfallCalls { .await .map_err(|_| NightfallContractError::TransactionError)?) } + + async fn get_proposal_receipt_status( + tx_hash: TxHash, + ) -> Result, NightfallContractError> { + let blockchain_client = get_blockchain_client_connection() + .await + .read() + .await + .get_client(); + let client = blockchain_client.root(); + let receipt = client + .get_transaction_receipt(tx_hash) + .await + .map_err(|e| NightfallContractError::ProviderError(e.to_string()))?; + + Ok(receipt.map(|receipt| receipt.status())) + } } diff --git a/nightfall_proposer/src/driven/nightfall_event.rs b/nightfall_proposer/src/driven/nightfall_event.rs index 5d545599..527c95fc 100644 --- a/nightfall_proposer/src/driven/nightfall_event.rs +++ b/nightfall_proposer/src/driven/nightfall_event.rs @@ -1,19 +1,27 @@ use crate::{ domain::entities::DepositDatawithFee, + domain::entities::{L1Ref, SyncState}, driven::{ db::mongo_db::StoredBlock, nightfall_client_transaction::process_deposit_transaction, }, - drivers::blockchain::nightfall_event_listener::get_synchronisation_status, - initialisation::{get_blockchain_client_connection, get_db_connection}, + drivers::blockchain::nightfall_event_listener::{ + get_synchronisation_status, is_listener_replay_catch_up_pending, + }, + initialisation::{ + begin_startup_replay_reset, get_blockchain_client_connection, get_db_connection, + }, ports::{ contracts::NightfallContract, - db::BlockStorageDB, + db::{BlockStorageDB, PendingBlockDB, SyncStateDB}, events::EventHandler, trees::{CommitmentTree, HistoricRootTree, NullifierTree}, }, - services::selected_transactions::reconcile_orphaned_selected_transactions, + services::assemble_block::{cleanup_selected_transactions, release_selected_transactions}, + services::selected_transactions::reconcile_active_client_transaction_lifecycle, + services::snapshot_scheduler::maybe_schedule_snapshot_for_applied_block, }; use alloy::primitives::{TxHash, I256}; +use alloy::rpc::types::Log; use alloy::{consensus::Transaction, sol_types::SolInterface}; use ark_bn254::Fr as Fr254; use ark_ff::BigInteger; @@ -23,7 +31,6 @@ use lib::{ error::EventHandlerError, get_fee_token_id, hex_conversion::HexConvertible, - merkle_trees::trees::IndexedTree, nf_client_proof::{Proof, ProvingEngine}, nf_token_id::to_nf_token_id_from_solidity, shared_entities::DepositData, @@ -38,6 +45,18 @@ use std::{ fmt::{Debug, Display}, }; use tokio::sync::{OnceCell, RwLock}; + +#[cfg(test)] +use std::{collections::HashSet, sync::Mutex}; + +fn merkle_tree_error_to_mongo( + error: lib::merkle_trees::trees::MerkleTreeError, +) -> mongodb::error::Error { + match error { + lib::merkle_trees::trees::MerkleTreeError::DatabaseError(db_error) => db_error, + other => mongodb::error::Error::custom(other.to_string()), + } +} // Define a mutable lazy static to hold the layer 2 blocknumber. We need this to // check if we're still in sync, but putting it in the context would mean passing it around too much pub async fn get_expected_layer2_blocknumber() -> &'static RwLock { @@ -47,6 +66,60 @@ pub async fn get_expected_layer2_blocknumber() -> &'static RwLock { .await } +#[cfg(test)] +static ENABLED_BLOCK_APPLY_FAILPOINTS: std::sync::OnceLock>> = + std::sync::OnceLock::new(); + +#[cfg(test)] +fn enabled_block_apply_failpoints() -> &'static Mutex> { + ENABLED_BLOCK_APPLY_FAILPOINTS.get_or_init(|| Mutex::new(HashSet::new())) +} + +fn maybe_fail_block_apply(_name: &str) -> Result<(), mongodb::error::Error> { + #[cfg(test)] + { + if enabled_block_apply_failpoints() + .lock() + .expect("block apply failpoint lock poisoned") + .contains(_name) + { + return Err(mongodb::error::Error::custom(format!( + "Simulated block apply failure at {_name}" + ))); + } + } + + Ok(()) +} + +#[cfg(test)] +struct TestBlockApplyFailpointGuard { + name: String, +} + +#[cfg(test)] +impl TestBlockApplyFailpointGuard { + fn enable(name: &str) -> Self { + enabled_block_apply_failpoints() + .lock() + .expect("block apply failpoint lock poisoned") + .insert(name.to_string()); + Self { + name: name.to_string(), + } + } +} + +#[cfg(test)] +impl Drop for TestBlockApplyFailpointGuard { + fn drop(&mut self) { + enabled_block_apply_failpoints() + .lock() + .expect("block apply failpoint lock poisoned") + .remove(&self.name); + } +} + #[derive(Debug)] pub enum ProcessBlockError { CouldNotStoreHistoricRoot, @@ -73,12 +146,16 @@ where E: ProvingEngine

, N: NightfallContract, { - async fn handle_event(&self, tx_hash: TxHash) -> Result<(), EventHandlerError> { + async fn handle_event(&self, log: Log) -> Result<(), EventHandlerError> { // we'll split out individual events here in case that's useful later + let tx_hash = log.transaction_hash.ok_or_else(|| { + EventHandlerError::IOError("Event log missing transaction_hash".to_string()) + })?; debug!("Handling event {self:?} for transaction {tx_hash:?}"); match &self { Nightfall::NightfallEvents::BlockProposed(filter) => { - process_nightfall_calldata::(tx_hash, filter.layer2_block_number).await? + process_nightfall_calldata::(tx_hash, filter.layer2_block_number, &log) + .await? } Nightfall::NightfallEvents::DepositEscrowed(filter) => { info!("Received DepositEscrowed event"); @@ -110,6 +187,7 @@ where pub async fn process_nightfall_calldata( transaction_hash: TxHash, block_number: I256, + log: &Log, ) -> Result<(), EventHandlerError> where P: Proof + Send + Serialize + Clone + Debug + Sync, @@ -132,7 +210,8 @@ where .map_err(|_| EventHandlerError::InvalidCalldata)?; if let Nightfall::NightfallCalls::propose_block(decode) = decoded { // OK to use unwrap because the smart contract has to provide a block number - process_propose_block_event::(decode, transaction_hash, block_number).await?; + process_propose_block_event::(decode, transaction_hash, block_number, log) + .await?; } } else { panic!("Transaction not found when looking up calldata"); @@ -140,10 +219,117 @@ where Ok(()) } +fn stored_block_from_pending_block( + pending_block: &crate::domain::entities::PendingBlock, + proposer_address: alloy::primitives::Address, +) -> Option { + let block = pending_block.block.as_ref()?; + + Some(StoredBlock { + layer2_block_number: pending_block.layer2_block_number, + commitments: block + .transactions + .iter() + .flat_map(|ntx| { + ntx.commitments + .iter() + .map(|c| c.to_hex_string()) + .collect::>() + }) + .collect(), + proposer_address, + }) +} + +async fn finalize_pending_block_after_applied_block

( + db: &Client, + pending_block: crate::domain::entities::PendingBlock, + our_address: alloy::primitives::Address, + applied_block: &StoredBlock, +) where + P: Proof, +{ + let cleanup_result = if let Some(pending_block_hash) = + stored_block_from_pending_block(&pending_block, our_address).map(|block| block.hash()) + { + if pending_block_hash == applied_block.hash() { + cleanup_selected_transactions::

( + db, + &pending_block.selected_deposits, + &pending_block.selected_client_transaction_hashes, + ) + .await + } else { + release_selected_transactions::

( + db, + &pending_block.selected_deposits, + &pending_block.selected_client_transaction_hashes, + ) + .await + } + } else { + release_selected_transactions::

( + db, + &pending_block.selected_deposits, + &pending_block.selected_client_transaction_hashes, + ) + .await + }; + + if let Err(error) = cleanup_result { + warn!( + "Keeping PendingBlock {} after canonical L2 block application because lifecycle \ + cleanup did not complete safely: {error}", + pending_block.layer2_block_number + ); + return; + } + + if db + .delete_pending_block(pending_block.layer2_block_number) + .await + .is_none() + { + warn!( + "PendingBlock {} matched canonical application cleanup but could not be deleted; \ + keeping persisted recovery state for later cleanup", + pending_block.layer2_block_number + ); + } +} + +async fn cleanup_mismatched_proposer_block_state( + db: &Client, + block_number: u64, +) -> Result<(), EventHandlerError> { + let db_for_cleanup = db.clone(); + let mut session = db.start_session().await.map_err(|_| { + EventHandlerError::IOError( + "Could not start MongoDB session for mismatch cleanup".to_string(), + ) + })?; + + session + .start_transaction() + .and_run2(async move |session| { + db_for_cleanup + .delete_block_by_number_with_session(block_number, session) + .await?; + Ok::<(), mongodb::error::Error>(()) + }) + .await + .map_err(|e| { + EventHandlerError::IOError(format!( + "Could not clean up mismatched proposer block state: {e}" + )) + }) +} + async fn process_propose_block_event( decode: Nightfall::propose_blockCall, transaction_hash: TxHash, layer_2_block_number_in_event: I256, + log: &Log, ) -> Result<(), EventHandlerError> where P: Proof, @@ -231,6 +417,7 @@ where .expect("I256 to u64 conversion failed"); // if proposer is out of sync, it won't have this block in db let current_block_stored = db.get_block_by_number(expected_block_number_u64).await; + let pending_block = db.get_pending_block(expected_block_number_u64).await; match current_block_stored { Some(current_block) => { @@ -244,8 +431,24 @@ where "Block hash mismatch. Expected {current_block_stored_hash}, got {block_store_pending_hash} in layer 2 block {layer_2_block_number_in_event}" ); - // Delete the invalid block and clear sync status - db.delete_block_by_number(expected_block_number_u64).await; + begin_startup_replay_reset(db).await.map_err(|error| { + EventHandlerError::IOError(format!( + "Could not persist startup replay reset marker before mismatch recovery: {error}" + )) + })?; + + if let Some(pending_block) = pending_block.as_ref() { + let _ = release_selected_transactions::

( + db, + &pending_block.selected_deposits, + &pending_block.selected_client_transaction_hashes, + ) + .await; + let _ = db.delete_pending_block(expected_block_number_u64).await; + } + + cleanup_mismatched_proposer_block_state(db, expected_block_number_u64).await?; + sync_status.clear_synchronised(); return Err(EventHandlerError::BlockHashError( @@ -262,8 +465,6 @@ where } } - *expected_onchain_block_number += I256::ONE; // move on to the next block - // warn that we're not synced with the blockchain if we're behind // before we used the event filter layer 2 block number // now we get the current_block_number from the blockchain @@ -273,12 +474,7 @@ where EventHandlerError::IOError("Could not retrieve current block number".to_string()) })?; - // if the current block number is exactly one, then we're automatically synchronised because we've seen one - // blockproposed event (or we wouldn't be here) and that must also be the only one - if current_block_number_in_contract == I256::ONE { - debug!("Synchronised with blockchain"); - sync_status.set_synchronised(); - } + let replay_catch_up_pending = is_listener_replay_catch_up_pending(); // next, we'll unpack the commitments and add them to the proposer's commitment tree // normally, we don't update the trees if we're the proposer, because we'll have done it when we proposed the block @@ -286,65 +482,71 @@ where // There's one more case, where this is the first block, so we must be synchronised in the sense that our block count is the // same as the blockchain's block count, but we've lost the commitment data. In this case, we need to update the trees too. // If we don't have the data from the first block, out commitment root will be zero. + let first_block_is_already_caught_up = + current_block_number_in_contract == I256::ONE && !replay_catch_up_pending; let commitment_root = >::get_root(db) .await .map_err(|_| { EventHandlerError::IOError("Could not retrieve commitment root".to_string()) })?; - if our_address != sender_address - || !sync_status.is_synchronised() - || commitment_root.0.is_zero() - { - let commitments = &blk - .transactions + let should_refresh_local_trees = our_address != sender_address + || (!sync_status.is_synchronised() && !first_block_is_already_caught_up) + || commitment_root.0.is_zero(); + let commitments = if should_refresh_local_trees { + blk.transactions .iter() .flat_map(|transaction| &transaction.commitments) .map(|u| FrBn254::try_from(*u).map(|f| f.into())) .collect::, _>>() - .expect("Could not convert commitments to U256"); - debug!( - "Adding {} commitments to commitment tree", - commitments.len() - ); - >::append_sub_trees(db, commitments, true) - .await - .map_err(|_| EventHandlerError::IOError("Could not store commitments".to_string()))?; - // and do the same with the nullifier tree - let nullifiers = blk - .transactions + .expect("Could not convert commitments to U256") + } else { + Vec::new() + }; + let nullifiers = if should_refresh_local_trees { + blk.transactions .iter() .flat_map(|transaction| &transaction.nullifiers) .map(|u| FrBn254::try_from(*u).map(|f| f.into())) .collect::, _>>() - .expect("Could not convert nullifiers to U256"); - debug!( - "Adding {} nullifiers to indexed Timber tree", - nullifiers.len() - ); - >::insert_leaves( - db, - &nullifiers, - >::TREE_NAME, - ) - .await - .map_err(|_| EventHandlerError::IOError("Could not store nullifiers".to_string()))?; - } - // and next,the commitments root (historic_root) is stored in the historic root tree + .expect("Could not convert nullifiers to U256") + } else { + Vec::new() + }; + let historic_root: Fr254 = FrBn254::try_from(blk.commitments_root) .map_err(|_| EventHandlerError::IOError("Could not convert to Fr254".to_string()))? .into(); + let sync_state_l1_ref = L1Ref { + block_number: log.block_number.ok_or_else(|| { + EventHandlerError::IOError("BlockProposed log missing block_number".to_string()) + })?, + tx_hash: log.transaction_hash.ok_or_else(|| { + EventHandlerError::IOError("BlockProposed log missing transaction_hash".to_string()) + })?, + log_index: log.log_index.ok_or_else(|| { + EventHandlerError::IOError("BlockProposed log missing log_index".to_string()) + })?, + }; + let db_for_transaction = db.clone(); + let commitments_for_transaction = commitments.clone(); + let nullifiers_for_transaction = nullifiers.clone(); + let block_for_transaction = store_block_pending.clone(); + let historic_root_for_transaction = historic_root; + let sync_state_l1_ref_for_transaction = sync_state_l1_ref.clone(); + let should_refresh_local_trees_for_transaction = should_refresh_local_trees; + let commitment_root = apply_proposer_block_transaction( + db_for_transaction, + should_refresh_local_trees_for_transaction, + commitments_for_transaction, + nullifiers_for_transaction, + historic_root_for_transaction, + block_for_transaction, + sync_state_l1_ref_for_transaction, + ) + .await?; - db.append_historic_commitment_root(&historic_root, true) - .await - .map_err(|_| EventHandlerError::IOError("Could not store historic root".to_string()))?; - debug!("Stored new commitments tree root in historic root timber tree: {historic_root}"); + *expected_onchain_block_number += I256::ONE; // move on to the next block after durable commit - // it's worth checking that the historic root agrees with what's in the commitment tree - let commitment_root = >::get_root(db) - .await - .map_err(|_| { - EventHandlerError::IOError("Could not retrieve commitment root".to_string()) - })?; if commitment_root != historic_root { error!( "Historic root does not match commitment tree root. Historic root: {historic_root}, Commitment tree root: {commitment_root}" @@ -356,18 +558,35 @@ where // see if we need to update the synchronisation status //This is a final safety check. Earlier we used event-level info to decide whether to sync. Now we consult the contract’s real-time state. - let delta = current_block_number_in_contract - layer_2_block_number_in_event - I256::ONE; + let latest_current_block_number_in_contract = + N::get_current_layer2_blocknumber().await.map_err(|_| { + EventHandlerError::IOError("Could not retrieve current block number".to_string()) + })?; + let delta = latest_current_block_number_in_contract - layer_2_block_number_in_event - I256::ONE; if delta != I256::ZERO { warn!("Synchronising - behind blockchain by {delta} layer 2 blocks "); sync_status.clear_synchronised(); + } else if replay_catch_up_pending { + debug!( + "Listener replay catch-up is still in progress after applying L2 block \ + {layer_2_block_number_in_event_u64}; deferring synchronised status until the \ + listener finishes draining historical events" + ); + sync_status.clear_synchronised(); } else { debug!("Synchronised with blockchain"); sync_status.set_synchronised(); } - // store the block in the db - // if db doesn't have the block, it will be stored - db.store_block(&store_block_pending).await; + if let Some(pending_block) = pending_block { + finalize_pending_block_after_applied_block::

( + db, + pending_block, + our_address, + &store_block_pending, + ) + .await; + } let reconciliation_block_number = current_block_number_in_contract .try_into() @@ -377,13 +596,107 @@ where drop(expected_onchain_block_number); if became_synchronised { - let _ = - reconcile_orphaned_selected_transactions::

(db, reconciliation_block_number).await; + let _ = reconcile_active_client_transaction_lifecycle::

(db, reconciliation_block_number) + .await; + } + + match get_blockchain_client_connection() + .await + .read() + .await + .get_client() + .get_block_number() + .await + { + Ok(current_l1_block) => { + maybe_schedule_snapshot_for_applied_block(db, current_l1_block).await; + } + Err(error) => { + warn!( + "Skipping automatic proposer snapshot scheduling after L2 block {layer_2_block_number_in_event_u64} because current L1 head could not be fetched: {error}" + ); + } } Ok(()) } +async fn apply_proposer_block_transaction( + db: Client, + should_refresh_local_trees: bool, + commitments: Vec, + nullifiers: Vec, + historic_root: Fr254, + block_for_transaction: StoredBlock, + sync_state_l1_ref: L1Ref, +) -> Result { + let mut session = db + .start_session() + .await + .map_err(|_| EventHandlerError::IOError("Could not start MongoDB session".to_string()))?; + session + .start_transaction() + .and_run2(async move |session| { + if should_refresh_local_trees { + debug!( + "Adding {} commitments to commitment tree", + commitments.len() + ); + >::append_sub_trees_with_session( + &db, + &commitments, + true, + session, + ) + .await + .map_err(merkle_tree_error_to_mongo)?; + debug!( + "Adding {} nullifiers to indexed Timber tree", + nullifiers.len() + ); + >::insert_nullifiers_with_session( + &db, + &nullifiers, + session, + ) + .await + .map_err(merkle_tree_error_to_mongo)?; + } + + db.append_historic_commitment_root_with_session(&historic_root, true, session) + .await + .map_err(merkle_tree_error_to_mongo)?; + debug!( + "Stored new commitments tree root in historic root timber tree: {historic_root}" + ); + + let commitment_root = + >::get_root_with_session(&db, session) + .await + .map_err(merkle_tree_error_to_mongo)?; + + db.store_block_with_session(&block_for_transaction, session) + .await?; + + let sync_state = SyncState::new( + block_for_transaction.layer2_block_number, + block_for_transaction.hash().to_hex_string(), + sync_state_l1_ref.clone(), + mongodb::bson::DateTime::now(), + ); + + db.update_sync_state_with_session(&sync_state, session) + .await?; + maybe_fail_block_apply("after_sync_state_update")?; + + Ok::(commitment_root) + }) + .await + .map_err(|e| { + EventHandlerError::IOError(format!("Could not apply proposer block transaction: {e}")) + }) +} + pub async fn process_deposit_escrowed_event( transaction_hash: TxHash, filter: &Nightfall::DepositEscrowed, @@ -457,7 +770,11 @@ where value: value_from_event, secret_hash, }; - let deposit_data = DepositDatawithFee { fee, deposit_data }; + let deposit_data = DepositDatawithFee { + fee, + deposit_data, + reserved: false, + }; process_deposit_transaction::(deposit_data) .await .map_err(|_| { @@ -470,3 +787,384 @@ where Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + domain::entities::{Block, DepositDatawithFee, PendingBlock, PendingBlockState, SyncState}, + driven::db::{ + mongo_db::{ensure_deposit_indexes, StoredBlock, DB, DEPOSIT_COLLECTION}, + snapshot::create_proposer_snapshot, + }, + ports::{ + db::{BlockStorageDB, PendingBlockDB, SyncStateDB, TransactionsDB}, + trees::{CommitmentTree, HistoricRootTree, NullifierTree}, + }, + }; + use alloy::primitives::{Address, Bytes, TxHash}; + use ark_ff::Zero; + use ark_serialize::SerializationError; + use lib::{ + hex_conversion::HexConvertible, + merkle_trees::trees::{MutableTree, TreeMetadata}, + nf_client_proof::Proof, + shared_entities::{DepositData, OnChainTransaction}, + tests_utils::{get_db_connection as get_test_db_connection, get_mongo}, + }; + use serde::{Deserialize, Serialize}; + use tokio::sync::{Mutex, OnceCell}; + + #[derive(Clone, Debug, Deserialize, Serialize)] + struct MockProof; + + impl Proof for MockProof { + fn compress_proof(&self) -> Result { + Ok(Bytes::new()) + } + + fn from_compressed(_compressed: Bytes) -> Result { + Ok(Self) + } + } + + async fn nightfall_event_test_lock() -> tokio::sync::MutexGuard<'static, ()> { + static LOCK: OnceCell> = OnceCell::const_new(); + LOCK.get_or_init(|| async { Mutex::new(()) }) + .await + .lock() + .await + } + + async fn persist_sync_state(client: &mongodb::Client, sync_state: &SyncState) { + let mut session = client.start_session().await.expect("start session"); + let client_for_write = client.clone(); + let sync_state_for_write = sync_state.clone(); + session + .start_transaction() + .and_run2(async move |session| { + client_for_write + .update_sync_state_with_session(&sync_state_for_write, session) + .await?; + Ok::<(), mongodb::error::Error>(()) + }) + .await + .expect("write sync_state"); + + let target_historic_root_sub_tree_count = sync_state + .last_applied_l2_block + .checked_add(2) + .expect("historic root count should not overflow in event tests"); + let mut historic_root_sub_tree_count = client + .database(DB) + .collection::>(&format!( + "{}_metadata", + >::TREE_NAME + )) + .find_one(mongodb::bson::doc! { "_id": 0 }) + .await + .expect("read historic root metadata") + .expect("historic root metadata should exist for event tests") + .sub_tree_count; + + while historic_root_sub_tree_count < target_historic_root_sub_tree_count { + let commitment_root = >::get_root(client) + .await + .expect("read commitment root"); + >::append_historic_commitment_root( + client, + &commitment_root, + true, + ) + .await + .expect("materialize historic root state for event test"); + historic_root_sub_tree_count += 1; + } + } + + async fn initialize_test_trees(client: &mongodb::Client) { + >::new_commitment_tree(client, 29, 3) + .await + .expect("create commitment tree"); + >::new_historic_root_tree(client, 32) + .await + .expect("create historic root tree"); + >::append_historic_commitment_root( + client, + &Fr254::zero(), + true, + ) + .await + .expect("append zero historic root"); + >::new_nullifier_tree(client, 29, 3) + .await + .expect("create nullifier tree"); + ensure_deposit_indexes(client) + .await + .expect("create deposit indexes"); + } + + async fn materialize_tree_state_for_block(client: &mongodb::Client, leaf: Fr254) { + >::insert_leaf( + client, + leaf, + true, + >::TREE_NAME, + ) + .await + .expect("append test commitment leaf"); + + let commitment_root = >::get_root(client) + .await + .expect("read commitment root"); + >::append_historic_commitment_root( + client, + &commitment_root, + true, + ) + .await + .expect("append historic root"); + } + + #[tokio::test] + async fn snapshots_created_after_pending_block_cleanup_exclude_consumed_deposits() { + let _lock = nightfall_event_test_lock().await; + let container = get_mongo().await; + let client = get_test_db_connection(&container).await; + + initialize_test_trees(&client).await; + + let selected_deposit = DepositDatawithFee { + fee: Fr254::from(3_u64), + deposit_data: DepositData { + nf_token_id: Fr254::from(11_u64), + nf_slot_id: Fr254::from(12_u64), + value: Fr254::from(13_u64), + secret_hash: Fr254::from(14_u64), + }, + reserved: true, + }; + >::set_mempool_deposits( + &client, + vec![selected_deposit], + ) + .await + .expect("store selected deposit"); + + let commitment = Fr254::from(21_u64); + let on_chain_transaction = OnChainTransaction { + commitments: [commitment, Fr254::zero(), Fr254::zero(), Fr254::zero()], + ..Default::default() + }; + let applied_block = StoredBlock { + layer2_block_number: 7, + commitments: on_chain_transaction + .commitments + .iter() + .map(|commitment| commitment.to_hex_string()) + .collect(), + proposer_address: Address::from([7_u8; 20]), + }; + client + .store_block(&applied_block) + .await + .expect("store applied block"); + materialize_tree_state_for_block(&client, commitment).await; + + persist_sync_state( + &client, + &SyncState::new( + applied_block.layer2_block_number, + applied_block.hash().to_hex_string(), + L1Ref { + block_number: 700, + tx_hash: TxHash::from([7_u8; 32]), + log_index: 7, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + + let pending_block = PendingBlock { + layer2_block_number: applied_block.layer2_block_number, + state: PendingBlockState::ReadyToPropose, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: Some(Block { + transactions: vec![on_chain_transaction], + ..Default::default() + }), + selected_deposits: vec![vec![selected_deposit]], + selected_client_transaction_hashes: Vec::new(), + }; + client + .store_pending_block(&pending_block) + .await + .expect("store pending block"); + + finalize_pending_block_after_applied_block::( + &client, + pending_block, + applied_block.proposer_address, + &applied_block, + ) + .await; + + assert_eq!( + client + .get_pending_block(applied_block.layer2_block_number) + .await, + None + ); + assert_eq!( + >::get_mempool_deposits(&client).await, + None + ); + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-event-snapshot-ordering-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create proposer snapshot"); + let deposits_manifest = manifest + .collections + .iter() + .find(|collection| collection.collection_name == DEPOSIT_COLLECTION) + .expect("snapshot should include deposits collection"); + assert_eq!( + deposits_manifest.document_count, 0, + "snapshot should not retain deposits consumed by the applied block" + ); + + tokio::fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot root"); + } + + #[tokio::test] + async fn failed_block_apply_does_not_advance_expected_layer2_blocknumber() { + let _lock = nightfall_event_test_lock().await; + let container = get_mongo().await; + let client = get_test_db_connection(&container).await; + + initialize_test_trees(&client).await; + *get_expected_layer2_blocknumber().await.write().await = + I256::try_from(9_u64).expect("test L2 block fits into I256"); + + let _failpoint = TestBlockApplyFailpointGuard::enable("after_sync_state_update"); + let error = apply_proposer_block_transaction( + client.clone(), + false, + Vec::new(), + Vec::new(), + Fr254::zero(), + StoredBlock { + layer2_block_number: 9, + commitments: Vec::new(), + proposer_address: Address::from([9_u8; 20]), + }, + L1Ref { + block_number: 900, + tx_hash: TxHash::from([9_u8; 32]), + log_index: 9, + }, + ) + .await + .expect_err("block apply should fail at failpoint"); + + let error_debug = format!("{error:?}"); + assert!( + matches!(&error, EventHandlerError::IOError(message) if message.contains("Could not apply proposer block transaction")), + "unexpected error from failed block apply: {error_debug}" + ); + assert_eq!( + *get_expected_layer2_blocknumber().await.read().await, + I256::try_from(9_u64).expect("test L2 block fits into I256"), + "failed block apply must not advance the in-memory expected L2 block number" + ); + assert_eq!(client.get_sync_state().await, None); + assert!(client.get_block_by_number(9).await.is_none()); + } + + #[tokio::test] + async fn mismatch_cleanup_preserves_sync_state_and_tree_coherence() { + let _lock = nightfall_event_test_lock().await; + let container = get_mongo().await; + let client = get_test_db_connection(&container).await; + + initialize_test_trees(&client).await; + let applied_commitment = Fr254::from(71_u64); + let applied_block = StoredBlock { + layer2_block_number: 7, + commitments: vec![applied_commitment.to_hex_string()], + proposer_address: Address::from([7_u8; 20]), + }; + client + .store_block(&applied_block) + .await + .expect("store applied block"); + materialize_tree_state_for_block(&client, applied_commitment).await; + + let sync_state = SyncState::new( + applied_block.layer2_block_number, + applied_block.hash().to_hex_string(), + L1Ref { + block_number: 700, + tx_hash: TxHash::from([7_u8; 32]), + log_index: 7, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &sync_state).await; + + let commitment_root_before = >::get_root(&client) + .await + .expect("read commitment root before mismatch cleanup"); + let historic_root_before = >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read historic root before mismatch cleanup"); + + let speculative_block = StoredBlock { + layer2_block_number: 8, + commitments: vec!["0xspeculative-mismatch".to_string()], + proposer_address: Address::from([8_u8; 20]), + }; + client + .store_block(&speculative_block) + .await + .expect("store speculative mismatched block"); + + cleanup_mismatched_proposer_block_state(&client, 8) + .await + .expect("cleanup should remove only the speculative mismatched block"); + + assert_eq!(client.get_sync_state().await, Some(sync_state)); + let retained_block = client + .get_block_by_number(7) + .await + .expect("applied block should still exist after mismatch cleanup"); + assert_eq!(retained_block.hash(), applied_block.hash()); + assert!(client.get_block_by_number(8).await.is_none()); + assert_eq!( + >::get_root(&client) + .await + .expect("read commitment root after mismatch cleanup"), + commitment_root_before + ); + assert_eq!( + >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read historic root after mismatch cleanup"), + historic_root_before + ); + } +} diff --git a/nightfall_proposer/src/drivers/blockchain/block_assembly.rs b/nightfall_proposer/src/drivers/blockchain/block_assembly.rs index f60f95b0..54d44da6 100644 --- a/nightfall_proposer/src/drivers/blockchain/block_assembly.rs +++ b/nightfall_proposer/src/drivers/blockchain/block_assembly.rs @@ -1,13 +1,18 @@ use crate::{ - domain::entities::Block, - drivers::blockchain::nightfall_event_listener::get_synchronisation_status, + domain::entities::{PendingBlock, PendingBlockState}, + drivers::blockchain::nightfall_event_listener::{ + get_synchronisation_status, is_listener_replay_catch_up_pending, + }, initialisation::{ - get_block_assembly_trigger, get_blockchain_client_connection, get_db_connection, + get_block_assembly_status, get_block_assembly_trigger, get_blockchain_client_connection, + }, + ports::{ + contracts::{NightfallContract, ProposeBlockOutcome}, + db::{BlockStorageDB, PendingBlockDB}, + proving::RecursiveProvingEngine, }, - ports::{contracts::NightfallContract, proving::RecursiveProvingEngine}, - services::{ - assemble_block::assemble_block, - selected_transactions::restore_selected_transactions_for_failed_block, + services::assemble_block::{ + assemble_block, cleanup_selected_transactions, release_selected_transactions, }, }; use alloy::{ @@ -29,13 +34,12 @@ use lib::{ use log::{debug, error, info, warn}; use nightfall_bindings::artifacts::RoundRobin; use std::{ - collections::VecDeque, error::Error, fmt::{Debug, Display, Formatter}, sync::Arc, time::Duration, }; -use tokio::sync::Mutex; +use tokio::sync::{Mutex, OnceCell}; #[derive(Debug)] pub enum BlockAssemblyError { @@ -121,6 +125,41 @@ impl From for BlockAssemblyError { BlockAssemblyError::ProvingError(format!("PlonkError: {e}")) } } +#[allow(clippy::large_enum_variant)] +enum PendingBlockQueueAction { + Completed, + Requeue(PendingBlock), +} + +const MAX_BROADCAST_PENDING_RECEIPT_CHECKS: u32 = 24; + +async fn get_pending_blocks_queue() -> &'static Mutex> { + static PENDING_BLOCKS: OnceCell>> = OnceCell::const_new(); + PENDING_BLOCKS + .get_or_init(|| async { Mutex::new(Vec::new()) }) + .await +} + +async fn replace_pending_blocks_queue(blocks: Vec) { + *get_pending_blocks_queue().await.lock().await = blocks; +} + +pub(crate) async fn clear_pending_blocks_queue() -> usize { + let mut blocks = get_pending_blocks_queue().await.lock().await; + let cleared = blocks.len(); + blocks.clear(); + cleared +} + +#[cfg(test)] +pub(crate) async fn push_pending_block_for_test(block: PendingBlock) { + get_pending_blocks_queue().await.lock().await.push(block); +} + +#[cfg(test)] +pub(crate) async fn pending_blocks_queue_len_for_test() -> usize { + get_pending_blocks_queue().await.lock().await.len() +} async fn check_l1_finality( client: &RootProvider, @@ -205,57 +244,404 @@ async fn check_l1_finality( } } -/// Process `pending_blocks`, stamping each block with the current on-chain -/// `layer2_block_number` immediately before submission. -/// -/// On transient RPC failure (or out-of-range block number), the remaining -/// blocks are left in the queue so the caller can retry on the next tick. -/// Submission failures (`N::propose_block`) preserve the pre-existing -/// behaviour of logging and dropping the offending block. -async fn propose_pending_blocks(blocks: &mut VecDeque) +async fn release_failed_pending_block

( + db: &mongodb::Client, + pending_block: &PendingBlock, +) -> Result<(), BlockAssemblyError> +where + P: Proof, +{ + release_selected_transactions::

( + db, + &pending_block.selected_deposits, + &pending_block.selected_client_transaction_hashes, + ) + .await?; + + if db + .delete_pending_block(pending_block.layer2_block_number) + .await + .is_none() + { + warn!( + "Pending block {} failed to propose and its persisted queue entry could not be removed", + pending_block.layer2_block_number + ); + } + + if db + .delete_block_by_number(pending_block.layer2_block_number) + .await + .is_none() + { + warn!( + "Pending block {} failed to propose and its speculative block record could not be removed", + pending_block.layer2_block_number + ); + } + + Ok(()) +} + +async fn propose_and_cleanup_pending_block_with_db( + db: &mongodb::Client, + pending_block: PendingBlock, +) -> Result where P: Proof, N: NightfallContract, { - while !blocks.is_empty() { - let current_l2_block_number = match N::get_current_layer2_blocknumber().await { - Ok(block_number) => block_number, - Err(e) => { + if pending_block.state != PendingBlockState::ReadyToPropose { + return Err(BlockAssemblyError::QueueError(format!( + "Pending block {} is not ready to propose", + pending_block.layer2_block_number + ))); + } + + let block = pending_block.block.clone().ok_or_else(|| { + BlockAssemblyError::QueueError(format!( + "Pending block {} is missing block data", + pending_block.layer2_block_number + )) + })?; + + let proposal_outcome = match N::propose_block(block).await { + Ok(outcome) => outcome, + Err(e) => { + if let Err(cleanup_error) = release_failed_pending_block::

(db, &pending_block).await + { + error!("Failed to release pending block after proposal error: {cleanup_error}"); + } + return Err(BlockAssemblyError::ContractError(format!( + "Block proposal failed: {e}" + ))); + } + }; + + match proposal_outcome { + ProposeBlockOutcome::Submitted { tx_hash } => { + info!( + "Pending block {} proposed successfully with L1 tx {tx_hash:?}", + pending_block.layer2_block_number + ); + } + ProposeBlockOutcome::NotBroadcast { reason } => { + if let Err(cleanup_error) = release_failed_pending_block::

(db, &pending_block).await + { error!( - "Failed to fetch current layer2 block number before proposing block: {e}. \ - Leaving {} block(s) queued for retry.", - blocks.len() + "Failed to release pending block after pre-broadcast failure: {cleanup_error}" ); - return; } - }; + return Err(BlockAssemblyError::ContractError(format!( + "Block proposal did not broadcast: {reason:?}" + ))); + } + ProposeBlockOutcome::BroadcastUnknown { tx_hash, reason } => { + let mut broadcast_pending_block = pending_block; + broadcast_pending_block.state = PendingBlockState::BroadcastPending; + broadcast_pending_block.broadcast_tx_hash = Some(tx_hash); + broadcast_pending_block.broadcast_receipt_checks = 0; - let block_number: u64 = match current_l2_block_number.try_into() { - Ok(block_number) => block_number, - Err(_) => { + if db + .store_pending_block(&broadcast_pending_block) + .await + .is_none() + { error!( - "Current layer2 block number is negative or out of range: \ - {current_l2_block_number}. Leaving {} block(s) queued.", - blocks.len() + "Failed to persist broadcast-pending state for block {} after ambiguous submission {tx_hash:?}", + broadcast_pending_block.layer2_block_number ); - return; } - }; - let mut block = blocks - .pop_front() - .expect("queue checked as non-empty above"); - block.block_number = block_number; - let failed_block = block.clone(); + warn!( + "Pending block {} reached broadcast-unknown state for tx {tx_hash:?}: {reason:?}", + broadcast_pending_block.layer2_block_number + ); + return Ok(PendingBlockQueueAction::Requeue(broadcast_pending_block)); + } + ProposeBlockOutcome::Reverted { tx_hash } => { + if let Err(cleanup_error) = release_failed_pending_block::

(db, &pending_block).await + { + error!("Failed to release pending block after known revert: {cleanup_error}"); + } + return Err(BlockAssemblyError::ContractError(format!( + "Block proposal transaction reverted for tx {tx_hash:?}" + ))); + } + } + + cleanup_selected_transactions::

( + db, + &pending_block.selected_deposits, + &pending_block.selected_client_transaction_hashes, + ) + .await?; - if let Err(e) = N::propose_block(block).await { - error!("Failed to propose block: {e}"); - let db = get_db_connection().await; - let _ = restore_selected_transactions_for_failed_block::

(db, &failed_block).await; + if db + .delete_pending_block(pending_block.layer2_block_number) + .await + .is_none() + { + warn!( + "Pending block {} was proposed, but its persisted queue entry could not be removed", + pending_block.layer2_block_number + ); + } + Ok(PendingBlockQueueAction::Completed) +} + +async fn resolve_broadcast_pending_block_with_db( + db: &mongodb::Client, + pending_block: PendingBlock, +) -> Result +where + P: Proof, + N: NightfallContract, +{ + let Some(tx_hash) = pending_block.broadcast_tx_hash else { + if let Err(cleanup_error) = release_failed_pending_block::

(db, &pending_block).await { + error!("Failed to release broadcast-pending block without tx hash: {cleanup_error}"); + } + return Err(BlockAssemblyError::QueueError(format!( + "Broadcast-pending block {} has no transaction hash", + pending_block.layer2_block_number + ))); + }; + + match N::get_proposal_receipt_status(tx_hash).await { + Ok(Some(true)) => { + info!( + "Broadcast-pending block {} was mined successfully in tx {tx_hash:?}", + pending_block.layer2_block_number + ); + cleanup_selected_transactions::

( + db, + &pending_block.selected_deposits, + &pending_block.selected_client_transaction_hashes, + ) + .await?; + + if db + .delete_pending_block(pending_block.layer2_block_number) + .await + .is_none() + { + warn!( + "Broadcast-pending block {} was mined, but its persisted queue entry could not be removed", + pending_block.layer2_block_number + ); + } + Ok(PendingBlockQueueAction::Completed) + } + Ok(Some(false)) => { + if let Err(cleanup_error) = release_failed_pending_block::

(db, &pending_block).await + { + error!("Failed to release broadcast-pending block after revert: {cleanup_error}"); + } + Err(BlockAssemblyError::ContractError(format!( + "Broadcast-pending block {} reverted in tx {tx_hash:?}", + pending_block.layer2_block_number + ))) + } + Ok(None) => { + let mut updated_block = pending_block; + updated_block.broadcast_receipt_checks = + updated_block.broadcast_receipt_checks.saturating_add(1); + + if updated_block.broadcast_receipt_checks >= MAX_BROADCAST_PENDING_RECEIPT_CHECKS { + warn!( + "Broadcast-pending block {} has no receipt for tx {tx_hash:?} after {} checks; releasing reserved transactions", + updated_block.layer2_block_number, + updated_block.broadcast_receipt_checks + ); + if let Err(cleanup_error) = + release_failed_pending_block::

(db, &updated_block).await + { + error!("Failed to release expired broadcast-pending block: {cleanup_error}"); + } + return Err(BlockAssemblyError::ContractError(format!( + "Broadcast-pending block {} never produced a receipt for tx {tx_hash:?}", + updated_block.layer2_block_number + ))); + } + + if db.store_pending_block(&updated_block).await.is_none() { + error!( + "Failed to persist receipt-check count for broadcast-pending block {}", + updated_block.layer2_block_number + ); + } + + info!( + "Broadcast-pending block {} still has no receipt for tx {tx_hash:?} ({}/{})", + updated_block.layer2_block_number, + updated_block.broadcast_receipt_checks, + MAX_BROADCAST_PENDING_RECEIPT_CHECKS + ); + Ok(PendingBlockQueueAction::Requeue(updated_block)) + } + Err(e) => { + warn!( + "Could not check receipt for broadcast-pending block {} tx {tx_hash:?}: {e}", + pending_block.layer2_block_number + ); + Ok(PendingBlockQueueAction::Requeue(pending_block)) } } } +async fn recover_pending_blocks

( + db: &mongodb::Client, +) -> Result, BlockAssemblyError> +where + P: Proof, +{ + let mut recovered_pending_blocks = db.get_all_pending_blocks().await.unwrap_or_default(); + let mut ready_pending_blocks = Vec::new(); + + for pending_block in recovered_pending_blocks.drain(..) { + match pending_block.state { + PendingBlockState::Reserved => { + if let Err(e) = release_selected_transactions::

( + db, + &pending_block.selected_deposits, + &pending_block.selected_client_transaction_hashes, + ) + .await + { + warn!( + "Failed to release reserved transactions for recovered block {}: {e}", + pending_block.layer2_block_number + ); + } + + if db + .delete_pending_block(pending_block.layer2_block_number) + .await + .is_none() + { + warn!( + "Recovered reserved block {} could not be removed from pending storage", + pending_block.layer2_block_number + ); + } + + let _ = db + .delete_block_by_number(pending_block.layer2_block_number) + .await; + } + PendingBlockState::ReadyToPropose | PendingBlockState::BroadcastPending => { + ready_pending_blocks.push(pending_block) + } + } + } + + ready_pending_blocks.sort_by_key(|pending_block| pending_block.layer2_block_number); + if ready_pending_blocks.len() > 1 { + return Err(BlockAssemblyError::QueueError( + "Expected at most one pending block in storage".to_string(), + )); + } + + Ok(ready_pending_blocks) +} + +async fn process_pending_blocks_for_proposal( + pending_blocks: &Mutex>, + phase: &str, +) where + P: Proof, + N: NightfallContract, +{ + if !get_synchronisation_status() + .await + .read() + .await + .is_synchronised() + || is_listener_replay_catch_up_pending() + { + warn!( + "Skipping pending block proposal because proposer recovery/desynchronisation or \ + replay catch-up is active" + ); + return; + } + if !get_block_assembly_status().await.read().await.is_running() { + warn!("Skipping pending block proposal because block assembly is paused"); + return; + } + + let drained: Vec<_> = { + let mut guard = pending_blocks.lock().await; + guard.drain(..).collect() + }; + + if drained.is_empty() { + return; + } + + info!( + "Finality checker: {phase}, processing {} pending block(s)", + drained.len() + ); + + let db = crate::initialisation::get_db_connection().await; + let mut blocks_to_requeue = Vec::new(); + + for pending_block in drained { + match pending_block.state { + PendingBlockState::ReadyToPropose => { + match propose_and_cleanup_pending_block_with_db::(db, pending_block.clone()) + .await + { + Ok(PendingBlockQueueAction::Completed) => {} + Ok(PendingBlockQueueAction::Requeue(updated_block)) => { + blocks_to_requeue.push(updated_block); + } + Err(e) => { + error!("Finality checker: propose_block failed: {e}"); + } + } + } + PendingBlockState::BroadcastPending => { + let Some(stored_pending_block) = db + .get_pending_block(pending_block.layer2_block_number) + .await + else { + info!( + "Finality checker: dropping resolved broadcast-pending block {}", + pending_block.layer2_block_number + ); + continue; + }; + + match resolve_broadcast_pending_block_with_db::(db, stored_pending_block) + .await + { + Ok(PendingBlockQueueAction::Completed) => {} + Ok(PendingBlockQueueAction::Requeue(updated_block)) => { + blocks_to_requeue.push(updated_block); + } + Err(e) => { + error!("Finality checker: broadcast-pending resolution failed: {e}"); + } + } + } + PendingBlockState::Reserved => { + warn!( + "Finality checker: reserved pending block {} should not be queued; dropping it", + pending_block.layer2_block_number + ); + } + } + } + + if !blocks_to_requeue.is_empty() { + let mut guard = pending_blocks.lock().await; + guard.extend(blocks_to_requeue); + } +} + // once called this function will trigger the block assembly process whenever // certain conditions are met // Any errors that propogate back up to here will cause a panic. @@ -310,8 +696,12 @@ where a.x509 ); + let db = crate::initialisation::get_db_connection().await; + let ready_pending_blocks = recover_pending_blocks::

(db).await?; + replace_pending_blocks_queue(ready_pending_blocks).await; + // Shared queue for blocks waiting for finality confirmation - let pending_blocks: Arc>> = Arc::new(Mutex::new(VecDeque::new())); + let pending_blocks = get_pending_blocks_queue().await; let confirmations_required = U64::from(12); let finality_check_interval = Duration::from_secs(5); @@ -319,7 +709,6 @@ where // Spawn the finality checking task let _finality_checker: tokio::task::JoinHandle> = { - let pending_blocks = Arc::clone(&pending_blocks); let rr = Arc::clone(&round_robin_instance); let blockchain_client = blockchain_client.clone(); tokio::spawn(async move { @@ -392,14 +781,11 @@ where } if last_finalized_turn == Some(onchain_start_block) { - let mut guard = pending_blocks.lock().await; - if !guard.is_empty() { - info!( - "Finality checker: current proposer turn {onchain_start_block} already finalized, proposing {} pending blocks", - guard.len() - ); - propose_pending_blocks::(&mut guard).await; - } + process_pending_blocks_for_proposal::( + pending_blocks, + "current proposer turn already finalized", + ) + .await; tokio::time::sleep(finality_check_interval).await; continue; @@ -471,14 +857,12 @@ where "ProposerRotated tx finalized: {tx_hash:?} (event block: {onchain_start_block})" ); last_finalized_turn = Some(onchain_start_block); - let mut guard = pending_blocks.lock().await; - if !guard.is_empty() { - info!( - "Finality checker: finalized & canonical rotation, proposing {} pending blocks", - guard.len() - ); - propose_pending_blocks::(&mut guard).await; - } + + process_pending_blocks_for_proposal::( + pending_blocks, + "finalized canonical rotation", + ) + .await; } Ok(false) => { debug!("Finality checker: rotation tx not yet finalized: {tx_hash:?}"); @@ -494,6 +878,16 @@ where }; // Main block assembly loop loop { + let has_pending = { + let blocks = pending_blocks.lock().await; + !blocks.is_empty() + }; + if has_pending { + debug!("Pending block already in flight; waiting before assembling another block"); + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + continue; + } + debug!("Checking proposer status..."); // Step 1: Get current proposer address from smart contract let current_proposer = match round_robin_instance @@ -550,8 +944,8 @@ where if current_proposer_after_trigger != our_address { info!( - "Proposer has changed after trigger. Skipping block assembly. New proposer is: {current_proposer_after_trigger:?}" - ); + "Proposer has changed after trigger. Skipping block assembly. New proposer is: {current_proposer_after_trigger:?}" + ); continue; } // Step 4: check if we're synchronised. @@ -566,8 +960,10 @@ where // if we're at block 0, we're automatically synchronised because no blocks have been made yet sync_status.set_synchronised(); } - if !sync_status.is_synchronised() { - warn!("We are not synchronised. We won't make blocks until we are"); + if !sync_status.is_synchronised() || is_listener_replay_catch_up_pending() { + warn!( + "We are not synchronised yet. We won't make blocks until replay catch-up completes" + ); continue; } debug!("Triggered block assembly"); @@ -578,7 +974,7 @@ where continue; } let block_result = assemble_block::(current_block_number.as_u64()).await; - let block = match block_result { + let pending_block = match block_result { Ok(block) => block, Err(e) => match e { BlockAssemblyError::InsufficientTransactions => continue, @@ -591,8 +987,785 @@ where // Add to pending blocks queue { let mut blocks = pending_blocks.lock().await; - blocks.push_back(block); + blocks.push(pending_block); info!("Added block to queue ({} pending)", blocks.len()); } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + domain::entities::{ + Block, ClientTransactionWithMetaData, DepositDatawithFee, PendingBlockState, + TxLifecycle, + }, + driven::db::mongo_db::StoredBlock, + ports::contracts::{BroadcastUnknownReason, ProposeBlockOutcome}, + ports::db::{PendingBlockDB, TransactionsDB}, + services::assemble_block::reserve_selected_transactions, + }; + use alloy::primitives::{TxHash, I256}; + use lib::{ + error::NightfallContractError, + plonk_prover::plonk_proof::PlonkProof, + shared_entities::DepositData, + tests_utils::{get_db_connection, get_mongo}, + }; + use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + use tokio::sync::Mutex as TokioMutex; + + struct FailingContract; + + #[async_trait::async_trait] + impl NightfallContract for FailingContract { + async fn propose_block( + _block: Block, + ) -> Result { + Ok(ProposeBlockOutcome::NotBroadcast { + reason: crate::ports::contracts::NotBroadcastReason::SendRawTransactionFailed, + }) + } + + async fn get_proposal_receipt_status( + _tx_hash: TxHash, + ) -> Result, NightfallContractError> { + Ok(None) + } + + async fn get_current_layer2_blocknumber() -> Result { + Ok(I256::ZERO) + } + } + + struct BroadcastUnknownContract; + + #[async_trait::async_trait] + impl NightfallContract for BroadcastUnknownContract { + async fn propose_block( + _block: Block, + ) -> Result { + Ok(ProposeBlockOutcome::BroadcastUnknown { + tx_hash: TxHash::from([7u8; 32]), + reason: BroadcastUnknownReason::ReceiptUnavailable, + }) + } + + async fn get_proposal_receipt_status( + _tx_hash: TxHash, + ) -> Result, NightfallContractError> { + Ok(None) + } + + async fn get_current_layer2_blocknumber() -> Result { + Ok(I256::ZERO) + } + } + + struct MissingReceiptContract; + + #[async_trait::async_trait] + impl NightfallContract for MissingReceiptContract { + async fn propose_block( + _block: Block, + ) -> Result { + panic!("MissingReceiptContract should only be used for receipt checks") + } + + async fn get_proposal_receipt_status( + _tx_hash: TxHash, + ) -> Result, NightfallContractError> { + Ok(None) + } + + async fn get_current_layer2_blocknumber() -> Result { + Ok(I256::ZERO) + } + } + + struct SuccessfulReceiptContract; + + #[async_trait::async_trait] + impl NightfallContract for SuccessfulReceiptContract { + async fn propose_block( + _block: Block, + ) -> Result { + panic!("SuccessfulReceiptContract should only be used for receipt checks") + } + + async fn get_proposal_receipt_status( + _tx_hash: TxHash, + ) -> Result, NightfallContractError> { + Ok(Some(true)) + } + + async fn get_current_layer2_blocknumber() -> Result { + Ok(I256::ZERO) + } + } + + struct RevertedReceiptContract; + + #[async_trait::async_trait] + impl NightfallContract for RevertedReceiptContract { + async fn propose_block( + _block: Block, + ) -> Result { + panic!("RevertedReceiptContract should only be used for receipt checks") + } + + async fn get_proposal_receipt_status( + _tx_hash: TxHash, + ) -> Result, NightfallContractError> { + Ok(Some(false)) + } + + async fn get_current_layer2_blocknumber() -> Result { + Ok(I256::ZERO) + } + } + + struct MockNightfallContract; + + fn mock_proposed_blocks() -> &'static AtomicUsize { + static PROPOSED_BLOCKS: std::sync::OnceLock = std::sync::OnceLock::new(); + PROPOSED_BLOCKS.get_or_init(|| AtomicUsize::new(0)) + } + + fn mock_current_l2_block() -> &'static AtomicU64 { + static CURRENT_L2_BLOCK: std::sync::OnceLock = std::sync::OnceLock::new(); + CURRENT_L2_BLOCK.get_or_init(|| AtomicU64::new(0)) + } + + #[async_trait::async_trait] + impl NightfallContract for MockNightfallContract { + async fn propose_block( + _block: Block, + ) -> Result { + mock_proposed_blocks().fetch_add(1, Ordering::SeqCst); + Ok(ProposeBlockOutcome::Submitted { + tx_hash: TxHash::from([1u8; 32]), + }) + } + + async fn get_proposal_receipt_status( + _tx_hash: TxHash, + ) -> Result, NightfallContractError> { + Ok(None) + } + + async fn get_current_layer2_blocknumber() -> Result { + Ok( + I256::try_from(mock_current_l2_block().load(Ordering::SeqCst)) + .expect("mock current l2 block should fit in I256"), + ) + } + } + + async fn block_assembly_test_lock() -> tokio::sync::MutexGuard<'static, ()> { + static LOCK: OnceCell> = OnceCell::const_new(); + LOCK.get_or_init(|| async { TokioMutex::new(()) }) + .await + .lock() + .await + } + + async fn reset_test_state() { + clear_pending_blocks_queue().await; + mock_proposed_blocks().store(0, Ordering::SeqCst); + mock_current_l2_block().store(0, Ordering::SeqCst); + get_block_assembly_status().await.write().await.resume(); + get_synchronisation_status() + .await + .write() + .await + .set_synchronised(); + } + + fn test_client_transaction(fee: u64) -> ClientTransactionWithMetaData { + ClientTransactionWithMetaData { + client_transaction: lib::shared_entities::ClientTransaction { + fee: ark_bn254::Fr::from(fee), + proof: PlonkProof::default(), + ..Default::default() + }, + lifecycle: TxLifecycle::Mempool, + hash: vec![fee as u32], + historic_roots: vec![ark_bn254::Fr::from(123u64)], + receipt_token: None, + } + } + + fn test_deposit(fee: u64) -> DepositDatawithFee { + DepositDatawithFee { + fee: ark_bn254::Fr::from(fee), + deposit_data: DepositData { + nf_token_id: ark_bn254::Fr::from(fee), + nf_slot_id: ark_bn254::Fr::from(fee), + value: ark_bn254::Fr::from(100u64), + secret_hash: ark_bn254::Fr::from(fee), + }, + reserved: false, + } + } + + async fn stored_reserved_pending_block( + db: &mongodb::Client, + block_number: u64, + state: PendingBlockState, + broadcast_tx_hash: Option, + broadcast_receipt_checks: u32, + ) -> PendingBlock { + let selected_deposits = vec![(1..=4).map(test_deposit).collect::>()]; + >::set_mempool_deposits( + db, + selected_deposits[0].clone(), + ) + .await; + + let selected_client_transactions = + (401..=404).map(test_client_transaction).collect::>(); + for tx in &selected_client_transactions { + db.store_transaction(tx.clone()).await.unwrap(); + } + + let selected_client_transaction_hashes = selected_client_transactions + .iter() + .map(|tx| tx.hash.clone()) + .collect::>(); + reserve_selected_transactions::( + db, + &selected_deposits, + &selected_client_transactions, + block_number, + ) + .await + .unwrap(); + + let pending_block = PendingBlock { + layer2_block_number: block_number, + state, + broadcast_tx_hash, + broadcast_receipt_checks, + block: Some(Block::default()), + selected_deposits: selected_deposits.clone(), + selected_client_transaction_hashes: selected_client_transaction_hashes.clone(), + }; + db.store_pending_block(&pending_block).await.unwrap(); + db.store_block(&StoredBlock { + layer2_block_number: block_number, + commitments: vec!["broadcast-pending".to_string()], + proposer_address: Address::from([6u8; 20]), + }) + .await + .unwrap(); + + pending_block + } + + #[tokio::test] + async fn test_proposal_failure_releases_mempool_and_clears_pending_state() { + let container = get_mongo().await; + let db = get_db_connection(&container).await; + + let selected_deposits = vec![(1..=4).map(test_deposit).collect::>()]; + >::set_mempool_deposits( + &db, + selected_deposits[0].clone(), + ) + .await; + + let selected_client_transactions = + (101..=104).map(test_client_transaction).collect::>(); + for tx in &selected_client_transactions { + db.store_transaction(tx.clone()).await.unwrap(); + } + + let selected_client_transaction_hashes = selected_client_transactions + .iter() + .map(|tx| tx.hash.clone()) + .collect::>(); + reserve_selected_transactions::( + &db, + &selected_deposits, + &selected_client_transactions, + 0, + ) + .await + .unwrap(); + + let pending_block = PendingBlock { + layer2_block_number: 0, + state: PendingBlockState::ReadyToPropose, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: Some(Block::default()), + selected_deposits: selected_deposits.clone(), + selected_client_transaction_hashes: selected_client_transaction_hashes.clone(), + }; + db.store_pending_block(&pending_block).await.unwrap(); + db.store_block(&StoredBlock { + layer2_block_number: 0, + commitments: vec!["test-commitment".to_string()], + proposer_address: Address::from([9u8; 20]), + }) + .await + .unwrap(); + + let result = propose_and_cleanup_pending_block_with_db::( + &db, + pending_block, + ) + .await; + + assert!(matches!( + result, + Err(BlockAssemblyError::ContractError(message)) + if message.contains("did not broadcast") + )); + + let available_deposits = + >::get_mempool_deposits(&db) + .await + .unwrap(); + assert_eq!(available_deposits.len(), 4); + + let available_client_transactions = + >::get_all_mempool_client_transactions( + &db, + ) + .await + .unwrap_or_default(); + assert_eq!(available_client_transactions.len(), 4); + + let pending_blocks = db.get_all_pending_blocks().await.unwrap_or_default(); + assert!(pending_blocks.is_empty()); + + let stored_blocks = db.get_all_blocks().await.unwrap_or_default(); + assert!(stored_blocks.is_empty()); + } + + #[tokio::test] + async fn test_recover_reserved_pending_block_releases_mempool_and_drops_orphaned_state() { + let container = get_mongo().await; + let db = get_db_connection(&container).await; + + let selected_deposits = vec![(1..=4).map(test_deposit).collect::>()]; + >::set_mempool_deposits( + &db, + selected_deposits[0].clone(), + ) + .await; + + let selected_client_transactions = + (201..=204).map(test_client_transaction).collect::>(); + for tx in &selected_client_transactions { + db.store_transaction(tx.clone()).await.unwrap(); + } + + let selected_client_transaction_hashes = selected_client_transactions + .iter() + .map(|tx| tx.hash.clone()) + .collect::>(); + reserve_selected_transactions::( + &db, + &selected_deposits, + &selected_client_transactions, + 0, + ) + .await + .unwrap(); + + let pending_block = PendingBlock { + layer2_block_number: 0, + state: PendingBlockState::Reserved, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: None, + selected_deposits: selected_deposits.clone(), + selected_client_transaction_hashes: selected_client_transaction_hashes.clone(), + }; + db.store_pending_block(&pending_block).await.unwrap(); + db.store_block(&StoredBlock { + layer2_block_number: 0, + commitments: vec!["orphaned-speculative-block".to_string()], + proposer_address: Address::from([4u8; 20]), + }) + .await + .unwrap(); + + let recovered = recover_pending_blocks::(&db).await.unwrap(); + + assert!(recovered.is_empty()); + + let available_deposits = + >::get_mempool_deposits(&db) + .await + .unwrap(); + assert_eq!(available_deposits.len(), 4); + + let available_client_transactions = + >::get_all_mempool_client_transactions( + &db, + ) + .await + .unwrap_or_default(); + assert_eq!(available_client_transactions.len(), 4); + + let pending_blocks = db.get_all_pending_blocks().await.unwrap_or_default(); + assert!(pending_blocks.is_empty()); + + let stored_blocks = db.get_all_blocks().await.unwrap_or_default(); + assert!(stored_blocks.is_empty()); + } + + #[tokio::test] + async fn test_broadcast_unknown_keeps_pending_state_and_reserved_inputs() { + let container = get_mongo().await; + let db = get_db_connection(&container).await; + + let selected_deposits = vec![(1..=4).map(test_deposit).collect::>()]; + >::set_mempool_deposits( + &db, + selected_deposits[0].clone(), + ) + .await; + + let selected_client_transactions = + (301..=304).map(test_client_transaction).collect::>(); + for tx in &selected_client_transactions { + db.store_transaction(tx.clone()).await.unwrap(); + } + + let selected_client_transaction_hashes = selected_client_transactions + .iter() + .map(|tx| tx.hash.clone()) + .collect::>(); + reserve_selected_transactions::( + &db, + &selected_deposits, + &selected_client_transactions, + 0, + ) + .await + .unwrap(); + + let pending_block = PendingBlock { + layer2_block_number: 0, + state: PendingBlockState::ReadyToPropose, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: Some(Block::default()), + selected_deposits: selected_deposits.clone(), + selected_client_transaction_hashes: selected_client_transaction_hashes.clone(), + }; + db.store_pending_block(&pending_block).await.unwrap(); + db.store_block(&StoredBlock { + layer2_block_number: 0, + commitments: vec!["broadcast-unknown".to_string()], + proposer_address: Address::from([6u8; 20]), + }) + .await + .unwrap(); + + let result = propose_and_cleanup_pending_block_with_db::< + PlonkProof, + BroadcastUnknownContract, + >(&db, pending_block) + .await + .unwrap(); + + assert!(matches!( + result, + PendingBlockQueueAction::Requeue(PendingBlock { + state: PendingBlockState::BroadcastPending, + .. + }) + )); + + let available_deposits = + >::get_mempool_deposits(&db) + .await + .unwrap_or_default(); + assert!(available_deposits.is_empty()); + + let available_client_transactions = + >::get_all_mempool_client_transactions( + &db, + ) + .await + .unwrap_or_default(); + assert!(available_client_transactions.is_empty()); + + let pending_blocks = db.get_all_pending_blocks().await.unwrap_or_default(); + assert_eq!(pending_blocks.len(), 1); + assert_eq!(pending_blocks[0].state, PendingBlockState::BroadcastPending); + assert_eq!( + pending_blocks[0].broadcast_tx_hash, + Some(TxHash::from([7u8; 32])) + ); + assert_eq!(pending_blocks[0].broadcast_receipt_checks, 0); + + let stored_blocks = db.get_all_blocks().await.unwrap_or_default(); + assert_eq!(stored_blocks.len(), 1); + } + + #[tokio::test] + async fn test_broadcast_pending_missing_receipt_requeues_and_increments_check_count() { + let container = get_mongo().await; + let db = get_db_connection(&container).await; + let tx_hash = TxHash::from([8u8; 32]); + let pending_block = stored_reserved_pending_block( + &db, + 0, + PendingBlockState::BroadcastPending, + Some(tx_hash), + 0, + ) + .await; + + let result = resolve_broadcast_pending_block_with_db::( + &db, + pending_block, + ) + .await + .unwrap(); + + let PendingBlockQueueAction::Requeue(updated_block) = result else { + panic!("missing receipt should keep the pending block queued"); + }; + assert_eq!(updated_block.broadcast_tx_hash, Some(tx_hash)); + assert_eq!(updated_block.broadcast_receipt_checks, 1); + + let pending_blocks = db.get_all_pending_blocks().await.unwrap_or_default(); + assert_eq!(pending_blocks.len(), 1); + assert_eq!(pending_blocks[0].broadcast_receipt_checks, 1); + + let available_deposits = + >::get_mempool_deposits(&db) + .await + .unwrap_or_default(); + assert!(available_deposits.is_empty()); + } + + #[tokio::test] + async fn test_broadcast_pending_missing_receipt_eventually_releases_reserved_inputs() { + let container = get_mongo().await; + let db = get_db_connection(&container).await; + let pending_block = stored_reserved_pending_block( + &db, + 0, + PendingBlockState::BroadcastPending, + Some(TxHash::from([9u8; 32])), + MAX_BROADCAST_PENDING_RECEIPT_CHECKS - 1, + ) + .await; + + let result = resolve_broadcast_pending_block_with_db::( + &db, + pending_block, + ) + .await; + + assert!(matches!( + result, + Err(BlockAssemblyError::ContractError(message)) + if message.contains("never produced a receipt") + )); + + let available_deposits = + >::get_mempool_deposits(&db) + .await + .unwrap(); + assert_eq!(available_deposits.len(), 4); + + let available_client_transactions = + >::get_all_mempool_client_transactions( + &db, + ) + .await + .unwrap_or_default(); + assert_eq!(available_client_transactions.len(), 4); + + let pending_blocks = db.get_all_pending_blocks().await.unwrap_or_default(); + assert!(pending_blocks.is_empty()); + + let stored_blocks = db.get_all_blocks().await.unwrap_or_default(); + assert!(stored_blocks.is_empty()); + } + + #[tokio::test] + async fn test_broadcast_pending_successful_receipt_cleans_up_reserved_inputs() { + let container = get_mongo().await; + let db = get_db_connection(&container).await; + let pending_block = stored_reserved_pending_block( + &db, + 0, + PendingBlockState::BroadcastPending, + Some(TxHash::from([10u8; 32])), + 3, + ) + .await; + + let result = + resolve_broadcast_pending_block_with_db::( + &db, + pending_block, + ) + .await + .unwrap(); + + assert!(matches!(result, PendingBlockQueueAction::Completed)); + + let available_deposits = + >::get_mempool_deposits(&db) + .await + .unwrap_or_default(); + assert!(available_deposits.is_empty()); + + let available_client_transactions = + >::get_all_mempool_client_transactions( + &db, + ) + .await + .unwrap_or_default(); + assert!(available_client_transactions.is_empty()); + + let pending_blocks = db.get_all_pending_blocks().await.unwrap_or_default(); + assert!(pending_blocks.is_empty()); + + let stored_blocks = db.get_all_blocks().await.unwrap_or_default(); + assert_eq!(stored_blocks.len(), 1); + } + + #[tokio::test] + async fn test_broadcast_pending_reverted_receipt_releases_reserved_inputs() { + let container = get_mongo().await; + let db = get_db_connection(&container).await; + let pending_block = stored_reserved_pending_block( + &db, + 0, + PendingBlockState::BroadcastPending, + Some(TxHash::from([11u8; 32])), + 3, + ) + .await; + + let result = + resolve_broadcast_pending_block_with_db::( + &db, + pending_block, + ) + .await; + + assert!(matches!( + result, + Err(BlockAssemblyError::ContractError(message)) + if message.contains("reverted") + )); + + let available_deposits = + >::get_mempool_deposits(&db) + .await + .unwrap(); + assert_eq!(available_deposits.len(), 4); + + let available_client_transactions = + >::get_all_mempool_client_transactions( + &db, + ) + .await + .unwrap_or_default(); + assert_eq!(available_client_transactions.len(), 4); + + let pending_blocks = db.get_all_pending_blocks().await.unwrap_or_default(); + assert!(pending_blocks.is_empty()); + + let stored_blocks = db.get_all_blocks().await.unwrap_or_default(); + assert!(stored_blocks.is_empty()); + } + + #[tokio::test] + async fn propose_pending_blocks_skips_when_recovery_is_active() { + let _lock = block_assembly_test_lock().await; + reset_test_state().await; + + let pending_block = PendingBlock { + layer2_block_number: 0, + state: PendingBlockState::ReadyToPropose, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: Some(Block::default()), + selected_deposits: Vec::new(), + selected_client_transaction_hashes: Vec::new(), + }; + + get_synchronisation_status() + .await + .write() + .await + .clear_synchronised(); + + push_pending_block_for_test(pending_block).await; + process_pending_blocks_for_proposal::( + get_pending_blocks_queue().await, + "test recovery pause", + ) + .await; + + assert_eq!(mock_proposed_blocks().load(Ordering::SeqCst), 0); + assert_eq!(pending_blocks_queue_len_for_test().await, 1); + } + + #[tokio::test] + async fn propose_pending_blocks_skips_when_block_assembly_is_paused() { + let _lock = block_assembly_test_lock().await; + reset_test_state().await; + + let pending_block = PendingBlock { + layer2_block_number: 0, + state: PendingBlockState::ReadyToPropose, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: Some(Block::default()), + selected_deposits: Vec::new(), + selected_client_transaction_hashes: Vec::new(), + }; + + get_block_assembly_status().await.write().await.pause(); + push_pending_block_for_test(pending_block).await; + + process_pending_blocks_for_proposal::( + get_pending_blocks_queue().await, + "test manual pause", + ) + .await; + + assert_eq!(mock_proposed_blocks().load(Ordering::SeqCst), 0); + assert_eq!(pending_blocks_queue_len_for_test().await, 1); + } + + #[tokio::test] + async fn clear_pending_blocks_queue_removes_enqueued_blocks() { + let _lock = block_assembly_test_lock().await; + reset_test_state().await; + + push_pending_block_for_test(PendingBlock { + layer2_block_number: 0, + state: PendingBlockState::ReadyToPropose, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: Some(Block::default()), + selected_deposits: Vec::new(), + selected_client_transaction_hashes: Vec::new(), + }) + .await; + assert_eq!(pending_blocks_queue_len_for_test().await, 1); + + let cleared = clear_pending_blocks_queue().await; + + assert_eq!(cleared, 1); + assert_eq!(pending_blocks_queue_len_for_test().await, 0); + } +} diff --git a/nightfall_proposer/src/drivers/blockchain/event_listener_manager.rs b/nightfall_proposer/src/drivers/blockchain/event_listener_manager.rs index e73f15dd..3d87e388 100644 --- a/nightfall_proposer/src/drivers/blockchain/event_listener_manager.rs +++ b/nightfall_proposer/src/drivers/blockchain/event_listener_manager.rs @@ -1,8 +1,11 @@ use crate::drivers::blockchain::nightfall_event_listener::start_event_listener; +use crate::initialisation::get_runtime_listener_start_block; use crate::ports::contracts::NightfallContract; use configuration::settings::get_settings; +use futures::FutureExt; use lib::nf_client_proof::{Proof, ProvingEngine}; use log::{info, warn}; +use std::panic::AssertUnwindSafe; use tokio::{ sync::{OnceCell, RwLock}, task::JoinHandle, @@ -24,11 +27,32 @@ where N: NightfallContract, { let s = get_settings(); - let genesis = s.genesis_block; - let max_attempts = s.nightfall_client.max_event_listener_attempts.unwrap_or(10); + let start_block = get_runtime_listener_start_block().await; + let max_attempts = s + .nightfall_proposer + .max_event_listener_attempts + .unwrap_or(10); tokio::spawn(async move { - let _ = start_event_listener::(genesis, max_attempts).await; // discard Result + let mut next_start_block = start_block; + loop { + match AssertUnwindSafe(start_event_listener::( + next_start_block, + max_attempts, + )) + .catch_unwind() + .await + { + Ok(()) => warn!( + "Event listener task exited after exhausting its retry budget; restarting supervisor loop shortly." + ), + Err(_) => warn!( + "Event listener task panicked; restarting supervisor loop shortly." + ), + } + next_start_block = get_runtime_listener_start_block().await; + sleep(Duration::from_secs(1)).await; + } }) } @@ -36,7 +60,16 @@ where pub async fn ensure_running, N: NightfallContract>() { let lock = listener_lock().await; let mut guard = lock.write().await; - if guard.is_none() { + let should_spawn = match guard.as_ref() { + None => true, + Some(handle) if handle.is_finished() => { + warn!("Event listener task had already stopped; spawning a replacement."); + true + } + Some(_) => false, + }; + + if should_spawn { *guard = Some(spawn_listener::().await); info!("Event listener started."); } diff --git a/nightfall_proposer/src/drivers/blockchain/nightfall_event_listener.rs b/nightfall_proposer/src/drivers/blockchain/nightfall_event_listener.rs index 120435f8..eeace9a9 100644 --- a/nightfall_proposer/src/drivers/blockchain/nightfall_event_listener.rs +++ b/nightfall_proposer/src/drivers/blockchain/nightfall_event_listener.rs @@ -1,18 +1,33 @@ use crate::{ - initialisation::{get_blockchain_client_connection, get_db_connection}, + domain::entities::{L1Ref, ProposerSnapshotManifest, RestoreJournalPhase, SyncState}, + driven::db::client_transaction_state::restore_all_selected_transactions_to_mempool, + driven::db::mongo_db::DB, + driven::db::snapshot::{ + acquire_proposer_state_maintenance_guard, cleanup_after_proposer_shadow_swap, + find_latest_valid_proposer_snapshot, recover_from_restore_journal, + restore_proposer_snapshot, + }, + driven::nightfall_event::get_expected_layer2_blocknumber, + drivers::blockchain::block_assembly::clear_pending_blocks_queue, + initialisation::{ + complete_startup_replay_reset, get_block_assembly_status, get_blockchain_client_connection, + get_db_connection, get_runtime_listener_resume_cursor, get_runtime_listener_start_block, + set_runtime_listener_resume_cursor, set_runtime_listener_start_block, + validate_live_proposer_state_consistency, + }, ports::{ contracts::NightfallContract, - db::TransactionsDB, - trees::{CommitmentTree, HistoricRootTree, NullifierTree}, + db::{PendingBlockDB, RestoreJournalDB, SyncStateDB, TransactionsDB}, }, services::process_events::process_events, - services::selected_transactions::reconcile_obviously_orphaned_selected_transactions, + services::selected_transactions::reconcile_active_client_transaction_lifecycle, + services::snapshot_scheduler::set_last_snapshot_l2_block, }; use alloy::{ - rpc::types::Filter, + primitives::I256, + rpc::types::{Filter, Log}, sol_types::{SolEvent, SolEventInterface}, }; -use ark_bn254::Fr as Fr254; use configuration::{addresses::get_addresses, settings::get_settings}; use futures::StreamExt; use futures::{future::BoxFuture, FutureExt}; @@ -26,12 +41,74 @@ use lib::{ use log::{debug, warn}; use mongodb::Client as MongoClient; use nightfall_bindings::artifacts::Nightfall; +use std::path::PathBuf; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + OnceLock, +}; use std::time::Duration; use tokio::{ sync::{OnceCell, RwLock}, time::sleep, }; +#[cfg(test)] +use std::{collections::HashSet, sync::Mutex}; + +#[cfg(test)] +static ENABLED_REPLAY_RESET_FAILPOINTS: std::sync::OnceLock>> = + std::sync::OnceLock::new(); + +#[cfg(test)] +fn enabled_replay_reset_failpoints() -> &'static Mutex> { + ENABLED_REPLAY_RESET_FAILPOINTS.get_or_init(|| Mutex::new(HashSet::new())) +} + +fn maybe_fail_replay_reset(_name: &str) -> Result<(), EventHandlerError> { + #[cfg(test)] + { + if enabled_replay_reset_failpoints() + .lock() + .expect("replay reset failpoint lock poisoned") + .contains(_name) + { + return Err(EventHandlerError::IOError(format!( + "Simulated replay reset failure at {_name}" + ))); + } + } + + Ok(()) +} + +#[cfg(test)] +struct TestReplayResetFailpointGuard { + name: String, +} + +#[cfg(test)] +impl TestReplayResetFailpointGuard { + fn enable(name: &str) -> Self { + enabled_replay_reset_failpoints() + .lock() + .expect("replay reset failpoint lock poisoned") + .insert(name.to_string()); + Self { + name: name.to_string(), + } + } +} + +#[cfg(test)] +impl Drop for TestReplayResetFailpointGuard { + fn drop(&mut self) { + enabled_replay_reset_failpoints() + .lock() + .expect("replay reset failpoint lock poisoned") + .remove(&self.name); + } +} + /// This function starts the event handler. It will attempt to restart the event handler in case of errors /// with an exponential backoff for a configurable number of attempts. If the event handler /// fails after the maximum number of attempts, it will log an error and send a notification (if configured) @@ -53,7 +130,12 @@ where loop { attempts += 1; - log::info!("Proposer event listener (attempt {attempts})..."); + let start_block = if attempts == 1 { + start_block + } else { + get_runtime_listener_start_block().await + }; + log::info!("Proposer event listener (attempt {attempts}) from L1 block {start_block}..."); let result = listen_for_events::(start_block).await; match result { @@ -65,6 +147,7 @@ where log::error!( "Proposer event listener terminated with error: {e:?}. Restarting in {backoff_delay:?}" ); + apply_listener_retry_runtime_state().await; if attempts >= max_attempts { log::error!("Proposer event listener: max attempts reached. Giving up."); if let Err(err) = notify_failure_proposer( @@ -91,6 +174,419 @@ async fn notify_failure_proposer(message: &str) -> Result<(), ()> { Ok(()) } +fn should_skip_replayed_log(cursor: Option<&L1Ref>, log: &Log) -> bool { + matches!( + (cursor, log.block_number, log.log_index), + (Some(cursor), Some(block_number), Some(log_index)) + if block_number == cursor.block_number && log_index <= cursor.log_index + ) +} + +fn listener_start_block_from_sync_state(sync_state: &SyncState) -> usize { + usize::try_from(sync_state.l1_ref.block_number) + .expect("Restored L1 block number does not fit into usize") +} + +async fn apply_restored_listener_runtime_state(sync_state: &SyncState) -> usize { + let next_listener_start_block = listener_start_block_from_sync_state(sync_state); + set_listener_replay_catch_up_pending(true); + set_runtime_listener_resume_cursor(Some(sync_state.l1_ref.clone())).await; + set_runtime_listener_start_block(next_listener_start_block).await; + next_listener_start_block +} + +async fn continue_from_restored_sync_state(sync_state: &SyncState) -> usize { + let next_expected_block = sync_state.last_applied_l2_block.saturating_add(1); + *get_expected_layer2_blocknumber().await.write().await = I256::try_from(next_expected_block) + .expect("Restored L2 block number does not fit into I256"); + set_last_snapshot_l2_block(sync_state.last_applied_l2_block).await; + get_synchronisation_status() + .await + .write() + .await + .clear_synchronised(); + let next_listener_start_block = apply_restored_listener_runtime_state(sync_state).await; + warn!( + "Recovered proposer snapshot state is live. Proposer will remain desynchronised until replay catches up from L1 block {} with next expected L2 block {}", + sync_state.l1_ref.block_number, + next_expected_block + ); + next_listener_start_block +} + +fn restored_sync_state_matches_manifest( + sync_state: &SyncState, + manifest: &ProposerSnapshotManifest, +) -> bool { + sync_state.last_applied_l2_block == manifest.last_applied_l2_block + && sync_state.fingerprint == manifest.fingerprint + && sync_state.l1_ref == manifest.l1_ref +} + +async fn recovered_snapshot_sync_state_if_available( + db: &MongoClient, + manifest: &ProposerSnapshotManifest, +) -> Result, String> { + if let Some(journal) = db.get_restore_journal().await { + if journal.phase != RestoreJournalPhase::SwapComplete { + return Ok(None); + } + + cleanup_after_proposer_shadow_swap(db) + .await + .map_err(|error| { + format!( + "restore journal recovery reached swap_complete but cleanup could not finish: {error}" + ) + })?; + } + + let Some(sync_state) = db.get_sync_state().await else { + return Ok(None); + }; + + if !restored_sync_state_matches_manifest(&sync_state, manifest) { + return Ok(None); + } + + validate_live_proposer_state_consistency(db) + .await + .map_err(|error| { + format!( + "restored sync_state matches the snapshot manifest, but live proposer state remains inconsistent: {error}" + ) + })?; + + Ok(Some(sync_state)) +} + +async fn restore_recovery_state_may_still_be_present(db: &MongoClient) -> Result { + if db.get_restore_journal().await.is_some() { + return Ok(true); + } + + let collection_names = db + .database(DB) + .list_collection_names() + .await + .map_err(|error| format!("could not inspect restore recovery collections: {error}"))?; + + Ok(collection_names + .iter() + .any(|name| name.starts_with("restore_backup__") || name.starts_with("restore_shadow__"))) +} + +async fn continue_after_failed_snapshot_restore

( + db: &MongoClient, + snapshot_dir: &std::path::Path, + manifest: &ProposerSnapshotManifest, + restore_error: &str, + replay_start_block: usize, +) -> usize +where + P: Proof, +{ + match recover_from_restore_journal(db).await { + Ok(()) => match recovered_snapshot_sync_state_if_available(db, manifest).await { + Ok(Some(sync_state)) => { + warn!( + "Snapshot restore from {} returned an error ({}), but restore journal recovery completed the requested snapshot successfully", + snapshot_dir.display(), + restore_error, + ); + continue_from_restored_sync_state(&sync_state).await + } + Ok(None) => { + warn!( + "Snapshot restore from {} failed: {}. Restore journal recovery did not leave the requested snapshot state live. Falling back to destructive reset+replay from L1 block {}", + snapshot_dir.display(), + restore_error, + replay_start_block + ); + continue_with_destructive_replay_fallback::

(db).await + } + Err(recovery_error) => { + continue_after_incomplete_restore_recovery::

( + db, + snapshot_dir, + restore_error, + &recovery_error, + replay_start_block, + ) + .await + } + }, + Err(recovery_error) => { + continue_after_incomplete_restore_recovery::

( + db, + snapshot_dir, + restore_error, + &recovery_error.to_string(), + replay_start_block, + ) + .await + } + } +} + +async fn continue_after_incomplete_restore_recovery

( + db: &MongoClient, + snapshot_dir: &std::path::Path, + restore_error: &str, + recovery_error: &str, + replay_start_block: usize, +) -> usize +where + P: Proof, +{ + match restore_recovery_state_may_still_be_present(db).await { + Ok(true) => panic!( + "Snapshot restore from {} failed: {}. Restore journal recovery also failed or \ + could not finalize safely: {}. restore_journal or restore backup/shadow state \ + may still be present. Manual intervention is required before destructive replay.", + snapshot_dir.display(), + restore_error, + recovery_error + ), + Ok(false) => { + warn!( + "Snapshot restore from {} failed: {}. Restore journal recovery also failed or \ + could not finalize safely: {}. No residual restore state remains, so \ + destructive reset+replay from L1 block {} is allowed.", + snapshot_dir.display(), + restore_error, + recovery_error, + replay_start_block + ); + continue_with_destructive_replay_fallback::

(db).await + } + Err(state_error) => panic!( + "Snapshot restore from {} failed: {}. Restore journal recovery also failed or \ + could not finalize safely: {}. Additionally, residual restore state could not be \ + inspected safely: {}. Manual intervention is required before destructive replay.", + snapshot_dir.display(), + restore_error, + recovery_error, + state_error + ), + } +} + +fn destructive_replay_start_block() -> usize { + get_settings().genesis_block +} + +async fn apply_destructive_replay_listener_runtime_state() -> usize { + let start_block = destructive_replay_start_block(); + set_listener_replay_catch_up_pending(true); + set_runtime_listener_resume_cursor(None).await; + set_runtime_listener_start_block(start_block).await; + start_block +} + +async fn continue_with_destructive_replay_fallback

(db: &MongoClient) -> usize +where + P: Proof, +{ + if let Err(reset_error) = reset_proposer_state_for_replay::

(db).await { + panic!( + "Proposer replay fallback aborted because reset could not complete safely: {reset_error}" + ); + } + set_last_snapshot_l2_block(0).await; + *get_expected_layer2_blocknumber().await.write().await = I256::ZERO; + apply_destructive_replay_listener_runtime_state().await +} + +async fn apply_listener_retry_runtime_state() { + set_listener_replay_catch_up_pending(true); + get_synchronisation_status() + .await + .write() + .await + .clear_synchronised(); + get_block_assembly_status().await.write().await.pause(); +} + +async fn apply_listener_caught_up_runtime_state() { + set_listener_replay_catch_up_pending(false); + get_synchronisation_status() + .await + .write() + .await + .set_synchronised(); + get_block_assembly_status().await.write().await.resume(); +} + +fn listener_replay_catch_up_pending_cell() -> &'static AtomicBool { + static LISTENER_REPLAY_CATCH_UP_PENDING: OnceLock = OnceLock::new(); + LISTENER_REPLAY_CATCH_UP_PENDING.get_or_init(|| AtomicBool::new(false)) +} + +pub(crate) fn is_listener_replay_catch_up_pending() -> bool { + listener_replay_catch_up_pending_cell().load(Ordering::SeqCst) +} + +pub(crate) fn set_listener_replay_catch_up_pending(pending: bool) { + listener_replay_catch_up_pending_cell().store(pending, Ordering::SeqCst); +} + +const LISTENER_CATCH_UP_RETRY_INTERVAL: Duration = Duration::from_secs(5); + +async fn reconcile_active_client_transactions_after_listener_catch_up

( + db: &MongoClient, + current_layer2_block_number: u64, +) where + P: Proof, +{ + match reconcile_active_client_transaction_lifecycle::

(db, current_layer2_block_number).await + { + Some(reclassified) if reclassified > 0 => { + warn!( + "Listener catch-up reclassified {reclassified} active client transaction(s) \ + against canonical StoredBlock state at L2 block \ + {current_layer2_block_number}" + ); + } + Some(_) => {} + None => { + warn!( + "Listener catch-up could not reconcile active client transaction lifecycle at \ + L2 block {current_layer2_block_number}" + ); + } + } +} + +async fn finalize_listener_catch_up() +where + P: Proof, + N: NightfallContract, +{ + let current_layer2_block_number = + match u64::try_from(*get_expected_layer2_blocknumber().await.read().await) { + Ok(block_number) => block_number, + Err(_) => { + warn!( + "Listener catch-up could not derive the current L2 block number from \ + expected_layer2_blocknumber; keeping proposer desynchronised until replay can \ + continue" + ); + return; + } + }; + + let current_onchain_layer2_block_number = match N::get_current_layer2_blocknumber().await { + Ok(block_number) => block_number, + Err(error) => { + warn!( + "Listener catch-up could not fetch the current on-chain L2 block number: \ + {error}. Keeping proposer desynchronised until replay can continue" + ); + return; + } + }; + let current_onchain_layer2_block_number = + match u64::try_from(current_onchain_layer2_block_number) { + Ok(block_number) => block_number, + Err(_) => { + warn!( + "Listener catch-up could not convert the current on-chain L2 block number \ + into u64; keeping proposer desynchronised until replay can continue" + ); + return; + } + }; + + if current_layer2_block_number != current_onchain_layer2_block_number { + warn!( + "Listener catch-up reached in-memory next expected L2 block \ + {current_layer2_block_number} but on-chain next expected L2 block is \ + {current_onchain_layer2_block_number}. Keeping proposer desynchronised until replay \ + fully catches up" + ); + return; + } + + let db = get_db_connection().await; + reconcile_active_client_transactions_after_listener_catch_up::

( + db, + current_layer2_block_number, + ) + .await; + apply_listener_caught_up_runtime_state().await; +} + +async fn retry_listener_catch_up_if_pending() +where + P: Proof, + N: NightfallContract, +{ + if is_listener_replay_catch_up_pending() { + finalize_listener_catch_up::().await; + } +} + +async fn advance_runtime_listener_state_from_log(log: &Log) { + let (Some(block_number), Some(tx_hash), Some(log_index)) = + (log.block_number, log.transaction_hash, log.log_index) + else { + return; + }; + let Ok(start_block) = usize::try_from(block_number) else { + return; + }; + + set_runtime_listener_resume_cursor(Some(L1Ref { + block_number, + tx_hash, + log_index, + })) + .await; + set_runtime_listener_start_block(start_block).await; +} + +async fn process_listener_log( + log: Log, + restart_start_block: usize, +) -> Result<(), EventHandlerError> +where + P: Proof, + E: ProvingEngine

, + N: NightfallContract, +{ + let event = match Nightfall::NightfallEvents::decode_log(&log.inner) { + Ok(event) => event, + Err(error) => { + warn!("Failed to decode log: {error:?}"); + return Ok(()); + } + }; + + match process_events::(event.data, log.clone()).await { + Ok(_) => { + advance_runtime_listener_state_from_log(&log).await; + Ok(()) + } + Err(EventHandlerError::MissingBlocks(n)) => { + warn!("Missing blocks. Last contiguous block was {n}. Restarting event listener"); + restart_event_listener::(restart_start_block).await; + Err(EventHandlerError::StreamTerminated) + } + Err(EventHandlerError::BlockHashError(expected, found)) => { + warn!( + "Block hash mismatch: expected {expected:?}, found {found:?}. Restarting event listener" + ); + restart_event_listener::(restart_start_block).await; + Err(EventHandlerError::StreamTerminated) + } + Err(error) => { + warn!("Event listener hit a recoverable processing error: {error:?}"); + Err(error) + } + } +} + // This function listens for events and processes them. It's started by the start_event_listener function pub async fn listen_for_events(start_block: usize) -> Result<(), EventHandlerError> where @@ -98,6 +594,7 @@ where E: ProvingEngine

, N: NightfallContract, { + set_listener_replay_catch_up_pending(true); let blockchain_client = get_blockchain_client_connection() .await .read() @@ -120,6 +617,7 @@ where .subscribe_logs(&events_filter) .await .map_err(|_| EventHandlerError::NoEventStream)?; + let replay_resume_cursor = get_runtime_listener_resume_cursor().await; { let latest_block = blockchain_client @@ -147,37 +645,10 @@ where }; log::info!("Found {} past events to process", past_events.len()); for evt in past_events { - let event = match Nightfall::NightfallEvents::decode_log(&evt.inner) { - Ok(e) => e, - Err(e) => { - warn!("Failed to decode log: {e:?}"); - continue; // Skip malformed events - } - }; - let result = process_events::(event.data, evt).await; - match result { - Ok(_) => continue, - Err(e) => { - match e { - // we're missing blocks, so we need to re-synchronise - EventHandlerError::MissingBlocks(n) => { - warn!("Missing blocks. Last contiguous block was {n}. Restarting event listener"); - restart_event_listener::(start_block).await; - return Err(EventHandlerError::StreamTerminated); - } - - EventHandlerError::BlockHashError(expected, found) => { - warn!( - "Block hash mismatch: expected {expected:?}, found {found:?}. Restarting event listener" - ); - restart_event_listener::(start_block).await; - return Err(EventHandlerError::StreamTerminated); - } - - _ => panic!("Error processing event: {e:?}"), - } - } + if should_skip_replayed_log(replay_resume_cursor.as_ref(), &evt) { + continue; } + process_listener_log::(evt, start_block).await?; } } else { println!( "Start block {start_block} is greater than latest block {latest_block}. No past events to process."); @@ -186,52 +657,142 @@ where let mut events_stream = events_subscription.into_stream(); - while let Some(log) = events_stream.next().await { - let event = match Nightfall::NightfallEvents::decode_log(&log.inner) { - Ok(e) => e, - Err(e) => { - warn!("Failed to decode log: {e:?}"); - continue; // Skip malformed events + loop { + match events_stream.next().now_or_never() { + Some(Some(log)) => { + process_listener_log::(log, start_block).await?; } - }; - let result = process_events::(event.data, log).await; - match result { - Ok(_) => continue, - Err(e) => { - match e { - // we're missing blocks, so we need to re-synchronise - EventHandlerError::MissingBlocks(n) => { - warn!("Missing blocks. Last contiguous block was {n}. Restarting event listener"); - restart_event_listener::(start_block).await; - return Err(EventHandlerError::StreamTerminated); - } + Some(None) => return Err(EventHandlerError::StreamTerminated), + None => break, + } + } - EventHandlerError::BlockHashError(expected, found) => { - warn!( - "Block hash mismatch: expected {expected:?}, found {found:?}. Restarting event listener" - ); - restart_event_listener::(start_block).await; - return Err(EventHandlerError::StreamTerminated); - } + finalize_listener_catch_up::().await; - _ => panic!("Error processing event: {e:?}"), - } + loop { + tokio::select! { + maybe_log = events_stream.next() => { + let Some(log) = maybe_log else { + return Err(EventHandlerError::StreamTerminated); + }; + process_listener_log::(log, start_block).await?; + retry_listener_catch_up_if_pending::().await; + } + _ = sleep(LISTENER_CATCH_UP_RETRY_INTERVAL), if is_listener_replay_catch_up_pending() => { + retry_listener_catch_up_if_pending::().await; } } } +} + +/// Returns the highest L2 block that is safe to target with snapshot restore during desync +/// recovery, based on the last contiguous block the proposer expected to have applied. +async fn latest_restorable_l2_block() -> usize { + let expected = *get_expected_layer2_blocknumber().await.read().await; + if expected <= I256::ZERO { + return 0; + } - Err(EventHandlerError::StreamTerminated) + expected.as_usize().checked_sub(1).unwrap_or_default() } -// We might need to restart the event listener if we fall out of sync and lose blocks -// This does not erase aleady synchronised data -pub async fn restart_event_listener(start_block: usize) +/// Cleans transient proposer-side state that should not survive a desync recovery attempt, +/// without touching the restored or replayed tree state. +/// +/// Known limitation: pending client transactions that have not reached `Selected` +/// are dropped during recovery and must be resubmitted by clients afterwards. +/// `Selected` transactions are released back to the mempool so replay can rebuild +/// canonical transaction lifecycle from scratch. +async fn cleanup_recovery_side_effects

(db: &MongoClient) +where + P: Proof, +{ + let removed_client_txs = TransactionsDB::

::remove_all_mempool_client_transactions(db).await; + + debug!( + "Mempool cleanup: removed {} client transactions.", + removed_client_txs.unwrap_or(0) + ); + if let Some(removed_client_txs) = removed_client_txs.filter(|count| *count > 0) { + warn!( + "Recovery dropped {removed_client_txs} pending proposer mempool client transaction(s); clients must resubmit them after recovery" + ); + } + + let restored_selected = restore_all_selected_transactions_to_mempool(db).await; + debug!( + "Selected transaction recovery after restart restored {} transaction(s) to the mempool.", + restored_selected.unwrap_or(0) + ); +} + +async fn clear_all_reserved_deposits_for_recovery

( + db: &MongoClient, +) -> Result +where + P: Proof, +{ + TransactionsDB::

::clear_all_mempool_deposit_reservations(db) + .await + .ok_or_else(|| { + EventHandlerError::IOError( + "Could not clear reserved deposits during proposer recovery".to_string(), + ) + }) +} + +#[cfg_attr(not(test), allow(dead_code))] +async fn cleanup_persisted_pending_blocks_for_recovery

( + db: &MongoClient, +) -> Result<(u64, u64), EventHandlerError> +where + P: Proof, +{ + let deleted_pending_blocks = db.delete_all_pending_blocks().await.ok_or_else(|| { + EventHandlerError::IOError( + "Could not delete persisted pending blocks during recovery".to_string(), + ) + })?; + let unreserved_deposits = clear_all_reserved_deposits_for_recovery::

(db).await?; + if deleted_pending_blocks > 0 || unreserved_deposits > 0 { + warn!( + "Recovery discarded {deleted_pending_blocks} persisted pending block(s) and cleared {unreserved_deposits} reserved deposit selection(s)" + ); + } + Ok((deleted_pending_blocks, unreserved_deposits)) +} + +/// Resets proposer state for the destructive replay fallback, including tree state and +/// persisted sync_state, before historical events are replayed. +async fn reset_proposer_state_for_replay

(db: &MongoClient) -> Result<(), EventHandlerError> +where + P: Proof, +{ + let _maintenance_guard = acquire_proposer_state_maintenance_guard().await; + maybe_fail_replay_reset("delete_sync_state_before_delete")?; + complete_startup_replay_reset(db) + .await + .map_err(EventHandlerError::IOError)?; + cleanup_recovery_side_effects::

(db).await; + Ok(()) +} + +/// Returns the configured root directory where proposer snapshots are discovered for restore. +async fn proposer_snapshot_root_dir() -> PathBuf { + PathBuf::from(&get_settings().nightfall_proposer.snapshot_root_dir) +} + +/// Restart the event listener after a desync (missing blocks or hash mismatch). +/// Attempts to restore from the most recent valid proposer snapshot first; falls +/// back to destructive reset+replay if no valid snapshot is found or the restore +/// itself fails. Only the destructive fallback erases already synchronised data. +pub async fn restart_event_listener(_start_block: usize) where P: Proof, E: ProvingEngine

, N: NightfallContract, { - // if we're restarting the event lister, we definitely shouldn't be in sync, so check that's the case + // if we're restarting the event listener, we definitely shouldn't be in sync, so check that's the case let sync_state = get_synchronisation_status() .await .read() @@ -240,41 +801,67 @@ where if sync_state { panic!("Restarting event listener while synchronised. This should not happen"); } - // clean the database and reset the trees - // this is a bit of a hack, but we need to reset the trees to get them back in sync - // with the blockchain. We should probably do this in a more elegant way, but this works for now - // and we can improve it later - { - let db = get_db_connection().await; - let _ = >::reset_tree(db).await; - let _ = >::reset_tree(db).await; - let _ = >::reset_tree(db).await; - // clean up the mempool, otherwise proposer gets duplicated transactions everytime it syncs - - let removed_deposits = TransactionsDB::

::remove_all_mempool_deposits(db).await; - let removed_client_txs = - TransactionsDB::

::remove_all_mempool_client_transactions(db).await; - - debug!( - "Mempool cleanup: removed {} deposits and {} client transactions.", - removed_deposits.unwrap_or(0), - removed_client_txs.unwrap_or(0) - ); - - let restored_selected = reconcile_obviously_orphaned_selected_transactions::

(db).await; - debug!( - "Selected transaction recovery after restart restored {} orphaned transaction(s).", - restored_selected.unwrap_or(0) + get_block_assembly_status().await.write().await.pause(); + let dropped_pending_blocks = clear_pending_blocks_queue().await; + if dropped_pending_blocks > 0 { + warn!( + "Recovery cleared {dropped_pending_blocks} queued block assembly candidate(s) before restore/replay" ); } - let settings = get_settings(); - let max_attempts = settings - .nightfall_proposer - .max_event_listener_attempts - .unwrap_or(10); - - start_event_listener::(start_block, max_attempts).await; + let db = get_db_connection().await; + let max_restorable_l2_block = latest_restorable_l2_block().await as u64; + let snapshot_root_dir = proposer_snapshot_root_dir().await; + let replay_start_block = destructive_replay_start_block(); + let next_listener_start_block = match find_latest_valid_proposer_snapshot( + &snapshot_root_dir, + max_restorable_l2_block, + ) + .await + { + Ok(Some((snapshot_dir, manifest))) => { + warn!( + "Desynchronised proposer state detected. Attempting snapshot restore from {} (L2 block {}) before falling back to reset+replay", + snapshot_dir.display(), + manifest.last_applied_l2_block + ); + match restore_proposer_snapshot(db, &snapshot_dir).await { + Ok(sync_state) => continue_from_restored_sync_state(&sync_state).await, + Err(error) => { + continue_after_failed_snapshot_restore::

( + db, + &snapshot_dir, + &manifest, + &error.to_string(), + replay_start_block, + ) + .await + } + } + } + Ok(None) => { + warn!( + "No valid proposer snapshot found under {} for L2 recovery point <= {}. Falling back to destructive reset+replay from L1 block {}", + snapshot_root_dir.display(), + max_restorable_l2_block, + replay_start_block + ); + continue_with_destructive_replay_fallback::

(db).await + } + Err(error) => { + warn!( + "Snapshot discovery failed under {}: {}. Falling back to destructive reset+replay from L1 block {}", + snapshot_root_dir.display(), + error, + replay_start_block + ); + continue_with_destructive_replay_fallback::

(db).await + } + }; + debug!( + "Prepared proposer listener recovery runtime state. Outer retry loop will resume from \ + L1 block {next_listener_start_block}" + ); } pub async fn get_synchronisation_status() -> &'static RwLock { @@ -283,3 +870,1109 @@ pub async fn get_synchronisation_status() -> &'static RwLock Result { + Ok(Bytes::new()) + } + + fn from_compressed(_compressed: Bytes) -> Result { + Ok(Self) + } + } + + async fn event_listener_test_lock() -> tokio::sync::MutexGuard<'static, ()> { + static LOCK: OnceCell> = OnceCell::const_new(); + LOCK.get_or_init(|| async { Mutex::new(()) }) + .await + .lock() + .await + } + + async fn persist_sync_state(client: &mongodb::Client, sync_state: &SyncState) { + let mut session = client.start_session().await.expect("start session"); + let client_for_write = client.clone(); + let sync_state_for_write = sync_state.clone(); + session + .start_transaction() + .and_run2(async move |session| { + client_for_write + .update_sync_state_with_session(&sync_state_for_write, session) + .await?; + Ok::<(), mongodb::error::Error>(()) + }) + .await + .expect("write sync_state"); + + let stored_block = client + .get_block_by_number(sync_state.last_applied_l2_block) + .await + .expect("stored block should exist for event listener test sync_state"); + + let commitment_metadata = client + .database(DB) + .collection::>(&format!( + "{}_metadata", + >::TREE_NAME + )) + .find_one(doc! { "_id": 0 }) + .await + .expect("read commitment metadata"); + if commitment_metadata + .as_ref() + .is_some_and(|metadata| metadata.sub_tree_count == 0) + && !stored_block.commitments.is_empty() + { + >::insert_leaf( + client, + Fr254::from(sync_state.last_applied_l2_block + 1), + true, + >::TREE_NAME, + ) + .await + .expect("materialize commitment tree state for event listener test"); + } + + let target_historic_root_sub_tree_count = sync_state + .last_applied_l2_block + .checked_add(2) + .expect("historic root count should not overflow in event listener tests"); + let mut historic_root_sub_tree_count = client + .database(DB) + .collection::>(&format!( + "{}_metadata", + >::TREE_NAME + )) + .find_one(doc! { "_id": 0 }) + .await + .expect("read historic root metadata") + .expect("historic root metadata should exist for event listener tests") + .sub_tree_count; + while historic_root_sub_tree_count < target_historic_root_sub_tree_count { + let commitment_root = >::get_root(client) + .await + .expect("read commitment root"); + >::append_historic_commitment_root( + client, + &commitment_root, + true, + ) + .await + .expect("materialize historic root state for event listener test"); + historic_root_sub_tree_count += 1; + } + } + + async fn initialize_test_trees(client: &mongodb::Client) { + >::new_commitment_tree(client, 29, 3) + .await + .expect("create commitment tree"); + >::new_historic_root_tree(client, 32) + .await + .expect("create historic root tree"); + >::append_historic_commitment_root( + client, + &Fr254::zero(), + true, + ) + .await + .expect("append zero historic root"); + >::new_nullifier_tree(client, 29, 3) + .await + .expect("create nullifier tree"); + ensure_deposit_indexes(client) + .await + .expect("create deposit indexes"); + } + + #[test] + fn should_skip_replayed_log_skips_lower_log_index_in_same_block() { + let cursor = L1Ref { + block_number: 100, + tx_hash: TxHash::from([1u8; 32]), + log_index: 8, + }; + let log = RpcLog { + block_number: Some(100), + log_index: Some(7), + ..RpcLog::default() + }; + + assert!(should_skip_replayed_log(Some(&cursor), &log)); + } + + #[test] + fn should_skip_replayed_log_skips_equal_log_index_in_same_block() { + let cursor = L1Ref { + block_number: 100, + tx_hash: TxHash::from([2u8; 32]), + log_index: 8, + }; + let log = RpcLog { + block_number: Some(100), + log_index: Some(8), + ..RpcLog::default() + }; + + assert!(should_skip_replayed_log(Some(&cursor), &log)); + } + + #[test] + fn should_skip_replayed_log_keeps_higher_log_index_in_same_block() { + let cursor = L1Ref { + block_number: 100, + tx_hash: TxHash::from([3u8; 32]), + log_index: 8, + }; + let log = RpcLog { + block_number: Some(100), + log_index: Some(9), + ..RpcLog::default() + }; + + assert!(!should_skip_replayed_log(Some(&cursor), &log)); + } + + #[test] + fn should_skip_replayed_log_keeps_logs_from_other_blocks() { + let cursor = L1Ref { + block_number: 100, + tx_hash: TxHash::from([4u8; 32]), + log_index: 8, + }; + let earlier_log = RpcLog { + block_number: Some(99), + log_index: Some(99), + ..RpcLog::default() + }; + let later_log = RpcLog { + block_number: Some(101), + log_index: Some(0), + ..RpcLog::default() + }; + + assert!(!should_skip_replayed_log(Some(&cursor), &earlier_log)); + assert!(!should_skip_replayed_log(Some(&cursor), &later_log)); + } + + #[test] + fn should_skip_replayed_log_keeps_logs_with_missing_log_index() { + let cursor = L1Ref { + block_number: 100, + tx_hash: TxHash::from([5u8; 32]), + log_index: 8, + }; + let log = RpcLog { + block_number: Some(100), + log_index: None, + ..RpcLog::default() + }; + + assert!(!should_skip_replayed_log(Some(&cursor), &log)); + } + + #[test] + fn should_skip_replayed_log_keeps_logs_with_missing_block_number() { + let cursor = L1Ref { + block_number: 100, + tx_hash: TxHash::from([6u8; 32]), + log_index: 8, + }; + let log = RpcLog { + block_number: None, + log_index: Some(8), + ..RpcLog::default() + }; + + assert!(!should_skip_replayed_log(Some(&cursor), &log)); + } + + #[test] + fn should_skip_replayed_log_keeps_logs_without_cursor() { + let log = RpcLog { + block_number: Some(100), + log_index: Some(8), + ..RpcLog::default() + }; + + assert!(!should_skip_replayed_log(None, &log)); + } + + #[tokio::test] + async fn apply_restored_listener_runtime_state_sets_resume_cursor_from_sync_state() { + let _lock = event_listener_test_lock().await; + set_listener_replay_catch_up_pending(false); + set_runtime_listener_resume_cursor(None).await; + set_runtime_listener_start_block(0).await; + + let sync_state = SyncState::new( + 8, + "fingerprint".to_string(), + L1Ref { + block_number: 100, + tx_hash: TxHash::from([7u8; 32]), + log_index: 8, + }, + mongodb::bson::DateTime::now(), + ); + + let start_block = apply_restored_listener_runtime_state(&sync_state).await; + + assert_eq!(start_block, 100); + assert_eq!(get_runtime_listener_start_block().await, 100); + assert_eq!( + get_runtime_listener_resume_cursor().await, + Some(sync_state.l1_ref.clone()) + ); + assert!(is_listener_replay_catch_up_pending()); + } + + #[tokio::test] + async fn apply_destructive_replay_listener_runtime_state_clears_resume_cursor() { + let _lock = event_listener_test_lock().await; + set_listener_replay_catch_up_pending(false); + set_runtime_listener_resume_cursor(Some(L1Ref { + block_number: 100, + tx_hash: TxHash::from([8u8; 32]), + log_index: 8, + })) + .await; + set_runtime_listener_start_block(100).await; + + let start_block = apply_destructive_replay_listener_runtime_state().await; + + assert_eq!(start_block, destructive_replay_start_block()); + assert_eq!(get_runtime_listener_start_block().await, start_block); + assert_eq!(get_runtime_listener_resume_cursor().await, None); + assert!(is_listener_replay_catch_up_pending()); + } + + #[tokio::test] + async fn advance_runtime_listener_state_from_log_updates_cursor_and_start_block() { + let _lock = event_listener_test_lock().await; + set_runtime_listener_resume_cursor(None).await; + set_runtime_listener_start_block(0).await; + + let log = RpcLog { + block_number: Some(321), + transaction_hash: Some(TxHash::from([9u8; 32])), + log_index: Some(12), + ..RpcLog::default() + }; + + advance_runtime_listener_state_from_log(&log).await; + + assert_eq!(get_runtime_listener_start_block().await, 321); + assert_eq!( + get_runtime_listener_resume_cursor().await, + Some(L1Ref { + block_number: 321, + tx_hash: TxHash::from([9u8; 32]), + log_index: 12, + }) + ); + } + + #[tokio::test] + async fn advance_runtime_listener_state_from_log_ignores_incomplete_logs() { + let _lock = event_listener_test_lock().await; + let original_cursor = L1Ref { + block_number: 400, + tx_hash: TxHash::from([4u8; 32]), + log_index: 4, + }; + set_runtime_listener_resume_cursor(Some(original_cursor.clone())).await; + set_runtime_listener_start_block(400).await; + + let incomplete_log = RpcLog { + block_number: Some(401), + transaction_hash: None, + log_index: Some(5), + ..RpcLog::default() + }; + + advance_runtime_listener_state_from_log(&incomplete_log).await; + + assert_eq!(get_runtime_listener_start_block().await, 400); + assert_eq!( + get_runtime_listener_resume_cursor().await, + Some(original_cursor) + ); + } + + #[tokio::test] + async fn apply_listener_retry_runtime_state_clears_sync_and_pauses_assembly() { + let _lock = event_listener_test_lock().await; + set_listener_replay_catch_up_pending(false); + get_synchronisation_status() + .await + .write() + .await + .set_synchronised(); + get_block_assembly_status().await.write().await.resume(); + + apply_listener_retry_runtime_state().await; + + assert!(!get_synchronisation_status() + .await + .read() + .await + .is_synchronised()); + assert!(!get_block_assembly_status().await.read().await.is_running()); + assert!(is_listener_replay_catch_up_pending()); + } + + #[tokio::test] + async fn apply_listener_caught_up_runtime_state_sets_sync_and_resumes_assembly() { + let _lock = event_listener_test_lock().await; + set_listener_replay_catch_up_pending(true); + get_synchronisation_status() + .await + .write() + .await + .clear_synchronised(); + get_block_assembly_status().await.write().await.pause(); + + apply_listener_caught_up_runtime_state().await; + + assert!(get_synchronisation_status() + .await + .read() + .await + .is_synchronised()); + assert!(get_block_assembly_status().await.read().await.is_running()); + assert!(!is_listener_replay_catch_up_pending()); + } + + fn test_selected_client_transaction( + seed: u32, + block_l2: u64, + ) -> ClientTransactionWithMetaData { + ClientTransactionWithMetaData { + client_transaction: ClientTransaction { + commitments: [ + Fr254::from(u64::from(seed) + 100), + Fr254::zero(), + Fr254::zero(), + Fr254::zero(), + ], + compressed_secrets: CompressedSecrets::default(), + proof: MockProof, + ..Default::default() + }, + lifecycle: TxLifecycle::Selected { block_l2 }, + hash: vec![seed, seed + 1, seed + 2], + historic_roots: vec![], + receipt_token: None, + } + } + + fn test_pending_deposit(seed: u64, reserved: bool) -> DepositDatawithFee { + DepositDatawithFee { + fee: Fr254::from(seed), + deposit_data: DepositData { + nf_token_id: Fr254::from(seed + 1), + nf_slot_id: Fr254::from(seed + 2), + value: Fr254::from(seed + 3), + secret_hash: Fr254::from(seed + 4), + }, + reserved, + } + } + + #[test] + fn restored_sync_state_matches_manifest_requires_exact_restore_identity() { + let sync_state = SyncState::new( + 11, + "fingerprint-11".to_string(), + L1Ref { + block_number: 1100, + tx_hash: TxHash::from([11u8; 32]), + log_index: 11, + }, + mongodb::bson::DateTime::now(), + ); + let manifest = ProposerSnapshotManifest::new( + "snapshot-11".to_string(), + mongodb::bson::DateTime::now(), + DB.to_string(), + &sync_state, + Vec::new(), + "overall".to_string(), + ); + + assert!(restored_sync_state_matches_manifest(&sync_state, &manifest)); + + let different_fingerprint = SyncState { + fingerprint: "different".to_string(), + ..sync_state.clone() + }; + assert!(!restored_sync_state_matches_manifest( + &different_fingerprint, + &manifest + )); + + let different_l1_ref = SyncState { + l1_ref: L1Ref { + log_index: 12, + ..sync_state.l1_ref.clone() + }, + ..sync_state.clone() + }; + assert!(!restored_sync_state_matches_manifest( + &different_l1_ref, + &manifest + )); + } + + #[tokio::test] + async fn recovered_snapshot_sync_state_if_available_finishes_swap_complete_cleanup() { + let _lock = event_listener_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_test_trees(&client).await; + let block = StoredBlock { + layer2_block_number: 25, + commitments: vec!["0xswap-complete-recovery".to_string()], + proposer_address: Address::from([25u8; 20]), + }; + client.store_block(&block).await.expect("store block"); + let sync_state = SyncState::new( + 25, + block.hash().to_hex_string(), + L1Ref { + block_number: 2500, + tx_hash: TxHash::from([25u8; 32]), + log_index: 25, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &sync_state).await; + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-event-listener-snapshot-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + swap_proposer_shadow_into_live(&client) + .await + .expect("swap snapshot into live"); + + let journal = client + .get_restore_journal() + .await + .expect("swap_complete journal should exist"); + assert_eq!(journal.phase, RestoreJournalPhase::SwapComplete); + + let recovered_sync_state = recovered_snapshot_sync_state_if_available(&client, &manifest) + .await + .expect("recover helper should succeed") + .expect("recover helper should surface restored sync_state"); + + assert_eq!(recovered_sync_state, sync_state); + assert_eq!(client.get_restore_journal().await, None); + } + + #[tokio::test] + async fn recovered_snapshot_sync_state_if_available_rejects_inconsistent_live_state() { + let _lock = event_listener_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_test_trees(&client).await; + let block = StoredBlock { + layer2_block_number: 31, + commitments: vec!["0xinconsistent-live-state".to_string()], + proposer_address: Address::from([31u8; 20]), + }; + client.store_block(&block).await.expect("store block"); + let sync_state = SyncState::new( + 31, + block.hash().to_hex_string(), + L1Ref { + block_number: 3100, + tx_hash: TxHash::from([31u8; 32]), + log_index: 31, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &sync_state).await; + + let manifest = ProposerSnapshotManifest::new( + "snapshot-31".to_string(), + mongodb::bson::DateTime::now(), + DB.to_string(), + &sync_state, + Vec::new(), + "overall".to_string(), + ); + + client + .database(DB) + .collection::(&format!( + "{}_metadata", + >::TREE_NAME + )) + .drop() + .await + .expect("drop nullifier metadata to simulate inconsistent live state"); + + let error = recovered_snapshot_sync_state_if_available(&client, &manifest) + .await + .expect_err("helper should reject matching sync_state when live state is inconsistent"); + + assert!(error.contains("live proposer state remains inconsistent")); + } + + #[tokio::test] + async fn continue_after_failed_snapshot_restore_aborts_when_recovery_leaves_restore_state() { + let _lock = event_listener_test_lock().await; + let _snapshot_lock = snapshot_test_lock().await; + clear_all_test_failpoints(); + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_test_trees(&client).await; + let snapshot_block = StoredBlock { + layer2_block_number: 31, + commitments: vec!["0xlistener-restore-abort".to_string()], + proposer_address: Address::from([31u8; 20]), + }; + client + .store_block(&snapshot_block) + .await + .expect("store snapshot block"); + let snapshot_sync_state = SyncState::new( + snapshot_block.layer2_block_number, + snapshot_block.hash().to_hex_string(), + L1Ref { + block_number: 3100, + tx_hash: TxHash::from([31u8; 32]), + log_index: 31, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &snapshot_sync_state).await; + + let snapshot_root = std::env::temp_dir().join(format!( + "nf4-event-listener-restart-abort-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + let manifest = create_proposer_snapshot(&client, &snapshot_root) + .await + .expect("create snapshot"); + let snapshot_dir = snapshot_root.join(&manifest.snapshot_id); + + let newer_live_block = StoredBlock { + layer2_block_number: 32, + commitments: Vec::new(), + proposer_address: Address::from([32u8; 20]), + }; + client + .store_block(&newer_live_block) + .await + .expect("store newer live block"); + let newer_live_sync_state = SyncState::new( + newer_live_block.layer2_block_number, + newer_live_block.hash().to_hex_string(), + L1Ref { + block_number: 3200, + tx_hash: TxHash::from([32u8; 32]), + log_index: 32, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &newer_live_sync_state).await; + + load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + swap_proposer_shadow_into_live(&client) + .await + .expect("swap snapshot into live"); + + client + .delete_block_by_number(snapshot_block.layer2_block_number) + .await + .expect("delete restored stored block"); + client + .database(DB) + .collection::(&format!("restore_backup__{PROPOSED_BLOCKS_COLLECTION}")) + .drop() + .await + .expect("drop required backup collection"); + + let client_for_restart = client.clone(); + let snapshot_dir_for_restart = snapshot_dir.clone(); + let manifest_for_restart = manifest.clone(); + let join_error = tokio::spawn(async move { + continue_after_failed_snapshot_restore::( + &client_for_restart, + &snapshot_dir_for_restart, + &manifest_for_restart, + "forced restore failure", + destructive_replay_start_block(), + ) + .await + }) + .await + .expect_err("restart helper should abort instead of entering destructive replay"); + + assert!(join_error.is_panic(), "expected panic, got {join_error:?}"); + assert!( + client.get_restore_journal().await.is_some(), + "stale restore_journal should remain for manual inspection" + ); + assert!( + client.get_sync_state().await.is_some(), + "destructive replay must not wipe sync_state after fail-closed abort" + ); + + tokio::fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); + } + + #[tokio::test] + async fn reset_proposer_state_for_replay_returns_error_when_tree_reset_fails() { + let _lock = event_listener_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_test_trees(&client).await; + let block = StoredBlock { + layer2_block_number: 30, + commitments: vec!["0xreset-tree".to_string()], + proposer_address: Address::from([30u8; 20]), + }; + client.store_block(&block).await.expect("store block"); + let sync_state = SyncState::new( + 30, + block.hash().to_hex_string(), + L1Ref { + block_number: 3000, + tx_hash: TxHash::from([30u8; 32]), + log_index: 30, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &sync_state).await; + + let _failpoint = + TestStartupReplayResetFailpointGuard::enable("reset_commitment_tree_before_drop"); + let error = reset_proposer_state_for_replay::(&client) + .await + .expect_err("tree reset failure should abort replay fallback"); + + assert!( + matches!(error, EventHandlerError::IOError(message) if message.contains("reset_commitment_tree_before_drop")) + ); + assert_eq!(client.get_sync_state().await, None); + + let metadata_count = client + .database(DB) + .collection::("Commitments_metadata") + .count_documents(doc! {}) + .await + .expect("count commitment metadata"); + assert!(metadata_count > 0); + } + + #[tokio::test] + async fn recovery_entry_clears_pending_block_queue() { + let _lock = event_listener_test_lock().await; + + clear_pending_blocks_queue().await; + get_block_assembly_status().await.write().await.resume(); + push_pending_block_for_test(PendingBlock { + layer2_block_number: 0, + state: PendingBlockState::ReadyToPropose, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: Some(Block::default()), + selected_deposits: Vec::new(), + selected_client_transaction_hashes: Vec::new(), + }) + .await; + assert_eq!(pending_blocks_queue_len_for_test().await, 1); + + get_synchronisation_status() + .await + .write() + .await + .clear_synchronised(); + get_block_assembly_status().await.write().await.pause(); + let dropped_pending_blocks = clear_pending_blocks_queue().await; + + assert_eq!(dropped_pending_blocks, 1); + assert_eq!(pending_blocks_queue_len_for_test().await, 0); + } + + #[tokio::test] + async fn cleanup_recovery_side_effects_preserves_pending_deposits() { + let _lock = event_listener_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_test_trees(&client).await; + let pending_deposit = DepositDatawithFee { + fee: Fr254::from(9u64), + deposit_data: DepositData { + nf_token_id: Fr254::from(41u64), + nf_slot_id: Fr254::from(42u64), + value: Fr254::from(43u64), + secret_hash: Fr254::from(44u64), + }, + reserved: false, + }; + >::set_mempool_deposits( + &client, + vec![pending_deposit], + ) + .await + .expect("store pending deposit"); + + cleanup_recovery_side_effects::(&client).await; + + let restored_deposits = + >::get_mempool_deposits(&client) + .await + .expect("pending deposits should remain after recovery cleanup"); + assert_eq!(restored_deposits, vec![pending_deposit]); + } + + #[tokio::test] + async fn cleanup_recovery_side_effects_restores_selected_transactions_to_mempool() { + let _lock = event_listener_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_test_trees(&client).await; + let selected_transaction = test_selected_client_transaction(77, 12); + client + .store_transaction(selected_transaction.clone()) + .await + .expect("store selected transaction"); + + cleanup_recovery_side_effects::(&client).await; + + assert!( + >::get_transaction( + &client, + &selected_transaction.hash, + ) + .await + .expect("selected transaction should still exist after cleanup") + .lifecycle + .is_mempool() + ); + } + + #[tokio::test] + async fn listener_catch_up_reconciliation_restores_orphaned_selected_transactions() { + let _lock = event_listener_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_test_trees(&client).await; + let selected_transaction = test_selected_client_transaction(88, 7); + client + .store_transaction(selected_transaction.clone()) + .await + .expect("store selected transaction"); + + reconcile_active_client_transactions_after_listener_catch_up::(&client, 8).await; + + assert!( + >::get_transaction( + &client, + &selected_transaction.hash, + ) + .await + .expect( + "selected transaction should still exist after listener catch-up reconciliation" + ) + .lifecycle + .is_mempool() + ); + } + + #[tokio::test] + async fn cleanup_persisted_pending_blocks_for_recovery_deletes_rows_and_unreserves_deposits() { + let _lock = event_listener_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_test_trees(&client).await; + let reserved_deposit = test_pending_deposit(50, true); + >::set_mempool_deposits( + &client, + vec![reserved_deposit], + ) + .await + .expect("store reserved deposit"); + client + .store_pending_block(&PendingBlock { + layer2_block_number: 50, + state: PendingBlockState::ReadyToPropose, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: None, + selected_deposits: vec![vec![reserved_deposit]], + selected_client_transaction_hashes: Vec::new(), + }) + .await + .expect("store stale pending block"); + + let (deleted_count, unreserved_count) = + cleanup_persisted_pending_blocks_for_recovery::(&client) + .await + .expect("cleanup stale persisted pending blocks"); + assert_eq!(deleted_count, 1); + assert_eq!(unreserved_count, 1); + assert!(client + .get_all_pending_blocks() + .await + .expect("read pending blocks after cleanup") + .is_empty()); + + let restored_deposits = + >::get_mempool_deposits(&client) + .await + .expect("reserved deposit should be visible again"); + assert_eq!( + restored_deposits, + vec![DepositDatawithFee { + reserved: false, + ..reserved_deposit + }] + ); + } + + #[tokio::test] + async fn cleanup_persisted_pending_blocks_for_recovery_unreserves_without_pending_rows() { + let _lock = event_listener_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_test_trees(&client).await; + let reserved_deposit = test_pending_deposit(60, true); + >::set_mempool_deposits( + &client, + vec![reserved_deposit], + ) + .await + .expect("store stranded reserved deposit"); + + let (deleted_count, unreserved_count) = + cleanup_persisted_pending_blocks_for_recovery::(&client) + .await + .expect("cleanup should clear stranded reservations"); + assert_eq!(deleted_count, 0); + assert_eq!(unreserved_count, 1); + + let restored_deposits = + >::get_mempool_deposits(&client) + .await + .expect("reserved deposit should be visible again"); + assert_eq!( + restored_deposits, + vec![DepositDatawithFee { + reserved: false, + ..reserved_deposit + }] + ); + } + + #[tokio::test] + async fn reset_proposer_state_for_replay_returns_error_when_sync_state_delete_fails() { + let _lock = event_listener_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_test_trees(&client).await; + let block = StoredBlock { + layer2_block_number: 31, + commitments: vec!["0xreset-delete".to_string()], + proposer_address: Address::from([31u8; 20]), + }; + client.store_block(&block).await.expect("store block"); + let sync_state = SyncState::new( + 31, + block.hash().to_hex_string(), + L1Ref { + block_number: 3100, + tx_hash: TxHash::from([31u8; 32]), + log_index: 31, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(&client, &sync_state).await; + + let _failpoint = TestReplayResetFailpointGuard::enable("delete_sync_state_before_delete"); + let error = reset_proposer_state_for_replay::(&client) + .await + .expect_err("sync_state delete failure should abort replay fallback"); + + assert!( + matches!(error, EventHandlerError::IOError(message) if message.contains("delete_sync_state_before_delete")) + ); + assert_eq!(client.get_sync_state().await, Some(sync_state)); + + let nullifier_metadata_count = client + .database(DB) + .collection::("Nullifiers_metadata") + .count_documents(doc! {}) + .await + .expect("count nullifier metadata"); + assert!(nullifier_metadata_count > 0); + } + + #[tokio::test] + async fn reset_proposer_state_for_replay_restores_zero_historic_root() { + let _lock = event_listener_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_test_trees(&client).await; + let block = StoredBlock { + layer2_block_number: 32, + commitments: vec!["0xreset-zero-root".to_string()], + proposer_address: Address::from([32u8; 20]), + }; + client.store_block(&block).await.expect("store block"); + persist_sync_state( + &client, + &SyncState::new( + 32, + block.hash().to_hex_string(), + L1Ref { + block_number: 3200, + tx_hash: TxHash::from([32u8; 32]), + log_index: 32, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + + reset_proposer_state_for_replay::(&client) + .await + .expect("replay reset should succeed"); + + let historic_root_metadata = client + .database(DB) + .collection::(&format!( + "{}_metadata", + >::TREE_NAME + )) + .find_one(doc! { "_id": 0 }) + .await + .expect("read historic root metadata") + .expect("historic root metadata should exist"); + assert_eq!( + historic_root_metadata + .get_i64("sub_tree_count") + .unwrap_or_default(), + 1, + "reset replay should restore the zero historic root invariant" + ); + assert_eq!(client.get_sync_state().await, None); + } + + #[tokio::test] + async fn reset_proposer_state_for_replay_removes_all_stale_proposed_blocks() { + let _lock = event_listener_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + initialize_test_trees(&client).await; + for block_number in [40_u64, 41_u64, 42_u64] { + let block = StoredBlock { + layer2_block_number: block_number, + commitments: vec![format!("0xstale-{block_number}")], + proposer_address: Address::from([block_number as u8; 20]), + }; + client.store_block(&block).await.expect("store stale block"); + if block_number == 42 { + persist_sync_state( + &client, + &SyncState::new( + block_number, + block.hash().to_hex_string(), + L1Ref { + block_number: 4200, + tx_hash: TxHash::from([42_u8; 32]), + log_index: 42, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + } + } + + reset_proposer_state_for_replay::(&client) + .await + .expect("replay reset should clear stale stored blocks"); + + assert!( + client + .get_all_blocks() + .await + .expect("read stored blocks after reset") + .is_empty(), + "replay reset should remove all stale stored blocks before replay begins" + ); + } +} diff --git a/nightfall_proposer/src/drivers/rest/client_transactions.rs b/nightfall_proposer/src/drivers/rest/client_transactions.rs index c69314d5..3eda9dc1 100644 --- a/nightfall_proposer/src/drivers/rest/client_transactions.rs +++ b/nightfall_proposer/src/drivers/rest/client_transactions.rs @@ -14,6 +14,7 @@ use lib::{ shared_entities::ClientTransaction, }; use log::{error, info}; + use warp::{hyper::StatusCode, path, Filter}; pub fn client_transaction( @@ -49,7 +50,7 @@ where handle_client_transaction_with(transaction, |transaction| async move { let result = process_nightfall_client_transaction::(transaction).await; match result { - Ok(_) => Ok(()), + Ok(receipt_token) => Ok(receipt_token), Err(e) => { error!("Error processing client transaction: {e}"); Err(warp::reject::custom( @@ -68,11 +69,15 @@ async fn handle_client_transaction_with( where P: Proof, F: FnOnce(ClientTransaction

) -> Fut, - Fut: Future>, + Fut: Future, warp::Rejection>>, { info!("Received client transaction"); - process(transaction).await?; - Ok(StatusCode::CREATED) + let receipt_token = process(transaction).await?; + let body = serde_json::json!({ "receipt_token": receipt_token }); + Ok(warp::reply::with_status( + warp::reply::json(&body), + StatusCode::CREATED, + )) } async fn handle_cancel_swap_request

( @@ -267,6 +272,7 @@ mod tests { deadline: Fr254::from(0u64), swap_side: Fr254::from(0u64), proof: MockProvingEngine::default_proof(), + receipt_token: None, } } @@ -304,7 +310,7 @@ mod tests { .and(warp::body::json()) .and_then(|transaction: ClientTransaction| async move { handle_client_transaction_with(transaction, |_tx| async { - Err(warp::reject::custom( + Err::, _>(warp::reject::custom( ProposerRejection::ClientTransactionFailed, )) }) diff --git a/nightfall_proposer/src/drivers/rest/mod.rs b/nightfall_proposer/src/drivers/rest/mod.rs index 901adce1..84372599 100644 --- a/nightfall_proposer/src/drivers/rest/mod.rs +++ b/nightfall_proposer/src/drivers/rest/mod.rs @@ -4,10 +4,14 @@ use crate::drivers::rest::{ client_transactions::{cancel_swap_request, client_transaction}, proposers::rotate_proposer, synchronisation::synchronisation, + transfer_receipts::{ + create_transfer_receipt, get_transfer_receipt, get_transfer_receipt_status, + }, }; use block_assembly::{ get_block_assembly_status_route, pause_block_assembly, resume_block_assembly, }; +use configuration::settings::WalletRole; use lib::{ health_check::health_route, nf_client_proof::{Proof, ProvingEngine}, @@ -26,6 +30,7 @@ pub mod block_data; pub mod client_transactions; pub mod proposers; pub mod synchronisation; +pub mod transfer_receipts; pub fn routes() -> impl Filter + Clone where @@ -40,12 +45,15 @@ where .or(add_proposer()) .or(remove_proposer()) .or(withdraw()) - .or(certification_validation_request()) + .or(certification_validation_request(WalletRole::Proposer)) .or(keys_validation_request()) .or(synchronisation()) .or(pause_block_assembly()) .or(resume_block_assembly()) .or(get_block_assembly_status_route()) + .or(create_transfer_receipt::

()) + .or(get_transfer_receipt_status::

()) + .or(get_transfer_receipt::

()) .recover(handle_rejection) } @@ -100,6 +108,26 @@ async fn handle_rejection(err: Rejection) -> Result Ok(reply::with_status( + "Transfer receipt creation failed", + warp::http::StatusCode::BAD_REQUEST, + )), + ProposerRejection::TransferReceiptNotFound => Ok(reply::with_status( + "Transfer receipt not found", + warp::http::StatusCode::NOT_FOUND, + )), + ProposerRejection::TransferReceiptTxNotFound => Ok(reply::with_status( + "Transfer receipt transaction not found", + warp::http::StatusCode::BAD_REQUEST, + )), + ProposerRejection::TransferReceiptConflict => Ok(reply::with_status( + "Transfer receipt already exists for this transaction", + warp::http::StatusCode::CONFLICT, + )), + ProposerRejection::TransferReceiptUnauthorized => Ok(reply::with_status( + "Invalid or missing receipt token for this transaction", + warp::http::StatusCode::UNAUTHORIZED, + )), } } else { Ok(reply::with_status( diff --git a/nightfall_proposer/src/drivers/rest/proposers.rs b/nightfall_proposer/src/drivers/rest/proposers.rs index afcd4fd4..e1ad9989 100644 --- a/nightfall_proposer/src/drivers/rest/proposers.rs +++ b/nightfall_proposer/src/drivers/rest/proposers.rs @@ -22,11 +22,10 @@ pub fn rotate_proposer() -> impl Filter Result { handle_rotate_proposer_with(|| async { // get a ManageProposers instance - let blockchain_client = get_blockchain_client_connection() - .await - .read() - .await - .get_client(); + let read_connection = get_blockchain_client_connection().await.read().await; + let blockchain_client = read_connection.get_client(); + let caller = read_connection.get_address(); + let wallet = read_connection.get_wallet_type().clone(); let verified = VerifiedContracts::resolve_and_verify_contract(blockchain_client.root(), get_addresses()) .await @@ -45,15 +44,8 @@ async fn handle_rotate_proposer() -> Result { warn!("Failed to fetch proposer count before rotation"); } } - // rotate the proposer - let signer = get_blockchain_client_connection() - .await - .read() - .await - .get_signer(); - let nonce = blockchain_client - .get_transaction_count(signer.address()) + .get_transaction_count(caller) .await .map_err(|e| { warn!("Failed to generate nonce during proposer rotation: {e}"); @@ -76,8 +68,8 @@ async fn handle_rotate_proposer() -> Result { .gas(gas_limit) .max_fee_per_gas(max_fee_per_gas) .max_priority_fee_per_gas(max_priority_fee_per_gas) - .chain_id(get_settings().network.chain_id) // Linea testnet chain ID - .build_raw_transaction((*signer).clone()) + .chain_id(get_settings().network.chain_id) + .build_raw_transaction(wallet) .await .map_err(|e| { warn!("Failed to build rotate_proposer transaction: {e}"); @@ -134,7 +126,7 @@ async fn handle_add_proposer(url: String) -> Result let read_connection = get_blockchain_client_connection().await.read().await; let blockchain_client = read_connection.get_client(); let caller = read_connection.get_address(); - let signer = read_connection.get_signer(); + let wallet = read_connection.get_wallet_type().clone(); let client = blockchain_client.root(); let verified = VerifiedContracts::resolve_and_verify_contract(client, get_addresses()) .await @@ -166,8 +158,8 @@ async fn handle_add_proposer(url: String) -> Result .gas(gas_limit) .max_fee_per_gas(max_fee_per_gas) .max_priority_fee_per_gas(max_priority_fee_per_gas) - .chain_id(get_settings().network.chain_id) // Linea testnet chain ID - .build_raw_transaction((*signer).clone()) + .chain_id(get_settings().network.chain_id) + .build_raw_transaction(wallet) .await .map_err(|e| { warn!("{e}"); @@ -210,7 +202,7 @@ async fn handle_remove_proposer() -> Result { let read_connection = get_blockchain_client_connection().await.read().await; let blockchain_client = read_connection.get_client(); let signer_address = read_connection.get_address(); - let signer = read_connection.get_signer(); + let wallet = read_connection.get_wallet_type().clone(); let client = blockchain_client.root(); let verified = VerifiedContracts::resolve_and_verify_contract(client, get_addresses()) .await @@ -262,8 +254,8 @@ async fn handle_remove_proposer() -> Result { .gas(gas_limit) .max_fee_per_gas(max_fee_per_gas) .max_priority_fee_per_gas(max_priority_fee_per_gas) - .chain_id(get_settings().network.chain_id) // Linea testnet chain ID - .build_raw_transaction((*signer).clone()) + .chain_id(get_settings().network.chain_id) + .build_raw_transaction(wallet) .await .map_err(|e| { warn!("{e}"); @@ -323,7 +315,7 @@ async fn handle_withdraw(amount: u64) -> Result { let read_connection = get_blockchain_client_connection().await.read().await; let blockchain_client = read_connection.get_client(); let caller = read_connection.get_address(); - let signer = read_connection.get_signer(); + let wallet = read_connection.get_wallet_type().clone(); let verified = VerifiedContracts::resolve_and_verify_contract(blockchain_client.root(), get_addresses()) .await @@ -354,8 +346,8 @@ async fn handle_withdraw(amount: u64) -> Result { .gas(gas_limit) .max_fee_per_gas(max_fee_per_gas) .max_priority_fee_per_gas(max_priority_fee_per_gas) - .chain_id(get_settings().network.chain_id) // Linea testnet chain ID - .build_raw_transaction((*signer).clone()) + .chain_id(get_settings().network.chain_id) + .build_raw_transaction(wallet) .await .map_err(|e| { warn!("{e}"); diff --git a/nightfall_proposer/src/drivers/rest/synchronisation.rs b/nightfall_proposer/src/drivers/rest/synchronisation.rs index 1d170b7e..7172e0e1 100644 --- a/nightfall_proposer/src/drivers/rest/synchronisation.rs +++ b/nightfall_proposer/src/drivers/rest/synchronisation.rs @@ -4,7 +4,9 @@ use warp::{ Filter, }; -use crate::drivers::blockchain::nightfall_event_listener::get_synchronisation_status; +use crate::drivers::blockchain::nightfall_event_listener::{ + get_synchronisation_status, is_listener_replay_catch_up_pending, +}; pub fn synchronisation( ) -> impl Filter + Clone { @@ -18,7 +20,8 @@ pub async fn handle_synchronisation() -> Result { .await .read() .await - .is_synchronised(); + .is_synchronised() + && !is_listener_replay_catch_up_pending(); if synchronised { Ok(reply::with_status( "Synchronised", @@ -35,12 +38,15 @@ pub async fn handle_synchronisation() -> Result { #[cfg(test)] mod tests { use super::*; - use crate::drivers::blockchain::nightfall_event_listener::get_synchronisation_status; + use crate::drivers::blockchain::nightfall_event_listener::{ + get_synchronisation_status, set_listener_replay_catch_up_pending, + }; use warp::http::StatusCode; #[tokio::test] async fn test_synchronisation_route_reports_both_states() { let filter = synchronisation(); + set_listener_replay_catch_up_pending(false); get_synchronisation_status() .await @@ -58,6 +64,19 @@ mod tests { "Synchronised" ); + set_listener_replay_catch_up_pending(true); + let catch_up_pending = warp::test::request() + .method("GET") + .path("/v1/synchronisation") + .reply(&filter) + .await; + assert_eq!(catch_up_pending.status(), StatusCode::OK); + assert_eq!( + std::str::from_utf8(catch_up_pending.body()).unwrap(), + "Not synchronised" + ); + + set_listener_replay_catch_up_pending(false); get_synchronisation_status() .await .write() diff --git a/nightfall_proposer/src/drivers/rest/transfer_receipts.rs b/nightfall_proposer/src/drivers/rest/transfer_receipts.rs new file mode 100644 index 00000000..b91280ab --- /dev/null +++ b/nightfall_proposer/src/drivers/rest/transfer_receipts.rs @@ -0,0 +1,397 @@ +use crate::{ + domain::{ + entities::{TransferReceipt, TransferReceiptStatus, TxHashBytes}, + error::ProposerRejection, + }, + initialisation::get_db_connection, + ports::db::{TransactionsDB, TransferReceiptDB, TransferReceiptStoreError}, +}; +use hmac::{Hmac, Mac}; +use lib::nf_client_proof::Proof; +use serde::{Deserialize, Serialize}; +use sha2::Sha256; +use std::time::{SystemTime, UNIX_EPOCH}; +use warp::{path, Filter, Rejection, Reply}; + +const MAX_RECEIPT_ID_LEN: usize = 64; +const CIPHERTEXT_V1_HEX_LEN: usize = 576; +const SUPPORTED_VERSION: u8 = 1; + +#[derive(Debug, PartialEq, Eq)] +enum ReceiptValidationError { + InvalidTxHash, + UnsupportedVersion, + InvalidCiphertext, +} + +impl ReceiptValidationError { + fn into_rejection(self) -> Rejection { + warp::reject::custom(ProposerRejection::TransferReceiptCreationFailed) + } +} + +#[derive(Debug, Deserialize)] +pub struct CreateTransferReceiptRequest { + pub tx_hash: String, + pub ciphertext: String, + pub version: Option, + pub receipt_token: String, +} + +#[derive(Debug, Serialize)] +pub struct CreateTransferReceiptResponse { + pub receipt_id: String, + pub status: TransferReceiptStatus, +} + +#[derive(Debug, Serialize)] +pub struct TransferReceiptStatusResponse { + pub receipt_id: String, + pub status: TransferReceiptStatus, +} + +pub fn create_transfer_receipt( +) -> impl Filter + Clone { + path!("v1" / "transfer-receipts") + .and(warp::post()) + .and(warp::body::content_length_limit(1024 * 64)) + .and(warp::body::json()) + .and_then(handle_create_transfer_receipt::

) +} + +pub fn get_transfer_receipt_status( +) -> impl Filter + Clone { + path!("v1" / "transfer-receipts" / String / "status") + .and(warp::get()) + .and(warp::path::end()) + .and_then(handle_get_transfer_receipt_status::

) +} + +pub fn get_transfer_receipt( +) -> impl Filter + Clone { + path!("v1" / "transfer-receipts" / String) + .and(warp::get()) + .and(warp::path::end()) + .and_then(handle_get_transfer_receipt::

) +} + +async fn handle_create_transfer_receipt( + request: CreateTransferReceiptRequest, +) -> Result { + let tx_hash = validate_tx_hash(request.tx_hash.trim()).map_err(|e| e.into_rejection())?; + let version = request.version.unwrap_or(SUPPORTED_VERSION); + let db = get_db_connection().await; + + // Verify the caller has authority to attach a receipt to this transaction + // BEFORE any idempotent early-return. Without this guard, a caller who + // knows the ciphertext can replay the endpoint with a wrong receipt_token + // and receive a 200 OK — bypassing auth entirely. + let tx_hash_vec = tx_hash.as_u32_vec(); + let tx_meta = >::get_transaction(db, &tx_hash_vec) + .await + .ok_or_else(|| warp::reject::custom(ProposerRejection::TransferReceiptTxNotFound))?; + + let stored_token = tx_meta.receipt_token.as_deref().unwrap_or(""); + if stored_token.is_empty() + || !constant_time_eq(stored_token.as_bytes(), request.receipt_token.as_bytes()) + { + return Err(warp::reject::custom( + ProposerRejection::TransferReceiptUnauthorized, + )); + } + + if let Some(mut existing) = db.get_transfer_receipt_by_tx_hash(&tx_hash).await { + if existing.ciphertext != request.ciphertext || existing.version != version { + return Err(warp::reject::custom( + ProposerRejection::TransferReceiptConflict, + )); + } + + refresh_status::

(db, &mut existing).await; + + let response = CreateTransferReceiptResponse { + receipt_id: existing.receipt_id.clone(), + status: existing.status, + }; + + return Ok(warp::reply::with_status( + warp::reply::json(&response), + warp::http::StatusCode::OK, + )); + } + + validate_new_receipt_fields(version, &request.ciphertext).map_err(|e| e.into_rejection())?; + + let now = unix_now() as i64; + let status = derive_status(tx_meta.lifecycle.block_l2()); + let receipt_id = derive_receipt_id(&request.receipt_token, &tx_hash); + let receipt = TransferReceipt { + receipt_id: receipt_id.clone(), + tx_hash, + ciphertext: request.ciphertext.clone(), + version, + status: status.clone(), + created_at_unix: now, + updated_at_unix: now, + }; + + match db.store_transfer_receipt(receipt).await { + Ok(()) => { + let response = CreateTransferReceiptResponse { + receipt_id: receipt_id.clone(), + status, + }; + + Ok(warp::reply::with_status( + warp::reply::json(&response), + warp::http::StatusCode::CREATED, + )) + } + Err(TransferReceiptStoreError::DuplicateKey) => { + if let Some(mut existing) = db.get_transfer_receipt_by_tx_hash(&tx_hash).await { + if existing.ciphertext != request.ciphertext || existing.version != version { + return Err(warp::reject::custom( + ProposerRejection::TransferReceiptConflict, + )); + } + + refresh_status::

(db, &mut existing).await; + + let response = CreateTransferReceiptResponse { + receipt_id: existing.receipt_id.clone(), + status: existing.status, + }; + + return Ok(warp::reply::with_status( + warp::reply::json(&response), + warp::http::StatusCode::OK, + )); + } + + Err(warp::reject::custom( + ProposerRejection::TransferReceiptCreationFailed, + )) + } + Err(TransferReceiptStoreError::Other(_)) => Err(warp::reject::custom( + ProposerRejection::TransferReceiptCreationFailed, + )), + } +} + +async fn handle_get_transfer_receipt( + receipt_id: String, +) -> Result { + if !is_valid_receipt_id(&receipt_id) { + return Err(warp::reject::custom( + ProposerRejection::TransferReceiptNotFound, + )); + } + + let db = get_db_connection().await; + let mut receipt = db + .get_transfer_receipt(&receipt_id) + .await + .ok_or_else(|| warp::reject::custom(ProposerRejection::TransferReceiptNotFound))?; + + refresh_status::

(db, &mut receipt).await; + + Ok(warp::reply::json(&receipt)) +} + +async fn handle_get_transfer_receipt_status( + receipt_id: String, +) -> Result { + if !is_valid_receipt_id(&receipt_id) { + return Err(warp::reject::custom( + ProposerRejection::TransferReceiptNotFound, + )); + } + + let db = get_db_connection().await; + let mut receipt = db + .get_transfer_receipt(&receipt_id) + .await + .ok_or_else(|| warp::reject::custom(ProposerRejection::TransferReceiptNotFound))?; + + refresh_status::

(db, &mut receipt).await; + + Ok(warp::reply::json(&TransferReceiptStatusResponse { + receipt_id: receipt.receipt_id, + status: receipt.status, + })) +} + +async fn refresh_status(db: &mongodb::Client, receipt: &mut TransferReceipt) { + let tx_hash = receipt.tx_hash.as_u32_vec(); + if let Some(tx_meta) = + >::get_transaction(db, &tx_hash).await + { + let status = derive_status(tx_meta.lifecycle.block_l2()); + if status != receipt.status { + let updated_at_unix = unix_now() as i64; + if db + .set_transfer_receipt_status(&receipt.receipt_id, status.clone(), updated_at_unix) + .await + .is_some() + { + receipt.status = status; + receipt.updated_at_unix = updated_at_unix; + } else { + log::warn!( + "Failed to persist receipt status refresh for id={}", + receipt.receipt_id + ); + } + } + } +} + +fn is_valid_receipt_id(id: &str) -> bool { + id.len() == MAX_RECEIPT_ID_LEN + && id + .chars() + .all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase()) +} + +fn derive_status(block_l2: Option) -> TransferReceiptStatus { + if block_l2.is_some() { + TransferReceiptStatus::IncludedL2 + } else { + TransferReceiptStatus::Pending + } +} + +fn unix_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +/// Derives a deterministic, unlinkable receipt ID using HMAC-SHA256. +/// +/// `receipt_id = HMAC-SHA256(key = receipt_token, data = DOMAIN || tx_hash_hex)` +/// +/// The same sender-generated `receipt_token` is stored by every proposer that +/// accepted the transaction, so this derivation always produces the same 64-char +/// hex ID on every proposer — allowing the receiver to query any of them without +/// needing to learn which proposer was used. +/// +/// The receipt_token is a secret known only to the sender and the proposer(s), +/// so the receipt_id cannot be linked to any on-chain data by an outside observer. +fn derive_receipt_id(receipt_token: &str, tx_hash: &TxHashBytes) -> String { + const DOMAIN: &str = "nf4:transfer-receipt:v1:"; + let mut mac = Hmac::::new_from_slice(receipt_token.as_bytes()) + .expect("HMAC accepts any key length"); + mac.update(DOMAIN.as_bytes()); + mac.update(tx_hash.as_hex().as_bytes()); + hex::encode(mac.finalize().into_bytes()) +} + +/// Constant-time byte comparison to prevent timing side-channels on token validation. +fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { + if a.len() != b.len() { + return false; + } + a.iter() + .zip(b.iter()) + .fold(0u8, |acc, (x, y)| acc | (x ^ y)) + == 0 +} + +fn validate_tx_hash(tx_hash: &str) -> Result { + TxHashBytes::from_hex(tx_hash).ok_or(ReceiptValidationError::InvalidTxHash) +} + +fn validate_new_receipt_fields( + version: u8, + ciphertext: &str, +) -> Result<(), ReceiptValidationError> { + if version != SUPPORTED_VERSION { + return Err(ReceiptValidationError::UnsupportedVersion); + } + + if !is_valid_receipt_ciphertext(version, ciphertext) { + return Err(ReceiptValidationError::InvalidCiphertext); + } + + Ok(()) +} + +fn is_valid_receipt_ciphertext(version: u8, ciphertext: &str) -> bool { + version == SUPPORTED_VERSION + && ciphertext.len() == CIPHERTEXT_V1_HEX_LEN + && ciphertext.chars().all(|c| c.is_ascii_hexdigit()) + && hex::decode(ciphertext).is_ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn derive_status_pending_if_no_block() { + assert_eq!(derive_status(None), TransferReceiptStatus::Pending); + } + + #[test] + fn derive_status_included_if_block() { + assert_eq!(derive_status(Some(1)), TransferReceiptStatus::IncludedL2); + } + + #[test] + fn ciphertext_v1_matches_receipt_format() { + let ciphertext = "ab".repeat(288); + assert_eq!(ciphertext.len(), 576); + } + + #[test] + fn valid_v1_receipt_ciphertext_accepted() { + assert!(is_valid_receipt_ciphertext(1, &"ab".repeat(288))); + } + + #[test] + fn extended_v1_ciphertext_rejected() { + assert!(!is_valid_receipt_ciphertext(1, &"ab".repeat(400))); + } + + #[test] + fn short_ciphertext_rejected() { + assert!(!is_valid_receipt_ciphertext(1, &"ab".repeat(100))); + } + + #[test] + fn oversized_ciphertext_rejected() { + assert!(!is_valid_receipt_ciphertext(1, &"ab".repeat(1025))); + } + + #[test] + fn odd_length_ciphertext_rejected() { + assert!(!is_valid_receipt_ciphertext(1, &"a".repeat(577))); + } + + #[test] + fn non_hex_ciphertext_rejected() { + assert!(!is_valid_receipt_ciphertext(1, &"zz".repeat(288))); + } + + #[test] + fn unsupported_ciphertext_version_rejected() { + assert!(!is_valid_receipt_ciphertext(2, &"ab".repeat(288))); + } + + #[test] + fn valid_receipt_id_accepted() { + assert!(is_valid_receipt_id(&"a".repeat(64))); + } + + #[test] + fn receipt_id_rejects_uppercase() { + assert!(!is_valid_receipt_id(&format!("{}{}", "a".repeat(63), "A"))); + } + + #[test] + fn receipt_id_rejects_wrong_length() { + assert!(!is_valid_receipt_id(&"a".repeat(63))); + } +} diff --git a/nightfall_proposer/src/initialisation/bootstrap.rs b/nightfall_proposer/src/initialisation/bootstrap.rs new file mode 100644 index 00000000..f5f0b46b --- /dev/null +++ b/nightfall_proposer/src/initialisation/bootstrap.rs @@ -0,0 +1,211 @@ +use super::{ + cleanup::resume_startup_replay_reset_if_needed, + consistency::{ + ahead_of_chain_error, missing_stored_block_error, + startup_legacy_transaction_migration_error, validate_startup_proposer_state_consistency, + validate_sync_state_against_block, validate_tree_state_against_sync_state, + }, + db::ensure_proposer_db_initialized, + get_raw_db_connection, + runtime_listener::{get_listener_start_block, set_runtime_listener_resume_cursor}, +}; +use crate::{ + driven::db::{ + client_transaction_state::backfill_legacy_client_transaction_lifecycle, + snapshot::recover_from_restore_journal, + }, + driven::nightfall_event::get_expected_layer2_blocknumber, + drivers::blockchain::nightfall_event_listener::get_synchronisation_status, + ports::{ + contracts::NightfallContract, + db::{BlockStorageDB, RestoreJournalDB, StartupReplayResetMarkerDB, SyncStateDB}, + }, + services::snapshot_scheduler::initialize_snapshot_scheduler_state, +}; +use alloy::primitives::I256; +use log::{info, warn}; +use mongodb::Client; + +pub(super) async fn backfill_legacy_client_transactions_at_startup( + client: &Client, +) -> Result<(), String> { + let stats = backfill_legacy_client_transaction_lifecycle(client) + .await + .map_err(|error| startup_legacy_transaction_migration_error(&error))?; + + if stats.total_backfilled() > 0 { + warn!( + "Backfilled {} legacy proposer client transaction lifecycle field(s) at startup \ + (mempool={}, selected={}, included={}, cancelled={}, dropped={})", + stats.total_backfilled(), + stats.mempool_backfilled, + stats.selected_backfilled, + stats.included_backfilled, + stats.cancelled_backfilled, + stats.dropped_backfilled + ); + } + + Ok(()) +} + +pub(super) async fn bootstrap_proposer_startup_state_with_db( + db: &Client, + initialize_snapshot_scheduler: bool, +) -> Result<(), String> +where + N: NightfallContract, +{ + backfill_legacy_client_transactions_at_startup(db).await?; + + if let Err(error) = recover_from_restore_journal(db).await { + if db.get_restore_journal().await.is_none() { + warn!( + "Proposer restore recovery completed by rolling back incomplete restore state before bootstrap: {error}" + ); + } else { + return Err(format!( + "Proposer restore recovery failed before bootstrap: {error}" + )); + } + } + if resume_startup_replay_reset_if_needed(db).await? { + warn!( + "Proposer startup resumed an interrupted startup replay reset before bootstrap validation" + ); + } + if initialize_snapshot_scheduler { + if let Err(error) = initialize_snapshot_scheduler_state().await { + warn!("Could not initialize proposer snapshot scheduler state: {error}"); + } + } + + validate_startup_proposer_state_consistency(db).await?; + + let onchain_next_block_i256 = N::get_current_layer2_blocknumber() + .await + .map_err(|e| format!("Could not fetch current L2 block number: {e}"))?; + + if onchain_next_block_i256 < I256::ZERO { + return Err(format!( + "Nightfall returned negative current L2 block number {onchain_next_block_i256}" + )); + } + + let onchain_next_block: u64 = onchain_next_block_i256 + .try_into() + .map_err(|_| "Current L2 block number does not fit into u64".to_string())?; + + let mut expected_block_number = get_expected_layer2_blocknumber().await.write().await; + let mut sync_status = get_synchronisation_status().await.write().await; + let mut listener_start_block = get_listener_start_block().await.write().await; + + *listener_start_block = configuration::settings::get_settings().genesis_block; + + match db.get_sync_state().await { + Some(sync_state) => { + let stored_block = db + .get_block_by_number(sync_state.last_applied_l2_block) + .await + .ok_or_else(|| missing_stored_block_error(sync_state.last_applied_l2_block))?; + + validate_sync_state_against_block(&sync_state, &stored_block)?; + validate_tree_state_against_sync_state(db, Some(&sync_state), Some(&stored_block)) + .await?; + + let next_expected_block = + sync_state + .last_applied_l2_block + .checked_add(1) + .ok_or_else(|| { + "last_applied_l2_block overflowed when computing next expected block" + .to_string() + })?; + + if next_expected_block > onchain_next_block { + return Err(ahead_of_chain_error( + next_expected_block, + onchain_next_block, + )); + } + + *expected_block_number = I256::try_from(next_expected_block) + .map_err(|_| "Next expected L2 block number does not fit into I256".to_string())?; + + let sync_state_l1_block_number: usize = + sync_state.l1_ref.block_number.try_into().map_err(|_| { + format!( + "sync_state L1 block number {} does not fit into usize", + sync_state.l1_ref.block_number + ) + })?; + *listener_start_block = sync_state_l1_block_number; + set_runtime_listener_resume_cursor(Some(sync_state.l1_ref.clone())).await; + + sync_status.clear_synchronised(); + if next_expected_block == onchain_next_block { + info!( + "Recovered proposer state at L2 tip {} from sync_state; proposer will remain desynchronised until replay from L1 block {} confirms catch-up", + sync_state.last_applied_l2_block, + sync_state.l1_ref.block_number + ); + } else { + info!( + "Recovered proposer state at L2 block {}, behind chain tip {}; listener replay will start from L1 block {}", + sync_state.last_applied_l2_block, + onchain_next_block.saturating_sub(1), + sync_state.l1_ref.block_number + ); + } + } + None => { + set_runtime_listener_resume_cursor(None).await; + validate_tree_state_against_sync_state(db, None, None).await?; + *expected_block_number = I256::ZERO; + if onchain_next_block == 0 { + sync_status.set_synchronised(); + info!("No proposer sync_state found and chain is at genesis; starting fresh"); + } else { + sync_status.clear_synchronised(); + warn!( + "No proposer sync_state found while chain is already at L2 block {}; replay is required", + onchain_next_block.saturating_sub(1) + ); + } + } + } + + Ok(()) +} + +pub async fn bootstrap_proposer_startup_state() -> Result<(), String> +where + N: NightfallContract, +{ + let db = get_raw_db_connection().await; + recover_then_initialize_proposer_db(db).await?; + bootstrap_proposer_startup_state_with_db::(db, true).await +} + +pub(super) async fn recover_then_initialize_proposer_db(db: &Client) -> Result<(), String> { + backfill_legacy_client_transactions_at_startup(db).await?; + if let Err(error) = recover_from_restore_journal(db).await { + if db.get_restore_journal().await.is_none() { + warn!( + "Proposer restore recovery completed by rolling back incomplete restore state before startup initialisation: {error}" + ); + } else { + return Err(format!( + "Proposer restore recovery failed before startup initialisation: {error}" + )); + } + } + if let Some(marker) = db.get_startup_replay_reset_marker().await { + warn!( + "Detected startup replay reset marker in phase {:?} during proposer startup initialisation; bootstrap will resume reset before validation", + marker.phase + ); + } + ensure_proposer_db_initialized(db).await; + Ok(()) +} diff --git a/nightfall_proposer/src/initialisation/cleanup.rs b/nightfall_proposer/src/initialisation/cleanup.rs new file mode 100644 index 00000000..83f510ba --- /dev/null +++ b/nightfall_proposer/src/initialisation/cleanup.rs @@ -0,0 +1,520 @@ +use super::consistency::startup_local_state_cleanup_error; +use crate::{ + domain::entities::{StartupReplayResetMarker, StartupReplayResetPhase, SyncState}, + driven::db::{ + client_transaction_state::{ + restore_all_included_transactions_to_mempool, + restore_all_selected_transactions_to_mempool, + restore_selected_transactions_to_mempool_after_block, + }, + mongo_db::{StoredBlock, DB, DEPOSIT_COLLECTION, PROPOSED_BLOCKS_COLLECTION}, + }, + ports::{ + db::{BlockStorageDB, PendingBlockDB, StartupReplayResetMarkerDB, SyncStateDB}, + trees::{CommitmentTree, HistoricRootTree, NullifierTree}, + }, +}; +use ark_bn254::Fr as Fr254; +use lib::merkle_trees::trees::{MutableTree, TreeMetadata}; +use mongodb::{ + bson::{doc, DateTime, Document}, + Client, +}; +use std::collections::HashSet; + +#[cfg(test)] +use std::sync::{Mutex, OnceLock}; + +#[cfg(test)] +static ENABLED_STARTUP_REPLAY_RESET_FAILPOINTS: OnceLock>> = OnceLock::new(); + +#[cfg(test)] +fn enabled_startup_replay_reset_failpoints() -> &'static Mutex> { + ENABLED_STARTUP_REPLAY_RESET_FAILPOINTS.get_or_init(|| Mutex::new(HashSet::new())) +} + +#[cfg(test)] +fn maybe_fail_startup_replay_reset(name: &str) -> Result<(), String> { + if enabled_startup_replay_reset_failpoints() + .lock() + .expect("startup replay reset failpoint lock poisoned") + .contains(name) + { + return Err(startup_local_state_cleanup_error(&format!( + "Simulated startup replay reset failure at {name}" + ))); + } + + Ok(()) +} + +#[cfg(test)] +pub(crate) struct TestStartupReplayResetFailpointGuard { + name: String, +} + +#[cfg(test)] +impl TestStartupReplayResetFailpointGuard { + pub(crate) fn enable(name: &str) -> Self { + enabled_startup_replay_reset_failpoints() + .lock() + .expect("startup replay reset failpoint lock poisoned") + .insert(name.to_string()); + Self { + name: name.to_string(), + } + } +} + +#[cfg(test)] +impl Drop for TestStartupReplayResetFailpointGuard { + fn drop(&mut self) { + enabled_startup_replay_reset_failpoints() + .lock() + .expect("startup replay reset failpoint lock poisoned") + .remove(&self.name); + } +} + +pub(super) async fn tree_sub_tree_count(client: &Client, tree_name: &str) -> Result { + let metadata_collection_name = format!("{tree_name}_metadata"); + let metadata = client + .database(DB) + .collection::>(&metadata_collection_name) + .find_one(mongodb::bson::doc! {}) + .await + .map_err(|error| format!("Could not read proposer tree metadata for {tree_name}: {error}"))? + .ok_or_else(|| { + super::consistency::tree_state_inconsistency_error(&format!( + "tree metadata for {tree_name} is missing." + )) + })?; + Ok(metadata.sub_tree_count) +} + +async fn collection_exists(client: &Client, collection_name: &str) -> Result { + let names = client + .database(DB) + .list_collection_names() + .await + .map_err(|error| format!("Could not list proposer collections: {error}"))?; + Ok(names.iter().any(|name| name == collection_name)) +} + +pub(super) async fn indexed_leaves_count(client: &Client, tree_name: &str) -> Result { + let collection_name = format!("{tree_name}_indexed_leaves"); + if !collection_exists(client, &collection_name).await? { + return Err(super::consistency::tree_state_inconsistency_error( + &format!("indexed leaves collection for {tree_name} is missing."), + )); + } + + client + .database(DB) + .collection::(&collection_name) + .count_documents(doc! {}) + .await + .map_err(|error| { + format!("Could not count proposer indexed leaves collection {collection_name}: {error}") + }) +} + +pub(super) async fn highest_stored_block_number(client: &Client) -> Result, String> { + client + .database(DB) + .collection::(PROPOSED_BLOCKS_COLLECTION) + .find_one(mongodb::bson::doc! {}) + .sort(mongodb::bson::doc! { "layer2_block_number": -1_i32 }) + .await + .map_err(|error| format!("Could not inspect proposer StoredBlocks: {error}")) + .map(|maybe_block| maybe_block.map(|block| block.layer2_block_number)) +} + +pub(super) async fn reserved_deposit_count(client: &Client) -> Result { + client + .database(DB) + .collection::(DEPOSIT_COLLECTION) + .count_documents(mongodb::bson::doc! { "reserved": true }) + .await + .map_err(|error| format!("Could not inspect proposer Deposits reservations: {error}")) +} + +pub(crate) async fn clear_all_reserved_deposits(client: &Client) -> Result { + client + .database(DB) + .collection::(DEPOSIT_COLLECTION) + .update_many( + doc! { "reserved": true }, + doc! { "$set": { "reserved": false } }, + ) + .await + .map(|result| result.modified_count) + .map_err(|error| format!("Could not clear proposer Deposits reservations: {error}")) +} + +async fn pending_block_numbers(client: &Client) -> Result, String> { + client + .get_all_pending_blocks() + .await + .ok_or_else(|| { + startup_local_state_cleanup_error( + "Could not inspect proposer PendingBlocks during startup cleanup.", + ) + }) + .map(|pending_blocks| { + pending_blocks + .into_iter() + .map(|pending_block| pending_block.layer2_block_number) + .collect() + }) +} + +fn pending_block_numbers_from_pending_blocks( + pending_blocks: &[crate::domain::entities::PendingBlock], +) -> HashSet { + pending_blocks + .iter() + .map(|pending_block| pending_block.layer2_block_number) + .collect() +} + +fn speculative_blocks_without_sync_state_are_cleanup_safe( + stored_blocks: &[StoredBlock], + pending_block_numbers: &HashSet, +) -> bool { + !stored_blocks.is_empty() + && stored_blocks + .iter() + .all(|stored_block| pending_block_numbers.contains(&stored_block.layer2_block_number)) +} + +pub(super) async fn startup_tree_replay_reset_candidate( + client: &Client, + sync_state: Option<&SyncState>, +) -> Result { + if client.get_startup_replay_reset_marker().await.is_some() { + return Ok(true); + } + + let pending_blocks = client.get_all_pending_blocks().await.ok_or_else(|| { + startup_local_state_cleanup_error( + "Could not inspect proposer PendingBlocks while evaluating startup tree recovery.", + ) + })?; + + match sync_state { + Some(sync_state) => { + let has_pending_state_ahead = pending_blocks.iter().any(|pending_block| { + pending_block.layer2_block_number > sync_state.last_applied_l2_block + }); + let has_stored_block_ahead = highest_stored_block_number(client) + .await? + .is_some_and(|highest_block| highest_block > sync_state.last_applied_l2_block); + Ok(has_pending_state_ahead || has_stored_block_ahead) + } + None => { + if !pending_blocks.is_empty() { + return Ok(true); + } + + let stored_blocks = client.get_all_blocks().await.ok_or_else(|| { + startup_local_state_cleanup_error( + "Could not inspect proposer StoredBlocks while evaluating startup tree recovery.", + ) + })?; + if stored_blocks.is_empty() { + return Ok(false); + } + + let pending_block_numbers = pending_block_numbers_from_pending_blocks(&pending_blocks); + Ok(speculative_blocks_without_sync_state_are_cleanup_safe( + &stored_blocks, + &pending_block_numbers, + )) + } + } +} + +async fn reset_trees_for_startup_replay(client: &Client) -> Result<(), String> { + #[cfg(test)] + maybe_fail_startup_replay_reset("reset_commitment_tree_before_drop")?; + + >::reset_mutable_tree( + client, + >::TREE_NAME, + ) + .await + .map_err(|error| { + startup_local_state_cleanup_error(&format!( + "Could not reset proposer commitment tree for startup replay reset: {error}" + )) + })?; + >::new_commitment_tree(client, 29, 3) + .await + .map_err(|error| { + startup_local_state_cleanup_error(&format!( + "Could not reinitialize proposer commitment tree for startup replay reset: {error}" + )) + })?; + + >::reset_mutable_tree( + client, + >::TREE_NAME, + ) + .await + .map_err(|error| { + startup_local_state_cleanup_error(&format!( + "Could not reset proposer historic root tree for startup replay reset: {error}" + )) + })?; + >::new_historic_root_tree(client, 32) + .await + .map_err(|error| { + startup_local_state_cleanup_error(&format!( + "Could not reinitialize proposer historic root tree for startup replay reset: {error}" + )) + })?; + >::append_historic_commitment_root( + client, + &Fr254::from(0u8), + true, + ) + .await + .map_err(|error| { + startup_local_state_cleanup_error(&format!( + "Could not restore zero historic root for startup replay reset: {error}" + )) + })?; + + >::reset_mutable_tree( + client, + >::TREE_NAME, + ) + .await + .map_err(|error| { + startup_local_state_cleanup_error(&format!( + "Could not reset proposer nullifier tree for startup replay reset: {error}" + )) + })?; + let indexed_collection = client + .database(DB) + .collection::("Nullifiers_indexed_leaves"); + if let Err(error) = indexed_collection.drop().await { + if !error.to_string().contains("ns not found") { + return Err(startup_local_state_cleanup_error(&format!( + "Could not reset proposer nullifier indexed leaves for startup replay reset: {error}" + ))); + } + } + >::new_nullifier_tree(client, 29, 3) + .await + .map_err(|error| { + startup_local_state_cleanup_error(&format!( + "Could not reinitialize proposer nullifier tree for startup replay reset: {error}" + )) + })?; + + Ok(()) +} + +async fn advance_startup_replay_reset_marker( + client: &Client, + marker: &StartupReplayResetMarker, + phase: StartupReplayResetPhase, +) -> Result { + let next_marker = marker.with_phase(phase, DateTime::now()); + client + .upsert_startup_replay_reset_marker(&next_marker) + .await + .map_err(|error| { + startup_local_state_cleanup_error(&format!( + "Could not update startup replay reset marker: {error}" + )) + })?; + Ok(next_marker) +} + +pub(crate) async fn begin_startup_replay_reset( + client: &Client, +) -> Result { + if let Some(marker) = client.get_startup_replay_reset_marker().await { + return Ok(marker); + } + + let marker = + StartupReplayResetMarker::new(StartupReplayResetPhase::DbCleanupPending, DateTime::now()); + client + .upsert_startup_replay_reset_marker(&marker) + .await + .map_err(|error| { + startup_local_state_cleanup_error(&format!( + "Could not persist startup replay reset marker: {error}" + )) + })?; + Ok(marker) +} + +async fn apply_startup_replay_db_cleanup( + client: &Client, + marker: &StartupReplayResetMarker, +) -> Result { + let mut session = client.start_session().await.map_err(|error| { + startup_local_state_cleanup_error(&format!( + "Could not start MongoDB session for startup replay reset: {error}" + )) + })?; + let client_for_cleanup = client.clone(); + let next_marker = marker.with_phase(StartupReplayResetPhase::DbCleanupApplied, DateTime::now()); + let marker_for_cleanup = next_marker.clone(); + session + .start_transaction() + .and_run2(async move |session| { + client_for_cleanup + .delete_sync_state_with_session(session) + .await?; + client_for_cleanup + .delete_all_blocks_with_session(session) + .await?; + client_for_cleanup + .delete_all_pending_blocks_with_session(session) + .await?; + client_for_cleanup + .upsert_startup_replay_reset_marker_with_session(&marker_for_cleanup, session) + .await?; + Ok::<(), mongodb::error::Error>(()) + }) + .await + .map_err(|error| { + startup_local_state_cleanup_error(&format!( + "Could not clear proposer canonical state for startup replay reset: {error}" + )) + })?; + + Ok(next_marker) +} + +pub(crate) async fn complete_startup_replay_reset(client: &Client) -> Result<(), String> { + let mut marker = begin_startup_replay_reset(client).await?; + + loop { + marker = match marker.phase { + StartupReplayResetPhase::DbCleanupPending => { + apply_startup_replay_db_cleanup(client, &marker).await? + } + StartupReplayResetPhase::DbCleanupApplied => { + reset_trees_for_startup_replay(client).await?; + advance_startup_replay_reset_marker( + client, + &marker, + StartupReplayResetPhase::AuxiliaryCleanupPending, + ) + .await? + } + StartupReplayResetPhase::AuxiliaryCleanupPending => { + cleanup_non_canonical_startup_state(client, None).await?; + client + .delete_startup_replay_reset_marker() + .await + .map_err(|error| { + startup_local_state_cleanup_error(&format!( + "Could not clear startup replay reset marker after successful recovery: {error}" + )) + })?; + return Ok(()); + } + }; + } +} + +pub(super) async fn resume_startup_replay_reset_if_needed(client: &Client) -> Result { + if client.get_startup_replay_reset_marker().await.is_none() { + return Ok(false); + } + + complete_startup_replay_reset(client).await?; + Ok(true) +} + +pub(super) async fn cleanup_non_canonical_startup_state( + client: &Client, + sync_state: Option<&SyncState>, +) -> Result<(), String> { + match sync_state { + Some(sync_state) => { + let _ = restore_selected_transactions_to_mempool_after_block( + client, + sync_state.last_applied_l2_block, + ) + .await + .map_err(|error| startup_local_state_cleanup_error(&error))?; + } + None => { + let _ = restore_all_selected_transactions_to_mempool(client) + .await + .map_err(|error| startup_local_state_cleanup_error(&error))?; + let _ = restore_all_included_transactions_to_mempool(client) + .await + .map_err(|error| startup_local_state_cleanup_error(&error))?; + } + } + + let _ = clear_all_reserved_deposits(client) + .await + .map_err(|error| startup_local_state_cleanup_error(&error))?; + + match sync_state { + Some(sync_state) => { + let stored_blocks = client.get_all_blocks().await.ok_or_else(|| { + startup_local_state_cleanup_error( + "Could not inspect proposer StoredBlocks during startup cleanup.", + ) + })?; + + for stored_block in stored_blocks.into_iter().filter(|stored_block| { + stored_block.layer2_block_number > sync_state.last_applied_l2_block + }) { + client + .delete_block_by_number(stored_block.layer2_block_number) + .await + .ok_or_else(|| { + startup_local_state_cleanup_error(&format!( + "Could not delete speculative StoredBlock {} during startup cleanup.", + stored_block.layer2_block_number + )) + })?; + } + } + None => { + let stored_blocks = client.get_all_blocks().await.ok_or_else(|| { + startup_local_state_cleanup_error( + "Could not inspect proposer StoredBlocks during startup cleanup.", + ) + })?; + if !stored_blocks.is_empty() { + let pending_block_numbers = pending_block_numbers(client).await?; + if !speculative_blocks_without_sync_state_are_cleanup_safe( + &stored_blocks, + &pending_block_numbers, + ) { + return Err(startup_local_state_cleanup_error( + "StoredBlocks exist without sync_state, but they are not fully explained by persisted PendingBlocks. Manual recovery is required.", + )); + } + } + + let _ = client.delete_all_blocks().await.ok_or_else(|| { + startup_local_state_cleanup_error( + "Could not delete speculative StoredBlocks during startup cleanup.", + ) + })?; + } + } + + let _ = client.delete_all_pending_blocks().await.ok_or_else(|| { + startup_local_state_cleanup_error( + "Could not delete persisted PendingBlocks during startup cleanup.", + ) + })?; + + Ok(()) +} diff --git a/nightfall_proposer/src/initialisation/consistency.rs b/nightfall_proposer/src/initialisation/consistency.rs new file mode 100644 index 00000000..e10618fe --- /dev/null +++ b/nightfall_proposer/src/initialisation/consistency.rs @@ -0,0 +1,400 @@ +use super::cleanup::{ + cleanup_non_canonical_startup_state, complete_startup_replay_reset, + highest_stored_block_number, indexed_leaves_count, reserved_deposit_count, + startup_tree_replay_reset_candidate, tree_sub_tree_count, +}; +use crate::{ + domain::entities::SyncState, + driven::db::{ + client_transaction_state::{ + selected_client_transaction_count, selected_client_transaction_count_after_block, + }, + mongo_db::StoredBlock, + }, + ports::{ + db::{BlockStorageDB, SyncStateDB}, + trees::{CommitmentTree, HistoricRootTree, NullifierTree}, + }, +}; +use ark_bn254::Fr as Fr254; +use lib::hex_conversion::HexConvertible; +use log::warn; +use mongodb::Client; + +pub(super) fn missing_stored_block_error(last_applied_l2_block: u64) -> String { + format!( + "Proposer startup aborted: sync_state references L2 block {last_applied_l2_block}, \ + but StoredBlock at height {last_applied_l2_block} is missing. Local proposer state \ + is inconsistent. Manual intervention is required before restart." + ) +} + +fn fingerprint_mismatch_error( + last_applied_l2_block: u64, + stored_fingerprint: &str, + sync_state_fingerprint: &str, +) -> String { + format!( + "Proposer startup aborted: sync_state references L2 block {last_applied_l2_block}, \ + but StoredBlock fingerprint ({stored_fingerprint}) does not match sync_state \ + fingerprint ({sync_state_fingerprint}). Local proposer state is inconsistent. \ + Manual intervention is required before restart." + ) +} + +fn unsupported_sync_state_schema_error(schema_version: u32) -> String { + format!( + "Proposer startup aborted: sync_state uses unsupported schema version \ + {schema_version}. Manual intervention is required before restart." + ) +} + +pub(super) fn ahead_of_chain_error(next_expected_block: u64, onchain_next_block: u64) -> String { + format!( + "Proposer startup aborted: local proposer state expects next L2 block \ + {next_expected_block}, but chain reports {onchain_next_block}. Local state is ahead \ + of chain. This may indicate an L1 reorg, chain rollback, or dev chain reset. \ + Manual recovery is required before restart." + ) +} + +pub(super) fn tree_state_inconsistency_error(details: &str) -> String { + format!( + "Proposer startup aborted: local proposer tree state is inconsistent with persisted \ + sync_state. {details} Manual recovery is required before restart." + ) +} + +fn stored_blocks_ahead_of_sync_state_error( + last_applied_l2_block: u64, + highest_stored_block: u64, +) -> String { + format!( + "Proposer startup aborted: highest StoredBlock ({highest_stored_block}) is ahead of \ + sync_state-applied block {last_applied_l2_block}. Local proposer state is \ + inconsistent. Manual recovery is required before restart." + ) +} + +fn reserved_deposits_ahead_of_sync_state_error( + last_applied_l2_block: u64, + reserved_deposit_count: u64, +) -> String { + format!( + "Proposer startup aborted: Deposits contains {reserved_deposit_count} reserved \ + selection(s) ahead of sync_state-applied block {last_applied_l2_block}. Local \ + proposer state is inconsistent. Manual recovery is required before restart." + ) +} + +fn selected_transactions_ahead_of_sync_state_error( + last_applied_l2_block: u64, + selected_transaction_count: u64, +) -> String { + format!( + "Proposer startup aborted: ClientTransactions contains {selected_transaction_count} \ + selected transaction(s) ahead of sync_state-applied block {last_applied_l2_block}. \ + Local proposer state is inconsistent. Manual recovery is required before restart." + ) +} + +pub(super) fn startup_local_state_cleanup_error(details: &str) -> String { + format!( + "Proposer startup aborted: non-canonical local proposer state cleanup failed. \ + {details} Manual recovery is required before restart." + ) +} + +pub(super) fn startup_legacy_transaction_migration_error(details: &str) -> String { + format!( + "Proposer startup aborted: legacy ClientTransactions lifecycle backfill failed. \ + {details} Manual recovery is required before restart." + ) +} + +fn expected_historic_root_sub_tree_count(last_applied_l2_block: u64) -> Result { + last_applied_l2_block.checked_add(2).ok_or_else(|| { + tree_state_inconsistency_error( + "sync_state last_applied_l2_block overflowed while validating historic roots.", + ) + }) +} + +pub(super) async fn validate_tree_state_against_sync_state( + client: &Client, + sync_state: Option<&SyncState>, + stored_block: Option<&StoredBlock>, +) -> Result<(), String> { + let commitment_sub_tree_count = tree_sub_tree_count( + client, + >::TREE_NAME, + ) + .await?; + let nullifier_sub_tree_count = + tree_sub_tree_count(client, >::TREE_NAME).await?; + let nullifier_indexed_leaf_count = + indexed_leaves_count(client, >::TREE_NAME).await?; + let historic_root_sub_tree_count = tree_sub_tree_count( + client, + >::TREE_NAME, + ) + .await?; + + match sync_state { + Some(sync_state) => { + let expected_historic_root_sub_tree_count = + expected_historic_root_sub_tree_count(sync_state.last_applied_l2_block)?; + + if historic_root_sub_tree_count <= 1 { + return Err(tree_state_inconsistency_error(&format!( + "sync_state records applied L2 block {}, but the historic root tree only \ + contains the zero leaf.", + sync_state.last_applied_l2_block, + ))); + } + + if historic_root_sub_tree_count > expected_historic_root_sub_tree_count { + return Err(tree_state_inconsistency_error(&format!( + "sync_state records applied L2 block {}, but the historic root tree \ + sub_tree_count {} is ahead of the maximum coherent value {}.", + sync_state.last_applied_l2_block, + historic_root_sub_tree_count, + expected_historic_root_sub_tree_count, + ))); + } + + if historic_root_sub_tree_count < expected_historic_root_sub_tree_count { + return Err(tree_state_inconsistency_error(&format!( + "sync_state records applied L2 block {}, but the historic root tree \ + sub_tree_count {} is behind the required coherent value {}.", + sync_state.last_applied_l2_block, + historic_root_sub_tree_count, + expected_historic_root_sub_tree_count, + ))); + } + + if commitment_sub_tree_count == 0 + && stored_block.is_some_and(|block| !block.commitments.is_empty()) + { + return Err(tree_state_inconsistency_error(&format!( + "sync_state records applied L2 block {}, but the commitment tree is empty \ + while the stored block at that height contains commitments.", + sync_state.last_applied_l2_block + ))); + } + + if nullifier_sub_tree_count == 0 { + return Err(tree_state_inconsistency_error(&format!( + "sync_state records applied L2 block {}, but the nullifier tree metadata \ + reports sub_tree_count 0.", + sync_state.last_applied_l2_block + ))); + } + + if nullifier_indexed_leaf_count == 0 { + return Err(tree_state_inconsistency_error(&format!( + "sync_state records applied L2 block {}, but the nullifier indexed leaves \ + collection is empty.", + sync_state.last_applied_l2_block + ))); + } + } + None => { + if commitment_sub_tree_count > 0 + || historic_root_sub_tree_count > 1 + || nullifier_sub_tree_count != 1 + || nullifier_indexed_leaf_count != 1 + { + return Err(tree_state_inconsistency_error(&format!( + "no proposer sync_state exists, but proposer trees are not empty \ + (commitment_sub_tree_count={commitment_sub_tree_count}, \ + historic_root_sub_tree_count={historic_root_sub_tree_count}, \ + nullifier_sub_tree_count={nullifier_sub_tree_count}, \ + nullifier_indexed_leaf_count={nullifier_indexed_leaf_count})." + ))); + } + } + } + + Ok(()) +} +pub(super) async fn validate_startup_proposer_state_consistency( + client: &Client, +) -> Result<(), String> { + let sync_state = client.get_sync_state().await; + + let initial_tree_validation = match sync_state.as_ref() { + Some(sync_state) => { + let stored_block = client + .get_block_by_number(sync_state.last_applied_l2_block) + .await + .ok_or_else(|| missing_stored_block_error(sync_state.last_applied_l2_block))?; + + validate_sync_state_against_block(sync_state, &stored_block)?; + validate_tree_state_against_sync_state(client, Some(sync_state), Some(&stored_block)) + .await + } + None => validate_tree_state_against_sync_state(client, None, None).await, + }; + + if let Err(error) = initial_tree_validation { + let should_reset_to_replay = error.contains("local proposer tree state is inconsistent") + && startup_tree_replay_reset_candidate(client, sync_state.as_ref()).await?; + if !should_reset_to_replay { + return Err(error); + } + + warn!( + "Startup found proposer tree state inconsistent with persisted sync_state while \ + speculative startup state was still present. Clearing local proposer canonical state \ + and resetting trees so listener replay can rebuild canonically: {error}" + ); + complete_startup_replay_reset(client).await?; + return validate_live_proposer_state_consistency(client).await; + } + + cleanup_non_canonical_startup_state(client, sync_state.as_ref()).await?; + validate_live_proposer_state_consistency(client).await +} + +async fn validate_snapshotted_live_proposer_state_with_sync_state( + client: &Client, + sync_state: Option<&SyncState>, +) -> Result<(), String> { + match sync_state { + Some(sync_state) => { + let stored_block = client + .get_block_by_number(sync_state.last_applied_l2_block) + .await + .ok_or_else(|| missing_stored_block_error(sync_state.last_applied_l2_block))?; + + validate_sync_state_against_block(sync_state, &stored_block)?; + validate_tree_state_against_sync_state(client, Some(sync_state), Some(&stored_block)) + .await?; + + if let Some(highest_stored_block) = highest_stored_block_number(client).await? { + if highest_stored_block > sync_state.last_applied_l2_block { + return Err(stored_blocks_ahead_of_sync_state_error( + sync_state.last_applied_l2_block, + highest_stored_block, + )); + } + } + + Ok(()) + } + None => { + validate_tree_state_against_sync_state(client, None, None).await?; + + if let Some(highest_stored_block) = highest_stored_block_number(client).await? { + return Err(stored_blocks_ahead_of_sync_state_error( + 0, + highest_stored_block, + )); + } + + Ok(()) + } + } +} + +pub(crate) async fn validate_snapshotted_live_proposer_state_consistency( + client: &Client, +) -> Result<(), String> { + let sync_state = client.get_sync_state().await; + validate_snapshotted_live_proposer_state_with_sync_state(client, sync_state.as_ref()).await +} + +pub(crate) async fn validate_live_proposer_state_consistency( + client: &Client, +) -> Result<(), String> { + let sync_state = client.get_sync_state().await; + validate_snapshotted_live_proposer_state_with_sync_state(client, sync_state.as_ref()).await?; + + match sync_state { + Some(sync_state) => { + let reserved_deposit_count = reserved_deposit_count(client).await?; + if reserved_deposit_count > 0 { + return Err(reserved_deposits_ahead_of_sync_state_error( + sync_state.last_applied_l2_block, + reserved_deposit_count, + )); + } + + let selected_transaction_count = selected_client_transaction_count_after_block( + client, + sync_state.last_applied_l2_block, + ) + .await?; + if selected_transaction_count > 0 { + return Err(selected_transactions_ahead_of_sync_state_error( + sync_state.last_applied_l2_block, + selected_transaction_count, + )); + } + + Ok(()) + } + None => { + validate_tree_state_against_sync_state(client, None, None).await?; + + if let Some(highest_stored_block) = highest_stored_block_number(client).await? { + return Err(stored_blocks_ahead_of_sync_state_error( + 0, + highest_stored_block, + )); + } + + let reserved_deposit_count = reserved_deposit_count(client).await?; + if reserved_deposit_count > 0 { + return Err(reserved_deposits_ahead_of_sync_state_error( + 0, + reserved_deposit_count, + )); + } + + let selected_transaction_count = selected_client_transaction_count(client).await?; + if selected_transaction_count > 0 { + return Err(selected_transactions_ahead_of_sync_state_error( + 0, + selected_transaction_count, + )); + } + + Ok(()) + } + } +} + +pub(super) fn validate_sync_state_against_block( + sync_state: &SyncState, + stored_block: &StoredBlock, +) -> Result<(), String> { + let stored_fingerprint = stored_block.hash().to_hex_string(); + if stored_fingerprint != sync_state.fingerprint { + return Err(fingerprint_mismatch_error( + sync_state.last_applied_l2_block, + &stored_fingerprint, + &sync_state.fingerprint, + )); + } + + if sync_state.schema_version != SyncState::SCHEMA_VERSION { + return Err(unsupported_sync_state_schema_error( + sync_state.schema_version, + )); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn expected_historic_root_sub_tree_count_tracks_last_applied_block_exactly() { + assert_eq!(expected_historic_root_sub_tree_count(0).unwrap(), 2); + assert_eq!(expected_historic_root_sub_tree_count(10).unwrap(), 12); + } +} diff --git a/nightfall_proposer/src/initialisation/db.rs b/nightfall_proposer/src/initialisation/db.rs new file mode 100644 index 00000000..36c81526 --- /dev/null +++ b/nightfall_proposer/src/initialisation/db.rs @@ -0,0 +1,81 @@ +use crate::ports::trees::{CommitmentTree, HistoricRootTree, NullifierTree}; +use ark_bn254::Fr as Fr254; +use ark_ff::Zero; +use lib::merkle_trees::trees::MutableTree; +use mongodb::Client; + +pub(super) async fn ensure_commitment_tree_initialized(client: &Client) { + if >::get_root(client) + .await + .is_ok() + { + return; + } + + >::new_commitment_tree(client, 29, 3) + .await + .expect("Could not create commitment tree"); +} + +pub(super) async fn ensure_nullifier_tree_initialized(client: &Client) { + if >::get_root( + client, + >::TREE_NAME, + ) + .await + .is_ok() + { + return; + } + + >::new_nullifier_tree(client, 29, 3) + .await + .expect("Could not create nullifier tree"); +} + +pub(super) async fn ensure_historic_root_tree_initialized(client: &Client) { + let zero_leaf = Fr254::from(0u8); + let root = match >::get_root( + client, + >::TREE_NAME, + ) + .await + { + Ok(root) => root, + Err(_) => { + >::new_historic_root_tree(client, 32) + .await + .expect("Could not create historic root tree"); + Fr254::zero() + } + }; + + let has_zero_leaf = + >::is_historic_root(client, &zero_leaf) + .await + .expect("Could not query historic root tree"); + + if has_zero_leaf { + return; + } + + if !root.is_zero() { + panic!("Historic root tree exists without zero leaf in a non-empty state"); + } + + >::append_historic_commitment_root(client, &zero_leaf, true) + .await + .expect("Couldn't insert zero leaf into the historic root tree"); +} + +pub(super) async fn ensure_proposer_db_initialized(client: &Client) { + ensure_commitment_tree_initialized(client).await; + ensure_historic_root_tree_initialized(client).await; + ensure_nullifier_tree_initialized(client).await; + crate::driven::db::mongo_db::ensure_deposit_indexes(client) + .await + .expect("Could not create deposit indexes"); + crate::driven::db::mongo_db::ensure_transfer_receipt_indexes(client) + .await + .expect("Could not create transfer receipt indexes"); +} diff --git a/nightfall_proposer/src/initialisation/mod.rs b/nightfall_proposer/src/initialisation/mod.rs new file mode 100644 index 00000000..b657c771 --- /dev/null +++ b/nightfall_proposer/src/initialisation/mod.rs @@ -0,0 +1,118 @@ +use super::driven::block_assembler::SmartTrigger; +use crate::{ + driven::block_assembler::BlockAssemblyStatus, + ports::block_assembly_trigger::BlockAssemblyTrigger, +}; +use ark_std::sync::Arc; +use configuration::settings::{get_settings, WalletRole}; +use lib::{ + blockchain_client::BlockchainClientConnection, nf_client_proof::Proof, wallets::LocalWsClient, +}; +use mongodb::Client; +use tokio::sync::{OnceCell, RwLock}; + +mod bootstrap; +mod cleanup; +mod consistency; +mod db; +mod runtime_listener; + +pub use bootstrap::bootstrap_proposer_startup_state; +#[cfg(test)] +pub(crate) use cleanup::TestStartupReplayResetFailpointGuard; +pub(crate) use cleanup::{begin_startup_replay_reset, complete_startup_replay_reset}; +pub(crate) use consistency::{ + validate_live_proposer_state_consistency, validate_snapshotted_live_proposer_state_consistency, +}; +pub use runtime_listener::{ + get_runtime_listener_resume_cursor, get_runtime_listener_start_block, + set_runtime_listener_resume_cursor, set_runtime_listener_start_block, +}; + +use db::ensure_proposer_db_initialized; + +#[cfg(test)] +use bootstrap::{bootstrap_proposer_startup_state_with_db, recover_then_initialize_proposer_db}; + +#[cfg(test)] +mod tests; + +async fn get_raw_db_connection() -> &'static Client { + static RAW_DB_CONNECTION: OnceCell = OnceCell::const_new(); + RAW_DB_CONNECTION + .get_or_init(|| async { + let uri = &get_settings().nightfall_proposer.db_url; + Client::with_uri_str(uri) + .await + .expect("Could not create database connection") + }) + .await +} + +async fn ensure_singleton_proposer_db_initialized(client: &Client) { + static DB_INITIALIZED: OnceCell<()> = OnceCell::const_new(); + DB_INITIALIZED + .get_or_init(|| async { + ensure_proposer_db_initialized(client).await; + }) + .await; +} + +/// This function is used to provide a singleton database connection across the entire application. +pub async fn get_db_connection() -> &'static Client { + let client = get_raw_db_connection().await; + ensure_singleton_proposer_db_initialized(client).await; + client +} + +/// This function is used to provide a singleton blockchain client connection across the entire application. +pub async fn get_blockchain_client_connection() -> &'static RwLock { + static BLOCKCHAIN_CLIENT_CONNECTION: OnceCell> = OnceCell::const_new(); + BLOCKCHAIN_CLIENT_CONNECTION + .get_or_init(|| async { + RwLock::new( + LocalWsClient::try_from_settings(get_settings(), WalletRole::Proposer) + .await + .expect("Could not create blockchain client connection"), + ) + }) + .await +} + +/// This function is used to provide a singleton trigger for block assembly across the entire application. +pub async fn get_block_assembly_trigger( +) -> &'static Arc> { + static BLOCK_ASSEMBLY_TRIGGER: OnceCell>> = + OnceCell::const_new(); + BLOCK_ASSEMBLY_TRIGGER + .get_or_init(|| async { + let status = get_block_assembly_status().await; + let db_client = get_db_connection().await; + let settings = get_settings(); + let initial_interval_secs = settings + .nightfall_proposer + .block_assembly_initial_interval_secs; + let max_wait_secs = settings.nightfall_proposer.block_assembly_max_wait_secs; + let target_fill_ratio = + settings.nightfall_proposer.block_assembly_target_fill_ratio as f32; + + let smart_trigger = SmartTrigger::

::new( + initial_interval_secs, + max_wait_secs, + status, + db_client, + target_fill_ratio, + ); + Arc::new(RwLock::new(smart_trigger)) + as Arc> + }) + .await +} + +/// This function is used to provide a singleton status for the BlockAssemblyTrigger across the entire application. +pub async fn get_block_assembly_status() -> &'static RwLock { + static BLOCK_ASSEMBLY_STATUS: OnceCell> = OnceCell::const_new(); + BLOCK_ASSEMBLY_STATUS + .get_or_init(|| async { RwLock::new(BlockAssemblyStatus::new()) }) + .await +} diff --git a/nightfall_proposer/src/initialisation/runtime_listener.rs b/nightfall_proposer/src/initialisation/runtime_listener.rs new file mode 100644 index 00000000..0a8d28cf --- /dev/null +++ b/nightfall_proposer/src/initialisation/runtime_listener.rs @@ -0,0 +1,33 @@ +use crate::domain::entities::L1Ref; +use configuration::settings::get_settings; +use tokio::sync::{OnceCell, RwLock}; + +pub(super) async fn get_listener_start_block() -> &'static RwLock { + static LISTENER_START_BLOCK: OnceCell> = OnceCell::const_new(); + LISTENER_START_BLOCK + .get_or_init(|| async { RwLock::new(get_settings().genesis_block) }) + .await +} + +pub(super) async fn get_listener_resume_cursor() -> &'static RwLock> { + static LISTENER_RESUME_CURSOR: OnceCell>> = OnceCell::const_new(); + LISTENER_RESUME_CURSOR + .get_or_init(|| async { RwLock::new(None) }) + .await +} + +pub async fn get_runtime_listener_start_block() -> usize { + *get_listener_start_block().await.read().await +} + +pub async fn set_runtime_listener_start_block(start_block: usize) { + *get_listener_start_block().await.write().await = start_block; +} + +pub async fn get_runtime_listener_resume_cursor() -> Option { + get_listener_resume_cursor().await.read().await.clone() +} + +pub async fn set_runtime_listener_resume_cursor(cursor: Option) { + *get_listener_resume_cursor().await.write().await = cursor; +} diff --git a/nightfall_proposer/src/initialisation/tests.rs b/nightfall_proposer/src/initialisation/tests.rs new file mode 100644 index 00000000..3c3bc82a --- /dev/null +++ b/nightfall_proposer/src/initialisation/tests.rs @@ -0,0 +1,2315 @@ +use super::*; +use crate::{ + domain::entities::{ + Block, ClientTransactionWithMetaData, DepositDatawithFee, L1Ref, PendingBlock, + PendingBlockState, RestoreJournalPhase, RestoreJournalStep, StartupReplayResetMarker, + StartupReplayResetPhase, SyncState, TxLifecycle, + }, + driven::db::{ + mongo_db::{StoredBlock, DB, DEPOSIT_COLLECTION}, + snapshot::{ + create_proposer_snapshot, load_proposer_snapshot_into_shadow, restore_proposer_snapshot, + }, + }, + driven::nightfall_event::get_expected_layer2_blocknumber, + drivers::blockchain::nightfall_event_listener::get_synchronisation_status, + ports::{ + contracts::NightfallContract, + db::{ + BlockStorageDB, PendingBlockDB, RestoreJournalDB, StartupReplayResetMarkerDB, + SyncStateDB, TransactionsDB, + }, + trees::{CommitmentTree, HistoricRootTree, NullifierTree}, + }, +}; +use alloy::primitives::{Address, Bytes, TxHash, I256}; +use ark_bn254::Fr as Fr254; +use ark_serialize::SerializationError; +use ark_std::Zero; +use lib::hex_conversion::HexConvertible; +use lib::merkle_trees::trees::{MutableTree, TreeMetadata}; +use lib::nf_client_proof::Proof; +use lib::shared_entities::{ClientTransaction, CompressedSecrets, DepositData}; +use lib::tests_utils::{get_db_connection, get_mongo}; +use mongodb::bson::{doc, Document}; +use serde::{Deserialize, Serialize}; +use std::{fs, path::PathBuf}; +use tokio::sync::{Mutex, OnceCell}; + +struct MockNightfallContract; + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +struct MockProof; + +impl Proof for MockProof { + fn compress_proof(&self) -> Result { + Ok(Bytes::new()) + } + + fn from_compressed(_compressed: Bytes) -> Result { + Ok(Self) + } +} + +#[derive(Debug, PartialEq)] +struct CapturedSyncState { + last_applied_l2_block: u64, + fingerprint: String, + l1_ref: L1Ref, + schema_version: u32, +} + +#[derive(Debug, PartialEq)] +struct CapturedStoredBlock { + layer2_block_number: u64, + commitments: Vec, + proposer_address: Address, +} + +#[derive(Debug, PartialEq)] +struct CapturedProposerState { + sync_state: CapturedSyncState, + blocks: Vec, + commitment_root: Fr254, + historic_root: Fr254, +} + +fn mock_onchain_next_block() -> &'static std::sync::atomic::AtomicU64 { + static ONCHAIN_NEXT_BLOCK: std::sync::OnceLock = + std::sync::OnceLock::new(); + ONCHAIN_NEXT_BLOCK.get_or_init(|| std::sync::atomic::AtomicU64::new(0)) +} + +impl MockNightfallContract { + fn set_onchain_next_block(block_number: u64) { + mock_onchain_next_block().store(block_number, std::sync::atomic::Ordering::SeqCst); + } +} + +#[async_trait::async_trait] +impl NightfallContract for MockNightfallContract { + async fn propose_block( + _block: crate::domain::entities::Block, + ) -> Result + { + unreachable!("propose_block is not used in bootstrap tests") + } + + async fn get_proposal_receipt_status( + _tx_hash: TxHash, + ) -> Result, lib::error::NightfallContractError> { + unreachable!("get_proposal_receipt_status is not used in bootstrap tests") + } + + async fn get_current_layer2_blocknumber() -> Result { + Ok( + I256::try_from(mock_onchain_next_block().load(std::sync::atomic::Ordering::SeqCst)) + .expect("mock block number fits into I256"), + ) + } +} + +async fn bootstrap_test_lock() -> tokio::sync::MutexGuard<'static, ()> { + static LOCK: OnceCell> = OnceCell::const_new(); + LOCK.get_or_init(|| async { Mutex::new(()) }) + .await + .lock() + .await +} + +async fn persist_sync_state(client: &mongodb::Client, sync_state: &SyncState) { + let mut session = client.start_session().await.expect("start session"); + let client_for_write = client.clone(); + let sync_state_for_write = sync_state.clone(); + session + .start_transaction() + .and_run2(async move |session| { + client_for_write + .update_sync_state_with_session(&sync_state_for_write, session) + .await?; + Ok::<(), mongodb::error::Error>(()) + }) + .await + .expect("write sync_state"); + + let stored_block = client + .get_block_by_number(sync_state.last_applied_l2_block) + .await + .expect("stored block should exist for test sync_state"); + + let commitment_metadata = client + .database(DB) + .collection::>(&format!( + "{}_metadata", + >::TREE_NAME + )) + .find_one(doc! { "_id": 0 }) + .await + .expect("read commitment metadata"); + if commitment_metadata + .as_ref() + .is_some_and(|metadata| metadata.sub_tree_count == 0) + && !stored_block.commitments.is_empty() + { + >::insert_leaf( + client, + Fr254::from(sync_state.last_applied_l2_block + 1), + true, + >::TREE_NAME, + ) + .await + .expect("materialize commitment tree state for bootstrap test"); + } + + let target_historic_root_sub_tree_count = sync_state + .last_applied_l2_block + .checked_add(2) + .expect("historic root count should not overflow in bootstrap tests"); + let Some(historic_root_metadata) = client + .database(DB) + .collection::>(&format!( + "{}_metadata", + >::TREE_NAME + )) + .find_one(doc! { "_id": 0 }) + .await + .expect("read historic root metadata") + else { + return; + }; + let mut historic_root_sub_tree_count = historic_root_metadata.sub_tree_count; + + while historic_root_sub_tree_count < target_historic_root_sub_tree_count { + let commitment_root = >::get_root(client) + .await + .expect("read commitment root"); + >::append_historic_commitment_root( + client, + &commitment_root, + true, + ) + .await + .expect("materialize historic root state for bootstrap test"); + historic_root_sub_tree_count += 1; + } +} + +async fn persist_startup_replay_reset_marker( + client: &mongodb::Client, + phase: StartupReplayResetPhase, +) -> StartupReplayResetMarker { + let marker = StartupReplayResetMarker::new(phase, mongodb::bson::DateTime::now()); + client + .upsert_startup_replay_reset_marker(&marker) + .await + .expect("write startup replay reset marker"); + marker +} + +async fn materialize_tree_state_for_block(client: &mongodb::Client, block_number: u64) { + let commitment_leaf = Fr254::from(block_number + 1); + >::insert_leaf( + client, + commitment_leaf, + true, + >::TREE_NAME, + ) + .await + .expect("append test commitment leaf"); + + let commitment_root = >::get_root(client) + .await + .expect("read commitment root"); + >::append_historic_commitment_root( + client, + &commitment_root, + true, + ) + .await + .expect("append historic root"); +} + +fn test_nullifier_batch(seed: u64) -> [Fr254; 8] { + [ + Fr254::from(seed), + Fr254::zero(), + Fr254::zero(), + Fr254::zero(), + Fr254::zero(), + Fr254::zero(), + Fr254::zero(), + Fr254::zero(), + ] +} + +async fn create_snapshot_fixture( + client: &mongodb::Client, + snapshot_root_prefix: &str, + layer2_block_number: u64, + l1_block_number: u64, +) -> (PathBuf, StoredBlock, SyncState) { + ensure_proposer_db_initialized(client).await; + + let stored_block = StoredBlock { + layer2_block_number, + commitments: vec![format!("0x{layer2_block_number:02x}")], + proposer_address: Address::from([layer2_block_number as u8; 20]), + }; + client + .store_block(&stored_block) + .await + .expect("store block for snapshot fixture"); + materialize_tree_state_for_block(client, layer2_block_number).await; + + let sync_state = SyncState::new( + stored_block.layer2_block_number, + stored_block.hash().to_hex_string(), + L1Ref { + block_number: l1_block_number, + tx_hash: TxHash::from([layer2_block_number as u8; 32]), + log_index: layer2_block_number, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(client, &sync_state).await; + + let snapshot_root = std::env::temp_dir().join(format!( + "{snapshot_root_prefix}-{}", + mongodb::bson::DateTime::now().timestamp_millis() + )); + create_proposer_snapshot(client, &snapshot_root) + .await + .expect("create snapshot fixture"); + + (snapshot_root, stored_block, sync_state) +} + +async fn set_live_sync_state( + client: &mongodb::Client, + layer2_block_number: u64, + commitment_tag: &str, + l1_block_number: u64, +) -> SyncState { + let stored_block = StoredBlock { + layer2_block_number, + commitments: vec![commitment_tag.to_string()], + proposer_address: Address::from([layer2_block_number as u8; 20]), + }; + client + .store_block(&stored_block) + .await + .expect("store live block"); + materialize_tree_state_for_block(client, layer2_block_number).await; + + let sync_state = SyncState::new( + stored_block.layer2_block_number, + stored_block.hash().to_hex_string(), + L1Ref { + block_number: l1_block_number, + tx_hash: TxHash::from([layer2_block_number as u8; 32]), + log_index: layer2_block_number, + }, + mongodb::bson::DateTime::now(), + ); + persist_sync_state(client, &sync_state).await; + sync_state +} + +fn test_reserved_deposit(seed: u64) -> DepositDatawithFee { + DepositDatawithFee { + fee: Fr254::from(seed), + deposit_data: DepositData { + nf_token_id: Fr254::from(seed + 100), + nf_slot_id: Fr254::from(seed + 101), + value: Fr254::from(seed + 102), + secret_hash: Fr254::from(seed + 103), + }, + reserved: true, + } +} + +fn test_selected_client_transaction( + seed: u32, + block_l2: u64, +) -> ClientTransactionWithMetaData { + ClientTransactionWithMetaData { + client_transaction: ClientTransaction { + commitments: [ + Fr254::from(u64::from(seed) + 200), + Fr254::zero(), + Fr254::zero(), + Fr254::zero(), + ], + compressed_secrets: CompressedSecrets::default(), + proof: MockProof, + ..Default::default() + }, + lifecycle: TxLifecycle::Selected { block_l2 }, + hash: vec![seed, seed + 1, seed + 2], + historic_roots: vec![], + receipt_token: None, + } +} + +fn legacy_document_from_transaction( + transaction: &ClientTransactionWithMetaData, + legacy_kind: &str, +) -> Document { + let mut document = + mongodb::bson::to_document(transaction).expect("serialize client transaction"); + document.remove("lifecycle"); + + match legacy_kind { + "selected_or_included" => { + document.insert("in_mempool", false); + document.insert("cancelled_explicitly", false); + document.insert( + "block_l2", + mongodb::bson::Bson::Int64( + i64::try_from(transaction.lifecycle.block_l2().expect("block_l2")) + .expect("block_l2 fits i64"), + ), + ); + } + other => panic!("unexpected legacy kind {other}"), + } + + document +} + +async fn capture_proposer_state( + client: &mongodb::Client, + block_numbers: &[u64], +) -> CapturedProposerState { + let sync_state = client + .get_sync_state() + .await + .expect("sync_state should exist for captured state"); + let mut blocks = Vec::with_capacity(block_numbers.len()); + for block_number in block_numbers { + let stored_block = client + .get_block_by_number(*block_number) + .await + .unwrap_or_else(|| panic!("missing stored block {block_number}")); + blocks.push(CapturedStoredBlock { + layer2_block_number: stored_block.layer2_block_number, + commitments: stored_block.commitments, + proposer_address: stored_block.proposer_address, + }); + } + + CapturedProposerState { + sync_state: CapturedSyncState { + last_applied_l2_block: sync_state.last_applied_l2_block, + fingerprint: sync_state.fingerprint, + l1_ref: sync_state.l1_ref, + schema_version: sync_state.schema_version, + }, + blocks, + commitment_root: >::get_root(client) + .await + .expect("read commitment root for captured state"), + historic_root: >::get_root( + client, + >::TREE_NAME, + ) + .await + .expect("read historic root for captured state"), + } +} + +fn snapshot_dir(snapshot_root: &PathBuf) -> PathBuf { + fs::read_dir(snapshot_root) + .expect("read snapshot root") + .next() + .expect("snapshot dir should exist") + .expect("read snapshot dir entry") + .path() +} + +async fn reset_runtime_bootstrap_state() { + *get_expected_layer2_blocknumber().await.write().await = I256::ZERO; + get_synchronisation_status() + .await + .write() + .await + .clear_synchronised(); + set_runtime_listener_start_block(0).await; + set_runtime_listener_resume_cursor(None).await; +} + +#[tokio::test] +async fn bootstrap_aborts_when_sync_state_exists_but_trees_are_torn_down() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + let stored_block = StoredBlock { + layer2_block_number: 5, + commitments: vec!["0xtorn-state".to_string()], + proposer_address: Address::from([5u8; 20]), + }; + client + .store_block(&stored_block) + .await + .expect("store torn-state block"); + persist_sync_state( + &client, + &SyncState::new( + stored_block.layer2_block_number, + stored_block.hash().to_hex_string(), + L1Ref { + block_number: 500, + tx_hash: TxHash::from([5u8; 32]), + log_index: 5, + }, + mongodb::bson::DateTime::now(), + ), + ) + .await; + + MockNightfallContract::set_onchain_next_block(6); + reset_runtime_bootstrap_state().await; + + let error = bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect_err("bootstrap should reject torn proposer tree state"); + + assert!( + error.contains("tree state is inconsistent") + && error.contains("Manual recovery is required"), + "unexpected bootstrap error: {error}" + ); +} + +#[tokio::test] +async fn bootstrap_aborts_when_sync_state_exists_but_nullifier_metadata_is_missing() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + set_live_sync_state(&client, 5, "0x05", 500).await; + client + .database(DB) + .collection::("Nullifiers_metadata") + .drop() + .await + .expect("drop nullifier metadata"); + + MockNightfallContract::set_onchain_next_block(6); + reset_runtime_bootstrap_state().await; + + let error = bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect_err("bootstrap should reject missing nullifier metadata with sync_state present"); + + assert!( + error.contains("tree state is inconsistent") + && error.contains("tree metadata for Nullifiers is missing"), + "unexpected bootstrap error: {error}" + ); +} + +#[tokio::test] +async fn bootstrap_aborts_when_sync_state_exists_but_nullifier_indexed_leaves_are_empty() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + set_live_sync_state(&client, 5, "0x05", 500).await; + client + .database(DB) + .collection::("Nullifiers_indexed_leaves") + .delete_many(doc! {}) + .await + .expect("empty nullifier indexed leaves"); + + MockNightfallContract::set_onchain_next_block(6); + reset_runtime_bootstrap_state().await; + + let error = bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect_err( + "bootstrap should reject empty nullifier indexed leaves with sync_state present", + ); + + assert!( + error.contains("tree state is inconsistent") + && error.contains("nullifier indexed leaves collection is empty"), + "unexpected bootstrap error: {error}" + ); +} + +#[tokio::test] +async fn bootstrap_cleans_stored_blocks_ahead_of_sync_state() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + set_live_sync_state(&client, 5, "0x05", 500).await; + client + .store_block(&StoredBlock { + layer2_block_number: 6, + commitments: vec!["0x06".to_string()], + proposer_address: Address::from([6u8; 20]), + }) + .await + .expect("store speculative block ahead of sync_state"); + + MockNightfallContract::set_onchain_next_block(6); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should clean stored blocks ahead of sync_state"); + + assert!(client.get_block_by_number(6).await.is_none()); +} + +#[tokio::test] +async fn bootstrap_aborts_when_no_sync_state_but_stored_blocks_exist() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + client + .store_block(&StoredBlock { + layer2_block_number: 0, + commitments: vec!["0x00".to_string()], + proposer_address: Address::from([0u8; 20]), + }) + .await + .expect("store block without sync_state"); + + MockNightfallContract::set_onchain_next_block(0); + reset_runtime_bootstrap_state().await; + + let error = bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect_err("bootstrap should fail closed on stored blocks without sync_state"); + + assert!( + error.contains("StoredBlocks exist without sync_state") + && error.contains("Manual recovery is required"), + "unexpected bootstrap error: {error}" + ); + assert!( + client.get_block_by_number(0).await.is_some(), + "stored block should remain because cleanup must not run after fail-closed validation" + ); +} + +#[tokio::test] +async fn bootstrap_cleans_first_pending_block_startup_state_without_sync_state() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + let canonical_commitment_root = >::get_root(&client) + .await + .expect("read canonical commitment root before cleanup"); + let canonical_historic_root = >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read canonical historic root before cleanup"); + + client + .store_pending_block(&PendingBlock { + layer2_block_number: 0, + state: PendingBlockState::ReadyToPropose, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: Some(Block::default()), + selected_deposits: Vec::new(), + selected_client_transaction_hashes: Vec::new(), + }) + .await + .expect("store first pending block"); + client + .store_block(&StoredBlock { + layer2_block_number: 0, + commitments: vec!["0x00".to_string()], + proposer_address: Address::from([0u8; 20]), + }) + .await + .expect("store first speculative block without sync_state"); + + MockNightfallContract::set_onchain_next_block(0); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should clean first pending startup state without sync_state"); + + assert_eq!(client.get_sync_state().await, None); + assert_eq!(client.get_pending_block(0).await, None); + assert!(client.get_block_by_number(0).await.is_none()); + assert_eq!( + >::get_root(&client) + .await + .expect("read commitment root after cleanup"), + canonical_commitment_root + ); + assert_eq!( + >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read historic root after cleanup"), + canonical_historic_root + ); +} + +#[tokio::test] +async fn bootstrap_resets_tree_state_for_first_pending_block_crash_before_sync_state() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + let zero_commitment_root = >::get_root(&client) + .await + .expect("read zero commitment root before reset"); + let zero_historic_root = >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read zero historic root before reset"); + let zero_nullifier_root = >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read zero nullifier root before reset"); + + client + .store_pending_block(&PendingBlock { + layer2_block_number: 0, + state: PendingBlockState::ReadyToPropose, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: Some(Block::default()), + selected_deposits: Vec::new(), + selected_client_transaction_hashes: Vec::new(), + }) + .await + .expect("store first pending block"); + client + .store_block(&StoredBlock { + layer2_block_number: 0, + commitments: vec!["0x00".to_string()], + proposer_address: Address::from([0u8; 20]), + }) + .await + .expect("store first speculative block without sync_state"); + materialize_tree_state_for_block(&client, 0).await; + >::insert_nullifiers(&client, &test_nullifier_batch(1)) + .await + .expect("materialize speculative nullifier state"); + + MockNightfallContract::set_onchain_next_block(0); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should reset speculative tree state and continue"); + + assert_eq!(client.get_sync_state().await, None); + assert_eq!(client.get_pending_block(0).await, None); + assert!(client.get_block_by_number(0).await.is_none()); + assert_eq!( + >::get_root(&client) + .await + .expect("read commitment root after replay reset"), + zero_commitment_root + ); + assert_eq!( + >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read historic root after replay reset"), + zero_historic_root + ); + assert_eq!( + >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read nullifier root after replay reset"), + zero_nullifier_root + ); + assert_eq!(client.get_startup_replay_reset_marker().await, None); +} + +#[tokio::test] +async fn bootstrap_resumes_startup_replay_reset_after_db_cleanup_applied_crash() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + let zero_commitment_root = >::get_root(&client) + .await + .expect("read zero commitment root before resume"); + let zero_historic_root = >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read zero historic root before resume"); + let zero_nullifier_root = >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read zero nullifier root before resume"); + + let selected_deposit = test_reserved_deposit(80); + let selected_transaction = test_selected_client_transaction(81, 0); + let included_transaction = ClientTransactionWithMetaData { + lifecycle: TxLifecycle::Included { block_l2: 0 }, + ..test_selected_client_transaction(91, 0) + }; + + >::set_mempool_deposits( + &client, + vec![selected_deposit], + ) + .await + .expect("store reserved deposit before replay reset resume"); + client + .store_transaction(selected_transaction.clone()) + .await + .expect("store selected transaction before replay reset resume"); + client + .store_transaction(included_transaction.clone()) + .await + .expect("store included transaction before replay reset resume"); + + materialize_tree_state_for_block(&client, 0).await; + >::insert_nullifiers(&client, &test_nullifier_batch(8)) + .await + .expect("materialize speculative nullifier state before replay reset resume"); + + persist_startup_replay_reset_marker(&client, StartupReplayResetPhase::DbCleanupApplied).await; + + MockNightfallContract::set_onchain_next_block(0); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should resume startup replay reset after db cleanup crash"); + + assert_eq!(client.get_startup_replay_reset_marker().await, None); + assert_eq!(client.get_sync_state().await, None); + assert_eq!( + >::get_transaction( + &client, + &selected_transaction.hash, + ) + .await + .expect("selected transaction should still exist after replay reset resume") + .lifecycle, + TxLifecycle::Mempool + ); + assert_eq!( + >::get_transaction( + &client, + &included_transaction.hash, + ) + .await + .expect("included transaction should still exist after replay reset resume") + .lifecycle, + TxLifecycle::Mempool + ); + assert_eq!( + client + .database(DB) + .collection::(DEPOSIT_COLLECTION) + .count_documents(doc! { "reserved": true }) + .await + .expect("count reserved deposits after replay reset resume"), + 0 + ); + assert_eq!( + >::get_root(&client) + .await + .expect("read commitment root after replay reset resume"), + zero_commitment_root + ); + assert_eq!( + >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read historic root after replay reset resume"), + zero_historic_root + ); + assert_eq!( + >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read nullifier root after replay reset resume"), + zero_nullifier_root + ); +} + +#[tokio::test] +async fn bootstrap_aborts_when_no_sync_state_but_trees_are_non_empty() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + materialize_tree_state_for_block(&client, 0).await; + + let commitment_root_before = >::get_root(&client) + .await + .expect("read commitment root before failed bootstrap"); + let historic_root_before = >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read historic root before failed bootstrap"); + + MockNightfallContract::set_onchain_next_block(0); + reset_runtime_bootstrap_state().await; + + let error = bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect_err("bootstrap should fail closed on trees without sync_state"); + + assert!( + error.contains("no proposer sync_state exists, but proposer trees are not empty"), + "unexpected bootstrap error: {error}" + ); + assert_eq!( + >::get_root(&client) + .await + .expect("read commitment root after failed bootstrap"), + commitment_root_before + ); + assert_eq!( + >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read historic root after failed bootstrap"), + historic_root_before + ); +} + +#[tokio::test] +async fn bootstrap_cleans_ready_to_propose_pending_block_startup_state() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + set_live_sync_state(&client, 5, "0x05", 500).await; + let canonical_commitment_root = >::get_root(&client) + .await + .expect("read canonical commitment root before cleanup"); + let canonical_historic_root = >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read canonical historic root before cleanup"); + let selected_deposit = test_reserved_deposit(5); + let selected_transaction = test_selected_client_transaction(51, 6); + let included_transaction = ClientTransactionWithMetaData { + lifecycle: TxLifecycle::Included { block_l2: 4 }, + ..test_selected_client_transaction(61, 4) + }; + let dropped_transaction = ClientTransactionWithMetaData { + lifecycle: TxLifecycle::Dropped, + ..test_selected_client_transaction(71, 6) + }; + >::set_mempool_deposits( + &client, + vec![selected_deposit], + ) + .await + .expect("store reserved deposit for ready pending block"); + client + .store_transaction(selected_transaction.clone()) + .await + .expect("store selected client transaction for ready pending block"); + client + .store_transaction(included_transaction.clone()) + .await + .expect("store included client transaction for ready pending block"); + client + .store_transaction(dropped_transaction.clone()) + .await + .expect("store dropped client transaction for ready pending block"); + + client + .store_pending_block(&PendingBlock { + layer2_block_number: 6, + state: PendingBlockState::ReadyToPropose, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: Some(Block::default()), + selected_deposits: vec![vec![selected_deposit]], + selected_client_transaction_hashes: vec![selected_transaction.hash.clone()], + }) + .await + .expect("store ready pending block"); + client + .store_block(&StoredBlock { + layer2_block_number: 6, + commitments: vec!["0x06".to_string()], + proposer_address: Address::from([6u8; 20]), + }) + .await + .expect("store speculative block for ready pending block"); + + MockNightfallContract::set_onchain_next_block(6); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should clean ready pending state"); + + assert_eq!(client.get_pending_block(6).await, None); + assert!(client.get_block_by_number(6).await.is_none()); + assert_eq!( + client + .database(DB) + .collection::(DEPOSIT_COLLECTION) + .count_documents(doc! { "reserved": true }) + .await + .expect("count reserved deposits after ready pending cleanup"), + 0 + ); + assert!( + >::get_transaction( + &client, + &selected_transaction.hash, + ) + .await + .expect("selected transaction should still exist after cleanup") + .lifecycle + .is_mempool() + ); + assert_eq!( + >::get_transaction( + &client, + &included_transaction.hash, + ) + .await + .expect("included transaction should still exist after cleanup") + .lifecycle, + TxLifecycle::Included { block_l2: 4 } + ); + assert_eq!( + >::get_transaction( + &client, + &dropped_transaction.hash, + ) + .await + .expect("dropped transaction should still exist after cleanup") + .lifecycle, + TxLifecycle::Dropped + ); + assert_eq!( + >::get_root(&client) + .await + .expect("read commitment root after cleanup"), + canonical_commitment_root + ); + assert_eq!( + >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read historic root after cleanup"), + canonical_historic_root + ); +} + +#[tokio::test] +async fn bootstrap_resets_tree_state_for_speculative_block_ahead_of_sync_state() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + let zero_commitment_root = >::get_root(&client) + .await + .expect("read zero commitment root before canonical setup"); + let zero_historic_root = >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read zero historic root before canonical setup"); + let zero_nullifier_root = >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read zero nullifier root before canonical setup"); + + set_live_sync_state(&client, 5, "0x05", 500).await; + client + .store_pending_block(&PendingBlock { + layer2_block_number: 6, + state: PendingBlockState::ReadyToPropose, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: Some(Block::default()), + selected_deposits: Vec::new(), + selected_client_transaction_hashes: Vec::new(), + }) + .await + .expect("store speculative pending block ahead of sync_state"); + client + .store_block(&StoredBlock { + layer2_block_number: 6, + commitments: vec!["0x06".to_string()], + proposer_address: Address::from([6u8; 20]), + }) + .await + .expect("store speculative block ahead of sync_state"); + materialize_tree_state_for_block(&client, 99).await; + >::insert_nullifiers(&client, &test_nullifier_batch(6)) + .await + .expect("materialize speculative nullifier state ahead of sync_state"); + + MockNightfallContract::set_onchain_next_block(6); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should reset local canonical state and continue via replay"); + + assert_eq!(client.get_sync_state().await, None); + assert!(client.get_block_by_number(5).await.is_none()); + assert!(client.get_block_by_number(6).await.is_none()); + assert_eq!(client.get_pending_block(6).await, None); + assert_eq!( + >::get_root(&client) + .await + .expect("read commitment root after startup replay reset"), + zero_commitment_root + ); + assert_eq!( + >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read historic root after startup replay reset"), + zero_historic_root + ); + assert_eq!( + >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read nullifier root after startup replay reset"), + zero_nullifier_root + ); + assert_eq!(client.get_startup_replay_reset_marker().await, None); +} + +#[tokio::test] +async fn bootstrap_resumes_startup_replay_reset_from_db_cleanup_pending_marker() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + let zero_commitment_root = >::get_root(&client) + .await + .expect("read zero commitment root before marker resume"); + let zero_historic_root = >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read zero historic root before marker resume"); + let zero_nullifier_root = >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read zero nullifier root before marker resume"); + + set_live_sync_state(&client, 5, "0x05", 500).await; + let selected_deposit = test_reserved_deposit(100); + let selected_transaction = test_selected_client_transaction(101, 6); + let included_transaction = ClientTransactionWithMetaData { + lifecycle: TxLifecycle::Included { block_l2: 6 }, + ..test_selected_client_transaction(111, 6) + }; + >::set_mempool_deposits( + &client, + vec![selected_deposit], + ) + .await + .expect("store reserved deposit before db cleanup pending resume"); + client + .store_transaction(selected_transaction.clone()) + .await + .expect("store selected transaction before db cleanup pending resume"); + client + .store_transaction(included_transaction.clone()) + .await + .expect("store included transaction before db cleanup pending resume"); + + client + .store_pending_block(&PendingBlock { + layer2_block_number: 6, + state: PendingBlockState::ReadyToPropose, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: Some(Block::default()), + selected_deposits: vec![vec![selected_deposit]], + selected_client_transaction_hashes: vec![selected_transaction.hash.clone()], + }) + .await + .expect("store speculative pending block before marker resume"); + client + .store_block(&StoredBlock { + layer2_block_number: 6, + commitments: vec!["0x06".to_string()], + proposer_address: Address::from([6u8; 20]), + }) + .await + .expect("store speculative block before marker resume"); + materialize_tree_state_for_block(&client, 99).await; + >::insert_nullifiers( + &client, + &test_nullifier_batch(10), + ) + .await + .expect("materialize nullifier state before db cleanup pending resume"); + + persist_startup_replay_reset_marker(&client, StartupReplayResetPhase::DbCleanupPending).await; + + MockNightfallContract::set_onchain_next_block(6); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should resume startup replay reset from db cleanup pending marker"); + + assert_eq!(client.get_startup_replay_reset_marker().await, None); + assert_eq!(client.get_sync_state().await, None); + assert!(client.get_block_by_number(5).await.is_none()); + assert!(client.get_block_by_number(6).await.is_none()); + assert_eq!(client.get_pending_block(6).await, None); + assert_eq!( + >::get_transaction( + &client, + &selected_transaction.hash, + ) + .await + .expect("selected transaction should still exist after db cleanup pending resume") + .lifecycle, + TxLifecycle::Mempool + ); + assert_eq!( + >::get_transaction( + &client, + &included_transaction.hash, + ) + .await + .expect("included transaction should still exist after db cleanup pending resume") + .lifecycle, + TxLifecycle::Mempool + ); + assert_eq!( + client + .database(DB) + .collection::(DEPOSIT_COLLECTION) + .count_documents(doc! { "reserved": true }) + .await + .expect("count reserved deposits after db cleanup pending resume"), + 0 + ); + assert_eq!( + >::get_root(&client) + .await + .expect("read commitment root after db cleanup pending resume"), + zero_commitment_root + ); + assert_eq!( + >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read historic root after db cleanup pending resume"), + zero_historic_root + ); + assert_eq!( + >::get_root( + &client, + >::TREE_NAME, + ) + .await + .expect("read nullifier root after db cleanup pending resume"), + zero_nullifier_root + ); +} + +#[tokio::test] +async fn bootstrap_resumes_startup_replay_reset_from_auxiliary_cleanup_pending_marker() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + let selected_deposit = test_reserved_deposit(120); + let selected_transaction = test_selected_client_transaction(121, 0); + let included_transaction = ClientTransactionWithMetaData { + lifecycle: TxLifecycle::Included { block_l2: 0 }, + ..test_selected_client_transaction(131, 0) + }; + + >::set_mempool_deposits( + &client, + vec![selected_deposit], + ) + .await + .expect("store reserved deposit before auxiliary cleanup resume"); + client + .store_transaction(selected_transaction.clone()) + .await + .expect("store selected transaction before auxiliary cleanup resume"); + client + .store_transaction(included_transaction.clone()) + .await + .expect("store included transaction before auxiliary cleanup resume"); + + persist_startup_replay_reset_marker(&client, StartupReplayResetPhase::AuxiliaryCleanupPending) + .await; + + MockNightfallContract::set_onchain_next_block(0); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should resume startup replay reset from auxiliary cleanup marker"); + + assert_eq!(client.get_startup_replay_reset_marker().await, None); + assert_eq!( + >::get_transaction( + &client, + &selected_transaction.hash, + ) + .await + .expect("selected transaction should still exist after auxiliary cleanup resume") + .lifecycle, + TxLifecycle::Mempool + ); + assert_eq!( + >::get_transaction( + &client, + &included_transaction.hash, + ) + .await + .expect("included transaction should still exist after auxiliary cleanup resume") + .lifecycle, + TxLifecycle::Mempool + ); + assert_eq!( + client + .database(DB) + .collection::(DEPOSIT_COLLECTION) + .count_documents(doc! { "reserved": true }) + .await + .expect("count reserved deposits after auxiliary cleanup resume"), + 0 + ); +} + +#[tokio::test] +async fn bootstrap_cleans_reserved_pending_block_recovery_state() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + set_live_sync_state(&client, 5, "0x05", 500).await; + let selected_deposit = test_reserved_deposit(7); + let selected_transaction = test_selected_client_transaction(71, 6); + >::set_mempool_deposits( + &client, + vec![selected_deposit], + ) + .await + .expect("store reserved deposit for reserved pending block"); + client + .store_transaction(selected_transaction.clone()) + .await + .expect("store selected client transaction for reserved pending block"); + + client + .store_pending_block(&PendingBlock { + layer2_block_number: 6, + state: PendingBlockState::Reserved, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: None, + selected_deposits: vec![vec![selected_deposit]], + selected_client_transaction_hashes: vec![vec![9, 9, 9]], + }) + .await + .expect("store reserved pending block"); + client + .store_block(&StoredBlock { + layer2_block_number: 6, + commitments: vec!["0x06".to_string()], + proposer_address: Address::from([6u8; 20]), + }) + .await + .expect("store speculative block for reserved pending block"); + + MockNightfallContract::set_onchain_next_block(6); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should clean reserved pending state"); + + assert_eq!(client.get_pending_block(6).await, None); + assert!(client.get_block_by_number(6).await.is_none()); + assert_eq!( + client + .database(DB) + .collection::(DEPOSIT_COLLECTION) + .count_documents(doc! { "reserved": true }) + .await + .expect("count reserved deposits after reserved pending cleanup"), + 0 + ); + assert!( + >::get_transaction( + &client, + &selected_transaction.hash, + ) + .await + .expect("selected transaction should still exist after cleanup") + .lifecycle + .is_mempool() + ); +} + +#[tokio::test] +async fn bootstrap_cleans_broadcast_pending_block_startup_state() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + set_live_sync_state(&client, 5, "0x05", 500).await; + let selected_deposit = test_reserved_deposit(9); + let selected_transaction = test_selected_client_transaction(91, 6); + >::set_mempool_deposits( + &client, + vec![selected_deposit], + ) + .await + .expect("store reserved deposit for broadcast pending block"); + client + .store_transaction(selected_transaction.clone()) + .await + .expect("store selected client transaction for broadcast pending block"); + + client + .store_pending_block(&PendingBlock { + layer2_block_number: 6, + state: PendingBlockState::BroadcastPending, + broadcast_tx_hash: Some(TxHash::from([9u8; 32])), + broadcast_receipt_checks: 3, + block: Some(Block::default()), + selected_deposits: vec![vec![selected_deposit]], + selected_client_transaction_hashes: vec![selected_transaction.hash.clone()], + }) + .await + .expect("store broadcast-pending block"); + client + .store_block(&StoredBlock { + layer2_block_number: 6, + commitments: vec!["0x06".to_string()], + proposer_address: Address::from([9u8; 20]), + }) + .await + .expect("store speculative block for broadcast-pending block"); + + MockNightfallContract::set_onchain_next_block(6); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should clean broadcast-pending state"); + + assert_eq!(client.get_pending_block(6).await, None); + assert!(client.get_block_by_number(6).await.is_none()); + assert_eq!( + client + .database(DB) + .collection::(DEPOSIT_COLLECTION) + .count_documents(doc! { "reserved": true }) + .await + .expect("count reserved deposits after broadcast cleanup"), + 0 + ); + assert!( + >::get_transaction( + &client, + &selected_transaction.hash, + ) + .await + .expect("selected transaction should still exist after cleanup") + .lifecycle + .is_mempool() + ); +} + +#[tokio::test] +async fn bootstrap_backfills_legacy_client_transaction_lifecycle_before_cleanup() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + let included_transaction = test_selected_client_transaction(201, 5); + let selected_transaction = test_selected_client_transaction(211, 6); + let canonical_commitment = + included_transaction.client_transaction.commitments[0].to_hex_string(); + set_live_sync_state(&client, 5, &canonical_commitment, 500).await; + client + .database(DB) + .collection::("ClientTransactions") + .insert_many(vec![ + legacy_document_from_transaction(&included_transaction, "selected_or_included"), + legacy_document_from_transaction(&selected_transaction, "selected_or_included"), + ]) + .await + .expect("store legacy lifecycle fixtures"); + + MockNightfallContract::set_onchain_next_block(6); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should backfill legacy lifecycle fields before cleanup"); + + assert_eq!( + >::get_transaction( + &client, + &included_transaction.hash, + ) + .await + .expect("included transaction should still exist after bootstrap") + .lifecycle, + TxLifecycle::Included { block_l2: 5 } + ); + assert!( + >::get_transaction( + &client, + &selected_transaction.hash, + ) + .await + .expect("selected transaction should still exist after bootstrap") + .lifecycle + .is_mempool(), + "startup cleanup should only demote the legacy Selected transaction after backfill" + ); +} + +#[tokio::test] +async fn bootstrap_preserves_selected_transactions_at_sync_state_height() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + let canonical_selected_transaction = test_selected_client_transaction(221, 5); + let speculative_selected_transaction = test_selected_client_transaction(231, 6); + client + .store_transaction(canonical_selected_transaction.clone()) + .await + .expect("store canonical selected transaction"); + client + .store_transaction(speculative_selected_transaction.clone()) + .await + .expect("store speculative selected transaction"); + set_live_sync_state(&client, 5, "0x05", 500).await; + + MockNightfallContract::set_onchain_next_block(6); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should preserve canonical selected transactions at sync_state height"); + + assert_eq!( + >::get_transaction( + &client, + &canonical_selected_transaction.hash, + ) + .await + .expect("canonical selected transaction should still exist after bootstrap") + .lifecycle, + TxLifecycle::Selected { block_l2: 5 } + ); + assert!( + >::get_transaction( + &client, + &speculative_selected_transaction.hash, + ) + .await + .expect("speculative selected transaction should still exist after bootstrap") + .lifecycle + .is_mempool(), + "startup cleanup should only demote Selected transactions ahead of sync_state" + ); +} + +#[tokio::test] +async fn bootstrap_cleans_reserved_deposits_ahead_of_sync_state_without_pending_block() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + set_live_sync_state(&client, 5, "0x05", 500).await; + >::set_mempool_deposits( + &client, + vec![DepositDatawithFee { + fee: Fr254::from(5u64), + deposit_data: DepositData { + nf_token_id: Fr254::from(51u64), + nf_slot_id: Fr254::from(52u64), + value: Fr254::from(53u64), + secret_hash: Fr254::from(54u64), + }, + reserved: true, + }], + ) + .await + .expect("store reserved deposit"); + + MockNightfallContract::set_onchain_next_block(6); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should clean reserved deposits ahead of sync_state"); + + assert_eq!( + client + .database(DB) + .collection::(DEPOSIT_COLLECTION) + .count_documents(doc! { "reserved": true }) + .await + .expect("count reserved deposits after cleanup"), + 0 + ); +} + +#[tokio::test] +async fn bootstrap_cleans_reserved_pending_block_with_mismatched_deposit_identities() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + set_live_sync_state(&client, 5, "0x05", 500).await; + let selected_deposit = DepositDatawithFee { + reserved: false, + ..test_reserved_deposit(11) + }; + let orphan_reserved_deposit = test_reserved_deposit(12); + >::set_mempool_deposits( + &client, + vec![selected_deposit, orphan_reserved_deposit], + ) + .await + .expect("store mismatched reserved deposits for reserved cleanup"); + + client + .store_pending_block(&PendingBlock { + layer2_block_number: 6, + state: PendingBlockState::Reserved, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: None, + selected_deposits: vec![vec![selected_deposit]], + selected_client_transaction_hashes: Vec::new(), + }) + .await + .expect("store reserved pending block"); + + MockNightfallContract::set_onchain_next_block(6); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should clean mismatched reserved deposit identities"); + + assert_eq!( + client + .database(DB) + .collection::(DEPOSIT_COLLECTION) + .count_documents(doc! { "reserved": true }) + .await + .expect("count reserved deposits after cleanup"), + 0 + ); + assert_eq!(client.get_pending_block(6).await, None); +} + +#[tokio::test] +async fn bootstrap_cleans_ready_pending_selected_client_transaction_hash_mismatch() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + set_live_sync_state(&client, 5, "0x05", 500).await; + let selected_deposit = test_reserved_deposit(13); + let selected_transaction = test_selected_client_transaction(131, 6); + let orphan_selected_transaction = test_selected_client_transaction(141, 6); + + >::set_mempool_deposits( + &client, + vec![selected_deposit], + ) + .await + .expect("store reserved deposit for ready pending state"); + client + .store_transaction(selected_transaction.clone()) + .await + .expect("store selected client transaction for ready pending block"); + client + .store_transaction(orphan_selected_transaction.clone()) + .await + .expect("store orphan selected transaction for ready pending block"); + + client + .store_pending_block(&PendingBlock { + layer2_block_number: 6, + state: PendingBlockState::ReadyToPropose, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: Some(Block::default()), + selected_deposits: vec![vec![selected_deposit]], + selected_client_transaction_hashes: vec![selected_transaction.hash.clone()], + }) + .await + .expect("store ready pending block with mismatched selected tx identities"); + client + .store_block(&StoredBlock { + layer2_block_number: 6, + commitments: vec!["0x06".to_string()], + proposer_address: Address::from([6u8; 20]), + }) + .await + .expect("store speculative block for ready pending block"); + + MockNightfallContract::set_onchain_next_block(6); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should clean mismatched selected transaction identities"); + + assert_eq!(client.get_pending_block(6).await, None); + assert!(client.get_block_by_number(6).await.is_none()); + assert!( + >::get_transaction( + &client, + &selected_transaction.hash, + ) + .await + .expect("selected transaction should still exist after cleanup") + .lifecycle + .is_mempool() + ); + assert!( + >::get_transaction( + &client, + &orphan_selected_transaction.hash, + ) + .await + .expect("orphan selected transaction should still exist after cleanup") + .lifecycle + .is_mempool() + ); +} + +#[tokio::test] +async fn bootstrap_cleans_ready_pending_selected_client_transaction_block_mismatch() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + ensure_proposer_db_initialized(&client).await; + set_live_sync_state(&client, 5, "0x05", 500).await; + let selected_deposit = test_reserved_deposit(15); + let selected_transaction = test_selected_client_transaction(151, 7); + + >::set_mempool_deposits( + &client, + vec![selected_deposit], + ) + .await + .expect("store reserved deposit for wrong-block selected transaction"); + client + .store_transaction(selected_transaction.clone()) + .await + .expect("store selected transaction with wrong block"); + + client + .store_pending_block(&PendingBlock { + layer2_block_number: 6, + state: PendingBlockState::ReadyToPropose, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: Some(Block::default()), + selected_deposits: vec![vec![selected_deposit]], + selected_client_transaction_hashes: vec![selected_transaction.hash.clone()], + }) + .await + .expect("store ready pending block"); + client + .store_block(&StoredBlock { + layer2_block_number: 6, + commitments: vec!["0x06".to_string()], + proposer_address: Address::from([6u8; 20]), + }) + .await + .expect("store speculative block for wrong-block selected tx"); + + MockNightfallContract::set_onchain_next_block(6); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should clean selected transaction bound to wrong block"); + + assert_eq!(client.get_pending_block(6).await, None); + assert!(client.get_block_by_number(6).await.is_none()); + assert!( + >::get_transaction( + &client, + &selected_transaction.hash, + ) + .await + .expect("selected transaction should still exist after cleanup") + .lifecycle + .is_mempool() + ); +} + +async fn rename_collection( + client: &mongodb::Client, + from: &str, + to: &str, +) -> Result<(), mongodb::error::Error> { + client + .database("admin") + .run_command(doc! { + "renameCollection": format!("{DB}.{from}"), + "to": format!("{DB}.{to}"), + "dropTarget": false, + }) + .await?; + Ok(()) +} + +fn ordered_restore_collections( + journal: &[crate::domain::entities::RestoreJournalCollection], +) -> Vec { + let mut ordered = journal.to_vec(); + ordered.sort_by(|left, right| { + match ( + left.live == crate::driven::db::mongo_db::SYNC_STATE_COLLECTION, + right.live == crate::driven::db::mongo_db::SYNC_STATE_COLLECTION, + ) { + (false, true) => std::cmp::Ordering::Less, + (true, false) => std::cmp::Ordering::Greater, + _ => left.live.cmp(&right.live), + } + }); + ordered +} + +async fn first_existing_live_restore_collection( + client: &mongodb::Client, + journal: &[crate::domain::entities::RestoreJournalCollection], +) -> (usize, crate::domain::entities::RestoreJournalCollection) { + let existing_collections = client + .database(DB) + .list_collection_names() + .await + .expect("list collections"); + ordered_restore_collections(journal) + .into_iter() + .enumerate() + .find(|(_, collection)| { + existing_collections + .iter() + .any(|name| name == &collection.live) + }) + .expect("at least one live restore collection should exist") +} + +async fn next_required_live_restore_collection( + client: &mongodb::Client, + journal: &[crate::domain::entities::RestoreJournalCollection], + after_index: usize, +) -> (usize, crate::domain::entities::RestoreJournalCollection) { + let existing_collections = client + .database(DB) + .list_collection_names() + .await + .expect("list collections"); + ordered_restore_collections(journal) + .into_iter() + .enumerate() + .skip(after_index + 1) + .find(|(_, collection)| { + existing_collections + .iter() + .any(|name| name == &collection.live) + && crate::driven::db::snapshot::required_proposer_snapshot_collection_names() + .iter() + .any(|name| name == &collection.live) + }) + .expect("expected another required live restore collection after the first processed one") +} + +#[tokio::test] +async fn bootstrap_cleans_loading_shadow_restore_journal_and_keeps_live_state() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + let (snapshot_root, _, _) = + create_snapshot_fixture(&client, "nf4-bootstrap-loading-shadow", 8, 800).await; + let snapshot_dir = snapshot_dir(&snapshot_root); + + let newer_live_sync_state = set_live_sync_state(&client, 9, "0xlive", 900).await; + let journal = load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + assert_eq!(journal.phase, RestoreJournalPhase::LoadingShadow); + + MockNightfallContract::set_onchain_next_block(10); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should clean loading shadow and continue"); + + assert_eq!(client.get_restore_journal().await, None); + assert_eq!( + client.get_sync_state().await, + Some(newer_live_sync_state.clone()) + ); + assert_eq!( + *get_expected_layer2_blocknumber().await.read().await, + I256::try_from(10_u64).expect("10 fits into I256") + ); + assert_eq!(get_runtime_listener_start_block().await, 900); + assert_eq!( + get_runtime_listener_resume_cursor().await, + Some(newer_live_sync_state.l1_ref.clone()) + ); + assert!(!get_synchronisation_status() + .await + .read() + .await + .is_synchronised()); + + let collection_names = client + .database(DB) + .list_collection_names() + .await + .expect("list collections"); + assert!(!collection_names + .iter() + .any(|name| name.starts_with("restore_shadow__"))); + assert!(!collection_names + .iter() + .any(|name| name.starts_with("restore_backup__"))); + + tokio::fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); +} + +#[tokio::test] +async fn bootstrap_resumes_swap_in_progress_restore_and_keeps_tip_desynchronised_until_replay() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + let (snapshot_root, snapshot_block, _) = + create_snapshot_fixture(&client, "nf4-bootstrap-swap-resume", 12, 1200).await; + let snapshot_dir = snapshot_dir(&snapshot_root); + + set_live_sync_state(&client, 13, "0xnewer-live", 1300).await; + let mut journal = load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + let ordered = ordered_restore_collections(&journal.collections); + let (first_live_index, first_live) = + first_existing_live_restore_collection(&client, &journal.collections).await; + rename_collection(&client, &first_live.live, &first_live.backup) + .await + .expect("simulate backup rename"); + journal.collections = ordered; + journal.phase = RestoreJournalPhase::SwapInProgress; + journal.current_index = Some(first_live_index as u32); + journal.current_step = Some(RestoreJournalStep::BackupCreated); + journal.updated_at = mongodb::bson::DateTime::now(); + client + .upsert_restore_journal(&journal) + .await + .expect("persist mid-swap journal"); + + MockNightfallContract::set_onchain_next_block(snapshot_block.layer2_block_number + 1); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should resume swap and continue"); + + let resumed = client + .get_restore_journal() + .await + .expect("swap_complete journal should remain for cleanup"); + assert_eq!(resumed.phase, RestoreJournalPhase::SwapComplete); + + let live_sync_state = client + .get_sync_state() + .await + .expect("restored sync_state should exist"); + assert_eq!(live_sync_state.last_applied_l2_block, 12); + assert_eq!(live_sync_state.l1_ref.block_number, 1200); + assert_eq!( + *get_expected_layer2_blocknumber().await.read().await, + I256::try_from(13_u64).expect("13 fits into I256") + ); + assert_eq!(get_runtime_listener_start_block().await, 1200); + assert_eq!( + get_runtime_listener_resume_cursor().await, + Some(live_sync_state.l1_ref.clone()) + ); + assert!(!get_synchronisation_status() + .await + .read() + .await + .is_synchronised()); + + tokio::fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); +} + +#[tokio::test] +async fn bootstrap_keeps_resumed_restore_desynchronised_when_one_block_behind_tip() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + let (snapshot_root, snapshot_block, snapshot_sync_state) = + create_snapshot_fixture(&client, "nf4-bootstrap-one-behind", 14, 1400).await; + let snapshot_dir = snapshot_dir(&snapshot_root); + + set_live_sync_state(&client, 15, "0xnewer-live", 1500).await; + let mut journal = load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + let ordered = ordered_restore_collections(&journal.collections); + let (first_live_index, first_live) = + first_existing_live_restore_collection(&client, &journal.collections).await; + rename_collection(&client, &first_live.live, &first_live.backup) + .await + .expect("simulate backup rename"); + journal.collections = ordered; + journal.phase = RestoreJournalPhase::SwapInProgress; + journal.current_index = Some(first_live_index as u32); + journal.current_step = Some(RestoreJournalStep::BackupCreated); + journal.updated_at = mongodb::bson::DateTime::now(); + client + .upsert_restore_journal(&journal) + .await + .expect("persist mid-swap journal"); + + MockNightfallContract::set_onchain_next_block(snapshot_block.layer2_block_number + 2); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should resume swap and remain desynchronised"); + + assert_eq!( + *get_expected_layer2_blocknumber().await.read().await, + I256::try_from(15_u64).expect("15 fits into I256") + ); + assert_eq!(get_runtime_listener_start_block().await, 1400); + assert_eq!( + get_runtime_listener_resume_cursor().await, + Some(snapshot_sync_state.l1_ref.clone()) + ); + assert!(!get_synchronisation_status() + .await + .read() + .await + .is_synchronised()); + + tokio::fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); +} + +#[tokio::test] +async fn bootstrap_continues_after_restore_rollback_clears_the_journal() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + let (snapshot_root, _, _) = + create_snapshot_fixture(&client, "nf4-bootstrap-rollback", 20, 2000).await; + let snapshot_dir = snapshot_dir(&snapshot_root); + + let newer_live_sync_state = set_live_sync_state(&client, 21, "0xnewer-live", 2100).await; + let mut journal = load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + let ordered = ordered_restore_collections(&journal.collections); + let (first_live_index, first_live) = + first_existing_live_restore_collection(&client, &journal.collections).await; + rename_collection(&client, &first_live.live, &first_live.backup) + .await + .expect("rename first live to backup"); + rename_collection(&client, &first_live.shadow, &first_live.live) + .await + .expect("rename first shadow to live"); + let (next_live_index, next_live) = + next_required_live_restore_collection(&client, &journal.collections, first_live_index) + .await; + + journal.collections = ordered; + journal.phase = RestoreJournalPhase::SwapInProgress; + journal.current_index = Some(next_live_index as u32); + journal.current_step = Some(RestoreJournalStep::BackupPending); + journal.updated_at = mongodb::bson::DateTime::now(); + client + .upsert_restore_journal(&journal) + .await + .expect("persist rollback fixture journal"); + + client + .database(DB) + .collection::(&next_live.live) + .drop() + .await + .expect("drop live collection to force rollback"); + + MockNightfallContract::set_onchain_next_block(22); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should continue after rollback recovery"); + + assert_eq!(client.get_restore_journal().await, None); + assert_eq!( + client.get_sync_state().await, + Some(newer_live_sync_state.clone()) + ); + assert_eq!( + *get_expected_layer2_blocknumber().await.read().await, + I256::try_from(22_u64).expect("22 fits into I256") + ); + assert_eq!(get_runtime_listener_start_block().await, 2100); + assert!(!get_synchronisation_status() + .await + .read() + .await + .is_synchronised()); + + tokio::fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); +} + +#[tokio::test] +async fn bootstrap_continues_after_swap_complete_rollback_and_restores_pre_snapshot_runtime_state() +{ + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + let (snapshot_root, _, _) = + create_snapshot_fixture(&client, "nf4-bootstrap-swap-complete-rollback", 22, 2200).await; + let snapshot_dir = snapshot_dir(&snapshot_root); + + let newer_live_sync_state = set_live_sync_state(&client, 23, "0xnewer-live", 2300).await; + load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + crate::driven::db::snapshot::swap_proposer_shadow_into_live(&client) + .await + .expect("swap shadow into live"); + + client + .delete_block_by_number(22) + .await + .expect("delete restored stored block to force swap_complete rollback"); + + MockNightfallContract::set_onchain_next_block(24); + reset_runtime_bootstrap_state().await; + + bootstrap_proposer_startup_state_with_db::(&client, false) + .await + .expect("bootstrap should continue after swap_complete rollback recovery"); + + assert_eq!(client.get_restore_journal().await, None); + assert_eq!( + client.get_sync_state().await, + Some(newer_live_sync_state.clone()) + ); + assert_eq!( + *get_expected_layer2_blocknumber().await.read().await, + I256::try_from(24_u64).expect("24 fits into I256") + ); + assert_eq!(get_runtime_listener_start_block().await, 2300); + assert!(!get_synchronisation_status() + .await + .read() + .await + .is_synchronised()); + + tokio::fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); +} + +#[tokio::test] +async fn startup_recovers_restore_journal_before_reinitializing_live_tree_collections() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + let (snapshot_root, snapshot_block, _) = + create_snapshot_fixture(&client, "nf4-bootstrap-ordering", 18, 1800).await; + let snapshot_dir = snapshot_dir(&snapshot_root); + + set_live_sync_state(&client, 19, "0xnewer-live", 1900).await; + let mut journal = load_proposer_snapshot_into_shadow(&client, &snapshot_dir) + .await + .expect("load snapshot into shadow"); + let ordered = ordered_restore_collections(&journal.collections); + let (first_live_index, first_live) = + first_existing_live_restore_collection(&client, &journal.collections).await; + rename_collection(&client, &first_live.live, &first_live.backup) + .await + .expect("simulate missing live collection mid-swap"); + journal.collections = ordered; + journal.phase = RestoreJournalPhase::SwapInProgress; + journal.current_index = Some(first_live_index as u32); + journal.current_step = Some(RestoreJournalStep::BackupCreated); + journal.updated_at = mongodb::bson::DateTime::now(); + client + .upsert_restore_journal(&journal) + .await + .expect("persist mid-swap journal"); + + recover_then_initialize_proposer_db(&client) + .await + .expect("startup recovery should complete before init recreates live trees"); + + let resumed = client + .get_restore_journal() + .await + .expect("swap_complete journal should remain after recovery"); + assert_eq!(resumed.phase, RestoreJournalPhase::SwapComplete); + + let restored_sync_state = client + .get_sync_state() + .await + .expect("restored sync_state should exist"); + assert_eq!( + restored_sync_state.last_applied_l2_block, + snapshot_block.layer2_block_number + ); + assert_eq!(restored_sync_state.l1_ref.block_number, 1800); + + tokio::fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot directory"); +} + +#[tokio::test] +async fn snapshot_restore_then_replay_reaches_canonical_state() { + let _lock = bootstrap_test_lock().await; + let container = get_mongo().await; + let client = get_db_connection(&container).await; + + let (snapshot_root, _, snapshot_sync_state) = + create_snapshot_fixture(&client, "nf4-restore-replay-e2e", 10, 1000).await; + let snapshot_dir = snapshot_dir(&snapshot_root); + + set_live_sync_state(&client, 11, "0x0b", 1100).await; + set_live_sync_state(&client, 12, "0x0c", 1200).await; + let canonical_state = capture_proposer_state(&client, &[10, 11, 12]).await; + + set_live_sync_state(&client, 12, "0xbad", 1299).await; + materialize_tree_state_for_block(&client, 99).await; + + let restored_sync_state = restore_proposer_snapshot(&client, &snapshot_dir).await; + assert_eq!( + restored_sync_state.expect("restore should succeed"), + snapshot_sync_state + ); + assert_eq!( + client + .get_sync_state() + .await + .expect("restore should reinstall snapshot sync_state"), + snapshot_sync_state + ); + + set_live_sync_state(&client, 11, "0x0b", 1100).await; + set_live_sync_state(&client, 12, "0x0c", 1200).await; + let replayed_state = capture_proposer_state(&client, &[10, 11, 12]).await; + + assert_eq!(replayed_state, canonical_state); + + tokio::fs::remove_dir_all(snapshot_root) + .await + .expect("cleanup snapshot root"); +} diff --git a/nightfall_proposer/src/lib.rs b/nightfall_proposer/src/lib.rs index 18d80238..83b04bb6 100644 --- a/nightfall_proposer/src/lib.rs +++ b/nightfall_proposer/src/lib.rs @@ -1,24 +1,35 @@ pub mod domain; pub mod driven; pub mod drivers; +pub mod initialisation; pub mod ports; pub mod services; -use ark_bn254::Fr as Fr254; +use ark_bn254::{Bn254, Fr as Fr254}; +use ark_serialize::CanonicalDeserialize; +use jf_plonk::nightfall::ipa_structs::ProvingKey; use jf_primitives::{ + pcs::prelude::UnivariateKzgPCS, poseidon::Poseidon, trees::{ imt::{IndexedMerkleTree, LeafDBEntry}, timber::Timber, }, }; +use lib::{ + rollup_circuit_checks::{find_file_with_path, get_configuration_keys_path}, + utils::{load_key_from_server, load_key_locally}, +}; +use log::warn; use std::{ collections::HashMap, - sync::{OnceLock, RwLock}, + sync::{Arc, OnceLock, RwLock}, }; + type AppendOnlyTree = Timber>; type NullifierTree = IndexedMerkleTree, HashMap>>; + /// This function is used so that we can work with one nullifier tree across the entire application. pub fn get_nullifier_tree() -> &'static RwLock { static IMT_TREE: OnceLock> = OnceLock::new(); @@ -41,110 +52,40 @@ pub fn get_historic_root_tree() -> &'static RwLock { }) } -pub mod initialisation { - - use super::driven::block_assembler::SmartTrigger; - use crate::{ - driven::block_assembler::BlockAssemblyStatus, - ports::{ - block_assembly_trigger::BlockAssemblyTrigger, - trees::{CommitmentTree, HistoricRootTree, NullifierTree}, - }, - }; - use ark_bn254::Fr as Fr254; - use ark_std::sync::Arc; - use configuration::settings::get_settings; - use lib::{ - blockchain_client::BlockchainClientConnection, nf_client_proof::Proof, - wallets::LocalWsClient, - }; - use mongodb::Client; - use tokio::sync::{OnceCell, RwLock}; - - /// This function is used to provide a singleton database connection across the entire application. - pub async fn get_db_connection() -> &'static Client { - static DB_CONNECTION: OnceCell = OnceCell::const_new(); - DB_CONNECTION - .get_or_init(|| async { - // select the proposer to use - let uri = &get_settings().nightfall_proposer.db_url; - let client = Client::with_uri_str(uri) - .await - .expect("Could not create database connection"); - // it's not enough just to connect to a database, we need to initialise some trees in it - >::new_commitment_tree(&client, 29, 3) - .await - .expect("Could not create commitment tree"); - >::new_historic_root_tree(&client, 32) - .await - .expect("Could not create historic root tree"); - >::new_nullifier_tree(&client, 29, 3) - .await - .expect("Could not create historic root tree"); - - >::append_historic_commitment_root( - &client, - &Fr254::from(0u8), - true, - ) - .await - .expect("Couldn't insert zero leaf into the historic root tree"); - client - }) - .await - } - - /// This function is used to provide a singleton blockchain client connection across the entire application. - pub async fn get_blockchain_client_connection() -> &'static RwLock { - static BLOCKCHAIN_CLIENT_CONNECTION: OnceCell> = - OnceCell::const_new(); - BLOCKCHAIN_CLIENT_CONNECTION - .get_or_init(|| async { - RwLock::new( - LocalWsClient::try_from_settings(get_settings()) - .await - .expect("Could not create blockchain client connection"), - ) - }) - .await - } - - /// This function is used to provide a singleton trigger for block assembly across the entire application. - pub async fn get_block_assembly_trigger( - ) -> &'static Arc> { - static BLOCK_ASSEMBLY_TRIGGER: OnceCell< - Arc>, - > = OnceCell::const_new(); - BLOCK_ASSEMBLY_TRIGGER - .get_or_init(|| async { - let status = get_block_assembly_status().await; - let db_client = get_db_connection().await; - let settings = get_settings(); - let initial_interval_secs = settings - .nightfall_proposer - .block_assembly_initial_interval_secs; - let max_wait_secs = settings.nightfall_proposer.block_assembly_max_wait_secs; - let target_fill_ratio = - settings.nightfall_proposer.block_assembly_target_fill_ratio as f32; - - let smart_trigger = SmartTrigger::

::new( - initial_interval_secs, - max_wait_secs, - status, - db_client, - target_fill_ratio, +/// This function is used to retrieve the deposit proving key. +pub fn get_deposit_proving_key() -> &'static Arc>> { + static PK: OnceLock>>> = OnceLock::new(); + PK.get_or_init(|| { + if let Some(path) = + get_configuration_keys_path().map(|path| path.join("deposit_proving_key")) + { + if let Some(source_file) = find_file_with_path(&path) { + if let Some(key_bytes) = load_key_locally(&source_file) { + let deposit_proving_key = + ProvingKey::>::deserialize_compressed_unchecked( + &*key_bytes, + ) + .expect("Could not deserialise deposit_proving_key"); + return Arc::new(deposit_proving_key); + } + warn!("Could not load deposit_proving_key from local file. Loading from server"); + } else { + warn!( + "Could not find local deposit_proving_key at {}. Loading from server", + path.display() ); - Arc::new(RwLock::new(smart_trigger)) - as Arc> - }) - .await - } + } + } else { + warn!("Configuration keys path not found. Loading deposit_proving_key from server"); + } - /// This function is used to provide a singleton status for the BlockAssemblyTrigger across the entire application. - pub async fn get_block_assembly_status() -> &'static RwLock { - static BLOCK_ASSEMBLY_STATUS: OnceCell> = OnceCell::const_new(); - BLOCK_ASSEMBLY_STATUS - .get_or_init(|| async { RwLock::new(BlockAssemblyStatus::new()) }) - .await - } + if let Some(key_bytes) = load_key_from_server("deposit_proving_key") { + let pk = ProvingKey::>::deserialize_compressed_unchecked( + &*key_bytes, + ) + .expect("Could not deserialise proving key"); + return Arc::new(pk); + } + panic!("Failed to load deposit_proving_key from both local and server"); + }) } diff --git a/nightfall_proposer/src/main.rs b/nightfall_proposer/src/main.rs index b01daa5d..d599c83d 100644 --- a/nightfall_proposer/src/main.rs +++ b/nightfall_proposer/src/main.rs @@ -6,6 +6,8 @@ use nightfall_proposer::drivers::blockchain::event_listener_manager::ensure_runn use nightfall_proposer::{ driven::{db::mongo_db::DB, mock_prover::MockProver, rollup_prover::RollupProver}, drivers::{blockchain::block_assembly::start_block_assembly, rest::routes}, + initialisation::bootstrap_proposer_startup_state, + services::snapshot_scheduler::run_snapshot_scheduler, }; use std::error::Error; @@ -22,10 +24,14 @@ async fn main() -> Result<(), Box> { settings.log_app_only, ); - // drop any existing database - let db_url = &settings.nightfall_proposer.db_url; - info!("Dropping database: {DB}"); - let _ = lib::utils::drop_database(db_url, DB).await; + info!("Bootstrapping proposer startup state from {DB}"); + bootstrap_proposer_startup_state::() + .await + .map_err(|e| format!("Proposer startup bootstrap failed: {e}"))?; + + // start the event listener + ensure_running::().await; + let _snapshot_scheduler_task = tokio::spawn(run_snapshot_scheduler()); let task_0 = if settings.mock_prover { info!("Using MockProver"); @@ -35,10 +41,6 @@ async fn main() -> Result<(), Box> { tokio::spawn(start_block_assembly::()) }; - // start the event listener - // ── start the (owned) event listener once ───────────────────────────────── - ensure_running::().await; - let routes = routes::(); let task_2 = tokio::spawn(warp::serve(routes).run(([0, 0, 0, 0], 3000))); info!("Starting warp server, block assembler and event_handler threads"); diff --git a/nightfall_proposer/src/ports/contracts.rs b/nightfall_proposer/src/ports/contracts.rs index 39bab43d..7374a966 100644 --- a/nightfall_proposer/src/ports/contracts.rs +++ b/nightfall_proposer/src/ports/contracts.rs @@ -1,14 +1,49 @@ //! This module contains the interface that a smart contract must work with to be classed as a Nightfall contract by a proposer. -use alloy::primitives::I256; +use alloy::primitives::{TxHash, I256}; use lib::error::NightfallContractError; use crate::domain::entities::Block; +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NotBroadcastReason { + TransactionPreparationFailed, + SendRawTransactionFailed, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BroadcastUnknownReason { + ReceiptUnavailable, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ProposeBlockOutcome { + Submitted { + tx_hash: TxHash, + }, + NotBroadcast { + reason: NotBroadcastReason, + }, + BroadcastUnknown { + tx_hash: TxHash, + reason: BroadcastUnknownReason, + }, + Reverted { + tx_hash: TxHash, + }, +} + #[async_trait::async_trait] pub trait NightfallContract { /// Proposes a block - async fn propose_block(block: Block) -> Result<(), NightfallContractError>; + async fn propose_block(block: Block) -> Result; + + /// Returns the receipt status for a proposal transaction. + /// + /// `Ok(None)` means the provider currently has no receipt for the transaction. + async fn get_proposal_receipt_status( + tx_hash: TxHash, + ) -> Result, NightfallContractError>; /// Gets the current layer 2 block number from the blockchain async fn get_current_layer2_blocknumber() -> Result; diff --git a/nightfall_proposer/src/ports/db.rs b/nightfall_proposer/src/ports/db.rs index 5445b3cd..d937a57d 100644 --- a/nightfall_proposer/src/ports/db.rs +++ b/nightfall_proposer/src/ports/db.rs @@ -1,5 +1,9 @@ use crate::{ - domain::entities::{ClientTransactionWithMetaData, DepositDatawithFee, HistoricRoot}, + domain::entities::{ + ClientTransactionWithMetaData, DepositDatawithFee, HistoricRoot, PendingBlock, + RestoreJournal, StartupReplayResetMarker, SyncState, TransferReceipt, + TransferReceiptStatus, TxHashBytes, + }, driven::db::mongo_db::StoredBlock, }; use ark_bn254::Fr as Fr254; @@ -12,10 +16,137 @@ use serde::{Deserialize, Serialize}; #[async_trait::async_trait] pub trait BlockStorageDB { async fn store_block(&self, block: &StoredBlock) -> Option<()>; + async fn store_block_with_session( + &self, + _block: &StoredBlock, + _session: &mut mongodb::ClientSession, + ) -> Result<(), mongodb::error::Error> { + Err(mongodb::error::Error::custom( + "store_block_with_session must be implemented with session-aware writes", + )) + } async fn get_block_by_number(&self, block_number: u64) -> Option; async fn get_all_blocks(&self) -> Option>; async fn delete_block_by_number(&self, block_number: u64) -> Option<()>; + async fn delete_all_blocks(&self) -> Option { + let blocks = self.get_all_blocks().await?; + let mut deleted = 0_u64; + for block in blocks { + self.delete_block_by_number(block.layer2_block_number) + .await?; + deleted += 1; + } + Some(deleted) + } + async fn delete_block_by_number_with_session( + &self, + _block_number: u64, + session: &mut mongodb::ClientSession, + ) -> Result<(), mongodb::error::Error> { + let _ = session; + Err(mongodb::error::Error::custom( + "delete_block_by_number_with_session must be implemented with session-aware deletes", + )) + } + async fn delete_all_blocks_with_session( + &self, + session: &mut mongodb::ClientSession, + ) -> Result { + let blocks = self + .get_all_blocks() + .await + .ok_or_else(|| mongodb::error::Error::custom("Could not list proposed blocks"))?; + let mut deleted = 0_u64; + for block in blocks { + self.delete_block_by_number_with_session(block.layer2_block_number, session) + .await?; + deleted += 1; + } + Ok(deleted) + } +} + +#[async_trait::async_trait] +pub trait SyncStateDB { + async fn update_sync_state_with_session( + &self, + state: &SyncState, + session: &mut mongodb::ClientSession, + ) -> Result<(), mongodb::error::Error>; + + async fn get_sync_state(&self) -> Option; + + async fn delete_sync_state_with_session( + &self, + _session: &mut mongodb::ClientSession, + ) -> Result<(), mongodb::error::Error> { + Err(mongodb::error::Error::custom( + "delete_sync_state_with_session must be implemented with session-aware deletes", + )) + } +} + +#[async_trait::async_trait] +pub trait RestoreJournalDB { + async fn upsert_restore_journal( + &self, + journal: &RestoreJournal, + ) -> Result<(), mongodb::error::Error>; + + async fn get_restore_journal(&self) -> Option; + + async fn delete_restore_journal(&self) -> Result<(), mongodb::error::Error>; +} + +#[async_trait::async_trait] +pub trait StartupReplayResetMarkerDB { + async fn upsert_startup_replay_reset_marker( + &self, + marker: &StartupReplayResetMarker, + ) -> Result<(), mongodb::error::Error>; + + async fn upsert_startup_replay_reset_marker_with_session( + &self, + _marker: &StartupReplayResetMarker, + _session: &mut mongodb::ClientSession, + ) -> Result<(), mongodb::error::Error> { + Err(mongodb::error::Error::custom( + "upsert_startup_replay_reset_marker_with_session must be implemented with session-aware writes", + )) + } + + async fn get_startup_replay_reset_marker(&self) -> Option; + + async fn delete_startup_replay_reset_marker(&self) -> Result<(), mongodb::error::Error>; } + +#[async_trait::async_trait] +pub trait PendingBlockDB { + async fn store_pending_block(&self, pending_block: &PendingBlock) -> Option<()>; + async fn get_pending_block(&self, block_number: u64) -> Option; + async fn get_all_pending_blocks(&self) -> Option>; + async fn delete_pending_block(&self, block_number: u64) -> Option<()>; + async fn delete_all_pending_blocks(&self) -> Option { + let pending_blocks = self.get_all_pending_blocks().await?; + let mut deleted = 0_u64; + for pending_block in pending_blocks { + self.delete_pending_block(pending_block.layer2_block_number) + .await?; + deleted += 1; + } + Some(deleted) + } + async fn delete_all_pending_blocks_with_session( + &self, + session: &mut mongodb::ClientSession, + ) -> Result { + let _ = session; + Err(mongodb::error::Error::custom( + "delete_all_pending_blocks_with_session must be implemented with session-aware deletes", + )) + } +} + /// Used to store transactions that are on chain. Can be queried to see if a nullifier or commitment is on chain. #[async_trait::async_trait] pub trait TransactionsDB<'a, P> { @@ -30,6 +161,9 @@ pub trait TransactionsDB<'a, P> { async fn get_all_selected_client_transactions( &self, ) -> Option, ClientTransactionWithMetaData

)>>; + async fn get_all_selected_or_included_client_transactions( + &self, + ) -> Option, ClientTransactionWithMetaData

)>>; async fn count_mempool_client_transactions(&self) -> Result; async fn count_mempool_swap_transactions( &self, @@ -54,10 +188,19 @@ pub trait TransactionsDB<'a, P> { transactions: &[ClientTransactionWithMetaData

], block_l2: u64, ) -> Option; + async fn mark_transactions_included_by_hashes( + &self, + transaction_hashes: &[Vec], + ) -> Option; async fn drop_transactions( &self, transactions: &[ClientTransactionWithMetaData

], ) -> Option; + async fn set_client_transactions_in_mempool_by_hashes( + &self, + transaction_hashes: &[Vec], + in_mempool: bool, + ) -> Option; async fn find_transaction( &self, tx: &ClientTransaction

, @@ -70,6 +213,12 @@ pub trait TransactionsDB<'a, P> { &self, used_deposits: Vec>, ) -> Option; + async fn set_mempool_deposits_reserved( + &self, + deposits: Vec>, + reserved: bool, + ) -> Option; + async fn clear_all_mempool_deposit_reservations(&self) -> Option; async fn remove_all_mempool_deposits(&self) -> Option; async fn remove_all_mempool_client_transactions(&self) -> Option; } @@ -178,3 +327,35 @@ pub trait MerkleTreeDB { async fn new_tree(&mut self, tree_height: u32, tree_name: &str) -> Result<(), Self::Error>; async fn get_tree_height(&self, tree_name: &str) -> Result; } + +/// Error type for transfer receipt storage operations. +#[derive(Debug)] +pub enum TransferReceiptStoreError { + /// Duplicate unique key (receipt_id or tx_hash already exists). + DuplicateKey, + /// Any other storage error. + Other(String), +} + +/// Trait for a DB that stores and retrieves transfer receipts. +#[async_trait::async_trait] +pub trait TransferReceiptDB { + async fn store_transfer_receipt( + &self, + receipt: TransferReceipt, + ) -> Result<(), TransferReceiptStoreError>; + + async fn get_transfer_receipt(&self, receipt_id: &str) -> Option; + + async fn get_transfer_receipt_by_tx_hash( + &self, + tx_hash: &TxHashBytes, + ) -> Option; + + async fn set_transfer_receipt_status( + &self, + receipt_id: &str, + status: TransferReceiptStatus, + updated_at_unix: i64, + ) -> Option<()>; +} diff --git a/nightfall_proposer/src/ports/events.rs b/nightfall_proposer/src/ports/events.rs index dac3e8f6..0722a10e 100644 --- a/nightfall_proposer/src/ports/events.rs +++ b/nightfall_proposer/src/ports/events.rs @@ -1,4 +1,4 @@ -use alloy::primitives::TxHash; +use alloy::rpc::types::Log; use lib::{ error::EventHandlerError, nf_client_proof::{Proof, ProvingEngine}, @@ -14,5 +14,5 @@ where E: ProvingEngine

, N: NightfallContract, { - async fn handle_event(&self, transaction_hash: TxHash) -> Result<(), EventHandlerError>; + async fn handle_event(&self, log: Log) -> Result<(), EventHandlerError>; } diff --git a/nightfall_proposer/src/ports/trees.rs b/nightfall_proposer/src/ports/trees.rs index 34878070..663c43b5 100644 --- a/nightfall_proposer/src/ports/trees.rs +++ b/nightfall_proposer/src/ports/trees.rs @@ -7,7 +7,7 @@ use configuration::settings::get_settings; use jf_primitives::{poseidon::PoseidonParams, trees::MembershipProof}; use lib::merkle_trees::trees::{IndexedTree, MerkleTreeError, MutableTree}; use log::debug; -use mongodb::Client; +use mongodb::{Client, ClientSession}; /// Trait defining the functionality of a commitment tree. #[async_trait::async_trait] @@ -26,6 +26,23 @@ where .await?; Ok(result) } + /// Add leaves into the tree using the provided MongoDB session. + async fn append_sub_trees_with_session( + &self, + leaves: &[F], + update_tree: bool, + session: &mut ClientSession, + ) -> Result { + let (result, _) = >::append_sub_trees_with_session( + self, + leaves, + update_tree, + Self::TREE_NAME, + Some(session), + ) + .await?; + Ok(result) + } /// Inserts leaves into the tree and returns information allowing us to verify in a circuit. async fn insert_for_circuit(&mut self, leaves: &[F]) -> Result; /// let's multiple sub trees be added in a single batch - it calls insert_subtree for each sub tree @@ -43,6 +60,10 @@ where } /// get the root of the tree async fn get_root(&self) -> Result; + /// get the root of the tree using the provided MongoDB session + async fn get_root_with_session(&self, session: &mut ClientSession) -> Result { + >::get_root_with_session(self, Self::TREE_NAME, Some(session)).await + } /// reset the tree async fn reset_tree(&self) -> Result<(), Self::Error> where @@ -97,6 +118,20 @@ where ) -> Result>::Error> { >::insert_leaves(self, nullifiers, Self::TREE_NAME).await } + /// inserts new nullifiers into the tree using the provided MongoDB session. + async fn insert_nullifiers_with_session( + &self, + nullifiers: &[F], + session: &mut ClientSession, + ) -> Result>::Error> { + >::insert_leaves_with_session( + self, + nullifiers, + Self::TREE_NAME, + Some(session), + ) + .await + } /// gets a non-inclusion proof for a nullifier in the tree. async fn get_non_membership_proof( &self, @@ -179,6 +214,23 @@ where .await?; Ok(result) } + /// Add a historic root using the provided MongoDB session. + async fn append_historic_commitment_root_with_session( + &self, + historic_commitment_root: &F, + update_tree: bool, + session: &mut ClientSession, + ) -> Result { + let (result, _) = >::append_sub_trees_with_session( + self, + &[*historic_commitment_root], + update_tree, + Self::TREE_NAME, + Some(session), + ) + .await?; + Ok(result) + } /// get a new historic root tree async fn new_historic_root_tree(&self, tree_height: u32) -> Result<(), Self::Error> { >::new_mutable_tree(self, tree_height, 0, Self::TREE_NAME).await diff --git a/nightfall_proposer/src/services/assemble_block.rs b/nightfall_proposer/src/services/assemble_block.rs index e850c94d..0b53c679 100644 --- a/nightfall_proposer/src/services/assemble_block.rs +++ b/nightfall_proposer/src/services/assemble_block.rs @@ -1,14 +1,17 @@ use crate::{ - domain::entities::{Block, ClientTransactionWithMetaData, DepositDatawithFee}, - driven::db::mongo_db::{StoredBlock, DB, PROPOSED_BLOCKS_COLLECTION}, + domain::entities::{ + Block, ClientTransactionWithMetaData, DepositDatawithFee, PendingBlock, PendingBlockState, + }, + driven::db::{mongo_db::StoredBlock, snapshot::acquire_proposer_state_maintenance_guard}, drivers::blockchain::block_assembly::BlockAssemblyError, initialisation::{get_blockchain_client_connection, get_db_connection}, ports::{ - db::{BlockStorageDB, TransactionsDB}, + db::{BlockStorageDB, PendingBlockDB, TransactionsDB}, proving::RecursiveProvingEngine, }, services::selected_transactions::reconcile_orphaned_selected_transactions, }; +use alloy::primitives::Address; use ark_bn254::Fr as Fr254; use ark_std::{collections::HashSet, Zero}; use jf_primitives::poseidon::{FieldHasher, Poseidon}; @@ -19,7 +22,7 @@ use lib::{ shared_entities::DepositData, utils::get_block_size, }; -use log::{info, warn}; +use log::{error, info, warn}; use std::cmp::Reverse; use tokio::time::Instant; @@ -178,30 +181,68 @@ pub(crate) fn transactions_to_include_in_block( // If we have more than block_size transactions, we'll only include the block_size - DepositDatas most valuable transactions mempool_transactions.unwrap_or_default() } -/// assemble_block is the main function that is called by the proposer to create a new block, -/// it fetches the necessary data from the database and the contract, then assembles the block -pub(crate) async fn assemble_block( - current_l2_block_number: u64, -) -> Result +async fn assemble_block_with_context( + db: &mongodb::Client, + block_size: usize, + layer2_block_number: u64, + proposer_address: Address, +) -> Result where P: Proof, R: RecursiveProvingEngine

+ Send + Sync + 'static, { - info!("Starting block assembly process"); - // initialise included_depositinfos_group, selected_client_transactions - let included_depositinfos_group; - let selected_client_transactions: Vec>; + let _maintenance_guard = acquire_proposer_state_maintenance_guard().await; + let result = prepare_block_data::

(db, block_size, layer2_block_number).await; + match &result { + Ok(_) => info!("Block data prepared successfully"), + Err(e) => warn!("Failed to prepare block data: {e:?}"), + } + + let (included_depositinfos_group, selected_client_transactions) = result?; + let selected_client_transaction_hashes = + selected_client_transaction_hashes(&selected_client_transactions); + + let reserved_pending_block = PendingBlock { + layer2_block_number, + state: PendingBlockState::Reserved, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: None, + selected_deposits: included_depositinfos_group.clone(), + selected_client_transaction_hashes: selected_client_transaction_hashes.clone(), + }; + + if db + .store_pending_block(&reserved_pending_block) + .await + .is_none() + { + return Err(BlockAssemblyError::QueueError( + "Failed to persist reserved pending block".to_string(), + )); + } + + if let Err(e) = reserve_selected_transactions::

( + db, + &included_depositinfos_group, + &selected_client_transactions, + layer2_block_number, + ) + .await { - info!("Getting DB connection"); - let db = get_db_connection().await; - info!("Preparing block data"); - let block_size = get_block_size()?; - let result = prepare_block_data::

(db, block_size, current_l2_block_number).await; - match &result { - Ok(_) => info!("Block data prepared successfully"), - Err(e) => warn!("Failed to prepare block data: {e:?}"), + if let Err(release_error) = release_selected_transactions::

( + db, + &included_depositinfos_group, + &selected_client_transaction_hashes, + ) + .await + { + error!( + "Failed to release selected transactions after reservation failure: {release_error}" + ); } - (included_depositinfos_group, selected_client_transactions) = result?; + let _ = db.delete_pending_block(layer2_block_number).await; + return Err(e); } // Convert DepositInfo into DepositData while maintaining nested structure @@ -237,35 +278,100 @@ where {withdraw_count} withdrawal(s), and {swap_count} swap transaction(s) ({} pair(s))", swap_count / 2 ); - let block = make_block::(included_deposits, selected_client_transactions).await?; - // save this block to Store block db + + let mut block = match make_block::(included_deposits, selected_client_transactions).await + { + Ok(block) => block, + Err(e) => { + if let Err(release_error) = release_selected_transactions::

( + db, + &included_depositinfos_group, + &selected_client_transaction_hashes, + ) + .await + { + error!( + "Failed to release reserved transactions after proving failure: {release_error}" + ); + } + let _ = db.delete_pending_block(layer2_block_number).await; + return Err(e); + } + }; + block.block_number = layer2_block_number; + + let store_block = make_stored_block(&block, layer2_block_number, proposer_address); + if db.store_block(&store_block).await.is_none() { + if let Err(release_error) = release_selected_transactions::

( + db, + &included_depositinfos_group, + &selected_client_transaction_hashes, + ) + .await + { + error!( + "Failed to release reserved transactions after speculative block storage failed: {release_error}" + ); + } + let _ = db.delete_pending_block(layer2_block_number).await; + return Err(BlockAssemblyError::QueueError( + "Failed to store speculative block".to_string(), + )); + } + + let pending_block = PendingBlock { + layer2_block_number, + state: PendingBlockState::ReadyToPropose, + broadcast_tx_hash: None, + broadcast_receipt_checks: 0, + block: Some(block), + selected_deposits: included_depositinfos_group, + selected_client_transaction_hashes, + }; + + if db.store_pending_block(&pending_block).await.is_none() { + let _ = db.delete_block_by_number(layer2_block_number).await; + if let Err(release_error) = release_selected_transactions::

( + db, + &pending_block.selected_deposits, + &pending_block.selected_client_transaction_hashes, + ) + .await + { + error!( + "Failed to release reserved transactions after pending block storage failed: {release_error}" + ); + } + let _ = db.delete_pending_block(layer2_block_number).await; + return Err(BlockAssemblyError::QueueError( + "Failed to persist pending block".to_string(), + )); + } + + Ok(pending_block) +} + +/// assemble_block is the main function that is called by the proposer to create a new block, +/// it fetches the necessary data from the database and the contract, then assembles the block +pub(crate) async fn assemble_block( + current_l2_block_number: u64, +) -> Result +where + P: Proof, + R: RecursiveProvingEngine

+ Send + Sync + 'static, +{ + info!("Starting block assembly process"); + info!("Getting DB connection"); let db = get_db_connection().await; + info!("Preparing block data"); + let block_size = get_block_size()?; let our_address = get_blockchain_client_connection() .await .read() .await .get_address(); - let store_block = StoredBlock { - layer2_block_number: current_l2_block_number, - commitments: block - .transactions - .iter() - .flat_map(|ntx| { - ntx.commitments - .iter() - .map(|c| c.to_hex_string()) - .collect::>() - }) - .collect(), - proposer_address: our_address, - }; - db.database(DB) - .collection::(PROPOSED_BLOCKS_COLLECTION) - .insert_one(store_block.clone()) - .await - .expect("Failed to insert block into database"); - Ok(block) + assemble_block_with_context::(db, block_size, current_l2_block_number, our_address).await } // this is where we compute the on chain block it's called by make_block @@ -544,15 +650,7 @@ where let deposit_count = used_deposits_info.len(); let reordered = place_swap_pairs_on_siblings(deposit_count, &selected_transactions); - // 9. Delete used deposits in mempool - >::remove_mempool_deposits(db, used_deposits_info.clone()) - .await; - - // 10. Mark selected client transactions as already chosen for this block. - db.mark_transactions_selected_for_block(&reordered, current_block_number) - .await; - - // 10b. Clear expired swaps from mempool + // Clear expired swaps from mempool; selected inputs are reserved after this function returns. if !expired_swaps.is_empty() { db.drop_transactions(&expired_swaps).await; } @@ -565,14 +663,324 @@ where Ok((used_deposits_info, reordered)) } +fn selected_client_transaction_hashes

( + selected_client_transactions: &[ClientTransactionWithMetaData

], +) -> Vec> { + selected_client_transactions + .iter() + .map(|tx| tx.hash.clone()) + .collect() +} + +fn make_stored_block( + block: &Block, + layer2_block_number: u64, + proposer_address: Address, +) -> StoredBlock { + StoredBlock { + layer2_block_number, + commitments: block + .transactions + .iter() + .flat_map(|ntx| { + ntx.commitments + .iter() + .map(|c| c.to_hex_string()) + .collect::>() + }) + .collect(), + proposer_address, + } +} + +pub(crate) async fn reserve_selected_transactions

( + db: &mongodb::Client, + selected_deposits: &[Vec], + selected_client_transactions: &[ClientTransactionWithMetaData

], + layer2_block_number: u64, +) -> Result<(), BlockAssemblyError> +where + P: Proof, +{ + >::set_mempool_deposits_reserved( + db, + selected_deposits.to_vec(), + true, + ) + .await + .ok_or_else(|| { + BlockAssemblyError::QueueError("Failed to reserve selected deposits".to_string()) + })?; + + let modified = db + .mark_transactions_selected_for_block(selected_client_transactions, layer2_block_number) + .await + .ok_or_else(|| { + BlockAssemblyError::QueueError( + "Failed to reserve selected client transactions".to_string(), + ) + })?; + + if modified != selected_client_transactions.len() as u64 { + let selected_client_transaction_hashes = + selected_client_transaction_hashes(selected_client_transactions); + let _ = release_selected_transactions::

( + db, + selected_deposits, + &selected_client_transaction_hashes, + ) + .await; + return Err(BlockAssemblyError::QueueError(format!( + "Reserved {modified} of {} selected client transactions", + selected_client_transactions.len() + ))); + } + + Ok(()) +} + +pub(crate) async fn release_selected_transactions

( + db: &mongodb::Client, + selected_deposits: &[Vec], + selected_client_transaction_hashes: &[Vec], +) -> Result<(), BlockAssemblyError> +where + P: Proof, +{ + >::set_mempool_deposits_reserved( + db, + selected_deposits.to_vec(), + false, + ) + .await + .ok_or_else(|| { + BlockAssemblyError::QueueError("Failed to release reserved deposits".to_string()) + })?; + + >::set_client_transactions_in_mempool_by_hashes( + db, + selected_client_transaction_hashes, + true, + ) + .await + .ok_or_else(|| { + BlockAssemblyError::QueueError("Failed to release reserved client transactions".to_string()) + })?; + Ok(()) +} + +pub(crate) async fn cleanup_selected_transactions

( + db: &mongodb::Client, + selected_deposits: &[Vec], + selected_client_transaction_hashes: &[Vec], +) -> Result<(), BlockAssemblyError> +where + P: Proof, +{ + >::remove_mempool_deposits(db, selected_deposits.to_vec()) + .await + .ok_or_else(|| { + BlockAssemblyError::QueueError( + "Failed to remove selected deposits from mempool".to_string(), + ) + })?; + + >::mark_transactions_included_by_hashes( + db, + selected_client_transaction_hashes, + ) + .await + .ok_or_else(|| { + BlockAssemblyError::QueueError( + "Failed to mark selected client transactions as included".to_string(), + ) + })?; + Ok(()) +} + #[cfg(test)] mod tests { use super::*; use crate::domain::entities::TxLifecycle; + use crate::driven::db::mongo_db::{DB, PROPOSED_BLOCKS_COLLECTION}; + use ark_bn254::Fq as Fq254; use lib::{ + nf_client_proof::PublicInputs, plonk_prover::plonk_proof::PlonkProof, tests_utils::{get_db_connection, get_mongo}, }; + use std::collections::HashSet; + + struct SuccessfulTestProver; + + impl RecursiveProvingEngine for SuccessfulTestProver { + type PreppedInfo = (); + type Error = BlockAssemblyError; + type RecursiveProof = Vec; + + async fn prepare_state_transition( + _deposit_transactions: &[(PlonkProof, PublicInputs)], + _transactions: &[ClientTransactionWithMetaData], + ) -> Result<(Self::PreppedInfo, [Fr254; 3]), Self::Error> { + Ok(( + (), + [Fr254::from(11u64), Fr254::from(12u64), Fr254::from(13u64)], + )) + } + + fn recursive_prove(_info: Self::PreppedInfo) -> Result { + Ok(vec![Fq254::from(99u64); 10]) + } + + fn create_deposit_proof( + _deposit_data: &[DepositData; 4], + _public_inputs: &mut PublicInputs, + ) -> Result { + Ok(PlonkProof::default()) + } + } + + struct FailingTestProver; + + impl RecursiveProvingEngine for FailingTestProver { + type PreppedInfo = (); + type Error = BlockAssemblyError; + type RecursiveProof = Vec; + + async fn prepare_state_transition( + _deposit_transactions: &[(PlonkProof, PublicInputs)], + _transactions: &[ClientTransactionWithMetaData], + ) -> Result<(Self::PreppedInfo, [Fr254; 3]), Self::Error> { + Err(BlockAssemblyError::ProvingError( + "intentional proving failure".to_string(), + )) + } + + fn recursive_prove(_info: Self::PreppedInfo) -> Result { + Ok(vec![Fq254::from(1u64); 10]) + } + + fn create_deposit_proof( + _deposit_data: &[DepositData; 4], + _public_inputs: &mut PublicInputs, + ) -> Result { + Ok(PlonkProof::default()) + } + } + + fn test_client_transaction(fee: u64) -> ClientTransactionWithMetaData { + ClientTransactionWithMetaData { + client_transaction: lib::shared_entities::ClientTransaction { + fee: Fr254::from(fee), + proof: PlonkProof::default(), + ..Default::default() + }, + lifecycle: TxLifecycle::Mempool, + hash: vec![fee as u32], + historic_roots: vec![Fr254::from(123)], + receipt_token: None, + } + } + + fn test_deposit(fee: u64) -> DepositDatawithFee { + DepositDatawithFee { + fee: Fr254::from(fee), + deposit_data: DepositData { + nf_token_id: Fr254::from(fee), + nf_slot_id: Fr254::from(fee), + value: Fr254::from(100u64), + secret_hash: Fr254::from(fee), + }, + reserved: false, + } + } + + async fn store_client_transactions( + db: &mongodb::Client, + fees: impl IntoIterator, + ) -> Vec> { + let transactions = fees + .into_iter() + .map(test_client_transaction) + .collect::>(); + for tx in &transactions { + db.store_transaction(tx.clone()).await.unwrap(); + } + transactions + } + + #[tokio::test] + async fn cleanup_selected_transactions_marks_client_transactions_included() { + let container = get_mongo().await; + let db = get_db_connection(&container).await; + let tx = test_client_transaction(42); + db.store_transaction(tx.clone()).await.unwrap(); + + reserve_selected_transactions::(&db, &[], std::slice::from_ref(&tx), 7) + .await + .unwrap(); + cleanup_selected_transactions::(&db, &[], std::slice::from_ref(&tx.hash)) + .await + .unwrap(); + + let stored: ClientTransactionWithMetaData = + db.get_transaction(&tx.hash).await.unwrap(); + assert_eq!(stored.lifecycle, TxLifecycle::Included { block_l2: 7 }); + assert_eq!( + >::count_mempool_client_transactions(&db) + .await + .unwrap(), + 0, + "included transactions must not be selectable again" + ); + assert!( + >::get_all_selected_client_transactions( + &db + ) + .await + .unwrap_or_default() + .is_empty(), + "included transactions should not look like in-flight reservations" + ); + } + + #[tokio::test] + async fn cleanup_selected_transactions_skips_non_selected_transactions() { + let container = get_mongo().await; + let db = get_db_connection(&container).await; + let selected_tx = test_client_transaction(52); + let mempool_tx = test_client_transaction(53); + db.store_transaction(selected_tx.clone()).await.unwrap(); + db.store_transaction(mempool_tx.clone()).await.unwrap(); + + reserve_selected_transactions::( + &db, + &[], + std::slice::from_ref(&selected_tx), + 8, + ) + .await + .unwrap(); + cleanup_selected_transactions::( + &db, + &[], + &[selected_tx.hash.clone(), mempool_tx.hash.clone()], + ) + .await + .unwrap(); + + let stored_selected: ClientTransactionWithMetaData = + db.get_transaction(&selected_tx.hash).await.unwrap(); + assert_eq!( + stored_selected.lifecycle, + TxLifecycle::Included { block_l2: 8 } + ); + + let stored_mempool: ClientTransactionWithMetaData = + db.get_transaction(&mempool_tx.hash).await.unwrap(); + assert!(stored_mempool.lifecycle.is_mempool()); + } + #[tokio::test] async fn test_prepare_block_data_simple_case() { // Prepare data: 44 deposit data in mempool, fee (1...240), 4 tx data, fee (241...244) @@ -595,6 +1003,7 @@ mod tests { value: Fr254::from(100u64), secret_hash: Fr254::from(i), }, + reserved: false, }) .collect(); @@ -613,6 +1022,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![i as u32], historic_roots: vec![Fr254::from(123)], + receipt_token: None, }) .collect(); @@ -660,14 +1070,29 @@ mod tests { "Incorrect number of client transactions included" ); - // **3. Check that the remaining 2 deposits are stored back in the mempool** + // Selection should not mutate the mempool. Cleanup happens after proposal/reconciliation. let remaining_deposits = { >::get_mempool_deposits(&db).await }; assert!( remaining_deposits .as_ref() - .is_none_or(|deposits| deposits.is_empty()), - "Remaining deposits are not empty" + .is_some_and(|deposits| deposits.len() == 240), + "Deposit mempool should remain unchanged until cleanup" + ); + let remaining_client_transactions = { + let mempool_client_transactions: Option< + Vec<(Vec, ClientTransactionWithMetaData)>, + > = db.get_all_mempool_client_transactions().await; + + transactions_to_include_in_block(mempool_client_transactions) + .into_iter() + .map(|(_, v)| v) + .collect::>>() + }; + assert_eq!( + remaining_client_transactions.len(), + 4, + "Client transaction mempool should remain unchanged until cleanup" ); } @@ -691,6 +1116,7 @@ mod tests { value: Fr254::from(i), secret_hash: Fr254::from(i), }, + reserved: false, }) .collect(); @@ -726,13 +1152,12 @@ mod tests { let remaining_deposits = { >::get_mempool_deposits(&db).await }; - // fee in the remaining deposit should be 1 let remain_deposits_fee: Vec = remaining_deposits.unwrap().iter().map(|d| d.fee).collect(); assert_eq!( - remain_deposits_fee, - vec![Fr254::from(1)], - "Remaining deposit fees do not match expected values" + remain_deposits_fee.len(), + 257, + "Deposit mempool should remain unchanged until cleanup" ); } @@ -758,6 +1183,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![i as u32], historic_roots: vec![Fr254::from(123)], + receipt_token: None, }) .collect(); @@ -802,10 +1228,10 @@ mod tests { .collect(); actual_remaining_client_fees.sort_by_key(|&fee| Reverse(fee)); - let expected_remaining_client_fees: Vec = (1..=10).rev().map(Fr254::from).collect(); + let expected_remaining_client_fees: Vec = (1..=74).rev().map(Fr254::from).collect(); assert_eq!( actual_remaining_client_fees, expected_remaining_client_fees, - "Remaining client transaction fees do not match expected values" + "Client transaction mempool should remain unchanged until cleanup" ); } @@ -831,6 +1257,7 @@ mod tests { value: Fr254::from(100u64), secret_hash: Fr254::from(i), }, + reserved: false, }) .collect(); @@ -850,6 +1277,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![i as u32], historic_roots: vec![Fr254::from(123)], + receipt_token: None, }) .collect(); @@ -901,9 +1329,10 @@ mod tests { .map(|deposit| deposit.fee) // Extracting only the fees .collect() }; - assert!( - actual_fees_deposit_remainning.is_empty(), - "Remaining deposit fees should be empty" + assert_eq!( + actual_fees_deposit_remainning.len(), + 4, + "Deposit mempool should remain unchanged until cleanup" ); let remaining_client = { @@ -916,17 +1345,116 @@ mod tests { .map(|(_, v)| v) .collect::>>() }; - let remaining_client_fees: Vec = remaining_client + let mut remaining_client_fees: Vec = remaining_client .iter() .map(|d| d.client_transaction.fee) .collect(); + remaining_client_fees.sort_by_key(|&fee| Reverse(fee)); + let expected_remaining_client_fees: Vec = (1..=64).rev().map(Fr254::from).collect(); assert_eq!( - remaining_client_fees, - vec![Fr254::from(1)], - "Remaining client transaction fees do not match expected values" + remaining_client_fees, expected_remaining_client_fees, + "Client transaction mempool should remain unchanged until cleanup" ); } + #[tokio::test] + async fn test_assemble_block_releases_reserved_transactions_on_proving_error() { + let container = get_mongo().await; + let db = get_db_connection(&container).await; + + >::set_mempool_deposits( + &db, + (1..=4).map(test_deposit).collect(), + ) + .await; + store_client_transactions(&db, 101..=104).await; + + let result = assemble_block_with_context::( + &db, + 64, + 0, + Address::from([7u8; 20]), + ) + .await; + + assert!(matches!( + result, + Err(BlockAssemblyError::ProvingError(message)) + if message == "intentional proving failure" + )); + + let available_deposits = + >::get_mempool_deposits(&db) + .await + .unwrap(); + assert_eq!(available_deposits.len(), 4); + + let available_client_transactions = + >::get_all_mempool_client_transactions( + &db, + ) + .await + .unwrap_or_default(); + assert_eq!(available_client_transactions.len(), 4); + + let pending_blocks = db.get_all_pending_blocks().await.unwrap_or_default(); + assert!(pending_blocks.is_empty()); + + let stored_blocks = db.get_all_blocks().await.unwrap_or_default(); + assert!(stored_blocks.is_empty()); + } + + #[tokio::test] + async fn test_assemble_block_skips_reserved_transactions_in_second_block() { + let container = get_mongo().await; + let db = get_db_connection(&container).await; + + store_client_transactions(&db, 1..=128).await; + + let first_block = assemble_block_with_context::( + &db, + 64, + 0, + Address::from([8u8; 20]), + ) + .await + .unwrap(); + let second_block = assemble_block_with_context::( + &db, + 64, + 1, + Address::from([8u8; 20]), + ) + .await + .unwrap(); + + let first_hashes = first_block + .selected_client_transaction_hashes + .iter() + .cloned() + .collect::>(); + let second_hashes = second_block + .selected_client_transaction_hashes + .iter() + .cloned() + .collect::>(); + + assert_eq!(first_hashes.len(), 64); + assert_eq!(second_hashes.len(), 64); + assert!(first_hashes.is_disjoint(&second_hashes)); + + let available_client_transactions = + >::get_all_mempool_client_transactions( + &db, + ) + .await + .unwrap_or_default(); + assert!(available_client_transactions.is_empty()); + + let pending_blocks = db.get_all_pending_blocks().await.unwrap_or_default(); + assert_eq!(pending_blocks.len(), 2); + } + #[tokio::test] async fn test_prepare_block_data_with_swaps() { let container = get_mongo().await; @@ -942,6 +1470,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![1], historic_roots: vec![], + receipt_token: None, }; let swap_txs: Vec> = vec![ @@ -957,6 +1486,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![2], historic_roots: vec![], + receipt_token: None, }, ClientTransactionWithMetaData { client_transaction: lib::shared_entities::ClientTransaction { @@ -970,6 +1500,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![3], historic_roots: vec![], + receipt_token: None, }, ]; @@ -1019,6 +1550,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![i as u32 + 1], historic_roots: vec![], + receipt_token: None, }) .collect(); @@ -1035,6 +1567,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![10 + i as u32], historic_roots: vec![], + receipt_token: None, }) .collect(); @@ -1088,6 +1621,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![i as u32 + 1], historic_roots: vec![], + receipt_token: None, }) .collect(); @@ -1153,6 +1687,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![20 + i as u32], historic_roots: vec![], + receipt_token: None, }) .collect(); @@ -1216,6 +1751,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![9, 9, 9, 9], historic_roots: vec![], + receipt_token: None, }; db.store_transaction(stale_tx.clone()).await.unwrap(); @@ -1245,6 +1781,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![1, 1, 1], historic_roots: vec![], + receipt_token: None, }; let cancelled_tx = ClientTransactionWithMetaData { client_transaction: lib::shared_entities::ClientTransaction { @@ -1255,6 +1792,7 @@ mod tests { lifecycle: TxLifecycle::Cancelled, hash: vec![2, 2, 2], historic_roots: vec![], + receipt_token: None, }; db.store_transaction(active_tx.clone()).await.unwrap(); @@ -1293,6 +1831,7 @@ mod tests { value: Fr254::from(100u64), secret_hash: Fr254::from(i as u64), }, + reserved: false, }) .collect(); >::set_mempool_deposits(&db, deposits).await; @@ -1311,6 +1850,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![50 + i as u32], historic_roots: vec![], + receipt_token: None, }) .collect(); @@ -1327,6 +1867,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![70 + i as u32], historic_roots: vec![], + receipt_token: None, }) .collect(); @@ -1384,6 +1925,7 @@ mod tests { value: Fr254::from(100u64), secret_hash: Fr254::from(i as u64), }, + reserved: false, }) .collect(); >::set_mempool_deposits(&db, deposits).await; @@ -1401,6 +1943,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![600 + i as u32], historic_roots: vec![], + receipt_token: None, }) .collect(); @@ -1462,6 +2005,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![200 + i as u32], historic_roots: vec![], + receipt_token: None, }) .collect(); @@ -1514,6 +2058,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![210], historic_roots: vec![], + receipt_token: None, }) .await .unwrap(); @@ -1564,6 +2109,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![300], historic_roots: vec![], + receipt_token: None, }, ClientTransactionWithMetaData { client_transaction: lib::shared_entities::ClientTransaction { @@ -1577,6 +2123,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![301], historic_roots: vec![], + receipt_token: None, }, ClientTransactionWithMetaData { client_transaction: lib::shared_entities::ClientTransaction { @@ -1590,6 +2137,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![302], historic_roots: vec![], + receipt_token: None, }, ]; @@ -1607,7 +2155,12 @@ mod tests { .count(); assert_eq!(swap_count, 2, "Exactly one swap pair should be selected"); - // One leg must remain in mempool. + let selected_deposits: Vec> = vec![]; + reserve_selected_transactions::(&db, &selected_deposits, &selected, 0) + .await + .unwrap(); + + // The selected pair is no longer selectable after reservation; the unmatched leg remains. let remaining_client = { let mempool_client_transactions: Option< Vec<(Vec, ClientTransactionWithMetaData)>, @@ -1648,6 +2201,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![400], historic_roots: vec![], + receipt_token: None, }, ClientTransactionWithMetaData { client_transaction: lib::shared_entities::ClientTransaction { @@ -1661,6 +2215,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![401], historic_roots: vec![], + receipt_token: None, }, ]; @@ -1713,6 +2268,7 @@ mod tests { lifecycle: TxLifecycle::Mempool, hash: vec![500 + i as u32], historic_roots: vec![], + receipt_token: None, }) .collect(); diff --git a/nightfall_proposer/src/services/mod.rs b/nightfall_proposer/src/services/mod.rs index d6164556..4e122ec3 100644 --- a/nightfall_proposer/src/services/mod.rs +++ b/nightfall_proposer/src/services/mod.rs @@ -2,3 +2,4 @@ pub mod assemble_block; pub mod process_client_transaction; pub mod process_events; pub mod selected_transactions; +pub mod snapshot_scheduler; diff --git a/nightfall_proposer/src/services/process_client_transaction.rs b/nightfall_proposer/src/services/process_client_transaction.rs index 07c1f2bd..cdcebc7d 100644 --- a/nightfall_proposer/src/services/process_client_transaction.rs +++ b/nightfall_proposer/src/services/process_client_transaction.rs @@ -7,7 +7,7 @@ use std::error::Error; pub async fn process_client_transaction( client_transaction: ClientTransaction

, -) -> Result<(), Box> +) -> Result, Box> where P: Proof, E: ProvingEngine

, @@ -15,6 +15,6 @@ where // This just calls through to the repository, because both the event processor and the REST API // need to call the following function. Thus, it can't really be located in services, otherwise we'd be // doing a reentrant call from repository to services, which is a bit of an odd pattern. - process_nightfall_client_transaction::(client_transaction).await?; - Ok(()) + let receipt_token = process_nightfall_client_transaction::(client_transaction).await?; + Ok(receipt_token) } diff --git a/nightfall_proposer/src/services/process_events.rs b/nightfall_proposer/src/services/process_events.rs index 0fee1d38..c20c79ce 100644 --- a/nightfall_proposer/src/services/process_events.rs +++ b/nightfall_proposer/src/services/process_events.rs @@ -16,5 +16,5 @@ where E: ProvingEngine

, N: NightfallContract, { - e.handle_event(log.transaction_hash.unwrap()).await + e.handle_event(log).await } diff --git a/nightfall_proposer/src/services/selected_transactions.rs b/nightfall_proposer/src/services/selected_transactions.rs index 9da40e59..017330da 100644 --- a/nightfall_proposer/src/services/selected_transactions.rs +++ b/nightfall_proposer/src/services/selected_transactions.rs @@ -1,10 +1,14 @@ use crate::{ - domain::entities::{Block, ClientTransactionWithMetaData}, - driven::db::mongo_db::StoredBlock, - ports::db::{BlockStorageDB, TransactionsDB}, + domain::entities::{Block, ClientTransactionWithMetaData, TxLifecycle}, + driven::db::{ + client_transaction_state::{lifecycle_bson, CLIENT_TRANSACTIONS_COLLECTION}, + mongo_db::{StoredBlock, DB}, + }, + ports::db::{BlockStorageDB, PendingBlockDB, TransactionsDB}, }; use ark_ff::Zero; use lib::{hex_conversion::HexConvertible, nf_client_proof::Proof}; +use mongodb::bson::doc; use std::collections::{HashMap, HashSet}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -34,9 +38,11 @@ fn block_contains_transaction

( block_commitments: &HashSet, transaction: &ClientTransactionWithMetaData

, ) -> bool { - transaction_commitments_hex(transaction) - .into_iter() - .all(|commitment| block_commitments.contains(&commitment)) + let transaction_commitments = transaction_commitments_hex(transaction); + !transaction_commitments.is_empty() + && transaction_commitments + .into_iter() + .all(|commitment| block_commitments.contains(&commitment)) } fn stored_block_commitments_map(stored_blocks: Vec) -> HashMap> { @@ -51,6 +57,16 @@ fn stored_block_commitments_map(stored_blocks: Vec) -> HashMap Option>> { + let pending_blocks = ::get_all_pending_blocks(db).await?; + Some( + pending_blocks + .into_iter() + .flat_map(|pending_block| pending_block.selected_client_transaction_hashes.into_iter()) + .collect(), + ) +} + fn classify_selected_transaction

( transaction: ClientTransactionWithMetaData

, stored_blocks: &HashMap>, @@ -143,15 +159,94 @@ where { let classified_transactions = get_classified_selected_transactions::

(db, current_layer2_block_number, true).await?; + let pending_hashes = pending_selected_transaction_hashes(db).await?; let orphaned_transactions = classified_transactions .into_iter() - .filter(|classified| classified.state == SelectedTransactionState::Orphaned) + .filter(|classified| { + classified.state == SelectedTransactionState::Orphaned + && !pending_hashes.contains(&classified.transaction.hash) + }) .map(|classified| classified.transaction) .collect::>(); restore_selected_transactions_grouped_by_block(db, orphaned_transactions).await } +fn reconcile_active_transaction_lifecycle

( + transaction: &ClientTransactionWithMetaData

, + stored_blocks: &HashMap>, + current_layer2_block_number: u64, +) -> Option { + let block_l2 = transaction.lifecycle.block_l2()?; + + let lifecycle = match stored_blocks.get(&block_l2) { + Some(block_commitments) if block_contains_transaction(block_commitments, transaction) => { + TxLifecycle::Included { block_l2 } + } + Some(_) => TxLifecycle::Mempool, + None if block_l2 < current_layer2_block_number => TxLifecycle::Mempool, + _ => TxLifecycle::Selected { block_l2 }, + }; + + Some(lifecycle) +} + +pub async fn reconcile_active_client_transaction_lifecycle

( + db: &mongodb::Client, + current_layer2_block_number: u64, +) -> Option +where + P: Proof, +{ + let active_transactions = + >::get_all_selected_or_included_client_transactions( + db, + ) + .await?; + let stored_blocks = stored_block_commitments_map( + ::get_all_blocks(db).await?, + ); + let collection = db + .database(DB) + .collection::(CLIENT_TRANSACTIONS_COLLECTION); + + let mut modified = 0_u64; + for (_, transaction) in active_transactions { + let Some(target_lifecycle) = reconcile_active_transaction_lifecycle( + &transaction, + &stored_blocks, + current_layer2_block_number, + ) else { + continue; + }; + + if transaction.lifecycle == target_lifecycle { + continue; + } + + let result = collection + .update_one( + doc! { + "hash": &transaction.hash, + "lifecycle.state": { "$in": ["selected", "included"] } + }, + doc! { + "$set": { + "lifecycle": lifecycle_bson(&target_lifecycle) + } + }, + ) + .await + .ok()?; + if result.modified_count != 1 { + return None; + } + modified += 1; + } + + Some(modified) +} + pub async fn reconcile_obviously_orphaned_selected_transactions

( db: &mongodb::Client, ) -> Option @@ -253,6 +348,7 @@ mod tests { lifecycle: TxLifecycle::Selected { block_l2 }, hash: vec![1, 2, 3], historic_roots: vec![], + receipt_token: None, } } @@ -406,6 +502,53 @@ mod tests { assert_eq!(stored.lifecycle, TxLifecycle::Mempool); } + #[tokio::test] + async fn replay_reconciliation_restores_stale_included_transaction_to_mempool() { + let container = get_mongo().await; + let db = get_db_connection(&container).await; + let tx = ClientTransactionWithMetaData { + lifecycle: TxLifecycle::Included { block_l2: 7 }, + ..sample_selected_transaction(Fr254::from(10u64), 7) + }; + db.store_transaction(tx.clone()).await.unwrap(); + db.store_block(&StoredBlock { + layer2_block_number: 7, + commitments: vec![Fr254::from(11u64).to_hex_string()], + proposer_address: Address::ZERO, + }) + .await + .unwrap(); + + let reconciled = reconcile_active_client_transaction_lifecycle::(&db, 8) + .await + .unwrap(); + + let stored: ClientTransactionWithMetaData = + db.get_transaction(&tx.hash).await.unwrap(); + assert_eq!(reconciled, 1); + assert_eq!(stored.lifecycle, TxLifecycle::Mempool); + } + + #[tokio::test] + async fn replay_reconciliation_demotes_future_included_transaction_back_to_selected() { + let container = get_mongo().await; + let db = get_db_connection(&container).await; + let tx = ClientTransactionWithMetaData { + lifecycle: TxLifecycle::Included { block_l2: 30 }, + ..sample_selected_transaction(Fr254::from(10u64), 30) + }; + db.store_transaction(tx.clone()).await.unwrap(); + + let reconciled = reconcile_active_client_transaction_lifecycle::(&db, 30) + .await + .unwrap(); + + let stored: ClientTransactionWithMetaData = + db.get_transaction(&tx.hash).await.unwrap(); + assert_eq!(reconciled, 1); + assert_eq!(stored.lifecycle, TxLifecycle::Selected { block_l2: 30 }); + } + #[tokio::test] async fn reconcile_does_not_restore_explicitly_cancelled_transactions() { let container = get_mongo().await; @@ -429,6 +572,7 @@ mod tests { lifecycle: TxLifecycle::Cancelled, hash: vec![9, 9, 9], historic_roots: vec![], + receipt_token: None, }; db.store_transaction(tx.clone()).await.unwrap(); diff --git a/nightfall_proposer/src/services/snapshot_scheduler.rs b/nightfall_proposer/src/services/snapshot_scheduler.rs new file mode 100644 index 00000000..2c9a6a16 --- /dev/null +++ b/nightfall_proposer/src/services/snapshot_scheduler.rs @@ -0,0 +1,610 @@ +use crate::driven::db::snapshot::{ + cleanup_orphaned_proposer_snapshot_temp_dirs, create_proposer_snapshot_unlocked, + find_latest_valid_proposer_snapshot, load_and_validate_snapshot_manifest, + try_acquire_proposer_state_maintenance_guard, SnapshotError, +}; +use crate::initialisation::{get_blockchain_client_connection, get_db_connection}; +use crate::ports::db::{RestoreJournalDB, SyncStateDB}; +use configuration::settings::get_settings; +use lib::blockchain_client::BlockchainClientConnection; +use log::{debug, info, warn}; +use mongodb::Client; +use std::path::{Path, PathBuf}; +use tokio::{ + fs, + sync::{OnceCell, RwLock}, + time::{sleep, Duration}, +}; + +const SNAPSHOT_SCHEDULER_POLL_INTERVAL: Duration = Duration::from_secs(5); + +async fn get_last_snapshot_l2_block() -> &'static RwLock { + static LAST_SNAPSHOT_L2_BLOCK: OnceCell> = OnceCell::const_new(); + LAST_SNAPSHOT_L2_BLOCK + .get_or_init(|| async { RwLock::new(0) }) + .await +} + +pub async fn set_last_snapshot_l2_block(last_snapshot_l2_block: u64) { + *get_last_snapshot_l2_block().await.write().await = last_snapshot_l2_block; +} + +fn snapshot_root_dir() -> PathBuf { + PathBuf::from(&get_settings().nightfall_proposer.snapshot_root_dir) +} + +async fn initialize_snapshot_scheduler_state_for_root( + root: &Path, + live_last_applied_l2_block: Option, +) -> Result<(), SnapshotError> { + cleanup_orphaned_proposer_snapshot_temp_dirs(root).await?; + let last_snapshot_l2_block = match live_last_applied_l2_block { + Some(last_applied_l2_block) => { + match find_latest_valid_proposer_snapshot(root, last_applied_l2_block).await? { + Some((_, manifest)) => manifest.last_applied_l2_block, + None => 0, + } + } + None => match find_latest_valid_proposer_snapshot(root, u64::MAX).await? { + Some((_, manifest)) => manifest.last_applied_l2_block, + None => 0, + }, + }; + + set_last_snapshot_l2_block(last_snapshot_l2_block).await; + Ok(()) +} + +pub fn maybe_should_snapshot( + current_l2_block: u64, + last_snapshot_l2_block: u64, + current_l1_block: u64, + applied_block_l1_block: u64, + snapshot_interval_l2_blocks: u64, + snapshot_min_l1_confirmations: u64, +) -> bool { + current_l2_block.saturating_sub(last_snapshot_l2_block) >= snapshot_interval_l2_blocks + && current_l1_block.saturating_sub(applied_block_l1_block) >= snapshot_min_l1_confirmations +} + +pub async fn initialize_snapshot_scheduler_state() -> Result<(), SnapshotError> { + let live_last_applied_l2_block = get_db_connection() + .await + .get_sync_state() + .await + .map(|sync_state| sync_state.last_applied_l2_block); + initialize_snapshot_scheduler_state_for_root(&snapshot_root_dir(), live_last_applied_l2_block) + .await +} + +async fn prune_old_snapshots_by_manifest( + snapshot_root_dir: &Path, + retention_count: usize, +) -> Result<(), SnapshotError> { + if retention_count == 0 || !fs::try_exists(snapshot_root_dir).await? { + return Ok(()); + } + + let mut entries = fs::read_dir(snapshot_root_dir).await?; + let mut snapshots = Vec::new(); + + while let Some(entry) = entries.next_entry().await? { + if !entry.file_type().await?.is_dir() { + continue; + } + + let path = entry.path(); + let manifest_path = path.join("manifest.json"); + if !fs::try_exists(&manifest_path).await? { + continue; + } + + let manifest = match load_and_validate_snapshot_manifest(&path).await { + Ok(manifest) => manifest, + Err(error) => { + warn!( + "Removing snapshot directory {} during prune because it is not a valid restore point: {}", + path.display(), + error + ); + fs::remove_dir_all(&path).await?; + continue; + } + }; + snapshots.push((path, manifest.last_applied_l2_block, manifest.snapshot_id)); + } + + snapshots.sort_by(|left, right| right.1.cmp(&left.1).then_with(|| right.2.cmp(&left.2))); + + for (path, _, _) in snapshots.into_iter().skip(retention_count) { + fs::remove_dir_all(&path).await?; + } + + Ok(()) +} + +async fn create_snapshot_task( + client: Client, + snapshot_root_dir: PathBuf, + retention_count: usize, + current_l1_block: u64, +) { + let _maintenance_guard = match try_acquire_proposer_state_maintenance_guard() { + Some(guard) => guard, + None => { + debug!( + "Skipping proposer snapshot creation because proposer maintenance is in progress" + ); + return; + } + }; + + if client.get_restore_journal().await.is_some() { + debug!( + "Skipping proposer snapshot creation because restore_journal indicates maintenance is still pending" + ); + return; + } + + let settings = &get_settings().nightfall_proposer; + let sync_state = match client.get_sync_state().await { + Some(sync_state) => sync_state, + None => { + debug!("Skipping proposer snapshot creation because sync_state is missing"); + return; + } + }; + let last_snapshot_l2_block = *get_last_snapshot_l2_block().await.read().await; + if !maybe_should_snapshot( + sync_state.last_applied_l2_block, + last_snapshot_l2_block, + current_l1_block, + sync_state.l1_ref.block_number, + settings.snapshot_interval_l2_blocks, + settings.snapshot_min_l1_confirmations, + ) { + debug!( + "Skipping proposer snapshot creation because eligibility changed after acquiring the maintenance guard" + ); + return; + } + + match create_proposer_snapshot_unlocked(&client, &snapshot_root_dir).await { + Ok(manifest) => { + set_last_snapshot_l2_block(manifest.last_applied_l2_block).await; + + if let Err(error) = + prune_old_snapshots_by_manifest(&snapshot_root_dir, retention_count).await + { + warn!( + "Proposer snapshot {} created, but pruning old snapshots failed: {}", + manifest.snapshot_id, error + ); + } else { + info!( + "Created proposer snapshot {} at L2 block {}", + manifest.snapshot_id, manifest.last_applied_l2_block + ); + } + } + Err(error) => { + warn!("Automatic proposer snapshot creation failed: {error}"); + } + } +} + +async fn maybe_schedule_snapshot_for_current_l1_head(client: &Client, current_l1_block: u64) { + let settings = &get_settings().nightfall_proposer; + if !settings.snapshot_enabled { + return; + } + + let sync_state = match client.get_sync_state().await { + Some(sync_state) => sync_state, + None => return, + }; + + let last_snapshot_l2_block = *get_last_snapshot_l2_block().await.read().await; + if !maybe_should_snapshot( + sync_state.last_applied_l2_block, + last_snapshot_l2_block, + current_l1_block, + sync_state.l1_ref.block_number, + settings.snapshot_interval_l2_blocks, + settings.snapshot_min_l1_confirmations, + ) { + return; + } + + let client = client.clone(); + let snapshot_root_dir = snapshot_root_dir(); + let retention_count = settings.snapshot_retention_count as usize; + + tokio::spawn(async move { + create_snapshot_task(client, snapshot_root_dir, retention_count, current_l1_block).await; + }); +} + +pub async fn maybe_schedule_snapshot_for_applied_block(client: &Client, current_l1_block: u64) { + maybe_schedule_snapshot_for_current_l1_head(client, current_l1_block).await; +} + +pub async fn run_snapshot_scheduler() { + loop { + let current_l1_block = match get_blockchain_client_connection() + .await + .read() + .await + .get_client() + .get_block_number() + .await + { + Ok(current_l1_block) => current_l1_block, + Err(error) => { + debug!( + "Skipping proposer snapshot scheduler poll because current L1 head could not be fetched: {error}" + ); + sleep(SNAPSHOT_SCHEDULER_POLL_INTERVAL).await; + continue; + } + }; + + let db = get_db_connection().await; + maybe_schedule_snapshot_for_current_l1_head(db, current_l1_block).await; + sleep(SNAPSHOT_SCHEDULER_POLL_INTERVAL).await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::domain::entities::{L1Ref, SnapshotCollectionManifest, SyncState}; + use crate::driven::db::mongo_db::{ + DEPOSIT_COLLECTION, PROPOSED_BLOCKS_COLLECTION, SYNC_STATE_COLLECTION, + }; + use crate::ports::trees::{CommitmentTree, HistoricRootTree, NullifierTree}; + use ark_bn254::Fr as Fr254; + use sha2::{Digest, Sha256}; + use tokio::sync::{Mutex, OnceCell as TokioOnceCell}; + + async fn scheduler_test_lock() -> tokio::sync::MutexGuard<'static, ()> { + static LOCK: TokioOnceCell> = TokioOnceCell::const_new(); + LOCK.get_or_init(|| async { Mutex::new(()) }) + .await + .lock() + .await + } + use mongodb::bson::DateTime; + + #[test] + fn maybe_should_snapshot_requires_interval_and_confirmations() { + assert!(maybe_should_snapshot(200, 100, 500, 480, 100, 12)); + assert!(!maybe_should_snapshot(199, 100, 500, 480, 100, 12)); + assert!(!maybe_should_snapshot(200, 100, 491, 480, 100, 12)); + } + + fn empty_file_checksum() -> String { + hex::encode(Sha256::digest([])) + } + + fn checksum_for_lines(lines: &[String]) -> String { + let mut checksum = Sha256::new(); + for line in lines { + checksum.update(line.as_bytes()); + checksum.update(b"\n"); + } + hex::encode(checksum.finalize()) + } + + fn compute_test_overall_checksum(collections: &[SnapshotCollectionManifest]) -> String { + let mut overall_checksum = Sha256::new(); + let mut collections_for_checksum: Vec<&SnapshotCollectionManifest> = + collections.iter().collect(); + collections_for_checksum + .sort_by(|left, right| left.collection_name.cmp(&right.collection_name)); + for collection in collections_for_checksum { + overall_checksum.update(collection.collection_name.as_bytes()); + overall_checksum.update(b"|"); + overall_checksum.update(collection.file_name.as_bytes()); + overall_checksum.update(b"|"); + overall_checksum.update(collection.document_count.to_le_bytes()); + overall_checksum.update(b"|"); + overall_checksum.update(collection.sha256.as_bytes()); + overall_checksum.update(b"\n"); + } + + hex::encode(overall_checksum.finalize()) + } + + async fn write_minimal_valid_snapshot( + snapshot_dir: &Path, + snapshot_id: &str, + last_applied_l2_block: u64, + ) { + let required_collections = vec![ + format!( + "{}_metadata", + >::TREE_NAME + ), + format!( + "{}_metadata", + >::TREE_NAME + ), + format!( + "{}_metadata", + >::TREE_NAME + ), + format!( + "{}_indexed_leaves", + >::TREE_NAME + ), + DEPOSIT_COLLECTION.to_string(), + PROPOSED_BLOCKS_COLLECTION.to_string(), + SYNC_STATE_COLLECTION.to_string(), + ]; + + let mut collections = Vec::new(); + for collection_name in required_collections { + let file_name = format!("{collection_name}.jsonl"); + let file_contents = if collection_name == SYNC_STATE_COLLECTION { + let sync_state = SyncState::new( + last_applied_l2_block, + format!("fingerprint-{last_applied_l2_block}"), + L1Ref { + block_number: last_applied_l2_block, + tx_hash: alloy::primitives::TxHash::from([last_applied_l2_block as u8; 32]), + log_index: 0, + }, + DateTime::now(), + ); + let bson = mongodb::bson::to_bson(&sync_state).expect("serialize sync_state"); + let line = bson.into_relaxed_extjson().to_string(); + vec![line] + } else { + Vec::new() + }; + let serialized = if file_contents.is_empty() { + String::new() + } else { + format!("{}\n", file_contents.join("\n")) + }; + fs::write(snapshot_dir.join(&file_name), serialized) + .await + .expect("write snapshot collection file"); + collections.push(SnapshotCollectionManifest { + collection_name, + file_name, + document_count: file_contents.len() as u64, + sha256: if file_contents.is_empty() { + empty_file_checksum() + } else { + checksum_for_lines(&file_contents) + }, + }); + } + + let manifest = crate::domain::entities::ProposerSnapshotManifest { + snapshot_id: snapshot_id.to_string(), + schema_version: crate::domain::entities::ProposerSnapshotManifest::SCHEMA_VERSION, + created_at: DateTime::now(), + storage_format: crate::domain::entities::ProposerSnapshotManifest::STORAGE_FORMAT + .to_string(), + database: "proposer".to_string(), + last_applied_l2_block, + fingerprint: format!("fingerprint-{last_applied_l2_block}"), + l1_ref: crate::domain::entities::L1Ref { + block_number: last_applied_l2_block, + tx_hash: alloy::primitives::TxHash::from([last_applied_l2_block as u8; 32]), + log_index: 0, + }, + overall_sha256: compute_test_overall_checksum(&collections), + collections, + }; + fs::write( + snapshot_dir.join("manifest.json"), + serde_json::to_vec(&manifest).expect("serialize manifest"), + ) + .await + .expect("write manifest"); + } + + #[tokio::test] + async fn prune_old_snapshots_keeps_only_most_recent_snapshots() { + let _scheduler_test_lock = scheduler_test_lock().await; + let root = std::env::temp_dir().join(format!( + "nf4-proposer-prune-test-{}", + DateTime::now().timestamp_millis() + )); + fs::create_dir_all(&root).await.expect("create root"); + + for index in 0..7_u64 { + let snapshot_dir = root.join(format!("snapshot-{index}")); + fs::create_dir_all(&snapshot_dir) + .await + .expect("create snapshot dir"); + write_minimal_valid_snapshot(&snapshot_dir, &format!("snapshot-{index}"), index).await; + } + + prune_old_snapshots_by_manifest(&root, 5) + .await + .expect("prune old snapshots"); + + let mut remaining = fs::read_dir(&root).await.expect("list root"); + let mut names = Vec::new(); + while let Some(entry) = remaining.next_entry().await.expect("read entry") { + names.push(entry.file_name().to_string_lossy().to_string()); + } + names.sort(); + + assert_eq!( + names, + vec![ + "snapshot-2".to_string(), + "snapshot-3".to_string(), + "snapshot-4".to_string(), + "snapshot-5".to_string(), + "snapshot-6".to_string(), + ] + ); + + fs::remove_dir_all(root).await.expect("cleanup prune root"); + } + + #[tokio::test] + async fn prune_old_snapshots_removes_invalid_newer_snapshot_before_retention() { + let _scheduler_test_lock = scheduler_test_lock().await; + let root = std::env::temp_dir().join(format!( + "nf4-proposer-prune-invalid-test-{}", + DateTime::now().timestamp_millis() + )); + fs::create_dir_all(&root).await.expect("create root"); + + for index in [10_u64, 11_u64] { + let snapshot_dir = root.join(format!("snapshot-{index}")); + fs::create_dir_all(&snapshot_dir) + .await + .expect("create snapshot dir"); + write_minimal_valid_snapshot(&snapshot_dir, &format!("snapshot-{index}"), index).await; + } + + let invalid_snapshot_dir = root.join("snapshot-12-invalid"); + fs::create_dir_all(&invalid_snapshot_dir) + .await + .expect("create invalid snapshot dir"); + let invalid_manifest = crate::domain::entities::ProposerSnapshotManifest { + snapshot_id: "snapshot-12-invalid".to_string(), + schema_version: crate::domain::entities::ProposerSnapshotManifest::SCHEMA_VERSION, + created_at: DateTime::now(), + storage_format: crate::domain::entities::ProposerSnapshotManifest::STORAGE_FORMAT + .to_string(), + database: "proposer".to_string(), + last_applied_l2_block: 12, + fingerprint: "fingerprint-12".to_string(), + l1_ref: crate::domain::entities::L1Ref { + block_number: 12, + tx_hash: alloy::primitives::TxHash::from([12; 32]), + log_index: 0, + }, + collections: Vec::new(), + overall_sha256: "definitely-not-the-empty-checksum".to_string(), + }; + fs::write( + invalid_snapshot_dir.join("manifest.json"), + serde_json::to_vec(&invalid_manifest).expect("serialize invalid manifest"), + ) + .await + .expect("write invalid manifest"); + + prune_old_snapshots_by_manifest(&root, 2) + .await + .expect("prune old snapshots"); + + let mut remaining = fs::read_dir(&root).await.expect("list root"); + let mut names = Vec::new(); + while let Some(entry) = remaining.next_entry().await.expect("read entry") { + names.push(entry.file_name().to_string_lossy().to_string()); + } + names.sort(); + + assert_eq!( + names, + vec!["snapshot-10".to_string(), "snapshot-11".to_string()] + ); + + fs::remove_dir_all(root) + .await + .expect("cleanup invalid prune root"); + } + + #[tokio::test] + async fn initialize_snapshot_scheduler_state_cleans_orphan_temp_dirs_and_tracks_latest() { + let _scheduler_test_lock = scheduler_test_lock().await; + let root = std::env::temp_dir().join(format!( + "nf4-proposer-scheduler-init-test-{}", + DateTime::now().timestamp_millis() + )); + fs::create_dir_all(&root).await.expect("create root"); + + fs::create_dir_all(root.join(".tmp-proposer-snapshot-orphan")) + .await + .expect("create orphan temp dir"); + + for index in [3_u64, 9_u64] { + let snapshot_dir = root.join(format!("snapshot-{index}")); + fs::create_dir_all(&snapshot_dir) + .await + .expect("create snapshot dir"); + write_minimal_valid_snapshot(&snapshot_dir, &format!("snapshot-{index}"), index).await; + } + + initialize_snapshot_scheduler_state_for_root(&root, None) + .await + .expect("initialize scheduler state"); + + let names: Vec = { + let mut entries = fs::read_dir(&root).await.expect("read root"); + let mut names = Vec::new(); + while let Some(entry) = entries.next_entry().await.expect("read entry") { + names.push(entry.file_name().to_string_lossy().to_string()); + } + names + }; + assert!(!names + .iter() + .any(|name| name.starts_with(".tmp-proposer-snapshot-"))); + assert_eq!(*get_last_snapshot_l2_block().await.read().await, 9); + + fs::remove_dir_all(root) + .await + .expect("cleanup scheduler init root"); + } + + #[tokio::test] + async fn initialize_snapshot_scheduler_state_uses_latest_disk_snapshot_not_ahead_of_live_sync_state( + ) { + let _scheduler_test_lock = scheduler_test_lock().await; + let root = std::env::temp_dir().join(format!( + "nf4-proposer-scheduler-live-state-test-{}", + DateTime::now().timestamp_millis() + )); + fs::create_dir_all(&root).await.expect("create root"); + + for index in [4_u64, 9_u64] { + let snapshot_dir = root.join(format!("snapshot-{index}")); + fs::create_dir_all(&snapshot_dir) + .await + .expect("create snapshot dir"); + write_minimal_valid_snapshot(&snapshot_dir, &format!("snapshot-{index}"), index).await; + } + + initialize_snapshot_scheduler_state_for_root(&root, Some(4)) + .await + .expect("initialize scheduler state from live sync_state"); + + assert_eq!(*get_last_snapshot_l2_block().await.read().await, 4); + + fs::remove_dir_all(root) + .await + .expect("cleanup scheduler live state root"); + } + + #[tokio::test] + async fn initialize_snapshot_scheduler_state_sets_zero_when_live_sync_state_has_no_snapshot() { + let _scheduler_test_lock = scheduler_test_lock().await; + let root = std::env::temp_dir().join(format!( + "nf4-proposer-scheduler-no-snapshot-test-{}", + DateTime::now().timestamp_millis() + )); + fs::create_dir_all(&root).await.expect("create root"); + + initialize_snapshot_scheduler_state_for_root(&root, Some(100)) + .await + .expect("initialize scheduler state with live sync_state but no snapshots"); + + assert_eq!(*get_last_snapshot_l2_block().await.read().await, 0); + + fs::remove_dir_all(root) + .await + .expect("cleanup scheduler no snapshot root"); + } +} diff --git a/nightfall_test.toml b/nightfall_test.toml index 0d19459d..d618cd1f 100644 --- a/nightfall_test.toml +++ b/nightfall_test.toml @@ -69,7 +69,7 @@ erc721_withdraw.value = "0x00" erc721_withdraw.fee = "0x03" erc721_withdraw.token_id = "0x01aa" -erc3525_deposit_1.value = "0x09" +erc3525_deposit_1.value = "0x02" erc3525_deposit_1.fee = "0x0a" erc3525_deposit_1.token_id = "0x07" @@ -77,15 +77,19 @@ erc3525_deposit_2.value = "0x0c" erc3525_deposit_2.fee = "0x0d" erc3525_deposit_2.token_id = "0x08" -erc3525_transfer_1.value = "0x03" +erc3525_transfer_1.value = "0x07" erc3525_transfer_1.fee = "0x0c" erc3525_transfer_1.token_id = "0x07" -erc3525_transfer_2.value = "0x04" -erc3525_transfer_2.fee = "0x01" +erc3525_transfer_same_slot.value = "0x02" +erc3525_transfer_same_slot.fee = "0x00" +erc3525_transfer_same_slot.token_id = "0x07" + +erc3525_transfer_2.value = "0x02" +erc3525_transfer_2.fee = "0x00" erc3525_transfer_2.token_id = "0x08" -erc3525_withdraw.value = "0x03" +erc3525_withdraw.value = "0x05" erc3525_withdraw.fee = "0x04" erc3525_withdraw.token_id = "0x07" diff --git a/nightfall_test/src/run_tests.rs b/nightfall_test/src/run_tests.rs index d6074680..d946cc2b 100644 --- a/nightfall_test/src/run_tests.rs +++ b/nightfall_test/src/run_tests.rs @@ -1,8 +1,12 @@ use crate::{ test::{ self, create_nf3_deposit_transaction, create_nf3_swap_request, - create_nf3_transfer_transaction, create_nf3_withdraw_transaction, get_key, - get_recipient_address, set_anvil_mining_interval, submit_swap_pair_and_assert_paired, + create_nf3_transfer_transaction, create_nf3_withdraw_transaction, + create_transfer_receipt_full, create_transfer_receipt_raw, + generate_realistic_receipt_ciphertext, get_key, get_recipient_address, + get_transfer_receipt_status, get_transfer_receipt_status_raw, + get_tx_hash_from_request_status, resolve_transfer_receipt, resolve_transfer_receipt_raw, + set_anvil_mining_interval, submit_swap_pair_and_assert_paired, verify_deposit_commitments_nf_token_id, wait_for_all_responses, wait_for_withdraws_on_chain, wait_on_chain, TokenType, }, @@ -80,6 +84,8 @@ pub async fn run_tests( let erc1155_deposit_1_token_id = test_settings.erc1155_deposit_1.token_id.clone(); let erc1155_withdraw_1_token_id = test_settings.erc1155_withdraw_1.token_id.clone(); + let erc3525_token_id_1 = test_settings.erc3525_deposit_1.token_id.clone(); + let erc3525_token_id_2 = test_settings.erc3525_deposit_2.token_id.clone(); // override the mining interval that may have been set in Anvil. If Anvil was set to automine, also turn that off let http_client = reqwest::Client::new(); @@ -171,6 +177,21 @@ pub async fn run_tests( .expect("balanceOf() call failed"); assert_eq!(my_balance, U256::from(2)); + let erc3525_slot_1 = erc3525_contract + .slotOf(U256::from_hex_string(&erc3525_token_id_1).unwrap()) + .call() + .await + .expect("slotOf() call failed"); + let erc3525_slot_2 = erc3525_contract + .slotOf(U256::from_hex_string(&erc3525_token_id_2).unwrap()) + .call() + .await + .expect("slotOf() call failed"); + assert_eq!( + erc3525_slot_1, erc3525_slot_2, + "ERC3525 integration test requires token ids to share a slot" + ); + //see if the NF4_LARGE_BLOCK_TEST environment variable is set to 'true' and run the large block test only if it is let ( client1_starting_balance, @@ -459,6 +480,9 @@ pub async fn run_tests( // Extract UUIDs and store expected token info for verification let transaction_ids: Vec = transaction_data.iter().map(|(uuid, _)| *uuid).collect(); + // Persist one deposit UUID so the receipt tests can later assert that + // non-transfer requests do NOT carry a receipt_token. + let deposit_uuid_for_receipt_test = transaction_ids[0]; // Build a lookup for later token validation let mut expected_token_data: HashMap> = HashMap::new(); @@ -702,14 +726,14 @@ pub async fn run_tests( let transaction_ids = try_join_all(transaction_ids).await.unwrap(); // wait for the responses to the transfer requests to come back to the webhook server - let transactions = wait_for_all_responses(&transaction_ids, responses.clone()) - .await - .into_iter() - .map(|(_, l)| { - serde_json::from_str::<(Value, Option)>(&l) + let transaction_responses = wait_for_all_responses(&transaction_ids, responses.clone()).await; + let transactions = transaction_responses + .iter() + .map(|(_, response_text)| { + serde_json::from_str::<(Value, Option)>(response_text) .expect("Failed to parse response") + .0 }) - .map(|l| l.0) .collect::>(); info!("Starting chain reorg, {} blocks reorged", 200); @@ -772,6 +796,8 @@ pub async fn run_tests( )); debug!("transaction_erc721_transfer has been created"); + // This ERC3525 transfer will use both token_id 7 and 8 as they have slot. + transaction_ids.push(create_nf3_transfer_transaction( zkp_key2.clone(), &http_client, @@ -781,15 +807,6 @@ pub async fn run_tests( )); debug!("transaction_erc3525_transfer_1 has been created"); - transaction_ids.push(create_nf3_transfer_transaction( - zkp_key2.clone(), - &http_client, - url.clone(), - TokenType::ERC3525, - test_settings.erc3525_transfer_2, - )); - debug!("transaction_erc3525_transfer_2 has been created"); - transaction_ids.push(create_nf3_transfer_transaction( zkp_key2.clone(), &http_client, @@ -833,6 +850,79 @@ pub async fn run_tests( .unwrap(); info!("Transfer commitments are now on-chain"); + info!("Sending same-slot ERC3525 transfer from client 2 back to client 1"); + let client2_transfer_url = Url::parse("http://client2:3000") + .unwrap() + .join("v1/transfer") + .unwrap(); + let same_slot_erc3525_transfer_id = create_nf3_transfer_transaction( + zkp_key.clone(), + &http_client, + client2_transfer_url.clone(), + TokenType::ERC3525, + test_settings.erc3525_transfer_same_slot, + ) + .await + .unwrap(); + + info!("Sending ERC3525 token 0x08 transfer from client 1 to client 2"); + let erc3525_transfer_2_id = create_nf3_transfer_transaction( + zkp_key2.clone(), + &http_client, + url.clone(), + TokenType::ERC3525, + test_settings.erc3525_transfer_2, + ) + .await + .unwrap(); + debug!("transaction_erc3525_transfer_2 has been created"); + + let erc3525_transfer_transactions = wait_for_all_responses( + &[same_slot_erc3525_transfer_id, erc3525_transfer_2_id], + responses.clone(), + ) + .await + .into_iter() + .map(|(id, l)| { + let transaction = serde_json::from_str::<(Value, Option)>(&l) + .expect("Failed to parse response") + .0; + (id, transaction) + }) + .collect::>(); + + let same_slot_erc3525_transaction = erc3525_transfer_transactions + .get(&same_slot_erc3525_transfer_id) + .expect("Missing same-slot ERC3525 transfer response"); + let erc3525_transfer_2_transaction = erc3525_transfer_transactions + .get(&erc3525_transfer_2_id) + .expect("Missing ERC3525 token 0x08 transfer response"); + + let same_slot_erc3525_commitment = Fr254::from_hex_string( + same_slot_erc3525_transaction["commitments"][0] + .as_str() + .expect("Missing same-slot ERC3525 commitment"), + ) + .unwrap(); + let erc3525_transfer_2_commitment = Fr254::from_hex_string( + erc3525_transfer_2_transaction["commitments"][0] + .as_str() + .expect("Missing ERC3525 token 0x08 transfer commitment"), + ) + .unwrap(); + + wait_on_chain( + &[same_slot_erc3525_commitment], + &settings.nightfall_client.url, + ) + .await + .unwrap(); + info!("Same-slot ERC3525 transfer commitment is now on-chain"); + wait_on_chain(&[erc3525_transfer_2_commitment], "http://client2:3000") + .await + .unwrap(); + info!("ERC3525 token 0x08 transfer commitment is now on-chain for client2"); + //check that the new balances are as expected let balance = get_erc20_balance( &http_client, @@ -846,6 +936,397 @@ pub async fn run_tests( let balance = get_erc20_balance(&http_client, Url::parse("http://client2:3000").unwrap()).await; info!("Balance of ERC20 tokens held as layer 2 commitments by client2: {balance}"); assert_eq!(balance, 20 + client2_starting_balance); + // ───────────────────────────────────────────────────────────────────────── + // Transfer Receipt Integration Tests + // ───────────────────────────────────────────────────────────────────────── + { + use ark_ff::PrimeField; + use nightfall_client::driven::primitives::kemdem_functions::receipt_kemdem_decrypt; + + let proposer_url = Url::parse(&settings.nightfall_proposer.url).unwrap(); + let client_base_url = Url::parse(&settings.nightfall_client.url).unwrap(); + + // ── Receipt Test 0: non-transfer requests must NOT have a receipt_token ── + info!("Receipt Test 0: Deposit request must not carry a receipt_token"); + { + let url = client_base_url + .join(&format!("v1/request/{deposit_uuid_for_receipt_test}")) + .unwrap(); + let resp = http_client + .get(url) + .send() + .await + .expect("GET /v1/request/{uuid} must succeed for deposit"); + assert!( + resp.status().is_success(), + "Expected 2xx for deposit request status, got {}", + resp.status() + ); + let body: serde_json::Value = resp + .json() + .await + .expect("Deposit request status body must be valid JSON"); + assert!( + body.get("receipt_token").is_none_or(|v| v.is_null()), + "Deposit request must not carry a receipt_token, got: {body}" + ); + } + info!("Receipt Test 0 passed: deposit request has no receipt_token"); + + let ctx = generate_realistic_receipt_ciphertext(); + assert_eq!(ctx.ciphertext_hex.len(), 576); + info!("Generated realistic receipt ciphertext (576 hex chars) via receipt_kemdem_encrypt"); + + // Obtain tx_hash via the client request status API — mirrors the wallet flow. + let request_metadata = get_tx_hash_from_request_status( + &http_client, + &client_base_url, + &transaction_ids[0].to_string(), + ) + .await + .expect("first transaction should have tx_hash and receipt_token in request status"); + let tx_hash = request_metadata + .tx_hash + .expect("request status tx_hash missing"); + let receipt_token = request_metadata + .receipt_token + .expect("request status receipt_token missing"); + + info!("Receipt Test 1: Create receipt — happy path"); + let (create_status, create_resp) = create_transfer_receipt_full( + &http_client, + &client_base_url, + &tx_hash, + &ctx.ciphertext_hex, + Some(1), + &receipt_token, + ) + .await + .expect("Receipt Test 1 failed"); + assert_eq!(create_status, 201); + assert_eq!(create_resp.receipt_id.len(), 64); + assert!(create_resp + .receipt_id + .chars() + .all(|c| c.is_ascii_hexdigit())); + // Transaction is already on-chain at this point, so status is included_l2 + assert_eq!(create_resp.status, "included_l2"); + + info!( + "Receipt Test 2: Fan-out verification — resolve from proposer after client submission" + ); + // The client POST in Test 1 fans the ciphertext out to all registered + // proposers. Resolving directly from proposer_url (the downstream + // fan-out target) confirms the ciphertext was actually replicated there. + let resolve_resp = + resolve_transfer_receipt(&http_client, &proposer_url, &create_resp.receipt_id) + .await + .expect("Receipt Test 2 failed"); + assert_eq!(resolve_resp.receipt_id, create_resp.receipt_id); + assert_eq!(resolve_resp.ciphertext, ctx.ciphertext_hex); + assert_eq!(resolve_resp.tx_hash, tx_hash); + assert_eq!(resolve_resp.version, 1); + assert!(resolve_resp.created_at_unix > 0); + assert!(resolve_resp.updated_at_unix > 0); + assert_eq!(resolve_resp.created_at_unix, resolve_resp.updated_at_unix); + + info!("Receipt Test 2b: Round-trip KEM-DEM decrypt"); + let ct_bytes = + hex::decode(&resolve_resp.ciphertext).expect("Stored ciphertext must be valid hex"); + let mut cipher_text_arr = [Fr254::default(); 9]; + for (i, slot) in cipher_text_arr.iter_mut().enumerate() { + let start = i * 32; + let end = start + 32; + *slot = Fr254::from_le_bytes_mod_order(&ct_bytes[start..end]); + } + let decrypted = receipt_kemdem_decrypt(ctx.recipient_private_key, &cipher_text_arr) + .expect("receipt_kemdem_decrypt should succeed"); + assert_eq!(decrypted.sender_public_key_x, ctx.input.sender_public_key_x); + assert_eq!(decrypted.sender_public_key_y, ctx.input.sender_public_key_y); + assert_eq!(decrypted.erc_address, ctx.input.erc_address); + assert_eq!(decrypted.token_type, lib::shared_entities::TokenType::ERC20); + assert_eq!(decrypted.token_id_or_value, ctx.input.token_id_or_value); + assert_eq!(decrypted.receiver_commitment, ctx.input.receiver_commitment); + info!("Round-trip decrypt verified: all ERC20 plaintext fields recovered correctly"); + + info!("Receipt Test 3: Get receipt status"); + let status_resp = + get_transfer_receipt_status(&http_client, &proposer_url, &create_resp.receipt_id) + .await + .expect("Receipt Test 3 failed"); + assert_eq!(status_resp.receipt_id, create_resp.receipt_id); + assert_eq!(status_resp.status, "included_l2"); + + info!("Receipt Test 4: Idempotent create"); + let (idempotent_status, idempotent_resp) = create_transfer_receipt_full( + &http_client, + &client_base_url, + &tx_hash, + &ctx.ciphertext_hex, + Some(1), + &receipt_token, + ) + .await + .expect("Receipt Test 4 failed"); + assert_eq!(idempotent_status, 200); + assert_eq!(idempotent_resp.receipt_id, create_resp.receipt_id); + // Replayed create must return refreshed status, not stale stored value + assert_eq!(idempotent_resp.status, "included_l2"); + + info!("Receipt Test 4b: Wrong receipt_token — fan-out returns 401"); + // The client fans the submission out to all registered proposers. Every + // proposer validates the receipt_token via constant-time HMAC comparison, + // so a wrong token must be rejected by all of them and the client must + // surface a 401 back to the caller. + let wrong_token = "00".repeat(32); // 64-char hex, wrong value + let err4b = create_transfer_receipt_raw( + &http_client, + &client_base_url, + &tx_hash, + &ctx.ciphertext_hex, + Some(1), + &wrong_token, + ) + .await + .expect_err("wrong receipt_token must be rejected"); + assert!( + err4b.to_string().contains("401"), + "expected 401 Unauthorized, got: {err4b}" + ); + info!("Wrong receipt_token correctly rejected with 401 through fan-out path"); + + info!("Receipt Test 5: Conflict on mismatched ciphertext"); + let mismatch_ctx = generate_realistic_receipt_ciphertext(); + let err5 = create_transfer_receipt_raw( + &http_client, + &client_base_url, + &tx_hash, + &mismatch_ctx.ciphertext_hex, + Some(1), + &receipt_token, + ) + .await + .expect_err("mismatched ciphertext must fail"); + assert!(err5.to_string().contains("409")); + info!("Mismatched ciphertext correctly rejected with 409"); + + info!("Receipt Test 6: Conflict on mismatched version"); + let err6 = create_transfer_receipt_raw( + &http_client, + &client_base_url, + &tx_hash, + &ctx.ciphertext_hex, + Some(99), + &receipt_token, + ) + .await + .expect_err("mismatched version must fail"); + assert!(err6.to_string().contains("409")); + info!("Mismatched version correctly rejected with 409"); + + info!("Receipt Test 7: Create with unknown tx_hash"); + let unknown_tx_hash: String = "00".repeat(32); + let err7 = create_transfer_receipt_raw( + &http_client, + &client_base_url, + &unknown_tx_hash, + &ctx.ciphertext_hex, + Some(1), + "placeholder-receipt-token", + ) + .await + .expect_err("unknown tx_hash must fail"); + assert!(err7.to_string().contains("400")); + + info!("Receipt Test 8: Resolve nonexistent receipt"); + let fake_id = "a".repeat(64); + let err8 = resolve_transfer_receipt_raw(&http_client, &proposer_url, &fake_id) + .await + .expect_err("missing receipt must fail"); + assert!(err8.to_string().contains("404")); + + info!("Receipt Test 9: Status for nonexistent receipt"); + let err9 = get_transfer_receipt_status_raw(&http_client, &proposer_url, &fake_id) + .await + .expect_err("missing receipt status must fail"); + assert!(err9.to_string().contains("404")); + + info!("Receipt Test 10: Resolve with invalid receipt_id format"); + let err10 = resolve_transfer_receipt_raw(&http_client, &proposer_url, "not-hex!") + .await + .expect_err("invalid receipt_id must fail"); + assert!(err10.to_string().contains("404")); + + info!("Receipt Test 11: Status with invalid receipt_id format"); + let err11 = get_transfer_receipt_status_raw(&http_client, &proposer_url, "not-hex!") + .await + .expect_err("invalid receipt status must fail"); + assert!(err11.to_string().contains("404")); + + info!("Receipt Test 12: Input validation — short tx_hash"); + let err12 = create_transfer_receipt_raw( + &http_client, + &client_base_url, + "abcd1234", + &ctx.ciphertext_hex, + Some(1), + "placeholder-receipt-token", + ) + .await + .expect_err("short tx_hash must fail"); + assert!(err12.to_string().contains("400")); + + info!("Receipt Test 13: Input validation — empty tx_hash"); + let err13 = create_transfer_receipt_raw( + &http_client, + &client_base_url, + "", + &ctx.ciphertext_hex, + Some(1), + "placeholder-receipt-token", + ) + .await + .expect_err("empty tx_hash must fail"); + assert!(err13.to_string().contains("400")); + + info!("Receipt Test 14: Input validation — non-hex tx_hash"); + let err14 = create_transfer_receipt_raw( + &http_client, + &client_base_url, + &"zz".repeat(32), + &ctx.ciphertext_hex, + Some(1), + "placeholder-receipt-token", + ) + .await + .expect_err("non-hex tx_hash must fail"); + assert!(err14.to_string().contains("400")); + + info!("Receipt Test 15: Input validation — short ciphertext"); + let err15 = create_transfer_receipt_raw( + &http_client, + &client_base_url, + &unknown_tx_hash, + &"ab".repeat(100), + Some(1), + "placeholder-receipt-token", + ) + .await + .expect_err("short ciphertext must fail"); + assert!(err15.to_string().contains("400")); + + info!("Receipt Test 16: Input validation — oversized ciphertext"); + let err16 = create_transfer_receipt_raw( + &http_client, + &client_base_url, + &unknown_tx_hash, + &"ab".repeat(1025), + Some(1), + "placeholder-receipt-token", + ) + .await + .expect_err("oversized ciphertext must fail"); + assert!(err16.to_string().contains("400")); + + info!("Receipt Test 17: Input validation — non-hex ciphertext"); + let err17 = create_transfer_receipt_raw( + &http_client, + &client_base_url, + &unknown_tx_hash, + &"zz".repeat(288), + Some(1), + "placeholder-receipt-token", + ) + .await + .expect_err("non-hex ciphertext must fail"); + assert!(err17.to_string().contains("400")); + + info!("Receipt Test 18: Input validation — odd-length ciphertext"); + let err18 = create_transfer_receipt_raw( + &http_client, + &client_base_url, + &unknown_tx_hash, + &"a".repeat(577), + Some(1), + "placeholder-receipt-token", + ) + .await + .expect_err("odd ciphertext must fail"); + assert!(err18.to_string().contains("400")); + + info!("Receipt Test 19: Input validation — unsupported version"); + let fresh_tx_hash_for_version = "ff".repeat(32); + let err19 = create_transfer_receipt_raw( + &http_client, + &client_base_url, + &fresh_tx_hash_for_version, + &ctx.ciphertext_hex, + Some(2), + "placeholder-receipt-token", + ) + .await + .expect_err("unsupported version must fail"); + assert!(err19.to_string().contains("400")); + + info!("Receipt Test 20: Create receipt for second valid tx_hash"); + if transactions.len() > 1 { + let request_metadata_2 = get_tx_hash_from_request_status( + &http_client, + &client_base_url, + &transaction_ids[1].to_string(), + ) + .await + .expect("second transaction should have tx_hash and receipt_token in request status"); + let tx_hash_2 = request_metadata_2 + .tx_hash + .expect("second request status tx_hash missing"); + let receipt_token_2 = request_metadata_2 + .receipt_token + .expect("second request status receipt_token missing"); + let ctx2 = generate_realistic_receipt_ciphertext(); + let (create_status_2, create_resp_2) = create_transfer_receipt_full( + &http_client, + &client_base_url, + &tx_hash_2, + &ctx2.ciphertext_hex, + Some(1), + &receipt_token_2, + ) + .await + .expect("second receipt create failed"); + assert_eq!(create_status_2, 201); + + let resolve_resp_2 = + resolve_transfer_receipt(&http_client, &proposer_url, &create_resp_2.receipt_id) + .await + .expect("second receipt resolve failed"); + + let ct_bytes_2 = hex::decode(&resolve_resp_2.ciphertext) + .expect("Stored ciphertext must be valid hex"); + let mut cipher_text_arr_2 = [Fr254::default(); 9]; + for (i, slot) in cipher_text_arr_2.iter_mut().enumerate() { + let start = i * 32; + let end = start + 32; + *slot = Fr254::from_le_bytes_mod_order(&ct_bytes_2[start..end]); + } + let decrypted_2 = + receipt_kemdem_decrypt(ctx2.recipient_private_key, &cipher_text_arr_2) + .expect("receipt_kemdem_decrypt should succeed for second receipt"); + assert_eq!( + decrypted_2.sender_public_key_x, + ctx2.input.sender_public_key_x + ); + assert_eq!(decrypted_2.token_id_or_value, ctx2.input.token_id_or_value); + info!( + "Second receipt created, resolved and decrypted: id={}", + create_resp_2.receipt_id + ); + } else { + info!("Second receipt test skipped: only one transaction available"); + } + + info!("All transfer receipt tests passed!"); + } // create swap requests (same party ordering in both legs) info!("Sending ERC20 swap transactions"); @@ -1303,11 +1784,14 @@ pub async fn run_tests( .await; let client2_erc20_after = get_erc20_balance(&http_client, Url::parse("http://client2:3000").unwrap()).await; - - assert_eq!(client1_erc3525_slot7_after, client1_erc3525_slot7_before); - assert_eq!(client1_erc3525_slot8_after, client1_erc3525_slot8_before); - assert_eq!(client2_erc3525_slot7_after, client2_erc3525_slot7_before); - assert_eq!(client2_erc3525_slot8_after, client2_erc3525_slot8_before); + assert_eq!( + client1_erc3525_slot7_after + client1_erc3525_slot8_after, + client1_erc3525_slot7_before + client1_erc3525_slot8_before + ); + assert_eq!( + client2_erc3525_slot7_after + client2_erc3525_slot8_after, + client2_erc3525_slot7_before + client2_erc3525_slot8_before + ); assert_eq!(client1_erc20_after, client1_erc20_before); assert_eq!(client2_erc20_after, client2_erc20_before); diff --git a/nightfall_test/src/test.rs b/nightfall_test/src/test.rs index f2df5401..50742269 100644 --- a/nightfall_test/src/test.rs +++ b/nightfall_test/src/test.rs @@ -38,17 +38,19 @@ use lib::{ nf_client_proof::{PrivateInputs, ProvingEngine, PublicInputs}, plonk_prover::plonk_proof::{PlonkProof, PlonkProvingEngine}, secret_hash::SecretHash, - shared_entities::{DepositSecret, Preimage, Salt}, + shared_entities::{ClientTransaction, DepositSecret, Preimage, Salt}, }; use log::{debug, info, warn}; use mongodb::bson::doc; use nf_curves::ed_on_bn254::{BabyJubjub as BabyJubJub, Fr as BJJScalar}; +use nf_curves::ed_on_bn254::{BabyJubjub, GENERATOR_X, GENERATOR_Y}; use nightfall_client::{ domain::{ entities::{CommitmentStatus, TokenData}, notifications::NotificationPayload, }, driven::db::mongo::CommitmentEntry, + driven::primitives::kemdem_functions::{receipt_kemdem_encrypt, ReceiptEncryptInput}, }; use nightfall_proposer::driven::db::mongo_db::{StoredBlock, DB, PROPOSED_BLOCKS_COLLECTION}; use num_bigint::BigUint; @@ -56,6 +58,7 @@ use reqwest::{ multipart::{Form, Part}, StatusCode, }; +use serde::{Deserialize, Serialize}; use serde_json::Value; use sha2::{Digest, Sha256}; use std::{ @@ -514,10 +517,7 @@ pub async fn verify_deposit_commitments_nf_token_id( assert!( match_found, "No matching expected token data found for UUID: {}\nActual ERC: {}\nActual TokenID: {}\nExpected entries: {:?}", - entry.uuid, - actual_erc_clean, - actual_token_id_clean, - expected_entries + entry.uuid, actual_erc_clean, actual_token_id_clean, expected_entries ); } @@ -575,12 +575,12 @@ pub fn forge_command(command: &[&str]) { ); } else { panic!( - "Command 'forge {:?}' executed with failing error code: {:?}\nStandard Output: {}\nStandard Error: {}", - command, - o.status.signal(), - String::from_utf8_lossy(&o.stdout), - String::from_utf8_lossy(&o.stderr) - ); + "Command 'forge {:?}' executed with failing error code: {:?}\nStandard Output: {}\nStandard Error: {}", + command, + o.status.signal(), + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ); } } Err(e) => { @@ -1452,7 +1452,7 @@ pub fn build_valid_transfer_inputs(rng: &mut impl Rng) -> (PublicInputs, Private let nf_token_id = Fr254::from(nf_token_id); // Retrieve the fee token ID and nightfall address - let nf_address = Address::from(rand::thread_rng().gen::<[u8; 20]>()); + let nf_address = Address::from(rand::thread_rng().r#gen::<[u8; 20]>()); // generate a 'random' fee token ID (we just use the keccak hash of 1) let fee_token_id = Fr254::from(BigUint::from_bytes_be(keccak256([1]).as_slice()) >> 4); @@ -1835,3 +1835,380 @@ mod tests { ); } } + +pub fn compute_tx_hash_from_response(response_json: &Value) -> Result, TestError> { + let normalized = normalize_transaction_response_json(response_json.clone())?; + let normalized_text = + serde_json::to_string(&normalized).map_err(|e| TestError::new(e.to_string()))?; + let client_transaction: ClientTransaction = + serde_json::from_str(&normalized_text).map_err(|e| TestError::new(e.to_string()))?; + client_transaction + .hash() + .map_err(|e| TestError::new(e.to_string())) +} + +pub fn compute_tx_hash_from_raw_response(response_text: &str) -> Result, TestError> { + let (response_json, _): (Value, Value) = + serde_json::from_str(response_text).map_err(|e| TestError::new(e.to_string()))?; + compute_tx_hash_from_response(&response_json) +} + +fn normalize_transaction_response_json(mut response_json: Value) -> Result { + normalize_hex_array_field(&mut response_json, "commitments")?; + normalize_hex_array_field(&mut response_json, "nullifiers")?; + + if let Some(compressed_secrets) = response_json.get_mut("compressed_secrets") { + normalize_hex_array_field(compressed_secrets, "cipher_text")?; + } + + Ok(response_json) +} + +/// Converts webhook hex-array fields into the concatenated hex-string shape used by +/// `ClientTransaction::hash`. The webhook presents array values in display order, while the +/// proposer-stored transaction hashes these fields in reverse order. +fn normalize_hex_array_field(parent: &mut Value, field_name: &str) -> Result<(), TestError> { + let Some(field) = parent.get_mut(field_name) else { + return Ok(()); + }; + + let Some(items) = field.as_array() else { + return Ok(()); + }; + + let joined = items + .iter() + .rev() + .map(|item| { + item.as_str().ok_or_else(|| { + TestError::new(format!( + "{field_name} should contain only hex-string elements" + )) + }) + }) + .collect::, _>>()? + .join(""); + + *field = Value::String(joined); + Ok(()) +} + +#[cfg(test)] +mod tx_hash_normalization_tests { + use super::*; + + #[test] + fn normalize_hex_array_field_reverses_and_joins_values() { + let mut value = serde_json::json!({ + "commitments": ["aa", "bb", "cc"], + }); + + normalize_hex_array_field(&mut value, "commitments").unwrap(); + + assert_eq!(value["commitments"], Value::String("ccbbaa".to_string())); + } + + #[test] + fn normalize_transaction_response_json_handles_nested_cipher_text() { + let value = serde_json::json!({ + "commitments": ["01", "02"], + "nullifiers": ["03", "04"], + "compressed_secrets": { + "cipher_text": ["05", "06"] + } + }); + + let normalized = normalize_transaction_response_json(value).unwrap(); + + assert_eq!(normalized["commitments"], Value::String("0201".to_string())); + assert_eq!(normalized["nullifiers"], Value::String("0403".to_string())); + assert_eq!( + normalized["compressed_secrets"]["cipher_text"], + Value::String("0605".to_string()) + ); + } + + #[test] + fn normalize_hex_array_field_rejects_non_string_elements() { + let mut value = serde_json::json!({ + "commitments": ["aa", 1], + }); + + assert!(normalize_hex_array_field(&mut value, "commitments").is_err()); + } +} + +#[derive(Debug, Serialize)] +pub struct CreateTransferReceiptRequest { + pub tx_hash: String, + pub ciphertext: String, + pub version: Option, + pub receipt_token: String, +} + +#[derive(Debug, Deserialize)] +pub struct CreateTransferReceiptResponse { + pub receipt_id: String, + pub status: String, +} + +#[derive(Debug, Deserialize)] +pub struct TransferReceiptResponse { + pub receipt_id: String, + pub tx_hash: String, + pub ciphertext: String, + pub version: u8, + pub status: String, + pub created_at_unix: i64, + pub updated_at_unix: i64, +} + +#[derive(Debug, Deserialize)] +pub struct TransferReceiptStatusResponse { + pub receipt_id: String, + pub status: String, +} + +#[derive(Debug, Deserialize)] +pub struct RequestStatusReceiptMetadata { + pub tx_hash: Option, + pub receipt_token: Option, +} + +pub struct ReceiptCiphertextContext { + pub ciphertext_hex: String, + pub recipient_private_key: BJJScalar, + pub input: ReceiptEncryptInput, +} + +pub fn generate_realistic_receipt_ciphertext() -> ReceiptCiphertextContext { + use ark_ff::UniformRand; + let rng = &mut ark_std::rand::thread_rng(); + + let recipient_private_key = BJJScalar::rand(rng); + let ephemeral_private_key = BJJScalar::rand(rng); + let public_point = TEAffine::::new(GENERATOR_X, GENERATOR_Y); + let recipient_public_key: TEAffine = (public_point * recipient_private_key).into(); + + let input = ReceiptEncryptInput { + sender_public_key_x: Fr254::rand(rng), + sender_public_key_y: Fr254::rand(rng), + erc_address: Fr254::rand(rng), + token_type: lib::shared_entities::TokenType::ERC20, + token_id_or_value: Fr254::rand(rng), // value for ERC20 + receiver_commitment: Fr254::rand(rng), + }; + + let cipher_text = receipt_kemdem_encrypt( + ephemeral_private_key, + recipient_public_key, + &input, + public_point, + ) + .expect("receipt_kemdem_encrypt should not fail"); + + let mut bytes = Vec::with_capacity(288); + for elem in &cipher_text { + bytes.extend_from_slice(&elem.into_bigint().to_bytes_le()); + } + assert_eq!(bytes.len(), 288); + let ciphertext_hex = hex::encode(&bytes); + assert_eq!(ciphertext_hex.len(), 576); + + ReceiptCiphertextContext { + ciphertext_hex, + recipient_private_key, + input, + } +} + +/// Submits a transfer receipt to the **client** (`POST /v1/transfer-receipts`). +/// The client fans out the ciphertext to every registered proposer, so the +/// receipt becomes queryable on any proposer via `GET /v1/transfer-receipts/{id}`. +pub async fn create_transfer_receipt_raw( + client: &reqwest::Client, + client_url: &Url, + tx_hash: &str, + ciphertext: &str, + version: Option, + receipt_token: &str, +) -> Result<(u16, String), TestError> { + let url = client_url + .join("v1/transfer-receipts") + .map_err(|e| TestError::new(e.to_string()))?; + let request = CreateTransferReceiptRequest { + tx_hash: tx_hash.to_string(), + ciphertext: ciphertext.to_string(), + version, + receipt_token: receipt_token.to_string(), + }; + let response = client + .post(url) + .json(&request) + .send() + .await + .map_err(|e| TestError::new(e.to_string()))?; + let status = response.status().as_u16(); + let body_text = response + .text() + .await + .map_err(|e| TestError::new(e.to_string()))?; + if !(200..300).contains(&status) { + return Err(TestError::new(format!("HTTP {status}: {body_text}"))); + } + Ok((status, body_text)) +} + +pub async fn create_transfer_receipt( + client: &reqwest::Client, + client_url: &Url, + tx_hash: &str, + ciphertext: &str, + version: Option, + receipt_token: &str, +) -> Result { + let (_, body_text) = create_transfer_receipt_raw( + client, + client_url, + tx_hash, + ciphertext, + version, + receipt_token, + ) + .await?; + serde_json::from_str(&body_text).map_err(|e| TestError::new(e.to_string())) +} + +pub async fn create_transfer_receipt_full( + client: &reqwest::Client, + client_url: &Url, + tx_hash: &str, + ciphertext: &str, + version: Option, + receipt_token: &str, +) -> Result<(u16, CreateTransferReceiptResponse), TestError> { + let (status, body_text) = create_transfer_receipt_raw( + client, + client_url, + tx_hash, + ciphertext, + version, + receipt_token, + ) + .await?; + let response = serde_json::from_str(&body_text).map_err(|e| TestError::new(e.to_string()))?; + Ok((status, response)) +} + +pub async fn resolve_transfer_receipt_raw( + client: &reqwest::Client, + proposer_url: &Url, + receipt_id: &str, +) -> Result<(u16, String), TestError> { + let url = proposer_url + .join(&format!("v1/transfer-receipts/{receipt_id}")) + .map_err(|e| TestError::new(e.to_string()))?; + let response = client + .get(url) + .send() + .await + .map_err(|e| TestError::new(e.to_string()))?; + let status = response.status().as_u16(); + let body_text = response + .text() + .await + .map_err(|e| TestError::new(e.to_string()))?; + if !(200..300).contains(&status) { + return Err(TestError::new(format!("HTTP {status}: {body_text}"))); + } + Ok((status, body_text)) +} + +pub async fn resolve_transfer_receipt( + client: &reqwest::Client, + proposer_url: &Url, + receipt_id: &str, +) -> Result { + let (_, body_text) = resolve_transfer_receipt_raw(client, proposer_url, receipt_id).await?; + serde_json::from_str(&body_text).map_err(|e| TestError::new(e.to_string())) +} + +pub async fn get_transfer_receipt_status_raw( + client: &reqwest::Client, + proposer_url: &Url, + receipt_id: &str, +) -> Result<(u16, String), TestError> { + let url = proposer_url + .join(&format!("v1/transfer-receipts/{receipt_id}/status")) + .map_err(|e| TestError::new(e.to_string()))?; + let response = client + .get(url) + .send() + .await + .map_err(|e| TestError::new(e.to_string()))?; + let status = response.status().as_u16(); + let body_text = response + .text() + .await + .map_err(|e| TestError::new(e.to_string()))?; + if !(200..300).contains(&status) { + return Err(TestError::new(format!("HTTP {status}: {body_text}"))); + } + Ok((status, body_text)) +} + +pub async fn get_transfer_receipt_status( + client: &reqwest::Client, + proposer_url: &Url, + receipt_id: &str, +) -> Result { + let (_, body_text) = get_transfer_receipt_status_raw(client, proposer_url, receipt_id).await?; + serde_json::from_str(&body_text).map_err(|e| TestError::new(e.to_string())) +} + +/// Retrieves the canonical proposer `tx_hash` and `receipt_token` for a submitted request by polling +/// `GET /v1/request/{uuid}` on the nightfall client. This mirrors how a wallet app +/// would obtain the tx_hash and receipt capability needed for receipt creation. +/// +/// Returns both values directly — pass them straight to the receipt API. +pub async fn get_tx_hash_from_request_status( + client: &reqwest::Client, + client_base_url: &Url, + request_uuid: &str, +) -> Result { + let url = client_base_url + .join(&format!("v1/request/{request_uuid}")) + .map_err(|e| TestError::new(e.to_string()))?; + + let resp = client + .get(url) + .send() + .await + .map_err(|e| TestError::new(format!("request status fetch failed: {e}")))?; + + if !resp.status().is_success() { + return Err(TestError::new(format!( + "request status returned {}", + resp.status() + ))); + } + + let body: RequestStatusReceiptMetadata = resp + .json() + .await + .map_err(|e| TestError::new(format!("request status body parse failed: {e}")))?; + + let tx_hash_hex = body.tx_hash.as_deref().ok_or_else(|| { + TestError::new("tx_hash not present in request status response".to_string()) + })?; + let receipt_token = body.receipt_token.as_deref().ok_or_else(|| { + TestError::new("receipt_token not present in request status response".to_string()) + })?; + + let _bytes = hex::decode(tx_hash_hex) + .map_err(|e| TestError::new(format!("tx_hash hex decode failed: {e}")))?; + Ok(RequestStatusReceiptMetadata { + tx_hash: Some(tx_hash_hex.to_string()), + receipt_token: Some(receipt_token.to_string()), + }) +} diff --git a/nightfall_test/src/test_settings.rs b/nightfall_test/src/test_settings.rs index ae76d8fd..268bef69 100644 --- a/nightfall_test/src/test_settings.rs +++ b/nightfall_test/src/test_settings.rs @@ -78,6 +78,7 @@ pub struct TestSettings { pub erc3525_deposit_1: TransactionDetails, pub erc3525_deposit_2: TransactionDetails, pub erc3525_transfer_1: TransactionDetails, + pub erc3525_transfer_same_slot: TransactionDetails, pub erc3525_transfer_2: TransactionDetails, pub erc3525_withdraw: TransactionDetails, pub erc1155_deposit_1: TransactionDetails,