diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d6bcbb0..2b0a6701 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,23 +34,18 @@ jobs: components: rustfmt - name: Run tests run: | - cargo check --all-targets --no-default-features --features tokio - cargo check --all-targets --no-default-features --features tokio,sparse - cargo check --all-targets --no-default-features --features tokio,sparse,cache - cargo check --all-targets --no-default-features --features async-std - cargo check --all-targets --no-default-features --features async-std,sparse - cargo check --all-targets --no-default-features --features async-std,sparse,cache - cargo test --no-default-features --features js_interop_tests,tokio - cargo test --no-default-features --features js_interop_tests,tokio,shared-core - cargo test --no-default-features --features js_interop_tests,tokio,sparse - cargo test --no-default-features --features js_interop_tests,tokio,sparse,cache - cargo test --no-default-features --features js_interop_tests,async-std - cargo test --no-default-features --features js_interop_tests,async-std,shared-core - cargo test --no-default-features --features js_interop_tests,async-std,sparse - cargo test --no-default-features --features js_interop_tests,async-std,sparse,cache - cargo test --benches --no-default-features --features tokio - cargo test --benches --no-default-features --features async-std - + cargo check --all-targets + cargo check --all-targets --all-features + cargo check --all-targets --no-default-features + cargo check --all-targets --no-default-features --features sparse + cargo check --all-targets --no-default-features --features sparse,cache + cargo test + cargo test --all-features + cargo test --no-default-features + cargo test --no-default-features --features js_interop_tests + cargo test --no-default-features --features js_interop_tests,sparse + cargo test --no-default-features --features js_interop_tests,sparse,cache + cargo test --benches --no-default-features test-windows: runs-on: windows-latest @@ -61,22 +56,17 @@ jobs: components: rustfmt - name: Run tests run: | - cargo check --all-targets --no-default-features --features tokio - cargo check --all-targets --no-default-features --features tokio,sparse - cargo check --all-targets --no-default-features --features tokio,sparse,cache - cargo check --all-targets --no-default-features --features async-std - cargo check --all-targets --no-default-features --features async-std,sparse - cargo check --all-targets --no-default-features --features async-std,sparse,cache - cargo test --no-default-features --features tokio - cargo test --no-default-features --features tokio,shared-core - cargo test --no-default-features --features tokio,sparse - cargo test --no-default-features --features tokio,sparse,cache - cargo test --no-default-features --features async-std - cargo test --no-default-features --features async-std,shared-core - cargo test --no-default-features --features async-std,sparse - cargo test --no-default-features --features async-std,sparse,cache - cargo test --benches --no-default-features --features tokio - cargo test --benches --no-default-features --features async-std + cargo check --all-targets + cargo check --all-targets --all-features + cargo check --all-targets --no-default-features + cargo check --all-targets --no-default-features --features sparse + cargo check --all-targets --no-default-features --features sparse,cache + cargo test + cargo test --all-features + cargo test --no-default-features + cargo test --no-default-features --features sparse + cargo test --no-default-features --features sparse,cache + cargo test --benches --no-default-features test-macos: runs-on: macos-latest @@ -88,22 +78,18 @@ jobs: components: rustfmt - name: Run tests run: | - cargo check --all-targets --no-default-features --features tokio - cargo check --all-targets --no-default-features --features tokio,sparse - cargo check --all-targets --no-default-features --features tokio,sparse,cache - cargo check --all-targets --no-default-features --features async-std - cargo check --all-targets --no-default-features --features async-std,sparse - cargo check --all-targets --no-default-features --features async-std,sparse,cache - cargo test --no-default-features --features js_interop_tests,tokio - cargo test --no-default-features --features js_interop_tests,tokio,shared-core - cargo test --no-default-features --features js_interop_tests,tokio,sparse - cargo test --no-default-features --features js_interop_tests,tokio,sparse,cache - cargo test --no-default-features --features js_interop_tests,async-std - cargo test --no-default-features --features js_interop_tests,async-std,shared-core - cargo test --no-default-features --features js_interop_tests,async-std,sparse - cargo test --no-default-features --features js_interop_tests,async-std,sparse,cache - cargo test --benches --no-default-features --features tokio - cargo test --benches --no-default-features --features async-std + cargo check --all-targets + cargo check --all-targets --all-features + cargo check --all-targets --no-default-features + cargo check --all-targets --no-default-features --features sparse + cargo check --all-targets --no-default-features --features sparse,cache + cargo test + cargo test --all-features + cargo test --no-default-features + cargo test --no-default-features --features js_interop_tests + cargo test --no-default-features --features js_interop_tests,sparse + cargo test --no-default-features --features js_interop_tests,sparse,cache + cargo test --benches --no-default-features build-extra: runs-on: ubuntu-latest @@ -115,24 +101,17 @@ jobs: targets: wasm32-unknown-unknown - name: Build WASM run: | - cargo build --target=wasm32-unknown-unknown --no-default-features --features tokio - cargo build --target=wasm32-unknown-unknown --no-default-features --features async-std + cargo build --target=wasm32-unknown-unknown --no-default-features - name: Build release run: | - cargo build --release --no-default-features --features tokio - cargo build --release --no-default-features --features tokio,sparse - cargo build --release --no-default-features --features tokio,sparse,cache - cargo build --release --no-default-features --features async-std - cargo build --release --no-default-features --features async-std,sparse - cargo build --release --no-default-features --features async-std,sparse,cache + cargo build --release --no-default-features + cargo build --release --no-default-features --features sparse + cargo build --release --no-default-features --features sparse,cache - name: Run examples run: | - cargo run --no-default-features --features tokio --example disk - cargo run --no-default-features --features async-std --example disk - cargo run --no-default-features --features tokio --example memory - cargo run --no-default-features --features async-std --example memory - cargo run --no-default-features --features tokio --example replication - cargo run --no-default-features --features async-std --example replication + cargo run --no-default-features --example disk + cargo run --no-default-features --example memory + cargo run --no-default-features --example replication lint: runs-on: ubuntu-latest diff --git a/Cargo.toml b/Cargo.toml index e831b78e..888dc69f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,21 +32,34 @@ flat-tree = "6" merkle-tree-stream = "0.12" pretty-hash = "0.4" rand = "0.8" -random-access-memory = "3" -random-access-storage = "5" sha2 = "0.10" futures = "0.3" crc32fast = "1" intmap = "2" moka = { version = "0.12", optional = true, features = ["sync"] } async-broadcast = { version = "0.7.1", optional = true } -async-lock = {version = "3.4.0", optional = true } +futures-lite = "2.6.1" [dependencies.hypercore_schema] version = "0.2.0" -[target.'cfg(not(target_arch = "wasm32"))'.dependencies] -random-access-disk = { version = "3", default-features = false } +[dependencies.hypercore-protocol] +optional = true +path = "../protocol/" + +[dependencies.hypercore_handshake] +version = "0.6.0" +optional = true + +[dependencies.random-access-storage] +version = "6.0.0-alpha" + +[dependencies.random-access-memory] +version = "4.0.0-alpha" + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.random-access-disk] +version = "4.0.0-alpha" +default-features = false [dev-dependencies] anyhow = "1.0.70" @@ -55,8 +68,9 @@ proptest-derive = "0.5.1" data-encoding = "2.2.0" remove_dir_all = "0.7.0" tempfile = "3.1.0" -async-std = { version = "1.12.0", features = ["attributes"] } -tokio = { version = "1.27.0", default-features = false, features = ["macros", "rt", "rt-multi-thread"] } +tokio = { version = "1.27.0", default-features = false, features = ["macros", "rt", "rt-multi-thread", "io-util", "time"] } +tokio-util = { version = "0.7", features = ["compat"] } +uint24le_framing = { version = "0.2.0" } tokio-test = "0.4" sha2 = "0.10" criterion = { version = "0.4", features = ["async_std", "async_tokio"] } @@ -64,12 +78,9 @@ test-log = { version = "0.2.11", default-features = false, features = ["trace"] tracing-subscriber = { version = "0.3.16", features = ["env-filter", "fmt"] } [features] -default = ["tokio", "sparse", "replication", "cache"] -replication = ["dep:async-broadcast"] -shared-core = ["replication", "dep:async-lock"] +default = ["sparse", "replication", "cache"] +replication = ["dep:async-broadcast", "dep:hypercore-protocol", "dep:hypercore_handshake"] sparse = ["random-access-disk/sparse"] -tokio = ["random-access-disk/tokio"] -async-std = ["random-access-disk/async-std"] cache = ["moka"] # Used only in interoperability tests under tests/js-interop which use the javascript version of hypercore # to verify that this crate works. To run them, use: diff --git a/benches/disk.rs b/benches/disk.rs index 3292df03..60d01c69 100644 --- a/benches/disk.rs +++ b/benches/disk.rs @@ -1,7 +1,5 @@ use std::time::{Duration, Instant}; -#[cfg(feature = "async-std")] -use criterion::async_executor::AsyncStdExecutor; use criterion::{Criterion, black_box, criterion_group, criterion_main}; use hypercore::{Hypercore, HypercoreBuilder, HypercoreError, Storage}; use tempfile::Builder as TempfileBuilder; @@ -10,11 +8,6 @@ fn bench_create_disk(c: &mut Criterion) { let mut group = c.benchmark_group("slow_call"); group.measurement_time(Duration::from_secs(20)); - #[cfg(feature = "async-std")] - group.bench_function("create_disk", move |b| { - b.to_async(AsyncStdExecutor) - .iter(|| create_hypercore("create")); - }); #[cfg(feature = "tokio")] group.bench_function("create_disk", move |b| { let rt = tokio::runtime::Runtime::new().unwrap(); @@ -51,10 +44,6 @@ fn bench_write_disk(c: &mut Criterion) { let mut group = c.benchmark_group("slow_call"); group.measurement_time(Duration::from_secs(20)); - #[cfg(feature = "async-std")] - group.bench_function("write disk", |b| { - b.to_async(AsyncStdExecutor).iter_custom(write_disk); - }); #[cfg(feature = "tokio")] group.bench_function("write disk", |b| { let rt = tokio::runtime::Runtime::new().unwrap(); @@ -76,10 +65,6 @@ fn bench_read_disk(c: &mut Criterion) { let mut group = c.benchmark_group("slow_call"); group.measurement_time(Duration::from_secs(20)); - #[cfg(feature = "async-std")] - group.bench_function("read disk", |b| { - b.to_async(AsyncStdExecutor).iter_custom(read_disk); - }); #[cfg(feature = "tokio")] group.bench_function("read disk", |b| { let rt = tokio::runtime::Runtime::new().unwrap(); @@ -104,10 +89,6 @@ fn bench_clear_disk(c: &mut Criterion) { let mut group = c.benchmark_group("slow_call"); group.measurement_time(Duration::from_secs(20)); - #[cfg(feature = "async-std")] - group.bench_function("clear disk", |b| { - b.to_async(AsyncStdExecutor).iter_custom(clear_disk); - }); #[cfg(feature = "tokio")] group.bench_function("clear disk", |b| { let rt = tokio::runtime::Runtime::new().unwrap(); diff --git a/benches/memory.rs b/benches/memory.rs index ac8015a5..89c22cdf 100644 --- a/benches/memory.rs +++ b/benches/memory.rs @@ -1,16 +1,13 @@ -use std::time::{Duration, Instant}; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; -#[cfg(feature = "async-std")] -use criterion::async_executor::AsyncStdExecutor; use criterion::{Criterion, black_box, criterion_group, criterion_main}; use hypercore::{Hypercore, HypercoreBuilder, HypercoreError, Storage}; use random_access_memory::RandomAccessMemory; fn bench_create_memory(c: &mut Criterion) { - #[cfg(feature = "async-std")] - c.bench_function("create memory", |b| { - b.to_async(AsyncStdExecutor).iter(|| create_hypercore(1024)); - }); #[cfg(feature = "tokio")] c.bench_function("create memory", |b| { let rt = tokio::runtime::Runtime::new().unwrap(); @@ -25,7 +22,7 @@ async fn create_hypercore(page_size: usize) -> Result let storage = Storage::open( |_| { Box::pin(async move { - Ok(Box::new(RandomAccessMemory::new(page_size)) as Box) + Ok(Arc::new(RandomAccessMemory::new(page_size)) as Arc) }) }, false, @@ -44,7 +41,7 @@ async fn create_hypercore(page_size: usize) -> Result let storage = Storage::open( |_| { Box::pin(async move { - Ok(Box::new(RandomAccessMemory::new(page_size)) as Box) + Ok(Arc::new(RandomAccessMemory::new(page_size)) as Arc) }) }, false, @@ -54,10 +51,6 @@ async fn create_hypercore(page_size: usize) -> Result } fn bench_write_memory(c: &mut Criterion) { - #[cfg(feature = "async-std")] - c.bench_function("write memory", |b| { - b.to_async(AsyncStdExecutor).iter_custom(write_memory); - }); #[cfg(feature = "tokio")] c.bench_function("write memory", |b| { let rt = tokio::runtime::Runtime::new().unwrap(); @@ -76,10 +69,6 @@ async fn write_memory(iters: u64) -> Duration { } fn bench_read_memory(c: &mut Criterion) { - #[cfg(feature = "async-std")] - c.bench_function("read memory", |b| { - b.to_async(AsyncStdExecutor).iter_custom(read_memory); - }); #[cfg(feature = "tokio")] c.bench_function("read memory", |b| { let rt = tokio::runtime::Runtime::new().unwrap(); @@ -101,10 +90,6 @@ async fn read_memory(iters: u64) -> Duration { } fn bench_clear_memory(c: &mut Criterion) { - #[cfg(feature = "async-std")] - c.bench_function("clear memory", |b| { - b.to_async(AsyncStdExecutor).iter_custom(clear_memory); - }); #[cfg(feature = "tokio")] c.bench_function("clear memory", |b| { let rt = tokio::runtime::Runtime::new().unwrap(); diff --git a/examples/disk.rs b/examples/disk.rs index d99b7a10..f92a4258 100644 --- a/examples/disk.rs +++ b/examples/disk.rs @@ -1,12 +1,8 @@ -#[cfg(feature = "async-std")] -use async_std::main as async_main; use hypercore::{HypercoreBuilder, HypercoreError, Storage}; use tempfile::Builder; -#[cfg(feature = "tokio")] -use tokio::main as async_main; /// Example about using an in-memory hypercore. -#[async_main] +#[tokio::main] async fn main() { // For the purposes of this example, first create a // temporary directory to hold hypercore. diff --git a/examples/memory.rs b/examples/memory.rs index a510ed6d..5ffe4569 100644 --- a/examples/memory.rs +++ b/examples/memory.rs @@ -1,11 +1,7 @@ -#[cfg(feature = "async-std")] -use async_std::main as async_main; use hypercore::{HypercoreBuilder, HypercoreError, Storage}; -#[cfg(feature = "tokio")] -use tokio::main as async_main; /// Example about using an in-memory hypercore. -#[async_main] +#[tokio::main] async fn main() { // Create a memory storage let storage = Storage::new_memory() diff --git a/examples/replication.rs b/examples/replication.rs index f2943796..c735342b 100644 --- a/examples/replication.rs +++ b/examples/replication.rs @@ -1,15 +1,11 @@ -#[cfg(feature = "async-std")] -use async_std::main as async_main; use hypercore::{Hypercore, HypercoreBuilder, HypercoreError, PartialKeypair, Storage}; use hypercore_schema::{RequestBlock, RequestUpgrade}; use tempfile::Builder; -#[cfg(feature = "tokio")] -use tokio::main as async_main; /// Example on how to replicate a (disk) hypercore to another (memory) hypercore. /// NB: The replication functions used here are low-level, built for use in the wire /// protocol. -#[async_main] +#[tokio::main] async fn main() { // For the purposes of this example, first create a // temporary directory to hold hypercore. @@ -99,7 +95,7 @@ async fn replicate_index( // Then the proof is verified and applied to the replicated party. assert!( replicated_hypercore - .verify_and_apply_proof(&proof) + .verify_and_apply_proof(proof) .await .expect("Verifying and applying proof failed") ); diff --git a/src/common/error.rs b/src/common/error.rs index 89ec0b37..dc53b400 100644 --- a/src/common/error.rs +++ b/src/common/error.rs @@ -1,4 +1,5 @@ use compact_encoding::EncodingError; +use random_access_storage::RandomAccessError; use thiserror::Error; use crate::Store; @@ -58,6 +59,10 @@ pub enum HypercoreError { #[source] source: std::io::Error, }, + + #[cfg(feature = "replication")] + #[error("hypercore_protocol Error")] + Protocol(#[from] hypercore_protocol::Error), } impl From for HypercoreError { @@ -76,3 +81,33 @@ impl From for HypercoreError { } } } + +impl From for HypercoreError { + fn from(value: RandomAccessError) -> Self { + map_random_access_err(value) + } +} + +pub(crate) fn map_random_access_err(err: RandomAccessError) -> HypercoreError { + match err { + RandomAccessError::IO { + return_code, + context, + source, + } => HypercoreError::IO { + context: Some(format!( + "RandomAccess IO error. Context: {context:?}, return_code: {return_code:?}", + )), + source, + }, + RandomAccessError::OutOfBounds { + offset, + end, + length, + } => HypercoreError::InvalidOperation { + context: format!( + "RandomAccess out of bounds. Offset: {offset}, end: {end:?}, length: {length}", + ), + }, + } +} diff --git a/src/core.rs b/src/core.rs deleted file mode 100644 index 3adc9a50..00000000 --- a/src/core.rs +++ /dev/null @@ -1,1181 +0,0 @@ -//! Hypercore's main abstraction. Exposes an append-only, secure log structure. -use ed25519_dalek::Signature; -use futures::future::Either; -use std::convert::TryFrom; -use std::fmt::Debug; -use tracing::instrument; - -#[cfg(feature = "cache")] -use crate::common::cache::CacheOptions; -use crate::{ - bitfield::Bitfield, - common::{BitfieldUpdate, HypercoreError, NodeByteRange, StoreInfo, ValuelessProof}, - crypto::{PartialKeypair, generate_signing_key}, - data::BlockStore, - oplog::{Header, MAX_OPLOG_ENTRIES_BYTE_SIZE, Oplog}, - storage::Storage, - tree::{MerkleTree, MerkleTreeChangeset}, -}; - -use hypercore_schema::{Proof, RequestBlock, RequestSeek, RequestUpgrade}; - -#[derive(Debug)] -pub(crate) struct HypercoreOptions { - pub(crate) key_pair: Option, - pub(crate) open: bool, - #[cfg(feature = "cache")] - pub(crate) node_cache_options: Option, -} - -impl HypercoreOptions { - pub(crate) fn new() -> Self { - Self { - key_pair: None, - open: false, - #[cfg(feature = "cache")] - node_cache_options: None, - } - } -} - -/// Hypercore is an append-only log structure. -#[derive(Debug)] -pub struct Hypercore { - pub(crate) key_pair: PartialKeypair, - pub(crate) storage: Storage, - pub(crate) oplog: Oplog, - pub(crate) tree: MerkleTree, - pub(crate) block_store: BlockStore, - pub(crate) bitfield: Bitfield, - skip_flush_count: u8, // autoFlush in Javascript - header: Header, - #[cfg(feature = "replication")] - events: crate::replication::events::Events, -} - -/// Response from append, matches that of the Javascript result -#[derive(Debug, PartialEq)] -pub struct AppendOutcome { - /// Length of the hypercore after append - pub length: u64, - /// Byte length of the hypercore after append - pub byte_length: u64, -} - -/// Info about the hypercore -#[derive(Debug, PartialEq)] -pub struct Info { - /// Length of the hypercore - pub length: u64, - /// Byte length of the hypercore - pub byte_length: u64, - /// Continuous length of entries in the hypercore with data - /// starting from index 0 - pub contiguous_length: u64, - /// Fork index. 0 if hypercore not forked. - pub fork: u64, - /// True if hypercore is writeable, false if read-only - pub writeable: bool, -} - -impl Hypercore { - /// Creates/opens new hypercore using given storage and options - pub(crate) async fn new( - mut storage: Storage, - mut options: HypercoreOptions, - ) -> Result { - let key_pair: Option = if options.open { - if options.key_pair.is_some() { - return Err(HypercoreError::BadArgument { - context: "Key pair can not be used when building an openable hypercore" - .to_string(), - }); - } - None - } else { - Some(options.key_pair.take().unwrap_or_else(|| { - let signing_key = generate_signing_key(); - PartialKeypair { - public: signing_key.verifying_key(), - secret: Some(signing_key), - } - })) - }; - - // Open/create oplog - let mut oplog_open_outcome = match Oplog::open(&key_pair, None)? { - Either::Right(value) => value, - Either::Left(instruction) => { - let info = storage.read_info(instruction).await?; - match Oplog::open(&key_pair, Some(info))? { - Either::Right(value) => value, - Either::Left(_) => { - return Err(HypercoreError::InvalidOperation { - context: "Could not open oplog".to_string(), - }); - } - } - } - }; - storage - .flush_infos(&oplog_open_outcome.infos_to_flush) - .await?; - - // Open/create tree - let mut tree = match MerkleTree::open( - &oplog_open_outcome.header.tree, - None, - #[cfg(feature = "cache")] - &options.node_cache_options, - )? { - Either::Right(value) => value, - Either::Left(instructions) => { - let infos = storage.read_infos(&instructions).await?; - match MerkleTree::open( - &oplog_open_outcome.header.tree, - Some(&infos), - #[cfg(feature = "cache")] - &options.node_cache_options, - )? { - Either::Right(value) => value, - Either::Left(_) => { - return Err(HypercoreError::InvalidOperation { - context: "Could not open tree".to_string(), - }); - } - } - } - }; - - // Create block store instance - let block_store = BlockStore::default(); - - // Open bitfield - let mut bitfield = match Bitfield::open(None) { - Either::Right(value) => value, - Either::Left(instruction) => { - let info = storage.read_info(instruction).await?; - match Bitfield::open(Some(info)) { - Either::Right(value) => value, - Either::Left(instruction) => { - let info = storage.read_info(instruction).await?; - match Bitfield::open(Some(info)) { - Either::Right(value) => value, - Either::Left(_) => { - return Err(HypercoreError::InvalidOperation { - context: "Could not open bitfield".to_string(), - }); - } - } - } - } - } - }; - - // Process entries stored only to the oplog and not yet flushed into bitfield or tree - if let Some(entries) = oplog_open_outcome.entries { - for entry in entries.iter() { - for node in &entry.tree_nodes { - tree.add_node(node.clone()); - } - - if let Some(bitfield_update) = &entry.bitfield { - bitfield.update(bitfield_update); - update_contiguous_length( - &mut oplog_open_outcome.header, - &bitfield, - bitfield_update, - ); - } - if let Some(tree_upgrade) = &entry.tree_upgrade { - // TODO: Generalize Either response stack - let mut changeset = - match tree.truncate(tree_upgrade.length, tree_upgrade.fork, None)? { - Either::Right(value) => value, - Either::Left(instructions) => { - let infos = storage.read_infos(&instructions).await?; - match tree.truncate( - tree_upgrade.length, - tree_upgrade.fork, - Some(&infos), - )? { - Either::Right(value) => value, - Either::Left(_) => { - return Err(HypercoreError::InvalidOperation { - context: format!( - "Could not truncate tree to length {}", - tree_upgrade.length - ), - }); - } - } - } - }; - changeset.ancestors = tree_upgrade.ancestors; - changeset.hash = Some(changeset.hash()); - changeset.signature = - Some(Signature::try_from(&*tree_upgrade.signature).map_err(|_| { - HypercoreError::InvalidSignature { - context: "Could not parse changeset signature".to_string(), - } - })?); - - // Update the header with this changeset to make in-memory value match that - // of the stored value. - oplog_open_outcome.oplog.update_header_with_changeset( - &changeset, - None, - &mut oplog_open_outcome.header, - )?; - - // TODO: Skip reorg hints for now, seems to only have to do with replication - // addReorgHint(header.hints.reorgs, tree, batch) - - // Commit changeset to in-memory tree - tree.commit(changeset)?; - } - } - } - - let oplog = oplog_open_outcome.oplog; - let header = oplog_open_outcome.header; - let key_pair = header.key_pair.clone(); - - Ok(Hypercore { - key_pair, - storage, - oplog, - tree, - block_store, - bitfield, - header, - skip_flush_count: 0, - #[cfg(feature = "replication")] - events: crate::replication::events::Events::new(), - }) - } - - /// Gets basic info about the Hypercore - pub fn info(&self) -> Info { - Info { - length: self.tree.length, - byte_length: self.tree.byte_length, - contiguous_length: self.header.hints.contiguous_length, - fork: self.tree.fork, - writeable: self.key_pair.secret.is_some(), - } - } - - /// Appends a data slice to the hypercore. - #[instrument(err, skip_all, fields(data_len = data.len()))] - pub async fn append(&mut self, data: &[u8]) -> Result { - self.append_batch(&[data]).await - } - - /// Appends a given batch of data slices to the hypercore. - #[instrument(err, skip_all, fields(batch_len = batch.as_ref().len()))] - pub async fn append_batch, B: AsRef<[A]>>( - &mut self, - batch: B, - ) -> Result { - let secret_key = match &self.key_pair.secret { - Some(key) => key, - None => return Err(HypercoreError::NotWritable), - }; - - if !batch.as_ref().is_empty() { - // Create a changeset for the tree - let mut changeset = self.tree.changeset(); - let mut batch_length: usize = 0; - for data in batch.as_ref().iter() { - batch_length += changeset.append(data.as_ref()); - } - changeset.hash_and_sign(secret_key); - - // Write the received data to the block store - let info = - self.block_store - .append_batch(batch.as_ref(), batch_length, self.tree.byte_length); - self.storage.flush_info(info).await?; - - // Append the changeset to the Oplog - let bitfield_update = BitfieldUpdate { - drop: false, - start: changeset.ancestors, - length: changeset.batch_length, - }; - let outcome = self.oplog.append_changeset( - &changeset, - Some(bitfield_update.clone()), - false, - &self.header, - )?; - self.storage.flush_infos(&outcome.infos_to_flush).await?; - self.header = outcome.header; - - // Write to bitfield - self.bitfield.update(&bitfield_update); - - // Contiguous length is known only now - update_contiguous_length(&mut self.header, &self.bitfield, &bitfield_update); - - // Commit changeset to in-memory tree - self.tree.commit(changeset)?; - - // Now ready to flush - if self.should_flush_bitfield_and_tree_and_oplog() { - self.flush_bitfield_and_tree_and_oplog(false).await?; - } - - #[cfg(feature = "replication")] - { - use tracing::trace; - - trace!(bitfield_update = ?bitfield_update, "Hppercore.append_batch emit DataUpgrade & Have"); - let _ = self.events.send(crate::replication::events::DataUpgrade {}); - let _ = self - .events - .send(crate::replication::events::Have::from(&bitfield_update)); - } - } - - // Return the new value - Ok(AppendOutcome { - length: self.tree.length, - byte_length: self.tree.byte_length, - }) - } - - #[cfg(feature = "replication")] - /// Subscribe to core events relevant to replication - pub fn event_subscribe(&self) -> async_broadcast::Receiver { - self.events.channel.new_receiver() - } - - /// Check if core has the block at the given `index` locally - #[instrument(ret, skip(self))] - pub fn has(&self, index: u64) -> bool { - self.bitfield.get(index) - } - - /// Read value at given index, if any. - #[instrument(err, skip(self))] - pub async fn get(&mut self, index: u64) -> Result>, HypercoreError> { - if !self.bitfield.get(index) { - #[cfg(feature = "replication")] - // if not in this core, emit Event::Get(index) - { - use tracing::trace; - - trace!(index = index, "Hppercore emit 'get' event"); - self.events.send_on_get(index); - } - return Ok(None); - } - - let byte_range = self.byte_range(index, None).await?; - - // TODO: Generalize Either response stack - let data = match self.block_store.read(&byte_range, None) { - Either::Right(value) => value, - Either::Left(instruction) => { - let info = self.storage.read_info(instruction).await?; - match self.block_store.read(&byte_range, Some(info)) { - Either::Right(value) => value, - Either::Left(_) => { - return Err(HypercoreError::InvalidOperation { - context: "Could not read block storage range".to_string(), - }); - } - } - } - }; - - Ok(Some(data.to_vec())) - } - - /// Clear data for entries between start and end (exclusive) indexes. - #[instrument(err, skip(self))] - pub async fn clear(&mut self, start: u64, end: u64) -> Result<(), HypercoreError> { - if start >= end { - // NB: This is what javascript does, so we mimic that here - return Ok(()); - } - // Write to oplog - let infos_to_flush = self.oplog.clear(start, end)?; - self.storage.flush_infos(&infos_to_flush).await?; - - // Set bitfield - self.bitfield.set_range(start, end - start, false); - - // Set contiguous length - if start < self.header.hints.contiguous_length { - self.header.hints.contiguous_length = start; - } - - // Find the biggest hole that can be punched into the data - let start = if let Some(index) = self.bitfield.last_index_of(true, start) { - index + 1 - } else { - 0 - }; - let end = if let Some(index) = self.bitfield.index_of(true, end) { - index - } else { - self.tree.length - }; - - // Find byte offset for first value - let mut infos: Vec = Vec::new(); - let clear_offset = match self.tree.byte_offset(start, None)? { - Either::Right(value) => value, - Either::Left(instructions) => { - let new_infos = self.storage.read_infos_to_vec(&instructions).await?; - infos.extend(new_infos); - match self.tree.byte_offset(start, Some(&infos))? { - Either::Right(value) => value, - Either::Left(_) => { - return Err(HypercoreError::InvalidOperation { - context: format!("Could not read offset for index {start} from tree"), - }); - } - } - } - }; - - // Find byte range for last value - let last_byte_range = self.byte_range(end - 1, Some(&infos)).await?; - - let clear_length = (last_byte_range.index + last_byte_range.length) - clear_offset; - - // Clear blocks - let info_to_flush = self.block_store.clear(clear_offset, clear_length); - self.storage.flush_info(info_to_flush).await?; - - // Now ready to flush - if self.should_flush_bitfield_and_tree_and_oplog() { - self.flush_bitfield_and_tree_and_oplog(false).await?; - } - - Ok(()) - } - - /// Access the key pair. - pub fn key_pair(&self) -> &PartialKeypair { - &self.key_pair - } - - /// Create a proof for given request - #[instrument(err, skip_all)] - pub async fn create_proof( - &mut self, - block: Option, - hash: Option, - seek: Option, - upgrade: Option, - ) -> Result, HypercoreError> { - let valueless_proof = self - .create_valueless_proof(block, hash, seek, upgrade) - .await?; - let value: Option> = if let Some(block) = valueless_proof.block.as_ref() { - let value = self.get(block.index).await?; - if value.is_none() { - // The data value requested in the proof can not be read, we return None here - // and let the party requesting figure out what to do. - return Ok(None); - } - value - } else { - None - }; - Ok(Some(valueless_proof.into_proof(value))) - } - - /// Verify and apply proof received from peer, returns true if changed, false if not - /// possible to apply. - #[instrument(skip_all)] - pub async fn verify_and_apply_proof(&mut self, proof: &Proof) -> Result { - if proof.fork != self.tree.fork { - return Ok(false); - } - let changeset = self.verify_proof(proof).await?; - if !self.tree.commitable(&changeset) { - return Ok(false); - } - - // In javascript there's _verifyExclusive and _verifyShared based on changeset.upgraded, but - // here we do only one. _verifyShared groups together many subsequent changesets into a single - // oplog push, and then flushes in the end only for the whole group. - let bitfield_update: Option = if let Some(block) = &proof.block.as_ref() { - let byte_offset = - match self - .tree - .byte_offset_in_changeset(block.index, &changeset, None)? - { - Either::Right(value) => value, - Either::Left(instructions) => { - let infos = self.storage.read_infos_to_vec(&instructions).await?; - match self.tree.byte_offset_in_changeset( - block.index, - &changeset, - Some(&infos), - )? { - Either::Right(value) => value, - Either::Left(_) => { - return Err(HypercoreError::InvalidOperation { - context: format!( - "Could not read offset for index {} from tree", - block.index - ), - }); - } - } - } - }; - - // Write the value to the block store - let info_to_flush = self.block_store.put(&block.value, byte_offset); - self.storage.flush_info(info_to_flush).await?; - - // Return a bitfield update for the given value - Some(BitfieldUpdate { - drop: false, - start: block.index, - length: 1, - }) - } else { - // Only from DataBlock can there be changes to the bitfield - None - }; - - // Append the changeset to the Oplog - let outcome = self.oplog.append_changeset( - &changeset, - bitfield_update.clone(), - false, - &self.header, - )?; - self.storage.flush_infos(&outcome.infos_to_flush).await?; - self.header = outcome.header; - - if let Some(bitfield_update) = &bitfield_update { - // Write to bitfield - self.bitfield.update(bitfield_update); - - // Contiguous length is known only now - update_contiguous_length(&mut self.header, &self.bitfield, bitfield_update); - } - - // Commit changeset to in-memory tree - self.tree.commit(changeset)?; - - // Now ready to flush - if self.should_flush_bitfield_and_tree_and_oplog() { - self.flush_bitfield_and_tree_and_oplog(false).await?; - } - - #[cfg(feature = "replication")] - { - if proof.upgrade.is_some() { - // Notify replicator if we receieved an upgrade - let _ = self.events.send(crate::replication::events::DataUpgrade {}); - } - - // Notify replicator if we receieved a bitfield update - if let Some(ref bitfield) = bitfield_update { - let _ = self - .events - .send(crate::replication::events::Have::from(bitfield)); - } - } - Ok(true) - } - - /// Used to fill the nodes field of a `RequestBlock` during - /// synchronization. - #[instrument(err, skip(self))] - pub async fn missing_nodes(&mut self, index: u64) -> Result { - self.missing_nodes_from_merkle_tree_index(index * 2).await - } - - /// Get missing nodes using a merkle tree index. Advanced variant of missing_nodex - /// that allow for special cases of searching directly from the merkle tree. - #[instrument(err, skip(self))] - pub async fn missing_nodes_from_merkle_tree_index( - &mut self, - merkle_tree_index: u64, - ) -> Result { - match self.tree.missing_nodes(merkle_tree_index, None)? { - Either::Right(value) => Ok(value), - Either::Left(instructions) => { - let mut instructions = instructions; - let mut infos: Vec = vec![]; - loop { - infos.extend(self.storage.read_infos_to_vec(&instructions).await?); - match self.tree.missing_nodes(merkle_tree_index, Some(&infos))? { - Either::Right(value) => { - return Ok(value); - } - Either::Left(new_instructions) => { - instructions = new_instructions; - } - } - } - } - } - } - - /// Makes the hypercore read-only by deleting the secret key. Returns true if the - /// hypercore was changed, false if the hypercore was already read-only. This is useful - /// in scenarios where a hypercore should be made immutable after initial values have - /// been stored. - #[instrument(err, skip_all)] - pub async fn make_read_only(&mut self) -> Result { - if self.key_pair.secret.is_some() { - self.key_pair.secret = None; - self.header.key_pair.secret = None; - // Need to flush clearing traces to make sure both oplog slots are cleared - self.flush_bitfield_and_tree_and_oplog(true).await?; - Ok(true) - } else { - Ok(false) - } - } - - async fn byte_range( - &mut self, - index: u64, - initial_infos: Option<&[StoreInfo]>, - ) -> Result { - match self.tree.byte_range(index, initial_infos)? { - Either::Right(value) => Ok(value), - Either::Left(instructions) => { - let mut instructions = instructions; - let mut infos: Vec = vec![]; - loop { - infos.extend(self.storage.read_infos_to_vec(&instructions).await?); - match self.tree.byte_range(index, Some(&infos))? { - Either::Right(value) => { - return Ok(value); - } - Either::Left(new_instructions) => { - instructions = new_instructions; - } - } - } - } - } - } - - async fn create_valueless_proof( - &mut self, - block: Option, - hash: Option, - seek: Option, - upgrade: Option, - ) -> Result { - match self.tree.create_valueless_proof( - block.as_ref(), - hash.as_ref(), - seek.as_ref(), - upgrade.as_ref(), - None, - )? { - Either::Right(value) => Ok(value), - Either::Left(instructions) => { - let mut instructions = instructions; - let mut infos: Vec = vec![]; - loop { - infos.extend(self.storage.read_infos_to_vec(&instructions).await?); - match self.tree.create_valueless_proof( - block.as_ref(), - hash.as_ref(), - seek.as_ref(), - upgrade.as_ref(), - Some(&infos), - )? { - Either::Right(value) => { - return Ok(value); - } - Either::Left(new_instructions) => { - instructions = new_instructions; - } - } - } - } - } - } - - /// Verify a proof received from a peer. Returns a changeset that should be - /// applied. - async fn verify_proof(&mut self, proof: &Proof) -> Result { - match self.tree.verify_proof(proof, &self.key_pair.public, None)? { - Either::Right(value) => Ok(value), - Either::Left(instructions) => { - let infos = self.storage.read_infos_to_vec(&instructions).await?; - match self - .tree - .verify_proof(proof, &self.key_pair.public, Some(&infos))? - { - Either::Right(value) => Ok(value), - Either::Left(_) => Err(HypercoreError::InvalidOperation { - context: "Could not verify proof from tree".to_string(), - }), - } - } - } - } - - fn should_flush_bitfield_and_tree_and_oplog(&mut self) -> bool { - if self.skip_flush_count == 0 - || self.oplog.entries_byte_length >= MAX_OPLOG_ENTRIES_BYTE_SIZE - { - self.skip_flush_count = 3; - true - } else { - self.skip_flush_count -= 1; - false - } - } - - async fn flush_bitfield_and_tree_and_oplog( - &mut self, - clear_traces: bool, - ) -> Result<(), HypercoreError> { - let infos = self.bitfield.flush(); - self.storage.flush_infos(&infos).await?; - let infos = self.tree.flush(); - self.storage.flush_infos(&infos).await?; - let infos = self.oplog.flush(&self.header, clear_traces)?; - self.storage.flush_infos(&infos).await?; - Ok(()) - } -} - -fn update_contiguous_length( - header: &mut Header, - bitfield: &Bitfield, - bitfield_update: &BitfieldUpdate, -) { - let end = bitfield_update.start + bitfield_update.length; - let mut c = header.hints.contiguous_length; - if bitfield_update.drop { - if c <= end && c > bitfield_update.start { - c = bitfield_update.start; - } - } else if c <= end && c >= bitfield_update.start { - c = end; - while bitfield.get(c) { - c += 1; - } - } - - if c != header.hints.contiguous_length { - header.hints.contiguous_length = c; - } -} - -#[cfg(test)] -pub(crate) mod tests { - use super::*; - - #[async_std::test] - async fn core_create_proof_block_only() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - - let proof = hypercore - .create_proof(Some(RequestBlock { index: 4, nodes: 2 }), None, None, None) - .await? - .unwrap(); - let block = proof.block.unwrap(); - assert_eq!(proof.upgrade, None); - assert_eq!(proof.seek, None); - assert_eq!(block.index, 4); - assert_eq!(block.nodes.len(), 2); - assert_eq!(block.nodes[0].index, 10); - assert_eq!(block.nodes[1].index, 13); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_block_and_upgrade() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - let proof = hypercore - .create_proof( - Some(RequestBlock { index: 4, nodes: 0 }), - None, - None, - Some(RequestUpgrade { - start: 0, - length: 10, - }), - ) - .await? - .unwrap(); - let block = proof.block.unwrap(); - let upgrade = proof.upgrade.unwrap(); - assert_eq!(proof.seek, None); - assert_eq!(block.index, 4); - assert_eq!(block.nodes.len(), 3); - assert_eq!(block.nodes[0].index, 10); - assert_eq!(block.nodes[1].index, 13); - assert_eq!(block.nodes[2].index, 3); - assert_eq!(upgrade.start, 0); - assert_eq!(upgrade.length, 10); - assert_eq!(upgrade.nodes.len(), 1); - assert_eq!(upgrade.nodes[0].index, 17); - assert_eq!(upgrade.additional_nodes.len(), 0); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_block_and_upgrade_and_additional() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - let proof = hypercore - .create_proof( - Some(RequestBlock { index: 4, nodes: 0 }), - None, - None, - Some(RequestUpgrade { - start: 0, - length: 8, - }), - ) - .await? - .unwrap(); - let block = proof.block.unwrap(); - let upgrade = proof.upgrade.unwrap(); - assert_eq!(proof.seek, None); - assert_eq!(block.index, 4); - assert_eq!(block.nodes.len(), 3); - assert_eq!(block.nodes[0].index, 10); - assert_eq!(block.nodes[1].index, 13); - assert_eq!(block.nodes[2].index, 3); - assert_eq!(upgrade.start, 0); - assert_eq!(upgrade.length, 8); - assert_eq!(upgrade.nodes.len(), 0); - assert_eq!(upgrade.additional_nodes.len(), 1); - assert_eq!(upgrade.additional_nodes[0].index, 17); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_block_and_upgrade_from_existing_state() -> Result<(), HypercoreError> - { - let mut hypercore = create_hypercore_with_data(10).await?; - let proof = hypercore - .create_proof( - Some(RequestBlock { index: 1, nodes: 0 }), - None, - None, - Some(RequestUpgrade { - start: 1, - length: 9, - }), - ) - .await? - .unwrap(); - let block = proof.block.unwrap(); - let upgrade = proof.upgrade.unwrap(); - assert_eq!(proof.seek, None); - assert_eq!(block.index, 1); - assert_eq!(block.nodes.len(), 0); - assert_eq!(upgrade.start, 1); - assert_eq!(upgrade.length, 9); - assert_eq!(upgrade.nodes.len(), 3); - assert_eq!(upgrade.nodes[0].index, 5); - assert_eq!(upgrade.nodes[1].index, 11); - assert_eq!(upgrade.nodes[2].index, 17); - assert_eq!(upgrade.additional_nodes.len(), 0); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_block_and_upgrade_from_existing_state_with_additional() - -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - let proof = hypercore - .create_proof( - Some(RequestBlock { index: 1, nodes: 0 }), - None, - None, - Some(RequestUpgrade { - start: 1, - length: 5, - }), - ) - .await? - .unwrap(); - let block = proof.block.unwrap(); - let upgrade = proof.upgrade.unwrap(); - assert_eq!(proof.seek, None); - assert_eq!(block.index, 1); - assert_eq!(block.nodes.len(), 0); - assert_eq!(upgrade.start, 1); - assert_eq!(upgrade.length, 5); - assert_eq!(upgrade.nodes.len(), 2); - assert_eq!(upgrade.nodes[0].index, 5); - assert_eq!(upgrade.nodes[1].index, 9); - assert_eq!(upgrade.additional_nodes.len(), 2); - assert_eq!(upgrade.additional_nodes[0].index, 13); - assert_eq!(upgrade.additional_nodes[1].index, 17); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_block_and_seek_1_no_upgrade() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - let proof = hypercore - .create_proof( - Some(RequestBlock { index: 4, nodes: 2 }), - None, - Some(RequestSeek { bytes: 8 }), - None, - ) - .await? - .unwrap(); - let block = proof.block.unwrap(); - assert_eq!(proof.seek, None); // seek included in block - assert_eq!(proof.upgrade, None); - assert_eq!(block.index, 4); - assert_eq!(block.nodes.len(), 2); - assert_eq!(block.nodes[0].index, 10); - assert_eq!(block.nodes[1].index, 13); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_block_and_seek_2_no_upgrade() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - let proof = hypercore - .create_proof( - Some(RequestBlock { index: 4, nodes: 2 }), - None, - Some(RequestSeek { bytes: 10 }), - None, - ) - .await? - .unwrap(); - let block = proof.block.unwrap(); - assert_eq!(proof.seek, None); // seek included in block - assert_eq!(proof.upgrade, None); - assert_eq!(block.index, 4); - assert_eq!(block.nodes.len(), 2); - assert_eq!(block.nodes[0].index, 10); - assert_eq!(block.nodes[1].index, 13); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_block_and_seek_3_no_upgrade() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - let proof = hypercore - .create_proof( - Some(RequestBlock { index: 4, nodes: 2 }), - None, - Some(RequestSeek { bytes: 13 }), - None, - ) - .await? - .unwrap(); - let block = proof.block.unwrap(); - let seek = proof.seek.unwrap(); - assert_eq!(proof.upgrade, None); - assert_eq!(block.index, 4); - assert_eq!(block.nodes.len(), 1); - assert_eq!(block.nodes[0].index, 10); - assert_eq!(seek.nodes.len(), 2); - assert_eq!(seek.nodes[0].index, 12); - assert_eq!(seek.nodes[1].index, 14); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_block_and_seek_to_tree_no_upgrade() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(16).await?; - let proof = hypercore - .create_proof( - Some(RequestBlock { index: 0, nodes: 4 }), - None, - Some(RequestSeek { bytes: 26 }), - None, - ) - .await? - .unwrap(); - let block = proof.block.unwrap(); - let seek = proof.seek.unwrap(); - assert_eq!(proof.upgrade, None); - assert_eq!(block.nodes.len(), 3); - assert_eq!(block.nodes[0].index, 2); - assert_eq!(block.nodes[1].index, 5); - assert_eq!(block.nodes[2].index, 11); - assert_eq!(seek.nodes.len(), 2); - assert_eq!(seek.nodes[0].index, 19); - assert_eq!(seek.nodes[1].index, 27); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_block_and_seek_with_upgrade() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - let proof = hypercore - .create_proof( - Some(RequestBlock { index: 4, nodes: 2 }), - None, - Some(RequestSeek { bytes: 13 }), - Some(RequestUpgrade { - start: 8, - length: 2, - }), - ) - .await? - .unwrap(); - let block = proof.block.unwrap(); - let seek = proof.seek.unwrap(); - let upgrade = proof.upgrade.unwrap(); - assert_eq!(block.index, 4); - assert_eq!(block.nodes.len(), 1); - assert_eq!(block.nodes[0].index, 10); - assert_eq!(seek.nodes.len(), 2); - assert_eq!(seek.nodes[0].index, 12); - assert_eq!(seek.nodes[1].index, 14); - assert_eq!(upgrade.nodes.len(), 1); - assert_eq!(upgrade.nodes[0].index, 17); - assert_eq!(upgrade.additional_nodes.len(), 0); - Ok(()) - } - - #[async_std::test] - async fn core_create_proof_seek_with_upgrade() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - let proof = hypercore - .create_proof( - None, - None, - Some(RequestSeek { bytes: 13 }), - Some(RequestUpgrade { - start: 0, - length: 10, - }), - ) - .await? - .unwrap(); - let seek = proof.seek.unwrap(); - let upgrade = proof.upgrade.unwrap(); - assert_eq!(proof.block, None); - assert_eq!(seek.nodes.len(), 4); - assert_eq!(seek.nodes[0].index, 12); - assert_eq!(seek.nodes[1].index, 14); - assert_eq!(seek.nodes[2].index, 9); - assert_eq!(seek.nodes[3].index, 3); - assert_eq!(upgrade.nodes.len(), 1); - assert_eq!(upgrade.nodes[0].index, 17); - assert_eq!(upgrade.additional_nodes.len(), 0); - Ok(()) - } - - #[async_std::test] - async fn core_verify_proof_invalid_signature() -> Result<(), HypercoreError> { - let mut hypercore = create_hypercore_with_data(10).await?; - // Invalid clone hypercore with a different public key - let mut hypercore_clone = create_hypercore_with_data(0).await?; - let proof = hypercore - .create_proof( - None, - Some(RequestBlock { index: 6, nodes: 0 }), - None, - Some(RequestUpgrade { - start: 0, - length: 10, - }), - ) - .await? - .unwrap(); - assert!( - hypercore_clone - .verify_and_apply_proof(&proof) - .await - .is_err() - ); - Ok(()) - } - - #[async_std::test] - async fn core_verify_and_apply_proof() -> Result<(), HypercoreError> { - let mut main = create_hypercore_with_data(10).await?; - let mut clone = create_hypercore_with_data_and_key_pair( - 0, - PartialKeypair { - public: main.key_pair.public, - secret: None, - }, - ) - .await?; - let index = 6; - let nodes = clone.missing_nodes(index).await?; - let proof = main - .create_proof( - None, - Some(RequestBlock { index, nodes }), - None, - Some(RequestUpgrade { - start: 0, - length: 10, - }), - ) - .await? - .unwrap(); - assert!(clone.verify_and_apply_proof(&proof).await?); - let main_info = main.info(); - let clone_info = clone.info(); - assert_eq!(main_info.byte_length, clone_info.byte_length); - assert_eq!(main_info.length, clone_info.length); - assert!(main.get(6).await?.is_some()); - assert!(clone.get(6).await?.is_none()); - - // Fetch data for index 6 and verify it is found - let index = 6; - let nodes = clone.missing_nodes(index).await?; - let proof = main - .create_proof(Some(RequestBlock { index, nodes }), None, None, None) - .await? - .unwrap(); - assert!(clone.verify_and_apply_proof(&proof).await?); - Ok(()) - } - - pub(crate) async fn create_hypercore_with_data( - length: u64, - ) -> Result { - let signing_key = generate_signing_key(); - create_hypercore_with_data_and_key_pair( - length, - PartialKeypair { - public: signing_key.verifying_key(), - secret: Some(signing_key), - }, - ) - .await - } - - pub(crate) async fn create_hypercore_with_data_and_key_pair( - length: u64, - key_pair: PartialKeypair, - ) -> Result { - let storage = Storage::new_memory().await?; - let mut hypercore = Hypercore::new( - storage, - HypercoreOptions { - key_pair: Some(key_pair), - open: false, - #[cfg(feature = "cache")] - node_cache_options: None, - }, - ) - .await?; - for i in 0..length { - hypercore.append(format!("#{}", i).as_bytes()).await?; - } - Ok(hypercore) - } -} diff --git a/src/core/inner.rs b/src/core/inner.rs new file mode 100644 index 00000000..950f75ee --- /dev/null +++ b/src/core/inner.rs @@ -0,0 +1,1203 @@ +use std::{ + future::Future, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; + +use ed25519_dalek::Signature; +use futures::future::Either; +use random_access_storage::BoxFuture; +use std::sync::Mutex; +use tracing::instrument; + +use crate::{ + bitfield::Bitfield, + common::{BitfieldUpdate, HypercoreError, NodeByteRange, StoreInfo, ValuelessProof}, + crypto::{PartialKeypair, generate_signing_key}, + data::BlockStore, + oplog::{Header, MAX_OPLOG_ENTRIES_BYTE_SIZE, Oplog, OplogCreateHeaderOutcome}, + storage::Storage, + tree::{MerkleTree, MerkleTreeChangeset}, +}; +use hypercore_schema::{Proof, RequestBlock, RequestSeek, RequestUpgrade}; + +use super::{AppendOutcome, HypercoreOptions, Info}; + +#[derive(Debug)] +pub(crate) struct HypercoreInnerInner { + pub(crate) key_pair: PartialKeypair, + pub(crate) storage: Storage, + pub(crate) oplog: Oplog, + pub(crate) tree: MerkleTree, + pub(crate) block_store: BlockStore, + pub(crate) bitfield: Bitfield, + pub(crate) skip_flush_count: u8, + pub(crate) header: Header, + #[cfg(feature = "replication")] + pub(crate) events: crate::replication::events::Events, +} + +impl HypercoreInnerInner { + pub(crate) async fn new( + storage: Storage, + mut options: HypercoreOptions, + ) -> Result { + let key_pair: Option = if options.open { + if options.key_pair.is_some() { + return Err(HypercoreError::BadArgument { + context: "Key pair can not be used when building an openable hypercore" + .to_string(), + }); + } + None + } else { + Some(options.key_pair.take().unwrap_or_else(|| { + let signing_key = generate_signing_key(); + PartialKeypair { + public: signing_key.verifying_key(), + secret: Some(signing_key), + } + })) + }; + + // Open/create oplog + let mut oplog_open_outcome = match Oplog::open(&key_pair, None)? { + Either::Right(value) => value, + Either::Left(instruction) => { + let info = storage.read_info(instruction).await?; + match Oplog::open(&key_pair, Some(info))? { + Either::Right(value) => value, + Either::Left(_) => { + return Err(HypercoreError::InvalidOperation { + context: "Could not open oplog".to_string(), + }); + } + } + } + }; + storage + .flush_infos(Vec::from(oplog_open_outcome.infos_to_flush)) + .await?; + + // Open/create tree + let mut tree = match MerkleTree::open( + &oplog_open_outcome.header.tree, + None, + #[cfg(feature = "cache")] + &options.node_cache_options, + )? { + Either::Right(value) => value, + Either::Left(instructions) => { + let infos = storage.read_infos(Vec::from(instructions)).await?; + match MerkleTree::open( + &oplog_open_outcome.header.tree, + Some(&infos), + #[cfg(feature = "cache")] + &options.node_cache_options, + )? { + Either::Right(value) => value, + Either::Left(_) => { + return Err(HypercoreError::InvalidOperation { + context: "Could not open tree".to_string(), + }); + } + } + } + }; + + // Create block store instance + let block_store = BlockStore::default(); + + // Open bitfield + let mut bitfield = match Bitfield::open(None) { + Either::Right(value) => value, + Either::Left(instruction) => { + let info = storage.read_info(instruction).await?; + match Bitfield::open(Some(info)) { + Either::Right(value) => value, + Either::Left(instruction) => { + let info = storage.read_info(instruction).await?; + match Bitfield::open(Some(info)) { + Either::Right(value) => value, + Either::Left(_) => { + return Err(HypercoreError::InvalidOperation { + context: "Could not open bitfield".to_string(), + }); + } + } + } + } + } + }; + + // Process entries stored only to the oplog and not yet flushed into bitfield or tree + if let Some(entries) = oplog_open_outcome.entries { + for entry in entries.iter() { + for node in &entry.tree_nodes { + tree.add_node(node.clone()); + } + + if let Some(bitfield_update) = &entry.bitfield { + bitfield.update(bitfield_update); + update_contiguous_length( + &mut oplog_open_outcome.header, + &bitfield, + bitfield_update, + ); + } + if let Some(tree_upgrade) = &entry.tree_upgrade { + let mut changeset = + match tree.truncate(tree_upgrade.length, tree_upgrade.fork, None)? { + Either::Right(value) => value, + Either::Left(instructions) => { + let infos = storage.read_infos(Vec::from(instructions)).await?; + match tree.truncate( + tree_upgrade.length, + tree_upgrade.fork, + Some(&infos), + )? { + Either::Right(value) => value, + Either::Left(_) => { + return Err(HypercoreError::InvalidOperation { + context: format!( + "Could not truncate tree to length {}", + tree_upgrade.length + ), + }); + } + } + } + }; + changeset.ancestors = tree_upgrade.ancestors; + changeset.hash = Some(changeset.hash()); + changeset.signature = + Some(Signature::try_from(&*tree_upgrade.signature).map_err(|_| { + HypercoreError::InvalidSignature { + context: "Could not parse changeset signature".to_string(), + } + })?); + + oplog_open_outcome.oplog.update_header_with_changeset( + &changeset, + None, + &mut oplog_open_outcome.header, + )?; + + tree.commit(changeset)?; + } + } + } + + let oplog = oplog_open_outcome.oplog; + let header = oplog_open_outcome.header; + let key_pair = header.key_pair.clone(); + + Ok(Self { + key_pair, + storage, + oplog, + tree, + block_store, + bitfield, + header, + skip_flush_count: 0, + #[cfg(feature = "replication")] + events: crate::replication::events::Events::new(), + }) + } + + pub(crate) fn info(&self) -> Info { + Info { + length: self.tree.length, + byte_length: self.tree.byte_length, + contiguous_length: self.header.hints.contiguous_length, + fork: self.tree.fork, + writeable: self.key_pair.secret.is_some(), + } + } + + pub(crate) fn key_pair(&self) -> &PartialKeypair { + &self.key_pair + } + + #[instrument(ret, skip(self))] + pub(crate) fn has(&self, index: u64) -> bool { + self.bitfield.get(index) + } + + #[cfg(feature = "replication")] + pub(crate) fn event_subscribe( + &self, + ) -> async_broadcast::Receiver { + self.events.channel.new_receiver() + } + + pub(crate) fn append_outcome(&self) -> AppendOutcome { + AppendOutcome { + length: self.tree.length, + byte_length: self.tree.byte_length, + } + } + + pub(crate) fn should_flush_bitfield_and_tree_and_oplog(&mut self) -> bool { + if self.skip_flush_count == 0 + || self.oplog.entries_byte_length >= MAX_OPLOG_ENTRIES_BYTE_SIZE + { + self.skip_flush_count = 3; + true + } else { + self.skip_flush_count -= 1; + false + } + } + + pub(crate) fn flush_bitfield_and_tree_and_oplog( + &mut self, + clear_traces: bool, + ) -> BoxFuture> { + let mut infos = vec![]; + infos.extend(self.bitfield.flush()); + infos.extend(self.tree.flush()); + match self.oplog.flush(&self.header, clear_traces) { + Ok(opinfo) => infos.extend(opinfo), + Err(e) => return Box::pin(async { Err(e) }), + } + + self.storage.flush_infos(infos) + } +} + +/// Shared slot for a background replicator, driven whenever `Hypercore::get` polls. +#[cfg(feature = "replication")] +pub(crate) type BackgroundFuture = Arc< + Mutex> + Send>>>>, +>; + +pub(crate) struct HypercoreInner { + pub(crate) inner: Arc>, + /// Replicator driven in-band by any `Hypercore::get` that must wait for a block. + #[cfg(feature = "replication")] + pub(crate) background: BackgroundFuture, +} + +impl std::fmt::Debug for HypercoreInner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HypercoreInner").finish_non_exhaustive() + } +} + +impl Clone for HypercoreInner { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + #[cfg(feature = "replication")] + background: self.background.clone(), + } + } +} + +impl HypercoreInner { + pub(crate) async fn new( + storage: Storage, + options: HypercoreOptions, + ) -> Result { + Ok(Self { + inner: Arc::new(Mutex::new( + HypercoreInnerInner::new(storage, options).await?, + )), + #[cfg(feature = "replication")] + background: Arc::new(Mutex::new(None)), + }) + } + pub(crate) fn info(&self) -> Info { + self.inner.lock().unwrap().info() + } + pub(crate) fn key_pair(&self) -> PartialKeypair { + self.inner.lock().unwrap().key_pair().clone() + } + + pub(crate) fn has(&self, index: u64) -> bool { + self.inner.lock().unwrap().has(index) + } + + #[cfg(feature = "replication")] + pub(crate) fn event_subscribe( + &self, + ) -> async_broadcast::Receiver { + self.inner.lock().unwrap().event_subscribe() + } + pub(crate) fn append_outcome(&self) -> AppendOutcome { + self.inner.lock().unwrap().append_outcome() + } + pub(crate) fn should_flush_bitfield_and_tree_and_oplog(&self) -> bool { + self.inner + .lock() + .unwrap() + .should_flush_bitfield_and_tree_and_oplog() + } + pub(crate) fn flush_bitfield_and_tree_and_oplog( + &self, + clear_traces: bool, + ) -> BoxFuture> { + self.inner + .lock() + .unwrap() + .flush_bitfield_and_tree_and_oplog(clear_traces) + } + + pub(crate) fn verify_proof(&self, proof: Proof) -> VerifyProofFuture { + VerifyProofFuture { + inner: self.inner.clone(), + proof, + infos: None, + pending_read: None, + } + } + pub(crate) fn missing_nodes_from_merkle_tree_index( + &self, + merkle_tree_index: u64, + ) -> MissingNodesFuture { + MissingNodesFuture { + inner: self.inner.clone(), + merkle_tree_index, + infos: Vec::new(), + pending_read: None, + } + } + pub(crate) fn verify_and_apply_proof(&self, proof: Proof) -> VerifyAndApplyProofFuture { + VerifyAndApplyProofFuture { + inner: self.inner.clone(), + proof, + changeset: None, + bitfield_update: None, + pending_header: None, + verify_fut: None, + byte_offset_infos: Vec::new(), + byte_offset_read_fut: None, + flush_block_fut: None, + flush_oplog_fut: None, + flush_all_fut: None, + } + } + + pub(crate) fn byte_range(&self, index: u64, initial_infos: Vec) -> ByteRangeFuture { + ByteRangeFuture { + inner: self.inner.clone(), + index, + infos: initial_infos, + pending_read: None, + } + } + + pub(crate) fn get(&self, index: u64) -> GetFuture { + GetFuture { + inner: self.inner.clone(), + index, + byte_range_fut: None, + byte_range: None, + block_read_fut: None, + #[cfg(feature = "replication")] + background: self.background.clone(), + #[cfg(feature = "replication")] + waiting: None, + } + } + + pub(crate) fn create_proof( + &self, + block: Option, + hash: Option, + seek: Option, + upgrade: Option, + ) -> CreateProofFuture { + CreateProofFuture { + inner: self.inner.clone(), + block, + hash, + seek, + upgrade, + valueless_proof_fut: None, + valueless_proof: None, + get_fut: None, + } + } +} + +pub(crate) struct CreateProofFuture { + inner: Arc>, + block: Option, + hash: Option, + seek: Option, + upgrade: Option, + // Phase 1: build the proof structure (without block data) + valueless_proof_fut: Option, + valueless_proof: Option, + // Phase 2: fetch the block value (only when proof.block is Some) + get_fut: Option, +} + +impl Future for CreateProofFuture { + type Output = Result, HypercoreError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + loop { + // Phase 2: fetch block value. + if let Some(fut) = this.get_fut.as_mut() { + match Pin::new(fut).poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(data)) => { + this.get_fut = None; + let vp = this.valueless_proof.take().unwrap(); + return match data { + // Block not present locally — can't serve the proof. + None => Poll::Ready(Ok(None)), + Some(bytes) => { + Poll::Ready(Ok(Some(vp.into_proof(Some(bytes.into_vec()))))) + } + }; + } + } + } + + // Phase 1: build the valueless proof. + if let Some(fut) = this.valueless_proof_fut.as_mut() { + match Pin::new(fut).poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(vp)) => { + this.valueless_proof_fut = None; + if let Some(block) = vp.block.as_ref() { + let index = block.index; + this.valueless_proof = Some(vp); + this.get_fut = Some(GetFuture { + inner: this.inner.clone(), + index, + byte_range_fut: None, + byte_range: None, + block_read_fut: None, + // CreateProofFuture serves local data only; no waiting. + #[cfg(feature = "replication")] + background: Arc::new(Mutex::new(None)), + #[cfg(feature = "replication")] + waiting: None, + }); + continue; + } + return Poll::Ready(Ok(Some(vp.into_proof(None)))); + } + } + } + + // Initial: start the valueless proof future. + this.valueless_proof_fut = Some(ValuelessProofFuture::new( + this.inner.clone(), + this.block.take(), + this.hash.take(), + this.seek.take(), + this.upgrade.take(), + Vec::new(), + None, + )); + } + } +} + +pub(crate) struct GetFuture { + inner: Arc>, + index: u64, + // Phase 1: resolve byte range + byte_range_fut: Option, + // Phase 2: read block from storage (only needed if block_store has no cached value) + byte_range: Option, + block_read_fut: Option>>, + // Replication: drive the background replicator while waiting for this block. + #[cfg(feature = "replication")] + background: BackgroundFuture, + #[cfg(feature = "replication")] + waiting: Option>, +} + +impl GetFuture { + /// Drive the background replicator and wait for a `Have` or `DataUpgrade` event. + /// Returns `Poll::Pending` while waiting, `Poll::Ready(Ok(None))` if no replicator + /// is attached or replication has finished without delivering the block. + #[cfg(feature = "replication")] + fn poll_background_and_wait( + &mut self, + cx: &mut Context<'_>, + ) -> Poll>, HypercoreError>> { + use crate::replication::events::Event; + + // Drive the background replicator. + let bg_done = { + let mut bg = self.background.lock().unwrap(); + match bg.as_mut() { + None => true, + Some(fut) => match fut.as_mut().poll(cx) { + Poll::Ready(Ok(())) => { + *bg = None; + true + } + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Pending => false, + }, + } + }; + + // Drain events; any Have/DataUpgrade means new data may be available. + if let Some(ref mut rx) = self.waiting { + use futures::Stream as _; + loop { + match Pin::new(&mut *rx).poll_next(cx) { + Poll::Ready(Some(Event::Have(_) | Event::DataUpgrade(_))) => { + self.waiting = None; + cx.waker().wake_by_ref(); + return Poll::Pending; + } + Poll::Ready(Some(Event::Get(_))) => {} + Poll::Ready(None) => return Poll::Ready(Ok(None)), + Poll::Pending => break, + } + } + } + + // Background finished without delivering the block. + if bg_done { + return Poll::Ready(Ok(None)); + } + + Poll::Pending + } +} + +impl Future for GetFuture { + type Output = Result>, HypercoreError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + // TODO: we really need to generalize the Either response stack + loop { + // Phase 2: storage read for block data (highest priority when active). + if let Some(fut) = this.block_read_fut.as_mut() { + match fut.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(info)) => { + this.block_read_fut = None; + let inner = this.inner.lock().unwrap(); + let byte_range = this.byte_range.as_ref().unwrap(); + return match inner.block_store.read(byte_range, Some(info)) { + Either::Right(data) => Poll::Ready(Ok(Some(data))), + Either::Left(_) => Poll::Ready(Err(HypercoreError::InvalidOperation { + context: "Could not read block storage range".to_string(), + })), + }; + } + } + } + + // Phase 1: resolve the byte range. + if let Some(fut) = this.byte_range_fut.as_mut() { + match Pin::new(fut).poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(byte_range)) => { + this.byte_range_fut = None; + let inner = this.inner.lock().unwrap(); + match inner.block_store.read(&byte_range, None) { + Either::Right(data) => return Poll::Ready(Ok(Some(data))), + Either::Left(instruction) => { + let storage = inner.storage.clone(); + this.block_read_fut = Some(storage.read_info(instruction)); + this.byte_range = Some(byte_range); + // Loop to poll block_read_fut immediately. + } + } + } + } + continue; + } + + // Initial: check bitfield; if block is missing, wait for replication. + { + let inner = this.inner.lock().unwrap(); + if !inner.bitfield.get(this.index) { + #[cfg(not(feature = "replication"))] + return Poll::Ready(Ok(None)); + + #[cfg(feature = "replication")] + { + // First miss: check whether a background replicator is attached. + if this.waiting.is_none() { + inner.events.send_on_get(this.index); + if this.background.lock().unwrap().is_none() { + // No replicator — return None immediately (original behaviour). + dbg!(); + return Poll::Ready(Ok(None)); + } + // Subscribe before emitting Get so we can't miss the Have reply. + let rx = inner.event_subscribe(); + drop(inner); + this.waiting = Some(rx); + } else { + drop(inner); + } + return this.poll_background_and_wait(cx); + } + } + } + this.byte_range_fut = Some(ByteRangeFuture { + inner: this.inner.clone(), + index: this.index, + infos: Vec::new(), + pending_read: None, + }); + // Loop to poll byte_range_fut immediately. + } + } +} + +pub(crate) struct MissingNodesFuture { + inner: Arc>, + merkle_tree_index: u64, + infos: Vec, + pending_read: Option, HypercoreError>>>, +} + +impl Future for MissingNodesFuture { + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + loop { + if let Some(fut) = this.pending_read.as_mut() { + match fut.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(new_infos)) => { + this.infos.extend(new_infos); + this.pending_read = None; + } + } + } + + let result = { + let inner = this.inner.lock().unwrap(); + let infos_opt = if this.infos.is_empty() { + None + } else { + Some(this.infos.as_slice()) + }; + inner.tree.missing_nodes(this.merkle_tree_index, infos_opt) + // Lock is dropped here. + }; + + match result { + Err(e) => return Poll::Ready(Err(e)), + Ok(Either::Right(value)) => return Poll::Ready(Ok(value)), + Ok(Either::Left(instructions)) => { + let storage = this.inner.lock().unwrap().storage.clone(); + this.pending_read = Some(storage.read_infos_to_vec(Vec::from(instructions))); + } + } + } + } +} + +pub(crate) struct ByteRangeFuture { + inner: Arc>, + index: u64, + infos: Vec, + pending_read: Option, HypercoreError>>>, +} + +impl Future for ByteRangeFuture { + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + loop { + if let Some(fut) = this.pending_read.as_mut() { + match fut.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(new_infos)) => { + this.infos.extend(new_infos); + this.pending_read = None; + } + } + } + + let result = { + let inner = this.inner.lock().unwrap(); + let infos_opt = if this.infos.is_empty() { + None + } else { + Some(this.infos.as_slice()) + }; + inner.tree.byte_range(this.index, infos_opt) + // Lock is dropped here. + }; + + match result { + Err(e) => return Poll::Ready(Err(e)), + Ok(Either::Right(value)) => return Poll::Ready(Ok(value)), + Ok(Either::Left(instructions)) => { + let storage = this.inner.lock().unwrap().storage.clone(); + this.pending_read = Some(storage.read_infos_to_vec(Vec::from(instructions))); + } + } + } + } +} + +pub(crate) struct VerifyProofFuture { + inner: Arc>, + proof: Proof, + // None = first attempt (no read done yet), Some = read completed + infos: Option>, + pending_read: Option, HypercoreError>>>, +} + +impl Future for VerifyProofFuture { + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + loop { + // Phase 1: if there's a pending storage read, drive it to completion. + if let Some(fut) = this.pending_read.as_mut() { + match fut.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(infos)) => { + this.infos = Some(infos); + this.pending_read = None; + // Fall through to retry verify_proof. + } + } + } + + // Phase 2: call tree.verify_proof synchronously under the lock. + let result = { + let inner = this.inner.lock().unwrap(); + let public_key = inner.key_pair.public; + let infos_opt = this.infos.as_deref(); + inner.tree.verify_proof(&this.proof, &public_key, infos_opt) + // Lock is dropped here. + }; + + match result { + Err(e) => return Poll::Ready(Err(e)), + Ok(Either::Right(value)) => return Poll::Ready(Ok(value)), + Ok(Either::Left(_)) if this.infos.is_some() => { + // We already read infos and still got Left — the proof can't + // be satisfied, which is an error. + return Poll::Ready(Err(HypercoreError::InvalidOperation { + context: "Could not verify proof from tree".to_string(), + })); + } + Ok(Either::Left(instructions)) => { + let storage = this.inner.lock().unwrap().storage.clone(); + this.pending_read = Some(storage.read_infos_to_vec(Vec::from(instructions))); + // Loop to poll the new future immediately. + } + } + } + } +} + +pub(crate) struct ValuelessProofFuture { + inner: Arc>, + block: Option, + hash: Option, + seek: Option, + upgrade: Option, + infos: Vec, + pending_read: Option, HypercoreError>>>, +} + +impl ValuelessProofFuture { + fn new( + inner: Arc>, + block: Option, + hash: Option, + seek: Option, + upgrade: Option, + infos: Vec, + pending_read: Option, HypercoreError>>>, + ) -> Self { + Self { + inner, + block, + hash, + seek, + upgrade, + infos, + pending_read, + } + } +} + +impl Future for ValuelessProofFuture { + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + // ValuelessProofFuture is Unpin (all fields are Unpin), so this is safe. + let this = self.get_mut(); + + loop { + // Phase 1: if there's a pending storage read, drive it to completion. + if let Some(fut) = this.pending_read.as_mut() { + match fut.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(new_infos)) => { + this.infos.extend(new_infos); + this.pending_read = None; + // Fall through to retry create_valueless_proof. + } + } + } + + // Phase 2: call tree.create_valueless_proof synchronously under the lock. + let result = { + let inner = this.inner.lock().unwrap(); + let infos_opt = if this.infos.is_empty() { + None + } else { + Some(this.infos.as_slice()) + }; + inner.tree.create_valueless_proof( + this.block.as_ref(), + this.hash.as_ref(), + this.seek.as_ref(), + this.upgrade.as_ref(), + infos_opt, + ) + // Lock is dropped here. + }; + + match result { + Err(e) => return Poll::Ready(Err(e)), + Ok(Either::Right(value)) => return Poll::Ready(Ok(value)), + Ok(Either::Left(instructions)) => { + // Need more nodes from storage. Clone storage (cheap Arc clone) + // outside the lock so we don't hold it across the async read. + let storage = this.inner.lock().unwrap().storage.clone(); + this.pending_read = Some(storage.read_infos_to_vec(Vec::from(instructions))); + // Loop to poll the new future immediately. + } + } + } + } +} + +pub(crate) struct VerifyAndApplyProofFuture { + inner: Arc>, + proof: Proof, + // Carried between phases + changeset: Option, + bitfield_update: Option, + pending_header: Option
, + // Phase 1: verify the proof + verify_fut: Option, + // Phase 2: read nodes for byte_offset_in_changeset (only if proof.block is Some) + byte_offset_infos: Vec, + byte_offset_read_fut: Option, HypercoreError>>>, + // Phase 3: flush block data to storage + flush_block_fut: Option>>, + // Phase 4: flush oplog + flush_oplog_fut: Option>>, + // Phase 5: flush bitfield+tree+oplog (conditional) + flush_all_fut: Option>>, +} + +impl VerifyAndApplyProofFuture { + // Run oplog.append_changeset synchronously under the lock and return the + // BoxFuture that flushes the resulting infos to storage. + fn start_flush_oplog( + inner: &Arc>, + changeset: &MerkleTreeChangeset, + pending_header: &mut Option
, + bitfield_update: &Option, + ) -> Result>, HypercoreError> { + let (storage, infos) = { + let mut guard = inner.lock().unwrap(); + let OplogCreateHeaderOutcome { + header, + infos_to_flush, + } = { + let HypercoreInnerInner { oplog, header, .. } = &mut *guard; + oplog.append_changeset(changeset, bitfield_update.clone(), false, header)? + }; + *pending_header = Some(header); + let storage = guard.storage.clone(); + (storage, infos_to_flush) + }; + Ok(storage.flush_infos(Vec::from(infos))) + } + + fn emit_events( + inner: &Arc>, + proof: &Proof, + bitfield_update: &Option, + ) { + #[cfg(feature = "replication")] + { + let inner = inner.lock().unwrap(); + if proof.upgrade.is_some() { + let _ = inner + .events + .send(crate::replication::events::DataUpgrade {}); + } + if let Some(bu) = bitfield_update { + let _ = inner + .events + .send(crate::replication::events::Have::from(bu)); + } + } + let _ = (inner, proof, bitfield_update); + } +} + +impl Future for VerifyAndApplyProofFuture { + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + loop { + // Phase 5: flush bitfield+tree+oplog. + if let Some(fut) = this.flush_all_fut.as_mut() { + match fut.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(())) => { + this.flush_all_fut = None; + Self::emit_events(&this.inner, &this.proof, &this.bitfield_update); + return Poll::Ready(Ok(true)); + } + } + } + + // Phase 4: flush oplog. + if let Some(fut) = this.flush_oplog_fut.as_mut() { + match fut.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(())) => { + this.flush_oplog_fut = None; + let maybe_flush = { + let mut inner = this.inner.lock().unwrap(); + inner.header = this.pending_header.take().unwrap(); + if let Some(ref bu) = this.bitfield_update { + inner.bitfield.update(bu); + let HypercoreInnerInner { + bitfield, header, .. + } = &mut *inner; + update_contiguous_length(header, bitfield, bu); + } + let changeset = this.changeset.take().unwrap(); + if let Err(e) = inner.tree.commit(changeset) { + return Poll::Ready(Err(e)); + } + if inner.should_flush_bitfield_and_tree_and_oplog() { + Some(inner.flush_bitfield_and_tree_and_oplog(false)) + } else { + None + } + }; + if let Some(fut) = maybe_flush { + this.flush_all_fut = Some(fut); + continue; + } + Self::emit_events(&this.inner, &this.proof, &this.bitfield_update); + return Poll::Ready(Ok(true)); + } + } + } + + // Phase 3: flush block data. + if let Some(fut) = this.flush_block_fut.as_mut() { + match fut.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(())) => { + this.flush_block_fut = None; + match Self::start_flush_oplog( + &this.inner, + this.changeset.as_ref().unwrap(), + &mut this.pending_header, + &this.bitfield_update, + ) { + Err(e) => return Poll::Ready(Err(e)), + Ok(fut) => this.flush_oplog_fut = Some(fut), + } + continue; + } + } + } + + // Phase 2: read nodes for byte_offset_in_changeset. + if let Some(fut) = this.byte_offset_read_fut.as_mut() { + match fut.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(infos)) => { + this.byte_offset_infos.extend(infos); + this.byte_offset_read_fut = None; + let block = this.proof.block.as_ref().unwrap(); + let changeset = this.changeset.as_ref().unwrap(); + let flush_fut = { + let inner = this.inner.lock().unwrap(); + let byte_offset = match inner.tree.byte_offset_in_changeset( + block.index, + changeset, + Some(&this.byte_offset_infos), + ) { + Err(e) => return Poll::Ready(Err(e)), + Ok(Either::Right(v)) => v, + Ok(Either::Left(_)) => { + return Poll::Ready(Err(HypercoreError::InvalidOperation { + context: format!( + "Could not read offset for index {} from tree", + block.index + ), + })); + } + }; + let info = inner.block_store.put(&block.value, byte_offset); + let storage = inner.storage.clone(); + drop(inner); + storage.flush_info(info) + }; + this.bitfield_update = Some(BitfieldUpdate { + drop: false, + start: block.index, + length: 1, + }); + this.flush_block_fut = Some(flush_fut); + continue; + } + } + } + + // Phase 1: verify the proof. + if let Some(fut) = this.verify_fut.as_mut() { + match Pin::new(fut).poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Ready(Ok(changeset)) => { + this.verify_fut = None; + { + let inner = this.inner.lock().unwrap(); + if !inner.tree.commitable(&changeset) { + return Poll::Ready(Ok(false)); + } + } + this.changeset = Some(changeset); + + if let Some(block) = this.proof.block.as_ref() { + let changeset = this.changeset.as_ref().unwrap(); + let next = { + let inner = this.inner.lock().unwrap(); + match inner.tree.byte_offset_in_changeset( + block.index, + changeset, + None, + ) { + Err(e) => return Poll::Ready(Err(e)), + Ok(Either::Right(byte_offset)) => { + let info = inner.block_store.put(&block.value, byte_offset); + let storage = inner.storage.clone(); + drop(inner); + let bu = BitfieldUpdate { + drop: false, + start: block.index, + length: 1, + }; + Either::Right((storage.flush_info(info), bu)) + } + Ok(Either::Left(instructions)) => { + let storage = inner.storage.clone(); + drop(inner); + Either::Left( + storage.read_infos_to_vec(Vec::from(instructions)), + ) + } + } + }; + match next { + Either::Right((flush_fut, bu)) => { + this.bitfield_update = Some(bu); + this.flush_block_fut = Some(flush_fut); + } + Either::Left(read_fut) => { + this.byte_offset_read_fut = Some(read_fut); + } + } + } else { + // No block — skip straight to oplog flush. + match Self::start_flush_oplog( + &this.inner, + this.changeset.as_ref().unwrap(), + &mut this.pending_header, + &this.bitfield_update, + ) { + Err(e) => return Poll::Ready(Err(e)), + Ok(fut) => this.flush_oplog_fut = Some(fut), + } + } + continue; + } + } + } + + // Initial: check fork, then start verify. + { + let inner = this.inner.lock().unwrap(); + if this.proof.fork != inner.tree.fork { + return Poll::Ready(Ok(false)); + } + } + this.verify_fut = Some(VerifyProofFuture { + inner: this.inner.clone(), + proof: this.proof.clone(), + infos: None, + pending_read: None, + }); + } + } +} + +pub(crate) fn update_contiguous_length( + header: &mut Header, + bitfield: &Bitfield, + bitfield_update: &BitfieldUpdate, +) { + let end = bitfield_update.start + bitfield_update.length; + let mut c = header.hints.contiguous_length; + if bitfield_update.drop { + if c <= end && c > bitfield_update.start { + c = bitfield_update.start; + } + } else if c <= end && c >= bitfield_update.start { + c = end; + while bitfield.get(c) { + c += 1; + } + } + + if c != header.hints.contiguous_length { + header.hints.contiguous_length = c; + } +} diff --git a/src/core/mod.rs b/src/core/mod.rs new file mode 100644 index 00000000..b31e3a91 --- /dev/null +++ b/src/core/mod.rs @@ -0,0 +1,850 @@ +//! Hypercore's main abstraction. Exposes an append-only, secure log structure. +pub(crate) mod inner; + +use futures::future::Either; +use tracing::instrument; + +#[cfg(feature = "cache")] +use crate::common::cache::CacheOptions; +use crate::{ + common::{BitfieldUpdate, HypercoreError, StoreInfo}, + core::inner::HypercoreInnerInner, + crypto::PartialKeypair, + storage::Storage, +}; +use hypercore_schema::{Proof, RequestBlock, RequestSeek, RequestUpgrade}; + +pub(crate) use inner::HypercoreInner; +use inner::update_contiguous_length; + +#[derive(Debug)] +pub(crate) struct HypercoreOptions { + pub(crate) key_pair: Option, + pub(crate) open: bool, + #[cfg(feature = "cache")] + pub(crate) node_cache_options: Option, +} + +impl HypercoreOptions { + pub(crate) fn new() -> Self { + Self { + key_pair: None, + open: false, + #[cfg(feature = "cache")] + node_cache_options: None, + } + } +} + +macro_rules! ininner { + ($self:expr) => { + $self.inner.inner.lock().unwrap() + }; +} + +/// Hypercore is an append-only log structure. +#[derive(Debug)] +pub struct Hypercore { + pub(crate) inner: HypercoreInner, +} + +/// Response from append, matches that of the Javascript result +#[derive(Debug, PartialEq)] +pub struct AppendOutcome { + /// Length of the hypercore after append + pub length: u64, + /// Byte length of the hypercore after append + pub byte_length: u64, +} + +/// Info about the hypercore +#[derive(Debug, PartialEq)] +pub struct Info { + /// Length of the hypercore + pub length: u64, + /// Byte length of the hypercore + pub byte_length: u64, + /// Continuous length of entries in the hypercore with data + /// starting from index 0 + pub contiguous_length: u64, + /// Fork index. 0 if hypercore not forked. + pub fork: u64, + /// True if hypercore is writeable, false if read-only + pub writeable: bool, +} + +impl Hypercore { + /// Creates/opens new hypercore using given storage and options + pub(crate) async fn new( + storage: Storage, + options: HypercoreOptions, + ) -> Result { + Ok(Hypercore { + inner: HypercoreInner::new(storage, options).await?, + }) + } + + /// Gets basic info about the Hypercore + pub fn info(&self) -> Info { + self.inner.info() + } + + /// Appends a data slice to the hypercore. + #[instrument(err, skip_all, fields(data_len = data.len()))] + pub async fn append(&mut self, data: &[u8]) -> Result { + self.append_batch(&[data]).await + } + + /// Appends a given batch of data slices to the hypercore. + #[instrument(err, skip_all, fields(batch_len = batch.as_ref().len()))] + pub async fn append_batch, B: AsRef<[A]>>( + &mut self, + batch: B, + ) -> Result { + let secret_key = match self.inner.key_pair().secret { + Some(key) => key, + None => return Err(HypercoreError::NotWritable), + }; + + if batch.as_ref().is_empty() { + return Ok(self.inner.append_outcome()); + } + // Create a changeset for the tree + let mut changeset = ininner!(self).tree.changeset(); + let mut batch_length: usize = 0; + for data in batch.as_ref().iter() { + batch_length += changeset.append(data.as_ref()); + } + changeset.hash_and_sign(&secret_key); + + // Write the received data to the block store + let byte_length = ininner!(self).tree.byte_length; + let info = + ininner!(self) + .block_store + .append_batch(batch.as_ref(), batch_length, byte_length); + { ininner!(self).storage.flush_info(info) }.await?; + + // Append the changeset to the Oplog + let bitfield_update = BitfieldUpdate { + drop: false, + start: changeset.ancestors, + length: changeset.batch_length, + }; + let outcome = { + let HypercoreInnerInner { oplog, header, .. } = &mut *ininner!(self); + oplog.append_changeset(&changeset, Some(bitfield_update.clone()), false, header)? + }; + { + ininner!(self) + .storage + .flush_infos(Vec::from(outcome.infos_to_flush)) + } + .await?; + ininner!(self).header = outcome.header; + + // Write to bitfield + ininner!(self).bitfield.update(&bitfield_update); + + // Contiguous length is known only now + { + let HypercoreInnerInner { + bitfield, header, .. + } = &mut *ininner!(self); + update_contiguous_length(header, bitfield, &bitfield_update); + } + + // Commit changeset to in-memory tree + ininner!(self).tree.commit(changeset)?; + + // Now ready to flush + if self.inner.should_flush_bitfield_and_tree_and_oplog() { + self.inner.flush_bitfield_and_tree_and_oplog(false).await?; + } + + #[cfg(feature = "replication")] + { + use tracing::trace; + + trace!(bitfield_update = ?bitfield_update, "Hppercore.append_batch emit DataUpgrade & Have"); + let _ = ininner!(self) + .events + .send(crate::replication::events::DataUpgrade {}); + let _ = ininner!(self) + .events + .send(crate::replication::events::Have::from(&bitfield_update)); + } + + Ok(self.inner.append_outcome()) + } + + #[cfg(feature = "replication")] + /// Subscribe to core events relevant to replication + pub fn event_subscribe(&self) -> async_broadcast::Receiver { + self.inner.event_subscribe() + } + + /// Check if core has the block at the given `index` locally + #[instrument(ret, skip(self))] + pub fn has(&self, index: u64) -> bool { + self.inner.has(index) + } + + /// Read value at given index, if any. + #[instrument(err, skip(self))] + pub async fn get(&self, index: u64) -> Result>, HypercoreError> { + Ok(self.inner.get(index).await?.map(|b| b.into_vec())) + } + + /// Clear data for entries between start and end (exclusive) indexes. + #[instrument(err, skip(self))] + pub async fn clear(&mut self, start: u64, end: u64) -> Result<(), HypercoreError> { + if start >= end { + // NB: This is what javascript does, so we mimic that here + return Ok(()); + } + // Write to oplog + let infos_to_flush = ininner!(self).oplog.clear(start, end)?; + { + ininner!(self) + .storage + .flush_infos(Vec::from(infos_to_flush)) + } + .await?; + + // Set bitfield + ininner!(self).bitfield.set_range(start, end - start, false); + + // Set contiguous length + if start < ininner!(self).header.hints.contiguous_length { + ininner!(self).header.hints.contiguous_length = start; + } + + // Find the biggest hole that can be punched into the data + let start = if let Some(index) = ininner!(self).bitfield.last_index_of(true, start) { + index + 1 + } else { + 0 + }; + let end = if let Some(index) = ininner!(self).bitfield.index_of(true, end) { + index + } else { + ininner!(self).tree.length + }; + + // Find byte offset for first value + let mut infos: Vec = Vec::new(); + let clear_offset = match { ininner!(self).tree.byte_offset(start, None)? } { + Either::Right(value) => value, + Either::Left(instructions) => { + let new_infos = { + ininner!(self) + .storage + .read_infos_to_vec(Vec::from(instructions)) + } + .await?; + infos.extend(new_infos); + match ininner!(self).tree.byte_offset(start, Some(&infos))? { + Either::Right(value) => value, + Either::Left(_) => { + return Err(HypercoreError::InvalidOperation { + context: format!("Could not read offset for index {start} from tree"), + }); + } + } + } + }; + + // Find byte range for last value + let last_byte_range = self.inner.byte_range(end - 1, infos).await?; + + let clear_length = (last_byte_range.index + last_byte_range.length) - clear_offset; + + // Clear blocks + let info_to_flush = ininner!(self).block_store.clear(clear_offset, clear_length); + { ininner!(self).storage.flush_info(info_to_flush) }.await?; + + // Now ready to flush + if self.inner.should_flush_bitfield_and_tree_and_oplog() { + self.inner.flush_bitfield_and_tree_and_oplog(false).await?; + } + + Ok(()) + } + + /// Access the key pair. + pub fn key_pair(&self) -> PartialKeypair { + self.inner.key_pair() + } + + /// Create a proof for given request + #[instrument(err, skip_all)] + pub async fn create_proof( + &self, + block: Option, + hash: Option, + seek: Option, + upgrade: Option, + ) -> Result, HypercoreError> { + self.inner.create_proof(block, hash, seek, upgrade).await + } + + /// Verify and apply proof received from peer, returns true if changed, false if not + /// possible to apply. + #[instrument(skip_all)] + pub async fn verify_and_apply_proof(&self, proof: Proof) -> Result { + self.inner.verify_and_apply_proof(proof).await + } + + #[allow(dead_code)] + async fn verify_and_apply_proof_old(&self, proof: &Proof) -> Result { + if proof.fork != ininner!(self).tree.fork { + return Ok(false); + } + let changeset = self.inner.verify_proof(proof.clone()).await?; + if !ininner!(self).tree.commitable(&changeset) { + return Ok(false); + } + + // In javascript there's _verifyExclusive and _verifyShared based on changeset.upgraded, but + // here we do only one. _verifyShared groups together many subsequent changesets into a single + // oplog push, and then flushes in the end only for the whole group. + let bitfield_update: Option = if let Some(block) = &proof.block.as_ref() { + let byte_offset = match { + ininner!(self) + .tree + .byte_offset_in_changeset(block.index, &changeset, None)? + } { + Either::Right(value) => value, + Either::Left(instructions) => { + let infos = { + ininner!(self) + .storage + .read_infos_to_vec(Vec::from(instructions)) + } + .await?; + match ininner!(self).tree.byte_offset_in_changeset( + block.index, + &changeset, + Some(&infos), + )? { + Either::Right(value) => value, + Either::Left(_) => { + return Err(HypercoreError::InvalidOperation { + context: format!( + "Could not read offset for index {} from tree", + block.index + ), + }); + } + } + } + }; + + // Write the value to the block store + let info_to_flush = ininner!(self).block_store.put(&block.value, byte_offset); + { ininner!(self).storage.flush_info(info_to_flush) }.await?; + + // Return a bitfield update for the given value + Some(BitfieldUpdate { + drop: false, + start: block.index, + length: 1, + }) + } else { + // Only from DataBlock can there be changes to the bitfield + None + }; + + // Append the changeset to the Oplog + let outcome = { + let HypercoreInnerInner { oplog, header, .. } = &mut *ininner!(self); + oplog.append_changeset(&changeset, bitfield_update.clone(), false, header)? + }; + { + ininner!(self) + .storage + .flush_infos(Vec::from(outcome.infos_to_flush)) + } + .await?; + ininner!(self).header = outcome.header; + + if let Some(bitfield_update) = &bitfield_update { + // Write to bitfield + ininner!(self).bitfield.update(bitfield_update); + + // Contiguous length is known only now + { + let HypercoreInnerInner { + bitfield, header, .. + } = &mut *ininner!(self); + update_contiguous_length(header, bitfield, bitfield_update); + } + } + + // Commit changeset to in-memory tree + ininner!(self).tree.commit(changeset)?; + + // Now ready to flush + if self.inner.should_flush_bitfield_and_tree_and_oplog() { + self.inner.flush_bitfield_and_tree_and_oplog(false).await?; + } + + #[cfg(feature = "replication")] + { + if proof.upgrade.is_some() { + // Notify replicator if we receieved an upgrade + let _ = ininner!(self) + .events + .send(crate::replication::events::DataUpgrade {}); + } + + // Notify replicator if we receieved a bitfield update + if let Some(ref bitfield) = bitfield_update { + let _ = ininner!(self) + .events + .send(crate::replication::events::Have::from(bitfield)); + } + } + Ok(true) + } + + /// Used to fill the nodes field of a `RequestBlock` during + /// synchronization. + #[instrument(err, skip(self))] + pub async fn missing_nodes(&self, index: u64) -> Result { + self.inner + .missing_nodes_from_merkle_tree_index(index * 2) + .await + } + + /// Get missing nodes using a merkle tree index. Advanced variant of missing_nodes + /// that allow for special cases of searching directly from the merkle tree. + #[instrument(err, skip(self))] + pub async fn missing_nodes_from_merkle_tree_index( + &self, + merkle_tree_index: u64, + ) -> Result { + self.inner + .missing_nodes_from_merkle_tree_index(merkle_tree_index) + .await + } + + /// Makes the hypercore read-only by deleting the secret key. Returns true if the + /// hypercore was changed, false if the hypercore was already read-only. This is useful + /// in scenarios where a hypercore should be made immutable after initial values have + /// been stored. + #[instrument(err, skip_all)] + pub async fn make_read_only(&mut self) -> Result { + if ininner!(self).key_pair.secret.is_some() { + ininner!(self).key_pair.secret = None; + ininner!(self).header.key_pair.secret = None; + // Need to flush clearing traces to make sure both oplog slots are cleared + { ininner!(self).flush_bitfield_and_tree_and_oplog(true) }.await?; + Ok(true) + } else { + Ok(false) + } + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use crate::crypto::{PartialKeypair, generate_signing_key}; + + #[tokio::test] + async fn core_create_proof_block_only() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + + let proof = hypercore + .create_proof(Some(RequestBlock { index: 4, nodes: 2 }), None, None, None) + .await? + .unwrap(); + let block = proof.block.unwrap(); + assert_eq!(proof.upgrade, None); + assert_eq!(proof.seek, None); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 2); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(block.nodes[1].index, 13); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_block_and_upgrade() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 4, nodes: 0 }), + None, + None, + Some(RequestUpgrade { + start: 0, + length: 10, + }), + ) + .await? + .unwrap(); + let block = proof.block.unwrap(); + let upgrade = proof.upgrade.unwrap(); + assert_eq!(proof.seek, None); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 3); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(block.nodes[1].index, 13); + assert_eq!(block.nodes[2].index, 3); + assert_eq!(upgrade.start, 0); + assert_eq!(upgrade.length, 10); + assert_eq!(upgrade.nodes.len(), 1); + assert_eq!(upgrade.nodes[0].index, 17); + assert_eq!(upgrade.additional_nodes.len(), 0); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_block_and_upgrade_and_additional() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 4, nodes: 0 }), + None, + None, + Some(RequestUpgrade { + start: 0, + length: 8, + }), + ) + .await? + .unwrap(); + let block = proof.block.unwrap(); + let upgrade = proof.upgrade.unwrap(); + assert_eq!(proof.seek, None); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 3); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(block.nodes[1].index, 13); + assert_eq!(block.nodes[2].index, 3); + assert_eq!(upgrade.start, 0); + assert_eq!(upgrade.length, 8); + assert_eq!(upgrade.nodes.len(), 0); + assert_eq!(upgrade.additional_nodes.len(), 1); + assert_eq!(upgrade.additional_nodes[0].index, 17); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_block_and_upgrade_from_existing_state() -> Result<(), HypercoreError> + { + let hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 1, nodes: 0 }), + None, + None, + Some(RequestUpgrade { + start: 1, + length: 9, + }), + ) + .await? + .unwrap(); + let block = proof.block.unwrap(); + let upgrade = proof.upgrade.unwrap(); + assert_eq!(proof.seek, None); + assert_eq!(block.index, 1); + assert_eq!(block.nodes.len(), 0); + assert_eq!(upgrade.start, 1); + assert_eq!(upgrade.length, 9); + assert_eq!(upgrade.nodes.len(), 3); + assert_eq!(upgrade.nodes[0].index, 5); + assert_eq!(upgrade.nodes[1].index, 11); + assert_eq!(upgrade.nodes[2].index, 17); + assert_eq!(upgrade.additional_nodes.len(), 0); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_block_and_upgrade_from_existing_state_with_additional() + -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 1, nodes: 0 }), + None, + None, + Some(RequestUpgrade { + start: 1, + length: 5, + }), + ) + .await? + .unwrap(); + let block = proof.block.unwrap(); + let upgrade = proof.upgrade.unwrap(); + assert_eq!(proof.seek, None); + assert_eq!(block.index, 1); + assert_eq!(block.nodes.len(), 0); + assert_eq!(upgrade.start, 1); + assert_eq!(upgrade.length, 5); + assert_eq!(upgrade.nodes.len(), 2); + assert_eq!(upgrade.nodes[0].index, 5); + assert_eq!(upgrade.nodes[1].index, 9); + assert_eq!(upgrade.additional_nodes.len(), 2); + assert_eq!(upgrade.additional_nodes[0].index, 13); + assert_eq!(upgrade.additional_nodes[1].index, 17); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_block_and_seek_1_no_upgrade() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 4, nodes: 2 }), + None, + Some(RequestSeek { bytes: 8 }), + None, + ) + .await? + .unwrap(); + let block = proof.block.unwrap(); + assert_eq!(proof.seek, None); // seek included in block + assert_eq!(proof.upgrade, None); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 2); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(block.nodes[1].index, 13); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_block_and_seek_2_no_upgrade() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 4, nodes: 2 }), + None, + Some(RequestSeek { bytes: 10 }), + None, + ) + .await? + .unwrap(); + let block = proof.block.unwrap(); + assert_eq!(proof.seek, None); // seek included in block + assert_eq!(proof.upgrade, None); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 2); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(block.nodes[1].index, 13); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_block_and_seek_3_no_upgrade() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 4, nodes: 2 }), + None, + Some(RequestSeek { bytes: 13 }), + None, + ) + .await? + .unwrap(); + let block = proof.block.unwrap(); + let seek = proof.seek.unwrap(); + assert_eq!(proof.upgrade, None); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 1); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(seek.nodes.len(), 2); + assert_eq!(seek.nodes[0].index, 12); + assert_eq!(seek.nodes[1].index, 14); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_block_and_seek_to_tree_no_upgrade() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(16).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 0, nodes: 4 }), + None, + Some(RequestSeek { bytes: 26 }), + None, + ) + .await? + .unwrap(); + let block = proof.block.unwrap(); + let seek = proof.seek.unwrap(); + assert_eq!(proof.upgrade, None); + assert_eq!(block.nodes.len(), 3); + assert_eq!(block.nodes[0].index, 2); + assert_eq!(block.nodes[1].index, 5); + assert_eq!(block.nodes[2].index, 11); + assert_eq!(seek.nodes.len(), 2); + assert_eq!(seek.nodes[0].index, 19); + assert_eq!(seek.nodes[1].index, 27); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_block_and_seek_with_upgrade() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 4, nodes: 2 }), + None, + Some(RequestSeek { bytes: 13 }), + Some(RequestUpgrade { + start: 8, + length: 2, + }), + ) + .await? + .unwrap(); + let block = proof.block.unwrap(); + let seek = proof.seek.unwrap(); + let upgrade = proof.upgrade.unwrap(); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 1); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(seek.nodes.len(), 2); + assert_eq!(seek.nodes[0].index, 12); + assert_eq!(seek.nodes[1].index, 14); + assert_eq!(upgrade.nodes.len(), 1); + assert_eq!(upgrade.nodes[0].index, 17); + assert_eq!(upgrade.additional_nodes.len(), 0); + Ok(()) + } + + #[tokio::test] + async fn core_create_proof_seek_with_upgrade() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + None, + None, + Some(RequestSeek { bytes: 13 }), + Some(RequestUpgrade { + start: 0, + length: 10, + }), + ) + .await? + .unwrap(); + let seek = proof.seek.unwrap(); + let upgrade = proof.upgrade.unwrap(); + assert_eq!(proof.block, None); + assert_eq!(seek.nodes.len(), 4); + assert_eq!(seek.nodes[0].index, 12); + assert_eq!(seek.nodes[1].index, 14); + assert_eq!(seek.nodes[2].index, 9); + assert_eq!(seek.nodes[3].index, 3); + assert_eq!(upgrade.nodes.len(), 1); + assert_eq!(upgrade.nodes[0].index, 17); + assert_eq!(upgrade.additional_nodes.len(), 0); + Ok(()) + } + + #[tokio::test] + async fn core_verify_proof_invalid_signature() -> Result<(), HypercoreError> { + let hypercore = create_hypercore_with_data(10).await?; + // Invalid clone hypercore with a different public key + let hypercore_clone = create_hypercore_with_data(0).await?; + let proof = hypercore + .create_proof( + None, + Some(RequestBlock { index: 6, nodes: 0 }), + None, + Some(RequestUpgrade { + start: 0, + length: 10, + }), + ) + .await? + .unwrap(); + assert!(hypercore_clone.verify_and_apply_proof(proof).await.is_err()); + Ok(()) + } + + #[tokio::test] + async fn core_verify_and_apply_proof() -> Result<(), HypercoreError> { + let main = create_hypercore_with_data(10).await?; + let clone = create_hypercore_with_data_and_key_pair( + 0, + PartialKeypair { + public: { ininner!(main).key_pair.public }, + secret: None, + }, + ) + .await?; + let index = 6; + let nodes = clone.missing_nodes(index).await?; + let proof = main + .create_proof( + None, + Some(RequestBlock { index, nodes }), + None, + Some(RequestUpgrade { + start: 0, + length: 10, + }), + ) + .await? + .unwrap(); + assert!(clone.verify_and_apply_proof(proof).await?); + let main_info = main.info(); + let clone_info = clone.info(); + assert_eq!(main_info.byte_length, clone_info.byte_length); + assert_eq!(main_info.length, clone_info.length); + assert!(main.get(6).await?.is_some()); + assert!(clone.get(6).await?.is_none()); + + // Fetch data for index 6 and verify it is found + let index = 6; + let nodes = clone.missing_nodes(index).await?; + let proof = main + .create_proof(Some(RequestBlock { index, nodes }), None, None, None) + .await? + .unwrap(); + assert!(clone.verify_and_apply_proof(proof).await?); + Ok(()) + } + + pub(crate) async fn create_hypercore_with_data( + length: u64, + ) -> Result { + let signing_key = generate_signing_key(); + create_hypercore_with_data_and_key_pair( + length, + PartialKeypair { + public: signing_key.verifying_key(), + secret: Some(signing_key), + }, + ) + .await + } + + pub(crate) async fn create_hypercore_with_data_and_key_pair( + length: u64, + key_pair: PartialKeypair, + ) -> Result { + let storage = Storage::new_memory().await?; + let mut hypercore = Hypercore::new( + storage, + HypercoreOptions { + key_pair: Some(key_pair), + open: false, + #[cfg(feature = "cache")] + node_cache_options: None, + }, + ) + .await?; + for i in 0..length { + hypercore.append(format!("#{}", i).as_bytes()).await?; + } + Ok(hypercore) + } +} diff --git a/src/lib.rs b/src/lib.rs index c2a4f4d0..d14dcd6e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,16 +1,12 @@ -#![forbid( - unsafe_code, - future_incompatible, - rust_2018_idioms, - rust_2018_compatibility, - missing_debug_implementations, - missing_docs -)] +#![forbid(unsafe_code, future_incompatible)] #![doc(test(attr(deny(warnings))))] #![warn( unreachable_pub, redundant_lifetimes, non_local_definitions, + missing_debug_implementations, + //missing_docs, + clippy::unused_async, clippy::needless_pass_by_value, clippy::needless_pass_by_ref_mut, clippy::enum_glob_use @@ -47,14 +43,9 @@ //! //! ## Example //! ```rust -//! # #[cfg(feature = "tokio")] //! # tokio_test::block_on(async { //! # example().await; //! # }); -//! # #[cfg(feature = "async-std")] -//! # async_std::task::block_on(async { -//! # example().await; -//! # }); //! # async fn example() { //! use hypercore::{HypercoreBuilder, Storage}; //! diff --git a/src/replication/events.rs b/src/replication/events.rs index 09f8320f..6d54752d 100644 --- a/src/replication/events.rs +++ b/src/replication/events.rs @@ -116,14 +116,14 @@ mod test { use super::*; use crate::replication::CoreMethodsError; - #[async_std::test] + #[tokio::test] async fn test_events() -> Result<(), CoreMethodsError> { let mut core = crate::core::tests::create_hypercore_with_data(0).await?; // Check that appending data emits a DataUpgrade and Have event let mut rx = core.event_subscribe(); - let handle = async_std::task::spawn(async move { + let handle = tokio::task::spawn(async move { let mut out = vec![]; loop { if out.len() == 2 { @@ -135,7 +135,7 @@ mod test { } }); core.append(b"foo").await?; - let (res, mut rx) = handle.await; + let (res, mut rx) = handle.await.unwrap(); assert!(matches!(res[0], Event::DataUpgrade(_))); assert!(matches!( res[1], @@ -150,7 +150,7 @@ mod test { // Check that Hypercore::get for missing data emits a Get event - let handle = async_std::task::spawn(async move { + let handle = tokio::task::spawn(async move { let mut out = vec![]; loop { if out.len() == 1 { @@ -162,7 +162,7 @@ mod test { } }); assert_eq!(core.get(1).await?, None); - let (res, rx) = handle.await; + let (res, rx) = handle.await.unwrap(); assert!(matches!( res[0], Event::Get(Get { diff --git a/src/replication/mod.rs b/src/replication/mod.rs index 35843736..25fdeace 100644 --- a/src/replication/mod.rs +++ b/src/replication/mod.rs @@ -1,19 +1,36 @@ //! Hypercore to Hypercore replication pub mod events; -#[cfg(feature = "shared-core")] -pub mod shared_core; -#[cfg(feature = "shared-core")] -pub use shared_core::SharedCore; +use std::{ + collections::{BTreeSet, VecDeque}, + future::Future, + io, + pin::Pin, + task::{Context, Poll}, +}; -use crate::{AppendOutcome, HypercoreError, Info, PartialKeypair}; +use futures::{ + Stream, StreamExt, + stream::{FuturesUnordered, SelectAll}, +}; +use hypercore_handshake::CipherTrait; +use hypercore_protocol::{ + Channel, Message, Protocol, discovery_key, + schema::{Data, Range, Request, Synchronize}, +}; +use tracing::{error, trace, warn}; -use hypercore_schema::{Proof, RequestBlock, RequestSeek, RequestUpgrade}; +use crate::{ + AppendOutcome, Hypercore, HypercoreError, Info, PartialKeypair, + core::inner::{ + CreateProofFuture, HypercoreInner, MissingNodesFuture, VerifyAndApplyProofFuture, + }, +}; +use hypercore_schema::{RequestBlock, RequestSeek, RequestUpgrade}; pub use events::Event; use async_broadcast::Receiver; -use std::future::Future; /// Methods related to just this core's information pub trait CoreInfo { @@ -39,7 +56,7 @@ pub trait ReplicationMethods: CoreInfo + Send { /// ref Core::verify_and_apply_proof fn verify_and_apply_proof( &self, - proof: &Proof, + proof: hypercore_schema::Proof, ) -> impl Future> + Send; /// ref Core::missing_nodes fn missing_nodes( @@ -53,7 +70,7 @@ pub trait ReplicationMethods: CoreInfo + Send { hash: Option, seek: Option, upgrade: Option, - ) -> impl Future, ReplicationMethodsError>> + Send; + ) -> impl Future, ReplicationMethodsError>> + Send; /// subscribe to core events fn event_subscribe(&self) -> impl Future>; } @@ -66,8 +83,7 @@ pub enum CoreMethodsError { HypercoreError(#[from] HypercoreError), } -/// Trait for things that consume [`crate::Hypercore`] can instead use this trait -/// so they can use all Hypercore-like things such as `SharedCore`. +/// Trait for things that consume [`crate::Hypercore`]. pub trait CoreMethods: CoreInfo { /// Check if the core has the block at the given index locally fn has(&self, index: u64) -> impl Future + Send; @@ -90,3 +106,617 @@ pub trait CoreMethods: CoreInfo { batch: B, ) -> impl Future> + Send; } + +// ── Remote bitfield ──────────────────────────────────────────────────────────── + +struct RemoteBitfield(BTreeSet); + +impl RemoteBitfield { + fn new() -> Self { + Self(BTreeSet::new()) + } + + fn get(&self, index: u64) -> bool { + self.0.contains(&index) + } + + fn set_range(&mut self, start: u64, length: u64, value: bool) { + for i in start..(start + length) { + if value { + self.0.insert(i); + } else { + self.0.remove(&i); + } + } + } +} + +// ── PeerState ────────────────────────────────────────────────────────────────── + +struct PeerState { + can_upgrade: bool, + remote_fork: u64, + remote_length: u64, + remote_bitfield: RemoteBitfield, + remote_can_upgrade: bool, + remote_uploading: bool, + remote_downloading: bool, + remote_synced: bool, + length_acked: u64, +} + +impl Default for PeerState { + fn default() -> Self { + Self { + can_upgrade: true, + remote_fork: 0, + remote_length: 0, + remote_bitfield: RemoteBitfield::new(), + remote_can_upgrade: false, + remote_uploading: true, + remote_downloading: true, + remote_synced: false, + length_acked: 0, + } + } +} + +// ── DataMeta ─────────────────────────────────────────────────────────────────── + +/// Saved from a Data message; used after verify_and_apply completes to decide +/// what to request next. +struct DataMeta { + has_upgrade: bool, + pre_length: u64, + remote_length: u64, + block_index: Option, +} + +// ── ChannelState ─────────────────────────────────────────────────────────────── + +struct ChannelState { + channel: Channel, + state: PeerState, + synced: bool, + + // Outgoing batches; drained one at a time through pending_send. + outgoing: VecDeque>, + pending_send: Option> + Send>>>, + + // Incoming Request → create_proof + pending_create_proof: Option, + pending_create_proof_id: u64, + pending_create_proof_fork: u64, + + // Incoming Data → verify_and_apply_proof + pending_verify_apply: Option, + pending_data_meta: Option, + + // Block request queue: each index needs a missing_nodes call before we + // can send the Request message. + pending_request_indices: VecDeque, + pending_missing_nodes: Option<(u64, MissingNodesFuture)>, + + // Core events (Get / Have / DataUpgrade) + core_events: async_broadcast::Receiver, +} + +impl ChannelState { + fn new(channel: Channel, core_events: async_broadcast::Receiver) -> Self { + Self { + channel, + state: PeerState::default(), + synced: false, + outgoing: VecDeque::new(), + pending_send: None, + pending_create_proof: None, + pending_create_proof_id: 0, + pending_create_proof_fork: 0, + pending_verify_apply: None, + pending_data_meta: None, + pending_request_indices: VecDeque::new(), + pending_missing_nodes: None, + core_events, + } + } + + fn poll( + &mut self, + cx: &mut Context<'_>, + inner: &HypercoreInner, + ) -> Poll> { + // ── Drive pending send ───────────────────────────────────────────────── + if let Some(ref mut fut) = self.pending_send { + match fut.as_mut().poll(cx) { + Poll::Ready(Ok(())) => self.pending_send = None, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e.into())), + Poll::Pending => {} + } + } + if self.pending_send.is_none() + && let Some(batch) = self.outgoing.pop_front() + { + let channel = self.channel.clone(); + self.pending_send = Some(Box::pin(async move { channel.send_batch(&batch).await })); + cx.waker().wake_by_ref(); + } + + // ── Initial sync ─────────────────────────────────────────────────────── + if !self.synced { + let info = inner.info(); + let remote_length = if info.fork == self.state.remote_fork { + self.state.remote_length + } else { + 0 + }; + let mut msgs = vec![Message::Synchronize(Synchronize { + fork: info.fork, + length: info.length, + remote_length, + can_upgrade: self.state.can_upgrade, + uploading: true, + downloading: true, + })]; + if info.contiguous_length > 0 { + msgs.push(Message::Range(Range { + drop: false, + start: 0, + length: info.contiguous_length, + })); + } + self.outgoing.push_back(msgs); + self.synced = true; + cx.waker().wake_by_ref(); + return Poll::Pending; + } + + // ── Poll core events ─────────────────────────────────────────────────── + loop { + match Pin::new(&mut self.core_events).poll_next(cx) { + Poll::Ready(Some(event)) => self.on_core_event(event, inner), + Poll::Ready(None) => break, + Poll::Pending => break, + } + } + + // ── Drive pending_missing_nodes ──────────────────────────────────────── + if let Some((index, ref mut fut)) = self.pending_missing_nodes { + match Pin::new(fut).poll(cx) { + Poll::Ready(Ok(nodes)) => { + self.pending_missing_nodes = None; + if self.state.remote_bitfield.get(index) && self.state.remote_length > index { + let info = inner.info(); + self.outgoing.push_back(vec![Message::Request(Request { + id: index + 1, + fork: info.fork, + block: Some(RequestBlock { index, nodes }), + hash: None, + seek: None, + upgrade: None, + manifest: false, + priority: 42, + })]); + cx.waker().wake_by_ref(); + } + if let Some(next) = self.pending_request_indices.pop_front() { + self.pending_missing_nodes = + Some((next, inner.missing_nodes_from_merkle_tree_index(next * 2))); + cx.waker().wake_by_ref(); + } + } + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Pending => {} + } + } else if let Some(index) = self.pending_request_indices.pop_front() { + self.pending_missing_nodes = + Some((index, inner.missing_nodes_from_merkle_tree_index(index * 2))); + cx.waker().wake_by_ref(); + } + + // ── Drive pending_create_proof ───────────────────────────────────────── + if let Some(ref mut fut) = self.pending_create_proof { + match Pin::new(fut).poll(cx) { + Poll::Ready(Ok(maybe_proof)) => { + let req_id = self.pending_create_proof_id; + let req_fork = self.pending_create_proof_fork; + self.pending_create_proof = None; + if let Some(proof) = maybe_proof { + self.outgoing.push_back(vec![Message::Data(Data { + request: req_id, + fork: req_fork, + hash: proof.hash, + block: proof.block, + seek: proof.seek, + upgrade: proof.upgrade, + })]); + cx.waker().wake_by_ref(); + } + } + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Pending => {} + } + } + + // ── Drive pending_verify_apply ───────────────────────────────────────── + if let Some(ref mut fut) = self.pending_verify_apply { + match Pin::new(fut).poll(cx) { + Poll::Ready(Ok(_applied)) => { + self.pending_verify_apply = None; + if let Some(meta) = self.pending_data_meta.take() { + let next_index = if meta.has_upgrade { + (meta.pre_length < meta.remote_length).then_some(meta.pre_length) + } else { + meta.block_index + .filter(|&i| i < meta.remote_length.saturating_sub(1)) + .map(|i| i + 1) + }; + if let Some(idx) = next_index { + self.pending_request_indices.push_front(idx); + cx.waker().wake_by_ref(); + } + } + } + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Pending => {} + } + } + + // ── Poll channel for incoming messages ───────────────────────────────── + match Pin::new(&mut self.channel).poll_next(cx) { + Poll::Ready(Some(message)) => { + self.on_message(message, inner); + cx.waker().wake_by_ref(); + } + Poll::Ready(None) => return Poll::Ready(Ok(())), + Poll::Pending => {} + } + + Poll::Pending + } + + fn on_message(&mut self, message: Message, inner: &HypercoreInner) { + match message { + Message::Synchronize(msg) => self.on_synchronize(&msg, inner), + Message::Request(msg) => { + if self.pending_create_proof.is_none() { + self.pending_create_proof_id = msg.id; + self.pending_create_proof_fork = msg.fork; + self.pending_create_proof = + Some(inner.create_proof(msg.block, msg.hash, msg.seek, msg.upgrade)); + } + } + Message::Data(msg) => { + if self.pending_verify_apply.is_none() { + let info = inner.info(); + self.pending_data_meta = Some(DataMeta { + has_upgrade: msg.upgrade.is_some(), + pre_length: info.length, + remote_length: self.state.remote_length, + block_index: msg.block.as_ref().map(|b| b.index), + }); + self.pending_verify_apply = + Some(inner.verify_and_apply_proof(msg.into_proof())); + } + } + Message::Range(Range { start, length, .. }) => { + self.state.remote_bitfield.set_range(start, length, true); + } + _ => {} + } + } + + fn on_synchronize(&mut self, msg: &Synchronize, inner: &HypercoreInner) { + let info = inner.info(); + let peer_length_changed = msg.length != self.state.remote_length; + let first_sync = !self.state.remote_synced; + let same_fork = msg.fork == info.fork; + + self.state.remote_fork = msg.fork; + self.state.remote_length = msg.length; + self.state.remote_can_upgrade = msg.can_upgrade; + self.state.remote_uploading = msg.uploading; + self.state.remote_downloading = msg.downloading; + self.state.remote_synced = true; + self.state.length_acked = if same_fork { msg.remote_length } else { 0 }; + + let mut messages = vec![]; + if first_sync { + messages.push(Message::Synchronize(Synchronize { + fork: info.fork, + length: info.length, + remote_length: self.state.remote_length, + can_upgrade: self.state.can_upgrade, + uploading: true, + downloading: true, + })); + } + if self.state.remote_length > info.length + && self.state.length_acked == info.length + && peer_length_changed + { + messages.push(Message::Request(Request { + id: 1, + fork: info.fork, + hash: None, + block: None, + seek: None, + upgrade: Some(RequestUpgrade { + start: info.length, + length: self.state.remote_length - info.length, + }), + manifest: false, + priority: 42, + })); + } + if !messages.is_empty() { + self.outgoing.push_back(messages); + } + } + + fn on_core_event(&mut self, event: events::Event, inner: &HypercoreInner) { + match event { + events::Event::Get(evt) => { + if self.state.remote_length > evt.index && self.state.remote_bitfield.get(evt.index) + { + self.pending_request_indices.push_back(evt.index); + } + } + events::Event::Have(evt) => { + self.outgoing.push_back(vec![Message::Range(Range { + drop: evt.drop, + start: evt.start, + length: evt.length, + })]); + } + events::Event::DataUpgrade(_) => { + let info = inner.info(); + self.outgoing + .push_back(vec![Message::Synchronize(Synchronize { + fork: info.fork, + length: info.length, + remote_length: self.state.remote_length, + downloading: true, + uploading: true, + can_upgrade: self.state.can_upgrade, + })]); + } + } + } +} + +// ── ConnectionReplicator ─────────────────────────────────────────────────────── + +/// Drives replication for a single peer connection. Used internally by +/// [`Replicator`]. +struct ConnectionReplicator { + inner: HypercoreInner, + protocol: Protocol, + discovery_key: [u8; 32], + public_key: [u8; 32], + pending_open: Option> + Send>>>, + channel_state: Option, +} + +impl ConnectionReplicator { + fn new(inner: HypercoreInner, stream: impl CipherTrait + 'static) -> Self { + let protocol = Protocol::new(Box::new(stream)); + let public_key = inner.key_pair().public.to_bytes(); + let discovery_key = discovery_key(&public_key); + Self { + inner, + protocol, + discovery_key, + public_key, + pending_open: None, + channel_state: None, + } + } + + fn poll_inner(&mut self, cx: &mut Context<'_>) -> Poll> { + if let Some(ref mut fut) = self.pending_open { + match fut.as_mut().poll(cx) { + Poll::Ready(Ok(())) => self.pending_open = None, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e.into())), + Poll::Pending => return Poll::Pending, + } + } + + match Pin::new(&mut self.protocol).poll_next(cx) { + Poll::Ready(Some(Ok(event))) => { + self.on_protocol_event(event); + cx.waker().wake_by_ref(); + } + Poll::Ready(Some(Err(e))) => return Poll::Ready(Err(e.into())), + Poll::Ready(None) => return Poll::Ready(Ok(())), + Poll::Pending => {} + } + + if let Some(ref mut cs) = self.channel_state { + match cs.poll(cx, &self.inner) { + Poll::Ready(Ok(())) => self.channel_state = None, + Poll::Ready(Err(e)) => return Poll::Ready(Err(e)), + Poll::Pending => {} + } + } + + Poll::Pending + } + + fn on_protocol_event(&mut self, event: hypercore_protocol::Event) { + match event { + hypercore_protocol::Event::Handshake(_) => { + if self.protocol.is_initiator() { + self.pending_open = Some(Box::pin(self.protocol.open(self.public_key))); + } + } + hypercore_protocol::Event::DiscoveryKey(dkey) => { + if self.discovery_key == dkey { + self.pending_open = Some(Box::pin(self.protocol.open(self.public_key))); + } else { + warn!("Got discovery key for different core: {dkey:?}"); + } + } + hypercore_protocol::Event::Channel(channel) => { + if self.discovery_key == *channel.discovery_key() { + let core_events = self.inner.event_subscribe(); + self.channel_state = Some(ChannelState::new(channel, core_events)); + } else { + error!("Wrong discovery key?"); + } + } + hypercore_protocol::Event::Close(_) => {} + _ => {} + } + } +} + +impl Future for ConnectionReplicator { + type Output = Result<(), HypercoreError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + self.get_mut().poll_inner(cx) + } +} + +// ── Replicator ───────────────────────────────────────────────────────────────── + +type BoxReplicatorStream = Pin + Send>>; + +/// Drives replication for one or more peer connections. +/// +/// Created by [`Hypercore::replicator`]. Add connections via +/// [`with_connection`](Replicator::with_connection) or whole connection streams +/// via [`with_connection_stream`](Replicator::with_connection_stream), then +/// `.await` to drive all replication. Resolves when all connections have closed +/// and all connection streams have ended. +pub struct Replicator { + inner: HypercoreInner, + active: FuturesUnordered, + pending: SelectAll, +} + +impl std::fmt::Debug for Replicator { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Replicator") + .field("active_count", &self.active.len()) + .finish_non_exhaustive() + } +} + +impl Replicator { + fn new(inner: HypercoreInner) -> Self { + Self { + inner, + active: FuturesUnordered::new(), + pending: SelectAll::new(), + } + } + + /// Add a single connection to replicate over. + pub fn with_connection(self, stream: impl CipherTrait + 'static) -> Self { + self.active + .push(ConnectionReplicator::new(self.inner.clone(), stream)); + self + } + + /// Add a stream of connections. Each connection yielded by the stream will + /// be replicated in parallel with all others. + /// + /// ```rust,ignore + /// core.replicator() + /// .with_connection_stream(swarm.connections().filter_map(|r| async move { + /// r.ok().map(|e| e.connection) + /// })) + /// .await?; + /// ``` + pub fn with_connection_stream(mut self, stream: S) -> Self + where + S: Stream + Send + 'static, + C: CipherTrait + 'static, + { + let inner = self.inner.clone(); + let boxed: BoxReplicatorStream = + Box::pin(stream.map(move |conn| ConnectionReplicator::new(inner.clone(), conn))); + self.pending.push(boxed); + self + } +} + +impl Future for Replicator { + type Output = Result<(), HypercoreError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + + // Drain pending connection streams → push new ConnectionReplicators into active. + loop { + let pending_result = Pin::new(&mut this.pending).poll_next(cx); + trace!( + "[replicator] pending.poll_next = {:?}", + match &pending_result { + Poll::Ready(Some(_)) => "Ready(Some(conn))", + Poll::Ready(None) => "Ready(None)", + Poll::Pending => "Pending", + } + ); + match pending_result { + Poll::Ready(Some(rep)) => this.active.push(rep), + Poll::Ready(None) | Poll::Pending => break, + } + } + + // Drive all active connection replicators. + loop { + match Pin::new(&mut this.active).poll_next(cx) { + Poll::Ready(Some(Ok(()))) => {} + Poll::Ready(Some(Err(e))) => return Poll::Ready(Err(e)), + Poll::Ready(None) | Poll::Pending => break, + } + } + + if this.pending.is_empty() && this.active.is_empty() { + Poll::Ready(Ok(())) + } else { + Poll::Pending + } + } +} + +// ── Hypercore::replicator / replicate ───────────────────────────────────────── +impl Hypercore { + /// Create a [`Replicator`] for this core. Add connections or connection + /// streams, then `.await` to drive all replication. + /// + /// ```rust,ignore + /// // Single connection + /// core.replicator().with_connection(stream).await?; + /// + /// // Stream of connections (e.g. from hyperswarm) + /// core.replicator() + /// .with_connection_stream(swarm.connections().filter_map(|r| async move { + /// r.ok().map(|e| e.connection) + /// })) + /// .await?; + /// ``` + pub fn replicator(&self) -> Replicator { + Replicator::new(self.inner.clone()) + } + + /// Shorthand for `self.replicator().with_connection(stream)`. + pub fn replicate(&self, stream: impl CipherTrait + 'static) -> Replicator { + self.replicator().with_connection(stream) + } + + /// Attach a replicator to this core so that [`Hypercore::get`] automatically + /// drives replication while waiting for missing blocks. + /// + /// Once attached, a `get` for a block that is not yet locally available will + /// block (without spinning) until replication delivers it, rather than + /// returning `None` immediately. + /// + /// Only one replicator can be attached at a time; calling this again replaces + /// the previous one. + pub fn attach_replicator(&self, replicator: Replicator) { + *self.inner.background.lock().unwrap() = Some(Box::pin(replicator)); + } +} diff --git a/src/replication/shared_core.rs b/src/replication/shared_core.rs deleted file mode 100644 index 80418062..00000000 --- a/src/replication/shared_core.rs +++ /dev/null @@ -1,190 +0,0 @@ -//! Implementation of a Hypercore that can have multiple owners. Along with implementations of all -//! the hypercore traits. -use crate::{AppendOutcome, Hypercore, Info, PartialKeypair}; -use async_broadcast::Receiver; -use async_lock::Mutex; -use hypercore_schema::{Proof, RequestBlock, RequestSeek, RequestUpgrade}; -use std::sync::Arc; - -use super::{ - CoreInfo, CoreMethods, CoreMethodsError, Event, ReplicationMethods, ReplicationMethodsError, -}; - -/// Hypercore that can have multiple owners -#[derive(Debug, Clone)] -pub struct SharedCore(pub Arc>); - -impl From for SharedCore { - fn from(core: Hypercore) -> Self { - SharedCore(Arc::new(Mutex::new(core))) - } -} -impl SharedCore { - /// Create a shared core from a [`Hypercore`] - pub fn from_hypercore(core: Hypercore) -> Self { - SharedCore(Arc::new(Mutex::new(core))) - } -} - -impl CoreInfo for SharedCore { - async fn info(&self) -> Info { - let core = &self.0.lock().await; - core.info() - } - - async fn key_pair(&self) -> PartialKeypair { - let core = &self.0.lock().await; - core.key_pair().clone() - } -} - -impl ReplicationMethods for SharedCore { - async fn verify_and_apply_proof(&self, proof: &Proof) -> Result { - Ok(self.0.lock().await.verify_and_apply_proof(proof).await?) - } - - async fn missing_nodes(&self, index: u64) -> Result { - Ok(self.0.lock().await.missing_nodes(index).await?) - } - - async fn create_proof( - &self, - block: Option, - hash: Option, - seek: Option, - upgrade: Option, - ) -> Result, ReplicationMethodsError> { - Ok(self - .0 - .lock() - .await - .create_proof(block, hash, seek, upgrade) - .await?) - } - - async fn event_subscribe(&self) -> Receiver { - self.0.lock().await.event_subscribe() - } -} - -impl CoreMethods for SharedCore { - async fn has(&self, index: u64) -> bool { - self.0.lock().await.has(index) - } - async fn get(&self, index: u64) -> Result>, CoreMethodsError> { - Ok(self.0.lock().await.get(index).await?) - } - - async fn append(&self, data: &[u8]) -> Result { - Ok(self.0.lock().await.append(data).await?) - } - - async fn append_batch, B: AsRef<[A]> + Send>( - &self, - batch: B, - ) -> Result { - Ok(self.0.lock().await.append_batch(batch).await?) - } -} - -#[cfg(test)] -mod tests { - - use super::*; - - use crate::core::tests::{create_hypercore_with_data, create_hypercore_with_data_and_key_pair}; - #[async_std::test] - async fn shared_core_methods() -> Result<(), CoreMethodsError> { - let core = crate::core::tests::create_hypercore_with_data(0).await?; - let core = SharedCore::from(core); - - // check CoreInfo - let info = core.info().await; - assert_eq!( - info, - crate::core::Info { - length: 0, - byte_length: 0, - contiguous_length: 0, - fork: 0, - writeable: true, - } - ); - - // key_pair is random, nothing to test here - let _kp = core.key_pair().await; - - // check CoreMethods - assert_eq!(core.has(0).await, false); - assert_eq!(core.get(0).await?, None); - let res = core.append(b"foo").await?; - assert_eq!( - res, - AppendOutcome { - length: 1, - byte_length: 3 - } - ); - assert_eq!(core.has(0).await, true); - assert_eq!(core.get(0).await?, Some(b"foo".into())); - let res = core.append_batch([b"hello", b"world"]).await?; - assert_eq!( - res, - AppendOutcome { - length: 3, - byte_length: 13 - } - ); - assert_eq!(core.has(2).await, true); - assert_eq!(core.get(2).await?, Some(b"world".into())); - Ok(()) - } - - #[async_std::test] - async fn shared_core_replication_methods() -> Result<(), ReplicationMethodsError> { - let main = create_hypercore_with_data(10).await?; - let clone = create_hypercore_with_data_and_key_pair( - 0, - PartialKeypair { - public: main.key_pair.public, - secret: None, - }, - ) - .await?; - - let main = SharedCore::from(main); - let clone = SharedCore::from(clone); - - let index = 6; - let nodes = clone.missing_nodes(index).await?; - let proof = main - .create_proof( - None, - Some(RequestBlock { index, nodes }), - None, - Some(RequestUpgrade { - start: 0, - length: 10, - }), - ) - .await? - .unwrap(); - assert!(clone.verify_and_apply_proof(&proof).await?); - let main_info = main.info().await; - let clone_info = clone.info().await; - assert_eq!(main_info.byte_length, clone_info.byte_length); - assert_eq!(main_info.length, clone_info.length); - assert!(main.get(6).await?.is_some()); - assert!(clone.get(6).await?.is_none()); - - // Fetch data for index 6 and verify it is found - let index = 6; - let nodes = clone.missing_nodes(index).await?; - let proof = main - .create_proof(Some(RequestBlock { index, nodes }), None, None, None) - .await? - .unwrap(); - assert!(clone.verify_and_apply_proof(&proof).await?); - Ok(()) - } -} diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 749192b9..aeab3b18 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -4,10 +4,10 @@ use futures::future::FutureExt; #[cfg(not(target_arch = "wasm32"))] use random_access_disk::RandomAccessDisk; use random_access_memory::RandomAccessMemory; -use random_access_storage::{RandomAccess, RandomAccessError}; -use std::fmt::Debug; +use random_access_storage::{BoxFuture, RandomAccess, RandomAccessError}; #[cfg(not(target_arch = "wasm32"))] use std::path::PathBuf; +use std::{fmt::Debug, sync::Arc}; use tracing::instrument; use crate::{ @@ -16,40 +16,16 @@ use crate::{ }; /// Supertrait for Storage -pub trait StorageTraits: RandomAccess + Debug {} -impl StorageTraits for T {} +pub trait StorageTraits: RandomAccess + Debug + Send + Sync {} +impl StorageTraits for T {} /// Save data to a desired storage backend. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Storage { - tree: Box, - data: Box, - bitfield: Box, - oplog: Box, -} - -pub(crate) fn map_random_access_err(err: RandomAccessError) -> HypercoreError { - match err { - RandomAccessError::IO { - return_code, - context, - source, - } => HypercoreError::IO { - context: Some(format!( - "RandomAccess IO error. Context: {context:?}, return_code: {return_code:?}", - )), - source, - }, - RandomAccessError::OutOfBounds { - offset, - end, - length, - } => HypercoreError::InvalidOperation { - context: format!( - "RandomAccess out of bounds. Offset: {offset}, end: {end:?}, length: {length}", - ), - }, - } + tree: Arc, + data: Arc, + bitfield: Arc, + oplog: Arc, } impl Storage { @@ -60,182 +36,182 @@ impl Storage { Store, ) -> std::pin::Pin< Box< - dyn std::future::Future< - Output = Result, RandomAccessError>, - > + Send, + dyn std::future::Future, RandomAccessError>> + + Send, >, >, { - let mut tree = create(Store::Tree).await.map_err(map_random_access_err)?; - let mut data = create(Store::Data).await.map_err(map_random_access_err)?; - let mut bitfield = create(Store::Bitfield) - .await - .map_err(map_random_access_err)?; - let mut oplog = create(Store::Oplog).await.map_err(map_random_access_err)?; + let tree: Arc = create(Store::Tree).await?; + let data: Arc = create(Store::Data).await?; + let bitfield: Arc = create(Store::Bitfield).await?; + let oplog: Arc = create(Store::Oplog).await?; if overwrite { - if tree.len().await.map_err(map_random_access_err)? > 0 { - tree.truncate(0).await.map_err(map_random_access_err)?; + if tree.len() > 0 { + tree.truncate(0).await?; } - if data.len().await.map_err(map_random_access_err)? > 0 { - data.truncate(0).await.map_err(map_random_access_err)?; + if data.len() > 0 { + data.truncate(0).await?; } - if bitfield.len().await.map_err(map_random_access_err)? > 0 { - bitfield.truncate(0).await.map_err(map_random_access_err)?; + if bitfield.len() > 0 { + bitfield.truncate(0).await?; } - if oplog.len().await.map_err(map_random_access_err)? > 0 { - oplog.truncate(0).await.map_err(map_random_access_err)?; + if oplog.len() > 0 { + oplog.truncate(0).await?; } } - let instance = Self { + Ok(Self { tree, data, bitfield, oplog, - }; - - Ok(instance) + }) } - /// Read info from store based on given instruction. Convenience method to `read_infos`. - pub(crate) async fn read_info( - &mut self, + /// Read info from store based on given instruction. + pub(crate) fn read_info( + &self, info_instruction: StoreInfoInstruction, - ) -> Result { - let mut infos = self.read_infos_to_vec(&[info_instruction]).await?; - Ok(infos - .pop() - .expect("Should have gotten one info with one instruction")) + ) -> BoxFuture> { + let fut = self.read_infos_to_vec(vec![info_instruction]); + Box::pin(async move { + Ok(fut + .await? + .pop() + .expect("Should have gotten one info with one instruction")) + }) } /// Read infos from stores based on given instructions - pub(crate) async fn read_infos( - &mut self, - info_instructions: &[StoreInfoInstruction], - ) -> Result, HypercoreError> { - let infos = self.read_infos_to_vec(info_instructions).await?; - Ok(infos.into_boxed_slice()) + pub(crate) fn read_infos( + &self, + info_instructions: Vec, + ) -> BoxFuture, HypercoreError>> { + let fut = self.read_infos_to_vec(info_instructions); + Box::pin(async move { Ok(fut.await?.into_boxed_slice()) }) } /// Reads infos but retains them as a Vec - pub(crate) async fn read_infos_to_vec( - &mut self, - info_instructions: &[StoreInfoInstruction], - ) -> Result, HypercoreError> { - if info_instructions.is_empty() { - return Ok(vec![]); - } - let mut current_store: Store = info_instructions[0].store.clone(); - let mut storage = self.get_random_access_mut(¤t_store); - let mut infos: Vec = Vec::with_capacity(info_instructions.len()); - for instruction in info_instructions.iter() { - if instruction.store != current_store { - current_store = instruction.store.clone(); - storage = self.get_random_access_mut(¤t_store); + pub(crate) fn read_infos_to_vec( + &self, + info_instructions: Vec, + ) -> BoxFuture, HypercoreError>> { + let storage = self.clone(); + let instructions = info_instructions; // TODO rm + Box::pin(async move { + if instructions.is_empty() { + return Ok(vec![]); } - match instruction.info_type { - StoreInfoType::Content => { - let read_length = match instruction.length { - Some(length) => length, - None => storage.len().await.map_err(map_random_access_err)?, - }; - let read_result = storage.read(instruction.index, read_length).await; - let info: StoreInfo = match read_result { - Ok(buf) => Ok(StoreInfo::new_content( + let mut current_store: Store = instructions[0].store.clone(); + let mut ra: Arc = storage.get_random_access(¤t_store).clone(); + let mut infos: Vec = Vec::with_capacity(instructions.len()); + for instruction in instructions.iter() { + if instruction.store != current_store { + current_store = instruction.store.clone(); + ra = storage.get_random_access(¤t_store).clone(); + } + match instruction.info_type { + StoreInfoType::Content => { + let read_length = match instruction.length { + Some(length) => length, + None => ra.len(), + }; + let read_result = ra.read(instruction.index, read_length).await; + let info: StoreInfo = match read_result { + Ok(buf) => Ok(StoreInfo::new_content( + instruction.store.clone(), + instruction.index, + &buf, + )), + Err(RandomAccessError::OutOfBounds { length, .. }) => { + if instruction.allow_miss { + Ok(StoreInfo::new_content_miss( + instruction.store.clone(), + instruction.index, + )) + } else { + Err(HypercoreError::InvalidOperation { + context: format!( + "Could not read from store {}, index {} / length {} is out of bounds for store length {}", + current_store, instruction.index, read_length, length + ), + }) + } + } + Err(e) => Err(HypercoreError::from(e)), + }?; + infos.push(info); + } + StoreInfoType::Size => { + let length = ra.len(); + infos.push(StoreInfo::new_size( instruction.store.clone(), instruction.index, - &buf, - )), - Err(RandomAccessError::OutOfBounds { length, .. }) => { - if instruction.allow_miss { - Ok(StoreInfo::new_content_miss( - instruction.store.clone(), - instruction.index, - )) - } else { - Err(HypercoreError::InvalidOperation { - context: format!( - "Could not read from store {}, index {} / length {} is out of bounds for store length {}", - current_store, instruction.index, read_length, length - ), - }) - } - } - Err(e) => Err(map_random_access_err(e)), - }?; - infos.push(info); - } - StoreInfoType::Size => { - let length = storage.len().await.map_err(map_random_access_err)?; - infos.push(StoreInfo::new_size( - instruction.store.clone(), - instruction.index, - length - instruction.index, - )); + length - instruction.index, + )); + } } } - } - Ok(infos) + Ok(infos) + }) } - /// Flush info to storage. Convenience method to `flush_infos`. - pub(crate) async fn flush_info(&mut self, slice: StoreInfo) -> Result<(), HypercoreError> { - self.flush_infos(&[slice]).await + /// Flush info to storage. + pub(crate) fn flush_info(&self, info: StoreInfo) -> BoxFuture> { + self.flush_infos(vec![info]) } /// Flush infos to storage - pub(crate) async fn flush_infos(&mut self, infos: &[StoreInfo]) -> Result<(), HypercoreError> { - if infos.is_empty() { - return Ok(()); - } - let mut current_store: Store = infos[0].store.clone(); - let mut storage = self.get_random_access_mut(¤t_store); - for info in infos.iter() { - if info.store != current_store { - current_store = info.store.clone(); - storage = self.get_random_access_mut(¤t_store); + pub(crate) fn flush_infos( + &self, + infos: Vec, + ) -> BoxFuture> { + let storage = self.clone(); + Box::pin(async move { + if infos.is_empty() { + return Ok(()); } - match info.info_type { - StoreInfoType::Content => { - if !info.miss { - if let Some(data) = &info.data { - storage - .write(info.index, data) - .await - .map_err(map_random_access_err)?; - } - } else { - storage - .del( + let mut current_store: Store = infos[0].store.clone(); + let mut ra: Arc = storage.get_random_access(¤t_store).clone(); + for info in infos.iter() { + if info.store != current_store { + current_store = info.store.clone(); + ra = storage.get_random_access(¤t_store).clone(); + } + match info.info_type { + StoreInfoType::Content => { + if !info.miss { + if let Some(data) = &info.data { + ra.write(info.index, data).await?; + } + } else { + ra.del( info.index, info.length.expect("When deleting, length must be given"), ) - .await - .map_err(map_random_access_err)?; + .await?; + } } - } - StoreInfoType::Size => { - if info.miss { - storage - .truncate(info.index) - .await - .map_err(map_random_access_err)?; - } else { - panic!("Flushing a size that isn't miss, is not supported"); + StoreInfoType::Size => { + if info.miss { + ra.truncate(info.index).await?; + } else { + panic!("Flushing a size that isn't miss, is not supported"); + } } } } - } - Ok(()) + Ok(()) + }) } - fn get_random_access_mut(&mut self, store: &Store) -> &mut Box { + fn get_random_access(&self, store: &Store) -> &Arc { match store { - Store::Tree => &mut self.tree, - Store::Data => &mut self.data, - Store::Bitfield => &mut self.bitfield, - Store::Oplog => &mut self.oplog, + Store::Tree => &self.tree, + Store::Data => &self.data, + Store::Bitfield => &self.bitfield, + Store::Oplog => &self.oplog, } } @@ -243,10 +219,8 @@ impl Storage { #[instrument(err)] pub async fn new_memory() -> Result { let create = |_| { - async { Ok(Box::new(RandomAccessMemory::default()) as Box) } - .boxed() + async { Ok(Arc::new(RandomAccessMemory::default()) as Arc) }.boxed() }; - // No reason to overwrite, as this is a new memory segment Self::open(create, false).await } @@ -264,8 +238,8 @@ impl Storage { Store::Oplog => "oplog", }; Ok( - Box::new(RandomAccessDisk::open(dir.as_path().join(name)).await?) - as Box, + Arc::new(RandomAccessDisk::open(dir.as_path().join(name)).await?) + as Arc, ) } .boxed() @@ -273,3 +247,67 @@ impl Storage { Self::open(storage, overwrite).await } } + +#[cfg(test)] +mod test { + use super::*; + use crate::common::{StoreInfo, StoreInfoInstruction}; + + #[tokio::test] + async fn test_futures_are_owned() -> Result<(), Box> { + let fut = { + let storage = Storage::new_memory().await?; + + let data = b"hello hypercore"; + + // Write to tree store + storage.flush_info(StoreInfo::new_content(Store::Tree, 0, data)) + }; + fut.await?; + + Ok(()) + } + + #[tokio::test] + async fn test_storage() -> Result<(), Box> { + let storage = Storage::new_memory().await?; + + let data = b"hello hypercore"; + + // Write to tree store + storage + .flush_info(StoreInfo::new_content(Store::Tree, 0, data)) + .await?; + + // Read it back + let info = storage + .read_info(StoreInfoInstruction::new_content( + Store::Tree, + 0, + data.len() as u64, + )) + .await?; + + assert_eq!(info.data.as_deref(), Some(data.as_slice())); + + // Write to two different stores, read back together + storage + .flush_infos(vec![ + StoreInfo::new_content(Store::Data, 0, b"block0"), + StoreInfo::new_content(Store::Bitfield, 0, b"bits"), + ]) + .await?; + + let infos = storage + .read_infos(vec![ + StoreInfoInstruction::new_content(Store::Data, 0, 6), + StoreInfoInstruction::new_content(Store::Bitfield, 0, 4), + ]) + .await?; + + assert_eq!(infos[0].data.as_deref(), Some(b"block0".as_slice())); + assert_eq!(infos[1].data.as_deref(), Some(b"bits".as_slice())); + + Ok(()) + } +} diff --git a/tests/core.rs b/tests/core.rs index 7b39a76b..d5283176 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -4,21 +4,15 @@ use anyhow::Result; use common::{create_hypercore, get_test_key_pair, open_hypercore, storage_contains_data}; use hypercore::{HypercoreBuilder, Storage}; use tempfile::Builder; -use test_log::test; -#[cfg(feature = "async-std")] -use async_std::test as async_test; -#[cfg(feature = "tokio")] -use tokio::test as async_test; - -#[test(async_test)] +#[tokio::test] async fn hypercore_new() -> Result<()> { let storage = Storage::new_memory().await?; let _hypercore = HypercoreBuilder::new(storage).build(); Ok(()) } -#[test(async_test)] +#[tokio::test] async fn hypercore_new_with_key_pair() -> Result<()> { let storage = Storage::new_memory().await?; let key_pair = get_test_key_pair(); @@ -29,7 +23,7 @@ async fn hypercore_new_with_key_pair() -> Result<()> { Ok(()) } -#[test(async_test)] +#[tokio::test] async fn hypercore_open_with_key_pair_error() -> Result<()> { let storage = Storage::new_memory().await?; let key_pair = get_test_key_pair(); @@ -44,7 +38,7 @@ async fn hypercore_open_with_key_pair_error() -> Result<()> { Ok(()) } -#[test(async_test)] +#[tokio::test] async fn hypercore_make_read_only() -> Result<()> { let dir = Builder::new() .prefix("hypercore_make_read_only") @@ -74,7 +68,7 @@ async fn hypercore_make_read_only() -> Result<()> { &write_key_pair.secret.as_ref().unwrap().to_bytes()[16..], )); - let mut hypercore = open_hypercore(&dir.path().to_string_lossy()).await?; + let hypercore = open_hypercore(&dir.path().to_string_lossy()).await?; assert_eq!(&hypercore.get(0).await?.unwrap(), b"Hello"); assert_eq!(&hypercore.get(1).await?.unwrap(), b"World!"); Ok(()) diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 5d02d737..92ceb348 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -5,12 +5,6 @@ use std::sync::Once; use anyhow::Result; use common::{create_hypercore, create_hypercore_hash, open_hypercore}; use js::{cleanup, install, js_run_step, prepare_test_set}; -use test_log::test; - -#[cfg(feature = "async-std")] -use async_std::test as async_test; -#[cfg(feature = "tokio")] -use tokio::test as async_test; const TEST_SET_JS_FIRST: &str = "jsfirst"; const TEST_SET_RS_FIRST: &str = "rsfirst"; @@ -24,7 +18,7 @@ fn init() { }); } -#[test(async_test)] +#[tokio::test] #[cfg_attr(not(feature = "js_interop_tests"), ignore)] async fn js_interop_js_first() -> Result<()> { init(); @@ -43,7 +37,7 @@ async fn js_interop_js_first() -> Result<()> { Ok(()) } -#[test(async_test)] +#[tokio::test] #[cfg_attr(not(feature = "js_interop_tests"), ignore)] async fn js_interop_rs_first() -> Result<()> { init(); diff --git a/tests/model.rs b/tests/model.rs index 86a74657..342bc0fe 100644 --- a/tests/model.rs +++ b/tests/model.rs @@ -44,13 +44,6 @@ proptest! { })] #[test] - #[cfg(feature = "async-std")] - fn implementation_matches_model(ops: Vec) { - assert!(async_std::task::block_on(assert_implementation_matches_model(ops))); - } - - #[test] - #[cfg(feature = "tokio")] fn implementation_matches_model(ops: Vec) { let rt = tokio::runtime::Runtime::new().unwrap(); assert!(rt.block_on(async { diff --git a/tests/replication.rs b/tests/replication.rs new file mode 100644 index 00000000..85768f96 --- /dev/null +++ b/tests/replication.rs @@ -0,0 +1,218 @@ +#![cfg(feature = "replication")] + +use hypercore::{Hypercore, HypercoreBuilder, PartialKeypair, Storage}; +use hypercore_handshake::{ + Cipher, CipherTrait, + state_machine::{SecStream, hc_specific::generate_keypair}, +}; +use std::time::Duration; +use tokio_util::compat::TokioAsyncReadCompatExt; +use uint24le_framing::Uint24LELengthPrefixedFraming; + +/// Create a pair of connected in-memory encrypted streams. +fn connected_pair() -> (impl CipherTrait + 'static, impl CipherTrait + 'static) { + let (a_b, b_a) = tokio::io::duplex(64 * 1024); + let a_b = Uint24LELengthPrefixedFraming::new(a_b.compat()); + let b_a = Uint24LELengthPrefixedFraming::new(b_a.compat()); + let keypair = generate_keypair().unwrap(); + let initiator = Cipher::new( + Some(Box::new(a_b)), + SecStream::new_initiator_xx(&[]).unwrap().into(), + ); + let responder = Cipher::new( + Some(Box::new(b_a)), + SecStream::new_responder_xx(&keypair, &[]).unwrap().into(), + ); + (initiator, responder) +} + +/// Create a writer (with data) and a blank reader sharing the same public key. +async fn make_writer_reader(data: &[&[u8]]) -> (Hypercore, Hypercore) { + let mut writer = HypercoreBuilder::new(Storage::new_memory().await.unwrap()) + .build() + .await + .unwrap(); + for chunk in data { + writer.append(chunk).await.unwrap(); + } + let public_key = writer.key_pair().public; + let reader = HypercoreBuilder::new(Storage::new_memory().await.unwrap()) + .key_pair(PartialKeypair { + public: public_key, + secret: None, + }) + .build() + .await + .unwrap(); + (writer, reader) +} + +/// Get block 0 from `reader`, with `writer`'s replicator spawned to drive the other end. +/// Uses `attach_replicator` so that `reader.get()` itself drives replication (structured +/// concurrency path). +async fn get_via_attached_replicator(writer: &Hypercore, reader: &Hypercore) -> Option> { + let (writer_stream, reader_stream) = connected_pair(); + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + reader.attach_replicator(reader.replicate(reader_stream)); + let block = tokio::time::timeout(Duration::from_secs(5), reader.get(0)) + .await + .expect("timed out waiting for attach_replicator get") + .unwrap(); + writer_rep.abort(); + block +} + +/// Poll until `core.info().contiguous_length >= expected`, with a 5-second timeout. +async fn wait_for_length(core: &Hypercore, expected: u64) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + let mut last_reported = u64::MAX; + loop { + let current = core.info().contiguous_length; + if current >= expected { + return; + } + if current != last_reported { + eprintln!("contiguous_length = {current} (waiting for {expected})"); + last_reported = current; + } + assert!( + tokio::time::Instant::now() < deadline, + "timed out: contiguous_length stuck at {current}, never reached {expected}" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } +} + +#[tokio::test] +async fn replicate_data_before_connect() { + let (writer, reader) = make_writer_reader(&[b"hello", b"world"]).await; + let (writer_stream, reader_stream) = connected_pair(); + + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + let reader_rep = tokio::spawn(reader.replicate(reader_stream)); + + wait_for_length(&reader, 2).await; + + assert_eq!(reader.get(0).await.unwrap(), Some(b"hello".to_vec())); + assert_eq!(reader.get(1).await.unwrap(), Some(b"world".to_vec())); + + writer_rep.abort(); + reader_rep.abort(); +} + +#[tokio::test] +async fn replicate_data_after_connect() { + let (mut writer, reader) = make_writer_reader(&[]).await; + let (writer_stream, reader_stream) = connected_pair(); + + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + let reader_rep = tokio::spawn(reader.replicate(reader_stream)); + + writer.append(b"late data").await.unwrap(); + + wait_for_length(&reader, 1).await; + + assert_eq!(reader.get(0).await.unwrap(), Some(b"late data".to_vec())); + + writer_rep.abort(); + reader_rep.abort(); +} + +/// Without an attached replicator, `get()` returns `None` immediately for a missing block. +#[tokio::test] +async fn get_returns_none_without_replicator() { + let (_, reader) = make_writer_reader(&[b"hello"]).await; + assert_eq!(reader.get(0).await.unwrap(), None); +} + +/// `attach_replicator` makes `reader.get()` drive replication itself — no spawn needed for +/// the reader side. +#[tokio::test] +async fn attach_replicator_drives_get() { + let (writer, reader) = make_writer_reader(&[b"hello", b"world"]).await; + assert_eq!( + get_via_attached_replicator(&writer, &reader).await, + Some(b"hello".to_vec()) + ); +} + +/// `attach_replicator` still works when the writer appends the block *after* the connection +/// is established. +#[tokio::test] +async fn attach_replicator_drives_get_late_data() { + let (mut writer, reader) = make_writer_reader(&[]).await; + let (writer_stream, reader_stream) = connected_pair(); + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + reader.attach_replicator(reader.replicate(reader_stream)); + + writer.append(b"late").await.unwrap(); + + let block = tokio::time::timeout(Duration::from_secs(5), reader.get(0)) + .await + .expect("timed out") + .unwrap(); + assert_eq!(block, Some(b"late".to_vec())); + writer_rep.abort(); +} + +/// Sequential gets — after `get(0)` resolves, `get(1)` and `get(2)` must also work. +#[tokio::test] +async fn attach_replicator_gets_sequential_blocks() { + let (writer, reader) = make_writer_reader(&[b"a", b"b", b"c"]).await; + let (writer_stream, reader_stream) = connected_pair(); + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + reader.attach_replicator(reader.replicate(reader_stream)); + + for (i, expected) in [b"a".as_slice(), b"b", b"c"].iter().enumerate() { + let block = tokio::time::timeout(Duration::from_secs(5), reader.get(i as u64)) + .await + .unwrap_or_else(|_| panic!("timed out on block {i}")) + .unwrap(); + assert_eq!(block.as_deref(), Some(*expected), "block {i}"); + } + writer_rep.abort(); +} + +/// Get a non-zero block directly without fetching earlier indices first. +/// Verifies that `Event::Get` fires with the requested index, not always 0. +#[tokio::test] +async fn attach_replicator_get_by_index() { + let data: Vec> = (0u8..5).map(|i| vec![i]).collect(); + let slices: Vec<&[u8]> = data.iter().map(|v| v.as_slice()).collect(); + let (writer, reader) = make_writer_reader(&slices).await; + let (writer_stream, reader_stream) = connected_pair(); + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + reader.attach_replicator(reader.replicate(reader_stream)); + + let block = tokio::time::timeout(Duration::from_secs(5), reader.get(4)) + .await + .expect("timed out") + .unwrap(); + assert_eq!(block, Some(vec![4u8])); + writer_rep.abort(); +} + +#[tokio::test] +async fn replicate_many_blocks() { + let data: Vec> = (0u8..10).map(|i| vec![i]).collect(); + let slices: Vec<&[u8]> = data.iter().map(|d| d.as_slice()).collect(); + + let (writer, reader) = make_writer_reader(&slices).await; + let (writer_stream, reader_stream) = connected_pair(); + + let writer_rep = tokio::spawn(writer.replicate(writer_stream)); + let reader_rep = tokio::spawn(reader.replicate(reader_stream)); + + wait_for_length(&reader, 10).await; + + for (i, expected) in data.iter().enumerate() { + assert_eq!( + reader.get(i as u64).await.unwrap().as_ref(), + Some(expected), + "block {i} mismatch" + ); + } + + writer_rep.abort(); + reader_rep.abort(); +}