diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 74562e3..db8d8df 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -58,26 +58,26 @@ jobs: command: fmt args: --all -- --check - clippy: - name: Clippy - runs-on: ubuntu-latest - needs: test - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Install Rust toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal - override: true - components: clippy - - name: Clippy check - uses: actions-rs/cargo@v1 - with: - command: clippy - # add --all-targets to include tests - args: --all-features --workspace -- -D warnings + # clippy: + # name: Clippy + # runs-on: ubuntu-latest + # needs: test + # steps: + # - name: Checkout repository + # uses: actions/checkout@v2 + # - name: Install Rust toolchain + # uses: actions-rs/toolchain@v1 + # with: + # toolchain: stable + # profile: minimal + # override: true + # components: clippy + # - name: Clippy check + # uses: actions-rs/cargo@v1 + # with: + # command: clippy + # # add --all-targets to include tests + # args: --all-features --workspace -- -D warnings docs: name: Docs diff --git a/Cargo.toml b/Cargo.toml index d14a2e1..e744243 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "im-lists" -version = "0.11.4" +version = "0.12.0" authors = ["mattwparas "] edition = "2021" readme = "README.md" @@ -15,6 +15,7 @@ categories = ["data-structures"] [dependencies] smallvec = { version = "1.10.0" } generic_singleton = "0.5.1" +allocator-api2 = "0.2.4" [dev-dependencies] criterion = "0.3.5" diff --git a/src/lib.rs b/src/lib.rs index 18db4f1..4d2b15f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,7 @@ pub mod handler; pub mod list; pub mod shared; +mod shared_vector; pub(crate) mod unrolled; /// Construct a [`List`](crate::list::List) from a sequence of elements diff --git a/src/list.rs b/src/list.rs index ddbef45..c7c425e 100644 --- a/src/list.rs +++ b/src/list.rs @@ -58,8 +58,8 @@ use crate::{ pub struct GenericList< T: Clone + 'static, P: PointerFamily = RcPointer, - const N: usize = 256, - const G: usize = 1, + const N: u32 = 256, + const G: u32 = 1, D: DropHandler = DefaultDropHandler, >(UnrolledList, PhantomData); @@ -74,12 +74,12 @@ pub type VList = GenericList; pub struct RawCell< T: Clone + 'static, P: PointerFamily, - const N: usize, - const G: usize, + const N: u32, + const G: u32, D: DropHandler>, >(*const UnrolledCell, PhantomData); -impl> Clone +impl> Clone for GenericList { fn clone(&self) -> Self { @@ -87,7 +87,7 @@ impl> +impl> GenericList { /// Construct an empty list. @@ -114,7 +114,7 @@ impl usize { - self.0.index() + self.0.index() as _ } #[doc(hidden)] @@ -145,12 +145,12 @@ impl usize { // Overflow is fine - this should give us a unique value? - self.0.elements_as_ptr_usize() + self.0.index() + self.0.elements_as_ptr_usize() + self.0.index() as usize } #[doc(hidden)] pub fn identity_tuple(&self) -> (usize, usize) { - (self.0.elements_as_ptr_usize(), self.0.index()) + (self.0.elements_as_ptr_usize(), self.0.index() as _) } // Check the next pointer. If the next pointer is the same, @@ -593,7 +593,7 @@ impl> Default +impl> Default for GenericList { fn default() -> Self { @@ -601,7 +601,7 @@ impl> Extend +impl> Extend for GenericList { fn extend>(&mut self, iter: I) { @@ -610,15 +610,15 @@ impl> - FromIterator for GenericList +impl> FromIterator + for GenericList { fn from_iter>(iter: I) -> Self { GenericList(iter.into_iter().collect(), PhantomData) } } -impl<'a, T: 'a + Clone, P: PointerFamily, const N: usize, const G: usize, D: DropHandler> +impl<'a, T: 'a + Clone, P: PointerFamily, const N: u32, const G: u32, D: DropHandler> FromIterator<&'a T> for GenericList { fn from_iter>(iter: I) -> Self { @@ -626,7 +626,7 @@ impl<'a, T: 'a + Clone, P: PointerFamily, const N: usize, const G: usize, D: Dro } } -impl> +impl> FromIterator> for GenericList { fn from_iter>>(iter: I) -> Self { @@ -639,7 +639,7 @@ impl> From> +impl> From> for GenericList { fn from(vec: Vec) -> Self { @@ -650,8 +650,8 @@ impl, > std::fmt::Debug for GenericList { @@ -665,8 +665,8 @@ pub struct Iter< 'a, T: Clone + 'static, P: PointerFamily, - const N: usize, - const G: usize, + const N: u32, + const G: u32, D: DropHandler>, >(IterWrapper<'a, T, P, N, G>, PhantomData); @@ -674,8 +674,8 @@ impl< 'a, T: Clone + 'static, P: PointerFamily, - const N: usize, - const G: usize, + const N: u32, + const G: u32, D: DropHandler>, > Iterator for Iter<'a, T, P, N, G, D> { @@ -705,8 +705,8 @@ impl< 'a, T: Clone, P: PointerFamily, - const N: usize, - const G: usize, + const N: u32, + const G: u32, D: DropHandler>, > IntoIterator for &'a GenericList { @@ -723,16 +723,16 @@ impl< pub struct ConsumingIter< T: Clone + 'static, P: PointerFamily, - const N: usize, - const G: usize, + const N: u32, + const G: u32, D: DropHandler>, >(ConsumingWrapper, PhantomData); impl< T: Clone, P: PointerFamily, - const N: usize, - const G: usize, + const N: u32, + const G: u32, D: DropHandler>, > Iterator for ConsumingIter { @@ -758,7 +758,7 @@ impl< } } -impl> IntoIterator +impl> IntoIterator for GenericList { type Item = T; @@ -774,8 +774,8 @@ impl< 'a, T: 'a + Clone, P: 'a + PointerFamily, - const N: usize, - const G: usize, + const N: u32, + const G: u32, D: 'a + DropHandler, > FromIterator<&'a GenericList> for GenericList { @@ -784,7 +784,7 @@ impl< } } -impl> From<&[T]> +impl> From<&[T]> for GenericList { fn from(vec: &[T]) -> Self { @@ -792,38 +792,28 @@ impl, - > PartialEq for GenericList +impl> + PartialEq for GenericList { fn eq(&self, other: &Self) -> bool { self.iter().eq(other.iter()) } } -impl> Eq +impl> Eq for GenericList { } -impl< - T: Clone + PartialOrd, - P: PointerFamily, - const N: usize, - const G: usize, - D: DropHandler, - > PartialOrd for GenericList +impl> + PartialOrd for GenericList { fn partial_cmp(&self, other: &Self) -> Option { self.iter().partial_cmp(other.iter()) } } -impl> Ord +impl> Ord for GenericList { fn cmp(&self, other: &Self) -> Ordering { @@ -831,7 +821,7 @@ impl> std::ops::Add +impl> std::ops::Add for GenericList { type Output = GenericList; @@ -845,8 +835,8 @@ impl>, > std::ops::Add for &GenericList { @@ -858,8 +848,8 @@ impl< } } -impl> - std::iter::Sum for GenericList +impl> std::iter::Sum + for GenericList { fn sum(it: I) -> Self where @@ -872,8 +862,8 @@ impl, > std::hash::Hash for GenericList { @@ -884,7 +874,7 @@ impl< } } -impl> +impl> std::ops::Index for GenericList { type Output = T; @@ -904,7 +894,7 @@ impl> Drop +impl> Drop for GenericList { fn drop(&mut self) { diff --git a/src/shared_vector/LICENSE.txt b/src/shared_vector/LICENSE.txt new file mode 100644 index 0000000..4d65d80 --- /dev/null +++ b/src/shared_vector/LICENSE.txt @@ -0,0 +1,24 @@ +This is module is a fork of the original library, found here: https://crates.io/crates/shared_vector +Every file in this module was originally licensed under this: + + +The MIT License (MIT) + +Copyright (c) 2013 Nicolas Silva + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/src/shared_vector/drain.rs b/src/shared_vector/drain.rs new file mode 100644 index 0000000..4c66025 --- /dev/null +++ b/src/shared_vector/drain.rs @@ -0,0 +1,155 @@ +// Most of the code in this file is copied from std::Vec's Drain implementation. + +use core::fmt; +use core::iter::FusedIterator; +use core::mem; +use core::ptr::{self, NonNull}; +use core::slice; + +use super::RawVector; + +/// A draining iterator for `Vector`. +/// +/// This `struct` is created by [`crate::shared_vector::Vector::drain`]. +/// See its documentation for more. +pub struct Drain<'a, T: 'a> { + /// Index of tail to preserve + pub(super) tail_start: usize, + /// Length of tail + pub(super) tail_len: usize, + /// Current remaining range to remove + pub(super) iter: slice::Iter<'a, T>, + pub(super) vec: NonNull>, +} + +impl fmt::Debug for Drain<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("Drain").field(&self.iter.as_slice()).finish() + } +} + +impl<'a, T> Drain<'a, T> { + /// Returns the remaining items of this iterator as a slice. + /// + /// # Examples + /// + /// ``` + /// let mut vec = vec!['a', 'b', 'c']; + /// let mut drain = vec.drain(..); + /// assert_eq!(drain.as_slice(), &['a', 'b', 'c']); + /// let _ = drain.next().unwrap(); + /// assert_eq!(drain.as_slice(), &['b', 'c']); + /// ``` + #[must_use] + pub fn as_slice(&self) -> &[T] { + self.iter.as_slice() + } +} + +impl<'a, T> AsRef<[T]> for Drain<'a, T> { + fn as_ref(&self) -> &[T] { + self.as_slice() + } +} + +unsafe impl Sync for Drain<'_, T> {} +unsafe impl Send for Drain<'_, T> {} + +impl Iterator for Drain<'_, T> { + type Item = T; + + #[inline] + fn next(&mut self) -> Option { + self.iter + .next() + .map(|elt| unsafe { ptr::read(elt as *const _) }) + } + + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } +} + +impl DoubleEndedIterator for Drain<'_, T> { + #[inline] + fn next_back(&mut self) -> Option { + self.iter + .next_back() + .map(|elt| unsafe { ptr::read(elt as *const _) }) + } +} + +impl Drop for Drain<'_, T> { + fn drop(&mut self) { + /// Moves back the un-`Drain`ed elements to restore the original `Vec`. + struct DropGuard<'r, 'a, T>(&'r mut Drain<'a, T>); + + impl<'r, 'a, T> Drop for DropGuard<'r, 'a, T> { + fn drop(&mut self) { + if self.0.tail_len > 0 { + unsafe { + let source_vec = self.0.vec.as_mut(); + // memmove back untouched tail, update to new length + let start = source_vec.len(); + let tail = self.0.tail_start; + if tail != start { + let src = source_vec.as_ptr().add(tail); + let dst = source_vec.as_mut_ptr().add(start); + ptr::copy(src, dst, self.0.tail_len); + } + source_vec.header.len = (start + self.0.tail_len) as u32; + } + } + } + } + + let iter = mem::replace(&mut self.iter, (&mut []).iter()); + let drop_len = iter.len(); + + let mut vec = self.vec; + + // if T::IS_ZST { + // // ZSTs have no identity, so we don't need to move them around, we only need to drop the correct amount. + // // this can be achieved by manipulating the Vec length instead of moving values out from `iter`. + // unsafe { + // let vec = vec.as_mut(); + // let old_len = vec.len(); + // vec.set_len(old_len + drop_len + self.tail_len); + // vec.truncate(old_len + self.tail_len); + // } + // return; + // } + + // ensure elements are moved back into their appropriate places, even when drop_in_place panics + let _guard = DropGuard(self); + + if drop_len == 0 { + return; + } + + // as_slice() must only be called when iter.len() is > 0 because + // it also gets touched by vec::Splice which may turn it into a dangling pointer + // which would make it and the vec pointer point to different allocations which would + // lead to invalid pointer arithmetic below. + let drop_ptr = iter.as_slice().as_ptr(); + + unsafe { + // drop_ptr comes from a slice::Iter which only gives us a &[T] but for drop_in_place + // a pointer with mutable provenance is necessary. Therefore we must reconstruct + // it from the original vec but also avoid creating a &mut to the front since that could + // invalidate raw pointers to it which some unsafe code might rely on. + let vec_ptr = vec.as_mut().as_mut_ptr(); + let drop_offset = sub_ptr(drop_ptr, vec_ptr); + let to_drop = ptr::slice_from_raw_parts_mut(vec_ptr.add(drop_offset), drop_len); + ptr::drop_in_place(to_drop); + } + } +} + +fn sub_ptr(a: *const T, b: *const T) -> usize { + debug_assert!(a >= b); + + (a as usize - b as usize) / mem::size_of::() +} + +impl FusedIterator for Drain<'_, T> {} diff --git a/src/shared_vector/intoiter.rs b/src/shared_vector/intoiter.rs new file mode 100644 index 0000000..963b957 --- /dev/null +++ b/src/shared_vector/intoiter.rs @@ -0,0 +1,229 @@ +use std::{ + mem, + ptr::{self, NonNull}, +}; + +use crate::shared_vector::alloc::{Allocator, Global}; +use crate::shared_vector::{RawVector, Vector}; + +pub struct IntoIter { + _buf: RawVector, // we don't actually care about this. Just need it to live. + iter: RawValIter, + pub(crate) allocator: A, +} + +impl Iterator for IntoIter { + type Item = T; + fn next(&mut self) -> Option { + self.iter.next() + } + fn size_hint(&self) -> (usize, Option) { + self.iter.size_hint() + } +} + +impl DoubleEndedIterator for IntoIter { + fn next_back(&mut self) -> Option { + self.iter.next_back() + } +} + +impl ExactSizeIterator for IntoIter {} + +impl Drop for IntoIter { + fn drop(&mut self) { + // drop any remaining elements + for _ in &mut *self {} + + unsafe { + self._buf.deallocate_no_drop(&self.allocator); + } + } +} + +impl IntoIterator for Vector { + type Item = T; + type IntoIter = IntoIter; + fn into_iter(self) -> IntoIter { + let (iter, buf) = unsafe { (RawValIter::new(&self), ptr::read(&self.raw)) }; + + mem::forget(self); + + IntoIter { + iter, + _buf: buf, + allocator: Global, + } + } +} + +struct RawValIter { + start: *const T, + end: *const T, +} + +impl RawValIter { + unsafe fn new(slice: &[T]) -> Self { + RawValIter { + start: slice.as_ptr(), + end: if mem::size_of::() == 0 { + ((slice.as_ptr() as usize) + slice.len()) as *const _ + } else if slice.len() == 0 { + slice.as_ptr() + } else { + slice.as_ptr().add(slice.len()) + }, + } + } +} + +impl Iterator for RawValIter { + type Item = T; + fn next(&mut self) -> Option { + if self.start == self.end { + None + } else { + unsafe { + if mem::size_of::() == 0 { + self.start = (self.start as usize + 1) as *const _; + Some(ptr::read(NonNull::::dangling().as_ptr())) + } else { + let old_ptr = self.start; + self.start = self.start.offset(1); + Some(ptr::read(old_ptr)) + } + } + } + } + + fn size_hint(&self) -> (usize, Option) { + let elem_size = mem::size_of::(); + let len = + (self.end as usize - self.start as usize) / if elem_size == 0 { 1 } else { elem_size }; + (len, Some(len)) + } +} + +impl DoubleEndedIterator for RawValIter { + fn next_back(&mut self) -> Option { + if self.start == self.end { + None + } else { + unsafe { + if mem::size_of::() == 0 { + self.end = (self.end as usize - 1) as *const _; + Some(ptr::read(NonNull::::dangling().as_ptr())) + } else { + self.end = self.end.offset(-1); + Some(ptr::read(self.end)) + } + } + } + } +} + +impl ExactSizeIterator for RawValIter {} + +#[cfg(test)] +mod tests { + use std::sync::atomic::AtomicUsize; + + #[test] + fn into_iter_test() { + struct Foo { + value: Box, + } + + impl Foo { + pub fn new(value: i32) -> Self { + Self { + value: Box::new(value), + } + } + } + + impl Drop for Foo { + fn drop(&mut self) {} + } + + let mut vector = crate::shared_vector::Vector::new(); + + for i in 0..=100 { + vector.push(Foo::new(i)); + } + + let resulting = vector.into_iter().collect::>(); + + let sum = resulting.into_iter().map(|x| *x.value).sum::(); + + assert_eq!(sum, 5050) + } + + #[test] + fn into_iter_drops_everything() { + static COUNTER: AtomicUsize = AtomicUsize::new(0); + + struct Foo { + value: Box, + } + + impl Foo { + pub fn new(value: i32) -> Self { + Self { + value: Box::new(value), + } + } + } + + impl Drop for Foo { + fn drop(&mut self) { + COUNTER.fetch_add(1, std::sync::atomic::Ordering::Acquire); + } + } + + let mut vector = crate::shared_vector::Vector::new(); + + for i in 0..=100 { + vector.push(Foo::new(i)); + } + + let resulting = vector.into_iter().collect::>(); + let sum = resulting.into_iter().map(|x| *x.value).sum::(); + assert_eq!(sum, 5050); + assert_eq!(COUNTER.load(std::sync::atomic::Ordering::Relaxed), 101); + } + + #[test] + fn into_iter_drops_everything_partial_usage() { + static COUNTER: AtomicUsize = AtomicUsize::new(0); + + struct Foo {} + + impl Foo { + pub fn new() -> Self { + Self {} + } + } + + impl Drop for Foo { + fn drop(&mut self) { + COUNTER.fetch_add(1, std::sync::atomic::Ordering::Acquire); + } + } + + let mut vector = crate::shared_vector::Vector::new(); + + for _ in 0..=100 { + vector.push(Foo::new()); + } + + let mut iter = vector.into_iter(); + + iter.next(); + iter.next(); + + drop(iter); + + assert_eq!(COUNTER.load(std::sync::atomic::Ordering::Relaxed), 101); + } +} diff --git a/src/shared_vector/mod.rs b/src/shared_vector/mod.rs new file mode 100644 index 0000000..9e8f880 --- /dev/null +++ b/src/shared_vector/mod.rs @@ -0,0 +1,138 @@ +mod drain; +mod intoiter; +mod raw; +mod shared; +mod splice; +mod vector; + +pub use raw::{AtomicRefCount, BufferSize, DefaultRefCount, RefCount}; + +#[allow(unused_imports)] +pub use shared::{AtomicSharedVector, RefCountedVector, SharedVector}; +pub use vector::{RawVector, Vector}; + +pub use intoiter::IntoIter; + +pub mod alloc { + pub use allocator_api2::alloc::{AllocError, Allocator, Global}; +} + +pub(crate) fn grow_amortized(len: usize, additional: usize) -> usize { + let required = len.saturating_add(additional); + let cap = len.saturating_add(len).max(required).max(8); + + const MAX: usize = BufferSize::MAX as usize; + + if cap > MAX { + if required <= MAX { + return required; + } + + panic!("Required allocation size is too large"); + } + + cap +} + +#[macro_export] +macro_rules! vector { + (@one@ $x:expr) => (1usize); + ($elem:expr; $n:expr) => ({ + $crate::shared_vector::Vector::from_elem($elem, $n) + }); + ($($x:expr),*$(,)*) => ({ + let count = 0usize $(+ $crate::vector!(@one@ $x))*; + let mut vec = $crate::shared_vector::Vector::with_capacity(count); + $(vec.push($x);)* + vec + }); + ([$($x:expr),*$(,)*] in $allocator:expr) => ({ + let count = 0usize $(+ $crate::vector!(@one@ $x))*; + let mut vec = $crate::shared_vector::Vector::try_with_capacity_in(count, $allocator).unwrap(); + $(vec.push($x);)* + vec + }); + ([$x:expr;$n:expr] in $allocator:expr) => ({ + let mut vec = $crate::Vector::try_with_capacity_in($n, $allocator).unwrap(); + for _ in 0..$n { vec.push($x.clone()); } + vec + }); +} + +#[macro_export] +macro_rules! rc_vector { + ($elem:expr; $n:expr) => ({ + let mut vec = $crate::shared_vector::SharedVector::with_capacity($n); + for _ in 0..$n { vec.push($elem.clone()); } + vec + }); + ($($x:expr),*$(,)*) => ({ + let count = 0usize $(+ $crate::vector!(@one@ $x))*; + let mut vec = $crate::shared_vector::SharedVector::with_capacity(count); + $(vec.push($x);)* + vec + }); + ([$($x:expr),*$(,)*] in $allocator:expr) => ({ + let count = 0usize $(+ $crate::vector!(@one@ $x))*; + let mut vec = $crate::SharedVector::try_with_capacity_in(count, $allocator).unwrap(); + $(vec.push($x);)* + vec + }); + ([$elem:expr;$n:expr] in $allocator:expr) => ({ + let mut vec = $crate::shared_vector::SharedVector::try_with_capacity_in($n, $allocator).unwrap(); + for _ in 0..$n { vec.push($elem.clone()); } + vec + }); +} + +#[macro_export] +macro_rules! arc_vector { + ($elem:expr; $n:expr) => ({ + let mut vec = $crate::shared_vector::AtomicSharedVector::with_capacity($n); + for _ in 0..$n { vec.push($elem.clone()); } + vec + }); + ($($x:expr),*$(,)*) => ({ + let count = 0usize $(+ $crate::vector!(@one@ $x))*; + let mut vec = $crate::shared_vector::AtomicSharedVector::with_capacity(count); + $(vec.push($x);)* + vec + }); + ([$($x:expr),*$(,)*] in $allocator:expr) => ({ + let count = 0usize $(+ $crate::vector!(@one@ $x))*; + let mut vec = $crate::shared_vector::AtomicSharedVector::try_with_capacity_in(count, $allocator).unwrap(); + $(vec.push($x);)* + vec + }); + ([$elem:expr;$n:expr] in $allocator:expr) => ({ + let mut vec = $crate::AtomicSharedVector::try_with_capacity_in($n, $allocator).unwrap(); + for _ in 0..$n { vec.push($elem.clone()); } + vec + }); +} + +#[test] +fn vector_macro() { + pub use allocator_api2::alloc::Global; + + let v1: Vector = vector![0, 1, 2, 3, 4, 5]; + let v2: Vector = vector![2; 4]; + let v3: Vector = vector!([6, 7] in Global); + assert_eq!(v1.as_slice(), &[0, 1, 2, 3, 4, 5]); + assert_eq!(v2.as_slice(), &[2, 2, 2, 2]); + assert_eq!(v3.as_slice(), &[6, 7]); + + let v1: SharedVector = rc_vector![0, 1, 2, 3, 4, 5]; + let v2: SharedVector = rc_vector![3; 5]; + let v3: SharedVector = rc_vector!([4; 3] in Global); + assert_eq!(v1.as_slice(), &[0, 1, 2, 3, 4, 5]); + assert_eq!(v2.as_slice(), &[3, 3, 3, 3, 3]); + assert_eq!(v3.as_slice(), &[4, 4, 4]); + + let v1: AtomicSharedVector = arc_vector![0, 1, 2, 3, 4, 5]; + let v2: AtomicSharedVector = arc_vector![1; 4]; + let v3: AtomicSharedVector = arc_vector![[3, 2, 1] in Global]; + assert_eq!(v1.as_slice(), &[0, 1, 2, 3, 4, 5]); + assert_eq!(v2.as_slice(), &[1, 1, 1, 1]); + assert_eq!(v3.as_slice(), &[3, 2, 1]); +} diff --git a/src/shared_vector/raw.rs b/src/shared_vector/raw.rs new file mode 100644 index 0000000..5653148 --- /dev/null +++ b/src/shared_vector/raw.rs @@ -0,0 +1,324 @@ +use core::alloc::Layout; +use core::cell::UnsafeCell; +use core::marker::PhantomData; +use core::mem; +use core::ptr::{self, NonNull}; +use core::sync::atomic::{ + AtomicI32, + Ordering::{Relaxed, Release}, +}; + +pub use crate::shared_vector::alloc::{AllocError, Allocator}; + +pub type BufferSize = u32; + +pub trait RefCount { + unsafe fn add_ref(&self); + unsafe fn release_ref(&self) -> bool; + fn new(count: i32) -> Self; + fn get(&self) -> i32; +} + +pub struct DefaultRefCount(UnsafeCell); +pub struct AtomicRefCount(AtomicI32); + +#[repr(C)] +#[derive(Clone)] +pub struct VecHeader { + pub cap: BufferSize, + pub len: BufferSize, +} + +impl VecHeader { + fn remaining_capacity(&self) -> u32 { + self.cap - self.len + } +} + +#[repr(C)] +pub struct Header { + pub(crate) vec: VecHeader, + pub(crate) ref_count: R, + pub(crate) allocator: A, +} + +impl RefCount for AtomicRefCount { + #[inline] + unsafe fn add_ref(&self) { + // Relaxed ordering is OK since the presence of the existing reference + // prevents threads from deleting the buffer. + self.0.fetch_add(1, Relaxed); + } + + #[inline] + unsafe fn release_ref(&self) -> bool { + self.0.fetch_sub(1, Release) == 1 + } + + #[inline] + fn new(val: i32) -> Self { + AtomicRefCount(AtomicI32::new(val)) + } + + #[inline] + fn get(&self) -> i32 { + self.0.load(Relaxed) + } +} + +impl RefCount for DefaultRefCount { + #[inline] + unsafe fn add_ref(&self) { + *self.0.get() += 1; + } + + #[inline] + unsafe fn release_ref(&self) -> bool { + let count = self.0.get(); + *count -= 1; + *count == 0 + } + + #[inline] + fn new(val: i32) -> Self { + DefaultRefCount(UnsafeCell::new(val)) + } + + #[inline] + fn get(&self) -> i32 { + unsafe { *self.0.get() } + } +} + +#[inline] +pub unsafe fn data_ptr(header: NonNull
) -> *mut T { + (header.as_ptr() as *mut u8).add(header_size::()) as *mut T +} + +pub(crate) const fn header_size() -> usize { + let a = mem::align_of::(); + let s = mem::size_of::
(); + let size = if a > s { a } else { s }; + + // Favor L1 cache line alignment for large structs. + let min = if mem::size_of::() < 64 { 16 } else { 64 }; + if size < min { + min + } else { + size + } +} + +pub fn buffer_layout(n: usize) -> Result { + let size = mem::size_of::().checked_mul(n).ok_or(AllocError)?; + let align = mem::align_of::
().max(mem::align_of::()); + let align = if mem::size_of::() < 64 { + align + } else { + align.max(64) + }; + let header_size = header_size::(); + + Layout::from_size_align(header_size + size, align).map_err(|_| AllocError) +} + +pub unsafe fn drop_items(mut ptr: *mut T, count: u32) { + for _ in 0..count { + core::ptr::drop_in_place(ptr); + ptr = ptr.add(1); + } +} + +pub unsafe fn dealloc(mut ptr: NonNull>, cap: BufferSize) { + let layout = buffer_layout::, T>(cap as usize).unwrap(); + let allocator = ptr::read(&ptr.as_mut().allocator); + allocator.deallocate(ptr.cast::(), layout); +} + +#[cold] +pub fn alloc_error_cold() -> AllocError { + AllocError +} + +#[repr(transparent)] +pub struct HeaderBuffer { + pub header: NonNull>, + _marker: PhantomData, +} + +impl HeaderBuffer { + pub unsafe fn from_raw(ptr: NonNull>) -> Self { + HeaderBuffer { + header: ptr, + _marker: PhantomData, + } + } + + #[inline] + pub unsafe fn as_mut(&mut self) -> &mut Header { + self.header.as_mut() + } + + #[inline] + pub unsafe fn as_ref(&self) -> &Header { + self.header.as_ref() + } + + #[inline] + pub unsafe fn as_ptr(&self) -> *mut Header { + self.header.as_ptr() + } + + #[inline] + pub fn allocator(&self) -> &A { + unsafe { &self.header.as_ref().allocator } + } +} + +pub unsafe fn move_data( + src_data: *mut T, + src_vec: &mut VecHeader, + dst_data: *mut T, + dst_vec: &mut VecHeader, +) { + debug_assert!(dst_vec.cap - dst_vec.len >= src_vec.len); + let len = src_vec.len; + if len > 0 { + unsafe { + let dst = dst_data.add(dst_vec.len as usize); + + let inital_dst_len = dst_vec.len; + dst_vec.len = inital_dst_len + len; + src_vec.len = 0; + + ptr::copy_nonoverlapping(src_data, dst, len as usize); + } + } +} + +pub unsafe fn extend_from_slice_assuming_capacity( + data: *mut T, + vec: &mut VecHeader, + slice: &[T], +) where + T: Clone, +{ + let len = slice.len() as u32; + debug_assert!(len <= vec.remaining_capacity()); + + let inital_len = vec.len; + + let mut ptr = data.add(inital_len as usize); + + for item in slice { + ptr::write(ptr, item.clone()); + ptr = ptr.add(1) + } + + vec.len += len; +} + +// Returns true if the iterator was emptied. +pub unsafe fn extend_within_capacity>( + data: *mut T, + vec: &mut VecHeader, + iter: &mut I, +) -> bool { + let inital_len = vec.len; + + let mut ptr = data.add(inital_len as usize); + + let mut count = 0; + let max = vec.remaining_capacity(); + let mut finished = false; + loop { + if count == max { + break; + } + if let Some(item) = iter.next() { + ptr::write(ptr, item); + ptr = ptr.add(1); + count += 1; + } else { + finished = true; + break; + } + } + + vec.len += count; + return finished; +} + +#[inline] +pub unsafe fn pop(data: *mut T, vec: &mut VecHeader) -> Option { + if vec.len == 0 { + return None; + } + + vec.len -= 1; + + Some(ptr::read(data.add(vec.len as usize))) +} + +#[inline(always)] +pub unsafe fn push_assuming_capacity(data: *mut T, vec: &mut VecHeader, val: T) { + let dst = data.add(vec.len as usize); + ptr::write(dst, val); + vec.len += 1; +} + +pub unsafe fn clear(data: *mut T, vec: &mut VecHeader) { + drop_items(data, vec.len); + vec.len = 0; +} + +pub fn assert_ref_count_layout() { + assert_eq!(mem::size_of::(), mem::size_of::()); + assert_eq!(mem::align_of::(), mem::align_of::()); +} + +#[inline(never)] +pub fn allocate_header_buffer( + mut cap: usize, + allocator: &A, +) -> Result<(NonNull, usize), AllocError> +where + A: Allocator, +{ + if cap == 0 { + cap = 16; + } + + if cap > BufferSize::MAX as usize { + return Err(alloc_error_cold()); + } + + let layout = buffer_layout::, T>(cap)?; + let allocation = allocator.allocate(layout)?; + let items_size = allocation.len() - header_size::, T>(); + let size_of = mem::size_of::(); + let real_capacity = if size_of == 0 { + cap + } else { + items_size / size_of + }; + + Ok((allocation.cast(), real_capacity)) +} + +pub unsafe fn header_from_data_ptr(data_ptr: NonNull) -> NonNull { + NonNull::new_unchecked((data_ptr.as_ptr() as *mut u8).sub(header_size::()) as *mut H) +} + +#[test] +fn buffer_layout_alignemnt() { + use allocator_api2::alloc::Global; + + type B = Box; + let layout = buffer_layout::, B>(2).unwrap(); + assert_eq!(layout.align(), mem::size_of::()); + + let atomic_layout = buffer_layout::, B>(2).unwrap(); + + assert_eq!(layout, atomic_layout); +} diff --git a/src/shared_vector/shared.rs b/src/shared_vector/shared.rs new file mode 100644 index 0000000..032f62b --- /dev/null +++ b/src/shared_vector/shared.rs @@ -0,0 +1,850 @@ +use core::fmt::Debug; +use core::ops::{Deref, DerefMut, Index, IndexMut}; +use core::ptr::NonNull; +use core::sync::atomic::Ordering; +use core::{mem, ptr}; + +use crate::shared_vector::alloc::{AllocError, Allocator, Global}; +use crate::shared_vector::raw; +use crate::shared_vector::raw::{BufferSize, HeaderBuffer}; +use crate::shared_vector::vector::{RawVector, Vector}; +use crate::shared_vector::{grow_amortized, AtomicRefCount, DefaultRefCount, RefCount}; + +/// A heap allocated, atomically reference counted, immutable contiguous buffer containing elements of type `T`. +/// +/// +/// +/// See [RefCountedVector]. +pub type AtomicSharedVector = RefCountedVector; + +/// A heap allocated, reference counted, immutable contiguous buffer containing elements of type `T`. +/// +/// +/// +/// See [RefCountedVector]. +pub type SharedVector = RefCountedVector; + +/// A heap allocated, reference counted, immutable contiguous buffer containing elements of type `T`. +/// +/// +/// +/// Similar in principle to `Arc<[T]>`. It can be converted into a `Vector` for +/// free if there is only a single reference to the RefCountedVector alive. +/// +/// # Copy-on-write "Immutable" vectors +/// +/// This type contains mutable methods like `push` and `pop`. These internally allocate a new buffer +/// if the buffer is not unique (there are more than one reference to it). When there is a single reference, +/// these mutable operation simply update the existing buffer. +/// +/// In other words, this type behaves like an [immutable (or persistent) data structure](https://en.wikipedia.org/wiki/Persistent_data_structure) +/// Actual mutability only happens under the hood as an optimization when a single reference exists. +#[repr(transparent)] +pub struct RefCountedVector { + pub(crate) inner: HeaderBuffer, +} + +impl RefCountedVector { + /// Creates an empty shared buffer without allocating memory. + #[inline] + pub fn new() -> RefCountedVector { + Self::try_with_capacity_in(0, Global).unwrap() + } + + /// Constructs a new, empty vector with at least the specified capacity. + #[inline] + pub fn with_capacity(cap: usize) -> RefCountedVector { + Self::try_with_capacity_in(cap, Global).unwrap() + } + + /// Clones the contents of a slice into a new vector. + #[inline] + pub fn from_slice(slice: &[T]) -> RefCountedVector + where + T: Clone, + { + Self::try_from_slice_in(slice, Global).unwrap() + } +} + +impl RefCountedVector { + /// Creates an empty vector without allocating memory. + pub fn new_in(allocator: A) -> Self { + Self::try_with_capacity_in(0, allocator).unwrap() + } + + /// Creates an empty pre-allocated vector with a given storage capacity. + pub fn with_capacity_in(cap: usize, allocator: A) -> Self { + Self::try_with_capacity_in(cap, allocator).unwrap() + } + + /// Tries to construct a new, empty vector with at least the specified capacity. + #[inline] + pub fn try_with_capacity_in(cap: usize, allocator: A) -> Result { + raw::assert_ref_count_layout::(); + unsafe { + let (ptr, cap) = raw::allocate_header_buffer::(cap, &allocator)?; + + ptr::write( + ptr.cast().as_ptr(), + raw::Header { + vec: raw::VecHeader { + cap: cap as BufferSize, + len: 0, + }, + ref_count: R::new(1), + allocator, + }, + ); + + Ok(RefCountedVector { + inner: HeaderBuffer::from_raw(ptr.cast()), + }) + } + } + + pub fn try_from_slice_in(slice: &[T], allocator: A) -> Result + where + T: Clone, + { + let mut v = Self::try_with_capacity_in(slice.len(), allocator)?; + + unsafe { + raw::extend_from_slice_assuming_capacity(v.data_ptr(), v.vec_header_mut(), slice); + } + + Ok(v) + } + + /// Returns `true` if the vector contains no elements. + #[inline] + pub fn is_empty(&self) -> bool { + self.vec_header().len == 0 + } + + /// Returns the number of elements in the vector, also referred to as its ‘length’. + #[inline] + pub fn len(&self) -> usize { + self.vec_header().len as usize + } + + /// Returns the total number of elements the vector can hold without reallocating. + #[inline] + pub fn capacity(&self) -> usize { + self.vec_header().cap as usize + } + + /// Returns number of elements that can be added without reallocating. + #[inline] + pub fn remaining_capacity(&self) -> usize { + let h = self.vec_header(); + (h.cap - h.len) as usize + } + + /// Returns a reference to the underlying allocator. + pub fn allocator(&self) -> &A { + self.inner.allocator() + } + + /// Creates a new reference without allocating. + /// + /// Equivalent to `Clone::clone`. + #[inline] + pub fn new_ref(&self) -> Self { + unsafe { + self.inner.as_ref().ref_count.add_ref(); + RefCountedVector { + inner: HeaderBuffer::from_raw(self.inner.header), + } + } + } + + /// Extracts a slice containing the entire vector. + #[inline] + pub fn as_slice(&self) -> &[T] { + unsafe { core::slice::from_raw_parts(self.data_ptr(), self.len()) } + } + + /// Returns true if this is the only existing handle to the buffer. + /// + /// When this function returns true, mutable methods and converting to a `Vector` + /// is very fast (does not involve additional memory allocations or copies). + #[inline] + pub fn is_unique(&self) -> bool { + unsafe { self.inner.as_ref().ref_count.get() == 1 } + } + + /// Clears the vector, removing all values. + pub fn clear(&mut self) + where + A: Clone, + { + if self.is_unique() { + unsafe { + raw::clear(self.data_ptr(), self.vec_header_mut()); + } + return; + } + + *self = + Self::try_with_capacity_in(self.capacity(), self.inner.allocator().clone()).unwrap(); + } + + /// Returns true if the two vectors share the same underlying storage. + pub fn ptr_eq(&self, other: &Self) -> bool { + self.inner.header == other.inner.header + } + + /// Allocates a duplicate of this buffer (infallible). + pub fn copy_buffer(&self) -> Self + where + T: Copy, + A: Clone, + { + self.try_copy_buffer().unwrap() + } + + /// Tries to allocate a duplicate of this buffer. + pub fn try_copy_buffer(&self) -> Result + where + T: Copy, + A: Clone, + { + unsafe { + let header = self.inner.as_ref(); + let len = header.vec.len; + let cap = header.vec.cap; + + if len > cap { + return Err(AllocError); + } + + let allocator = header.allocator.clone(); + let mut clone = Self::try_with_capacity_in(cap as usize, allocator)?; + + if len > 0 { + core::ptr::copy_nonoverlapping(self.data_ptr(), clone.data_ptr(), len as usize); + clone.vec_header_mut().len = len; + } + + Ok(clone) + } + } + + #[inline] + pub fn data_ptr(&self) -> *mut T { + unsafe { + (self.inner.as_ptr() as *mut u8).add(raw::header_size::, T>()) + as *mut T + } + } + + // SAFETY: call this only if the vector is unique. + pub(crate) unsafe fn vec_header_mut(&mut self) -> &mut raw::VecHeader { + &mut self.inner.as_mut().vec + } + + pub(crate) fn vec_header(&self) -> &raw::VecHeader { + unsafe { &self.inner.as_ref().vec } + } +} + +/// Mutable methods that can cause the vector to be cloned and therefore require both the items and +/// the allocator to be cloneable. +impl RefCountedVector { + /// Converts this RefCountedVector into an immutable one, allocating a new copy if there are other references. + #[inline] + pub fn into_unique(mut self) -> Vector { + self.ensure_unique(); + + unsafe { + let data = NonNull::new_unchecked(self.data_ptr()); + let header = self.vec_header().clone(); + let allocator = self.inner.as_ref().allocator.clone(); + + mem::forget(self); + + Vector { + raw: RawVector { data, header }, + allocator, + } + } + } + + /// Appends an element to the back of a collection. + /// + /// # Panics + /// + /// Panics if the new capacity exceeds `u32::MAX` bytes. + pub fn push(&mut self, val: T) { + self.reserve(1); + unsafe { + raw::push_assuming_capacity(self.data_ptr(), &mut self.vec_header_mut(), val); + } + } + + /// Removes the last element from the vector and returns it, or `None` if it is empty. + pub fn pop(&mut self) -> Option { + self.ensure_unique(); + + unsafe { raw::pop(self.data_ptr(), &mut self.vec_header_mut()) } + } + + /// Removes an element from the vector and returns it. + /// + /// The removed element is replaced by the last element of the vector. + /// + /// # Panics + /// + /// Panics if index is out of bounds. + #[inline] + pub fn swap_remove(&mut self, idx: usize) -> T { + self.ensure_unique(); + + let len = self.len(); + assert!(idx < len); + + unsafe { + let data_ptr = self.data_ptr(); + let ptr = data_ptr.add(idx); + let item = ptr::read(ptr); + + let last_idx = len - 1; + if idx != last_idx { + let last_ptr = data_ptr.add(last_idx); + ptr::write(ptr, ptr::read(last_ptr)); + } + + self.vec_header_mut().len = last_idx as BufferSize; + + item + } + } + + /// Appends an element if there is sufficient spare capacity, otherwise an error is returned + /// with the element. + /// + /// Like other mutable operations, this method may reallocate if the vector is not unique. + /// However it will not reallocate when there’s insufficient capacity. + /// The caller should use reserve or try_reserve to ensure that there is enough capacity. + pub fn push_within_capacity(&mut self, val: T) -> Result<(), T> { + if self.remaining_capacity() == 0 { + return Err(val); + } + + self.ensure_unique(); + unsafe { + raw::push_assuming_capacity(self.data_ptr(), &mut self.vec_header_mut(), val); + } + + Ok(()) + } + + /// Clones and appends the contents of the slice to the back of a collection. + pub fn extend_from_slice(&mut self, slice: &[T]) { + self.reserve(slice.len()); + unsafe { + raw::extend_from_slice_assuming_capacity(self.data_ptr(), self.vec_header_mut(), slice); + } + } + + /// Appends the contents of an iterator to the back of a collection. + pub fn extend(&mut self, data: impl IntoIterator) { + let mut iter = data.into_iter(); + + let (min, max) = iter.size_hint(); + self.reserve(max.unwrap_or(min)); + + unsafe { + if raw::extend_within_capacity(self.data_ptr(), self.vec_header_mut(), &mut iter) { + return; + } + } + + for item in iter { + self.push(item); + } + } + + /// Ensures this shared vector uniquely owns its storage, allocating a new copy + /// If there are other references to it. + /// + /// In principle this is mostly useful internally to provide safe mutable methods + /// as it does not observaly affect most of the shared vector behavior, however + /// it has a few niche use cases, for example to provoke copies earlier for more + /// predictable performance or in some unsafe endeavors. + #[inline] + pub fn ensure_unique(&mut self) { + if !self.is_unique() { + *self = self.try_clone_buffer(None).unwrap(); + } + } + + /// Extracts a mutable slice containing the entire vector. + /// + /// Like other mutable methods, this will clone the vector's storage + /// if it is not unique to ensure safe mutations. + #[inline] + pub fn as_mut_slice(&mut self) -> &mut [T] + where + T: Clone, + A: Clone, + { + self.ensure_unique(); + unsafe { core::slice::from_raw_parts_mut(self.data_ptr(), self.len()) } + } + + /// Allocates a duplicate of this buffer (infallible). + pub fn clone_buffer(&self) -> Self + where + T: Clone, + A: Clone, + { + self.try_clone_buffer(None).unwrap() + } + + fn try_clone_buffer(&self, new_cap: Option) -> Result + where + T: Clone, + A: Clone, + { + unsafe { + let header = self.inner.as_ref(); + let len = header.vec.len; + let cap = if let Some(cap) = new_cap { + cap + } else { + header.vec.cap + }; + let allocator = header.allocator.clone(); + + if len > cap { + return Err(AllocError); + } + + let mut clone = Self::try_with_capacity_in(cap as usize, allocator)?; + + raw::extend_from_slice_assuming_capacity( + clone.data_ptr(), + clone.vec_header_mut(), + self.as_slice(), + ); + + Ok(clone) + } + } + + /// Ensures the vector can be safely mutated and has enough extra capacity to + /// add `additional` more items. + /// + /// This will allocate new storage for the vector if the vector is not unique or if + /// the capacity is not sufficient to accomodate `self.len() + additional` items. + /// The vector may reserve more space to speculatively avoid frequent reallocations. + #[inline] + pub fn reserve(&mut self, additional: usize) { + let is_unique = self.is_unique(); + let enough_capacity = self.remaining_capacity() >= additional; + + if !is_unique || !enough_capacity { + // Hopefully the least common case. + self.try_realloc_additional(is_unique, enough_capacity, additional) + .unwrap(); + } + } + + /// Tries to reserve at least `additional` extra elements to be inserted in the given vector. + /// + /// The vector may reserve more space to speculatively avoid frequent reallocations. + /// After calling try_reserve, capacity will be greater than or equal to `self.len() + additional` + /// if it returns `Ok(())`. + /// Does nothing if capacity is already sufficient. This method preserves the contents even if an + /// error occurs. + pub fn try_reserve(&mut self, additional: usize) -> Result<(), AllocError> { + let is_unique = self.is_unique(); + let enough_capacity = self.remaining_capacity() >= additional; + + if !is_unique || !enough_capacity { + // Hopefully the least common case. + self.try_realloc_additional(is_unique, enough_capacity, additional)?; + } + + Ok(()) + } + + /// Reserves the minimum capacity for at least `additional` elements to be inserted in the given vector. + /// + /// Unlike `reserve`, this will not deliberately over-allocate to speculatively avoid frequent allocations. + /// After calling `try_reserve_exact`, capacity will be greater than or equal to `self.len() + additional` if + /// it returns `Ok(())`. + /// This will also allocate if the vector is not unique. + /// Does nothing if the capacity is already sufficient and the vector is unique. + /// + /// Note that the allocator may give the collection more space than it requests. Therefore, capacity can not + /// be relied upon to be precisely minimal. Prefer `try_reserve` if future insertions are expected. + pub fn reserve_exact(&mut self, additional: usize) { + self.try_reserve_exact(additional).unwrap(); + } + + /// Tries to reserve the minimum capacity for at least `additional` elements to be inserted in the given vector. + /// + /// Unlike `try_reserve`, this will not deliberately over-allocate to speculatively avoid frequent allocations. + /// After calling `reserve_exact`, capacity will be greater than or equal to `self.len() + additional`. + /// This will also allocate if the vector is not unique. + /// Does nothing if the capacity is already sufficient and the vector is unique. + /// + /// Note that the allocator may give the collection more space than it requests. Therefore, capacity can not + /// be relied upon to be precisely minimal. Prefer `try_reserve` if future insertions are expected. + pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), AllocError> { + let is_unique = self.is_unique(); + let enough_capacity = self.remaining_capacity() >= additional; + + if !is_unique || !enough_capacity { + // Hopefully the least common case. + self.try_realloc_with_capacity(is_unique, additional)?; + } + + Ok(()) + } + + /// Shrinks the capacity of the vector with a lower bound. + /// + /// The capacity will remain at least as large as both the length and the supplied value. + /// If the current capacity is less than the lower limit, this is a no-op. + pub fn shrink_to(&mut self, min_capacity: usize) { + let min_capacity = min_capacity.max(self.len()); + if self.capacity() <= min_capacity { + return; + } + + let is_unique = self.is_unique(); + self.try_realloc_with_capacity(is_unique, min_capacity) + .unwrap(); + } + + /// Shrinks the capacity of the vector as much as possible. + pub fn shrink_to_fit(&mut self) { + self.shrink_to(self.len()) + } + + /// Moves all the elements of `other` into `self`, leaving `other` empty. + /// + /// If `other is not unique, the elements are cloned instead of moved. + pub fn append(&mut self, other: &mut Self) { + self.reserve(other.len()); + + unsafe { + if other.is_unique() { + // Fast path: memcpy + raw::move_data( + other.data_ptr(), + &mut other.inner.header.as_mut().vec, + self.data_ptr(), + &mut self.inner.as_mut().vec, + ) + } else { + // Slow path, clone each item. + raw::extend_from_slice_assuming_capacity( + self.data_ptr(), + self.vec_header_mut(), + other.as_slice(), + ); + + *other = + Self::try_with_capacity_in(other.capacity(), self.inner.allocator().clone()) + .unwrap(); + } + } + } + + #[cold] + fn try_realloc_additional( + &mut self, + is_unique: bool, + enough_capacity: bool, + additional: usize, + ) -> Result<(), AllocError> { + let new_cap = if enough_capacity { + self.capacity() + } else { + grow_amortized(self.len(), additional) + }; + + self.try_realloc_with_capacity(is_unique, new_cap) + } + + #[cold] + fn try_realloc_with_capacity( + &mut self, + is_unique: bool, + new_cap: usize, + ) -> Result<(), AllocError> { + let allocator = self.inner.allocator().clone(); + if is_unique && self.capacity() > 0 { + // The buffer is not large enough, we'll have to create a new one, however we + // know that we have the only reference to it so we'll move the data with + // a simple memcpy instead of cloning it. + + unsafe { + use crate::shared_vector::raw::{buffer_layout, Header}; + let old_cap = self.capacity(); + let old_header = self.inner.header; + let old_layout = buffer_layout::, T>(old_cap).unwrap(); + let new_layout = buffer_layout::, T>(new_cap).unwrap(); + + let new_alloc = if new_layout.size() >= old_layout.size() { + allocator.grow(old_header.cast(), old_layout, new_layout) + } else { + allocator.shrink(old_header.cast(), old_layout, new_layout) + }?; + + self.inner.header = new_alloc.cast(); + self.inner.as_mut().vec.cap = new_cap as BufferSize; + + return Ok(()); + } + } + + // The slowest path, we pay for both the new allocation and the need to clone + // each item one by one. + let mut new_vec = Self::try_with_capacity_in(new_cap, allocator)?; + new_vec.extend_from_slice(self.as_slice()); + + mem::swap(self, &mut new_vec); + + Ok(()) + } + + // TODO: remove this one? + /// Returns the concatenation of two vectors. + pub fn concatenate(mut self, mut other: Self) -> Self + where + T: Clone, + A: Clone, + { + self.append(&mut other); + + self + } +} + +impl Drop for RefCountedVector { + fn drop(&mut self) { + unsafe { + if self.inner.as_ref().ref_count.release_ref() { + let header = self.vec_header().clone(); + // See the implementation of std Arc for the need to use this fence. Note that + // we only need it for the atomic reference counted version but I don't expect + // this to make a measurable difference. + core::sync::atomic::fence(Ordering::Acquire); + + raw::drop_items(self.data_ptr(), header.len); + raw::dealloc::(self.inner.header, header.cap); + } + } + } +} + +unsafe impl Send for AtomicSharedVector {} + +unsafe impl Sync for AtomicSharedVector {} + +impl Clone for RefCountedVector { + fn clone(&self) -> Self { + self.new_ref() + } +} + +impl, R: RefCount, A: Allocator> PartialEq> + for RefCountedVector +{ + fn eq(&self, other: &Self) -> bool { + self.ptr_eq(other) || self.as_slice() == other.as_slice() + } +} + +impl, R: RefCount, A: Allocator> PartialEq<&[T]> for RefCountedVector { + fn eq(&self, other: &&[T]) -> bool { + self.as_slice() == *other + } +} + +impl AsRef<[T]> for RefCountedVector { + fn as_ref(&self) -> &[T] { + self.as_slice() + } +} + +impl Default for RefCountedVector { + fn default() -> Self { + Self::new() + } +} + +impl<'a, T, R: RefCount, A: Allocator> IntoIterator for &'a RefCountedVector { + type Item = &'a T; + type IntoIter = core::slice::Iter<'a, T>; + fn into_iter(self) -> core::slice::Iter<'a, T> { + self.as_slice().iter() + } +} + +impl<'a, T: Clone, R: RefCount, A: Allocator + Clone> IntoIterator + for &'a mut RefCountedVector +{ + type Item = &'a mut T; + type IntoIter = core::slice::IterMut<'a, T>; + fn into_iter(self) -> core::slice::IterMut<'a, T> { + self.as_mut_slice().iter_mut() + } +} + +impl Index for RefCountedVector +where + R: RefCount, + A: Allocator, + I: core::slice::SliceIndex<[T]>, +{ + type Output = >::Output; + fn index(&self, index: I) -> &Self::Output { + self.as_slice().index(index) + } +} + +impl IndexMut for RefCountedVector +where + T: Clone, + R: RefCount, + A: Allocator + Clone, + I: core::slice::SliceIndex<[T]>, +{ + fn index_mut(&mut self, index: I) -> &mut Self::Output { + self.as_mut_slice().index_mut(index) + } +} + +impl Deref for RefCountedVector { + type Target = [T]; + fn deref(&self) -> &[T] { + self.as_slice() + } +} + +impl DerefMut for RefCountedVector { + fn deref_mut(&mut self) -> &mut [T] { + self.as_mut_slice() + } +} + +impl Debug for RefCountedVector { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> { + self.as_slice().fmt(f) + } +} + +impl From> for SharedVector { + fn from(vector: Vector) -> Self { + vector.into_shared() + } +} + +impl From> for AtomicSharedVector { + fn from(vector: Vector) -> Self { + vector.into_shared_atomic() + } +} + +// In order to give us a chance to catch leaks and double-frees, test with values that implement drop. +#[cfg(test)] +fn num(val: u32) -> Box { + Box::new(val) +} + +#[test] +fn basic_shared() { + basic_shared_impl::(); + basic_shared_impl::(); + + fn basic_shared_impl() { + let mut a: RefCountedVector, R> = RefCountedVector::with_capacity(64); + a.push(num(1)); + a.push(num(2)); + + let mut b = a.new_ref(); + b.push(num(4)); + + a.push(num(3)); + + assert_eq!(a.as_slice(), &[num(1), num(2), num(3)]); + assert_eq!(b.as_slice(), &[num(1), num(2), num(4)]); + + let popped = a.pop(); + assert_eq!(a.as_slice(), &[num(1), num(2)]); + assert_eq!(popped, Some(num(3))); + + let mut b2 = b.new_ref(); + let popped = b2.pop(); + assert_eq!(b2.as_slice(), &[num(1), num(2)]); + assert_eq!(popped, Some(num(4))); + + println!("concatenate"); + let c = a.concatenate(b2); + assert_eq!(c.as_slice(), &[num(1), num(2), num(1), num(2)]); + } +} + +#[test] +fn empty_buffer() { + let _: AtomicSharedVector = AtomicSharedVector::new(); + let _: AtomicSharedVector = AtomicSharedVector::new(); + + let _: SharedVector<()> = SharedVector::new(); + let _: SharedVector<()> = SharedVector::new(); + + let _: AtomicSharedVector<()> = AtomicSharedVector::new(); + let _: AtomicSharedVector<()> = AtomicSharedVector::new(); + + let _: Vector<()> = Vector::new(); +} + +#[test] +#[rustfmt::skip] +fn grow() { + let mut a = Vector::with_capacity(0); + + a.push(num(1)); + a.push(num(2)); + a.push(num(3)); + + a.extend_from_slice(&[num(4), num(5), num(6), num(7), num(8), num(9), num(10), num(12), num(12), num(13), num(14), num(15), num(16), num(17), num(18)]); + + assert_eq!( + a.as_slice(), + &[num(1), num(2), num(3), num(4), num(5), num(6), num(7), num(8), num(9), num(10), num(12), num(12), num(13), num(14), num(15), num(16), num(17), num(18)] + ); + + let mut b = SharedVector::new(); + b.push(num(1)); + b.push(num(2)); + b.push(num(3)); + + assert_eq!(b.as_slice(), &[num(1), num(2), num(3)]); + + let mut b = AtomicSharedVector::new(); + b.push(num(1)); + b.push(num(2)); + b.push(num(3)); + + assert_eq!(b.as_slice(), &[num(1), num(2), num(3)]); +} + +#[test] +fn ensure_unique_empty() { + let mut v: SharedVector = SharedVector::new(); + v.ensure_unique(); +} + +#[test] +fn shrink_to_zero() { + let mut v: SharedVector = SharedVector::new(); + v.shrink_to(0); +} diff --git a/src/shared_vector/splice.rs b/src/shared_vector/splice.rs new file mode 100644 index 0000000..c212645 --- /dev/null +++ b/src/shared_vector/splice.rs @@ -0,0 +1,140 @@ +use crate::shared_vector::alloc::{Allocator, Global}; +use core::ptr::{self}; +use core::slice::{self}; + +use crate::shared_vector::drain::Drain; + +/// A splicing iterator for `Vec`. +/// +/// This struct is created by [`Vec::splice()`]. +/// See its documentation for more. +/// +/// # Example +/// +/// ``` +/// let mut v = vec![0, 1, 2]; +/// let new = [7, 8]; +/// let iter: std::vec::Splice<_> = v.splice(1.., new); +/// ``` +#[derive(Debug)] +pub struct Splice<'a, I: Iterator + 'a, A: Allocator + 'a = Global> { + pub(crate) drain: Drain<'a, I::Item>, + pub(crate) replace_with: I, + pub(crate) allocator: &'a A, +} + +impl Iterator for Splice<'_, I, A> { + type Item = I::Item; + + fn next(&mut self) -> Option { + self.drain.next() + } + + fn size_hint(&self) -> (usize, Option) { + self.drain.size_hint() + } +} + +impl DoubleEndedIterator for Splice<'_, I, A> { + fn next_back(&mut self) -> Option { + self.drain.next_back() + } +} + +impl ExactSizeIterator for Splice<'_, I, A> {} + +impl Drop for Splice<'_, I, A> { + fn drop(&mut self) { + self.drain.by_ref().for_each(drop); + // At this point draining is done and the only remaining tasks are splicing + // and moving things into the final place. + // Which means we can replace the slice::Iter with pointers that won't point to deallocated + // memory, so that Drain::drop is still allowed to call iter.len(), otherwise it would break + // the ptr.sub_ptr contract. + self.drain.iter = (&[]).iter(); + + unsafe { + if self.drain.tail_len == 0 { + self.drain + .vec + .as_mut() + .extend(self.allocator, self.replace_with.by_ref()); + return; + } + + // First fill the range left by drain(). + if !self.drain.fill(&mut self.replace_with) { + return; + } + + // There may be more elements. Use the lower bound as an estimate. + // FIXME: Is the upper bound a better guess? Or something else? + let (lower_bound, _upper_bound) = self.replace_with.size_hint(); + if lower_bound > 0 { + self.drain.move_tail(self.allocator, lower_bound); + if !self.drain.fill(&mut self.replace_with) { + return; + } + } + + // Collect any remaining elements. + // This is a zero-length vector which does not allocate if `lower_bound` was exact. + let mut collected = self + .replace_with + .by_ref() + .collect::>() + .into_iter(); + // Now we have an exact count. + if collected.len() > 0 { + self.drain.move_tail(self.allocator, collected.len()); + let filled = self.drain.fill(&mut collected); + debug_assert!(filled); + debug_assert_eq!(collected.len(), 0); + } + } + // Let `Drain::drop` move the tail back if necessary and restore `vec.len`. + } +} + +/// Private helper methods for `Splice::drop` +impl Drain<'_, T> { + /// The range from `self.vec.len` to `self.tail_start` contains elements + /// that have been moved out. + /// Fill that range as much as possible with new elements from the `replace_with` iterator. + /// Returns `true` if we filled the entire range. (`replace_with.next()` didn’t return `None`.) + unsafe fn fill>(&mut self, replace_with: &mut I) -> bool { + let vec = unsafe { self.vec.as_mut() }; + let range_start = vec.header.len as usize; + let range_end = self.tail_start as usize; + let range_slice = unsafe { + slice::from_raw_parts_mut(vec.as_mut_ptr().add(range_start), range_end - range_start) + }; + + for place in range_slice { + if let Some(new_item) = replace_with.next() { + unsafe { ptr::write(place, new_item) }; + vec.header.len += 1; + } else { + return false; + } + } + true + } + + /// Makes room for inserting more elements before the tail. + unsafe fn move_tail(&mut self, allocator: &A, additional: usize) { + let vec = unsafe { self.vec.as_mut() }; + let len = self.tail_start + self.tail_len; + unsafe { + vec.try_reserve(allocator, len + additional).unwrap(); + } + + let new_tail_start = self.tail_start + additional; + unsafe { + let src = vec.as_ptr().add(self.tail_start); + let dst = vec.as_mut_ptr().add(new_tail_start); + ptr::copy(src, dst, self.tail_len); + } + self.tail_start = new_tail_start; + } +} diff --git a/src/shared_vector/vector.rs b/src/shared_vector/vector.rs new file mode 100644 index 0000000..8a94bf6 --- /dev/null +++ b/src/shared_vector/vector.rs @@ -0,0 +1,1636 @@ +use core::fmt::Debug; +use core::ops::RangeBounds; +use core::ops::{Deref, DerefMut, Index, IndexMut}; +use core::ptr::NonNull; +use core::{mem, ptr}; + +use crate::shared_vector::alloc::{AllocError, Allocator, Global}; +use crate::shared_vector::drain::Drain; +use crate::shared_vector::raw::{ + self, buffer_layout, move_data, AtomicRefCount, BufferSize, Header, HeaderBuffer, RefCount, + VecHeader, +}; +use crate::shared_vector::shared::{AtomicSharedVector, SharedVector}; +use crate::shared_vector::splice::Splice; +use crate::shared_vector::{grow_amortized, DefaultRefCount}; + +/// A heap allocated, mutable contiguous buffer containing elements of type `T`, with manual deallocation. +/// +/// +/// +/// See also `Vector`. +/// +/// This container is similar to this crate's `Vector` data structure with two key difference: +/// - It does store an allocator field. Instead, all methods that require interacting with an allocator are +/// marked unsafe and take the allocator as parameter. +/// - `RawVector`'s `Drop` implementation does not automatically deallocate the memory. Instead the memory +/// must be manually deallocated via the `deallocate` method. Dropping a raw vector without deallocating it +/// silently leaks the memory. +/// +/// `Vector` is implemented as a thin wrapper around this type. +/// +/// # Use cases +/// +/// In most cases, `Vector` is more appropriate. However in some situations it can be beneficial to not +/// store the allocator in the container. In complex data structures that contain many vectors, for example, +/// it may be preferable to store the allocator once at the root of the data structure than multiple times +/// in each of the internally managed vectors. +pub struct RawVector { + pub(crate) data: NonNull, + pub(crate) header: VecHeader, +} + +impl RawVector { + /// Creates an empty, unallocated raw vector. + pub fn new() -> Self { + RawVector { + data: NonNull::dangling(), + header: VecHeader { len: 0, cap: 0 }, + } + } + + /// Creates an empty pre-allocated vector with a given storage capacity. + /// + /// Does not allocate memory if `cap` is zero. + pub fn try_with_capacity( + allocator: &A, + cap: usize, + ) -> Result, AllocError> { + if cap == 0 { + return Ok(RawVector::new()); + } + + unsafe { + let (base_ptr, cap) = raw::allocate_header_buffer::(cap, allocator)?; + let data = NonNull::new_unchecked(raw::data_ptr::, T>( + base_ptr.cast(), + )); + Ok(RawVector { + data, + header: VecHeader { + cap: cap as BufferSize, + len: 0, + }, + }) + } + } + + pub fn try_from_slice(allocator: &A, data: &[T]) -> Result + where + T: Clone, + { + let mut v = Self::try_with_capacity(allocator, data.len())?; + unsafe { + v.extend_from_slice(allocator, data); + } + + Ok(v) + } + + /// Creates a raw vector with `n` clones of `elem`. + pub fn try_from_elem(allocator: &A, elem: T, n: usize) -> Result + where + T: Clone, + { + if n == 0 { + return Ok(Self::new()); + } + + let mut v = Self::try_with_capacity(allocator, n)?; + unsafe { + for _ in 0..(n - 1) { + v.push(allocator, elem.clone()) + } + + v.push(allocator, elem); + } + + Ok(v) + } + + /// Clears and deallocates this raw vector, leaving it in its unallocated state. + /// + /// It is safe (no-op) to call `deallocate` on a vector that is already in its unallocated state. + /// + /// # Safety + /// + /// The provided allocator must be the one this raw vector was created with. + pub unsafe fn deallocate(&mut self, allocator: &A) { + if self.header.cap == 0 { + return; + } + + self.clear(); + + self.deallocate_buffer(allocator); + + self.data = NonNull::dangling(); + self.header.cap = 0; + self.header.len = 0; + } + + pub unsafe fn deallocate_no_drop(&mut self, allocator: &A) { + if self.header.cap == 0 { + return; + } + + self.clear_without_drop(); + self.deallocate_buffer(allocator); + + self.data = NonNull::dangling(); + self.header.cap = 0; + self.header.len = 0; + } + + #[inline] + /// Returns `true` if the vector contains no elements. + pub fn is_empty(&self) -> bool { + self.header.len == 0 + } + + #[inline] + /// Returns the number of elements in the vector, also referred to as its ‘length’. + pub fn len(&self) -> usize { + self.header.len as usize + } + + #[inline] + /// Returns the total number of elements the vector can hold without reallocating. + pub fn capacity(&self) -> usize { + self.header.cap as usize + } + + /// Returns number of elements that can be added without reallocating. + #[inline] + pub fn remaining_capacity(&self) -> usize { + (self.header.cap - self.header.len) as usize + } + + #[inline] + fn data_ptr(&self) -> *mut T { + self.data.as_ptr() + } + + #[inline] + pub fn as_slice(&self) -> &[T] { + unsafe { core::slice::from_raw_parts(self.data_ptr(), self.len()) } + } + + #[inline] + pub fn as_mut_slice(&mut self) -> &mut [T] { + unsafe { core::slice::from_raw_parts_mut(self.data_ptr(), self.len()) } + } + + /// Clears the vector, removing all values. + pub fn clear(&mut self) { + unsafe { raw::clear(self.data_ptr(), &mut self.header) } + } + + pub fn clear_without_drop(&mut self) { + self.header.len = 0; + } + + unsafe fn base_ptr(&self, _allocator: &A) -> NonNull { + debug_assert!(self.header.cap > 0); + raw::header_from_data_ptr::, T>(self.data).cast() + } + + /// Appends an element to the back of a collection. + /// + /// # Safety + /// + /// The provided allocator must be the one this raw vector was created with. + /// + /// # Panics + /// + /// Panics if the new capacity exceeds `u32::MAX` bytes. + #[inline] + pub unsafe fn push(&mut self, allocator: &A, val: T) { + if self.header.len == self.header.cap { + self.try_realloc_additional(allocator, 1).unwrap(); + } + + raw::push_assuming_capacity(self.data_ptr(), &mut self.header, val); + } + + /// Appends an element if there is sufficient spare capacity, otherwise an error is returned + /// with the element. + /// + /// Unlike push this method will not reallocate when there’s insufficient capacity. + /// The caller should use reserve or try_reserve to ensure that there is enough capacity. + #[inline] + pub fn push_within_capacity(&mut self, val: T) -> Result<(), T> { + if self.header.len == self.header.cap { + return Err(val); + } + + unsafe { + let dst = self.data_ptr().add(self.header.len as usize); + self.header.len += 1; + ptr::write(dst, val); + } + + Ok(()) + } + + /// Removes the last element from the vector and returns it, or `None` if it is empty. + #[inline] + pub fn pop(&mut self) -> Option { + unsafe { raw::pop(self.data_ptr(), &mut self.header) } + } + + /// Removes and returns the element at position `index` within the vector, + /// shifting all elements after it to the left. + /// + /// # Panics + /// + /// Panics if `index` is out of bounds. + /// + pub fn remove(&mut self, index: usize) -> T { + #[cold] + #[inline(never)] + #[track_caller] + fn assert_failed(index: usize, len: usize) -> ! { + panic!("removal index (is {index}) should be < len (is {len})"); + } + + let len = self.len(); + if index >= len { + assert_failed(index, len); + } + unsafe { + // infallible + let ret; + { + // the place we are taking from. + let ptr = self.as_mut_ptr().add(index); + // copy it out, unsafely having a copy of the value on + // the stack and in the vector at the same time. + ret = ptr::read(ptr); + + // Shift everything down to fill in that spot. + ptr::copy(ptr.add(1), ptr, len - index - 1); + } + self.header.len = len as u32 - 1; + ret + } + } + + /// Removes an element from the vector and returns it. + /// + /// The removed element is replaced by the last element of the vector. + /// + /// # Panics + /// + /// Panics if index is out of bounds. + #[inline] + pub fn swap_remove(&mut self, idx: usize) -> T { + let len = self.len(); + assert!(idx < len); + + unsafe { + let ptr = self.data_ptr().add(idx); + let item = ptr::read(ptr); + + let last_idx = len - 1; + if idx != last_idx { + let last_ptr = self.data_ptr().add(last_idx); + ptr::write(ptr, ptr::read(last_ptr)); + } + + self.header.len -= 1; + + item + } + } + + /// Inserts an element at position `index` within the vector, shifting all + /// elements after it to the right. + /// + /// # Panics + /// + /// Panics if `index > len`. + pub unsafe fn insert(&mut self, allocator: &A, index: usize, element: T) { + #[cold] + #[inline(never)] + fn assert_failed(index: usize, len: usize) -> ! { + panic!("insertion index (is {index}) should be <= len (is {len})"); + } + + unsafe { + // space for the new element + if self.header.len == self.header.cap { + self.try_reserve(allocator, 1).unwrap(); + } + + let len = self.len(); + + // infallible + // The spot to put the new value + { + let p = self.as_mut_ptr().add(index); + if index < len { + // Shift everything over to make space. (Duplicating the + // `index`th element into two consecutive places.) + ptr::copy(p, p.add(1), len - index); + } else if index == len { + // No elements need shifting. + } else { + assert_failed(index, len); + } + // Write it in, overwriting the first copy of the `index`th + // element. + ptr::write(p, element); + } + self.header.len += 1; + } + } + + /// Clones and appends the contents of the slice to the back of a collection. + /// + /// # Safety + /// + /// The provided allocator must be the one this raw vector was created with. + pub unsafe fn extend_from_slice(&mut self, allocator: &A, slice: &[T]) + where + T: Clone, + { + self.try_reserve(allocator, slice.len()).unwrap(); + unsafe { + raw::extend_from_slice_assuming_capacity(self.data_ptr(), &mut self.header, slice); + } + } + + /// Moves all the elements of `other` into `self`, leaving `other` empty. + /// + /// # Safety + /// + /// The provided allocator must be the one this raw vector was created with. + pub unsafe fn append(&mut self, allocator: &A, other: &mut Self) + where + T: Clone, + { + if other.is_empty() { + return; + } + + self.try_reserve(allocator, other.len()).unwrap(); + + unsafe { + move_data( + other.data_ptr(), + &mut other.header, + self.data_ptr(), + &mut self.header, + ); + } + } + + /// Appends the contents of an iterator to the back of a collection. + /// + /// # Safety + /// + /// The provided allocator must be the one this raw vector was created with. + pub unsafe fn extend( + &mut self, + allocator: &A, + data: impl IntoIterator, + ) { + let mut iter = data.into_iter(); + let (min, max) = iter.size_hint(); + self.try_reserve(allocator, max.unwrap_or(min)).unwrap(); + unsafe { + self.extend_within_capacity(&mut iter); + + for item in iter { + self.push(allocator, item); + } + } + } + + unsafe fn extend_within_capacity(&mut self, iter: &mut impl Iterator) { + let n = self.remaining_capacity() as BufferSize; + + let mut ptr = self.data_ptr().add(self.len()); + let mut count = 0; + + unsafe { + for item in iter { + if count == n { + break; + } + ptr::write(ptr, item); + ptr = ptr.add(1); + count += 1; + } + self.header.len += count; + } + } + + /// Allocate a clone of this buffer. + /// + /// The provided allocator does not need to be the one this raw vector was created with. + /// The returned raw vector is considered to be created with the provided allocator. + pub fn clone_buffer(&self, allocator: &A) -> Self + where + T: Clone, + { + self.clone_buffer_with_capacity(allocator, self.len()) + } + + /// Allocate a clone of this buffer with a different capacity + /// + /// The capacity must be at least as large as the buffer's length. + pub fn clone_buffer_with_capacity(&self, allocator: &A, cap: usize) -> Self + where + T: Clone, + { + let mut clone = Self::try_with_capacity(allocator, cap.max(self.len())).unwrap(); + + unsafe { + raw::extend_from_slice_assuming_capacity( + clone.data_ptr(), + &mut clone.header, + self.as_slice(), + ); + } + + clone + } + + // Note: Marking this #[inline(never)] is a pretty large regression in the push benchmark. + #[cold] + unsafe fn try_realloc_additional( + &mut self, + allocator: &A, + additional: usize, + ) -> Result<(), AllocError> { + let new_cap = grow_amortized(self.len(), additional); + if new_cap < self.len() { + return Err(AllocError); + } + + self.try_realloc_with_capacity(allocator, new_cap) + } + + #[cold] + unsafe fn try_realloc_with_capacity( + &mut self, + allocator: &A, + new_cap: usize, + ) -> Result<(), AllocError> { + type R = DefaultRefCount; + + unsafe { + if new_cap == 0 { + self.deallocate_buffer(allocator); + } + + let new_layout = buffer_layout::, T>(new_cap).unwrap(); + + let new_alloc = if self.header.cap == 0 { + allocator.allocate(new_layout)? + } else { + let old_cap = self.capacity(); + let old_ptr = self.base_ptr(allocator); + let old_layout = buffer_layout::, T>(old_cap).unwrap(); + let new_layout = buffer_layout::, T>(new_cap).unwrap(); + + if new_layout.size() >= old_layout.size() { + allocator.grow(old_ptr, old_layout, new_layout) + } else { + allocator.shrink(old_ptr, old_layout, new_layout) + }? + }; + + let new_data_ptr = + crate::shared_vector::raw::data_ptr::, T>(new_alloc.cast()); + self.data = NonNull::new_unchecked(new_data_ptr); + self.header.cap = new_cap as u32; + } + + Ok(()) + } + + // Deallocates the memory, does not drop the vector's content. + unsafe fn deallocate_buffer(&mut self, allocator: &A) { + let layout = buffer_layout::, T>(self.capacity()).unwrap(); + let ptr = self.base_ptr(allocator); + + allocator.deallocate(ptr, layout); + + self.header.cap = 0; + self.header.len = 0; + self.data = NonNull::dangling(); + } + + /// Tries to reserve at least enough space for `additional` extra items. + /// + /// # Safety + /// + /// The provided allocator must be the one this raw vector was created with. + #[inline] + pub unsafe fn try_reserve( + &mut self, + allocator: &A, + additional: usize, + ) -> Result<(), AllocError> { + if self.remaining_capacity() < additional { + self.try_realloc_additional(allocator, additional)?; + } + + Ok(()) + } + + /// Tries to reserve the minimum capacity for at least `additional` elements to be inserted in the given vector. + /// + /// Unlike `try_reserve`, this will not deliberately over-allocate to speculatively avoid frequent allocations. + /// After calling `reserve_exact`, capacity will be greater than or equal to `self.len() + additional`. + /// This will also allocate if the vector is not unique. + /// Does nothing if the capacity is already sufficient and the vector is unique. + /// + /// Note that the allocator may give the collection more space than it requests. Therefore, capacity can not + /// be relied upon to be precisely minimal. Prefer `try_reserve` if future insertions are expected. + /// + /// # Safety + /// + /// The provided allocator must be the one this raw vector was created with. + pub unsafe fn try_reserve_exact( + &mut self, + allocator: &A, + additional: usize, + ) -> Result<(), AllocError> { + if self.remaining_capacity() >= additional { + return Ok(()); + } + + self.try_realloc_with_capacity(allocator, self.len() + additional) + } + + /// Shrinks the capacity of the vector with a lower bound. + /// + /// The capacity will remain at least as large as both the length and the supplied value. + /// If the current capacity is less than the lower limit, this is a no-op. + /// + /// # Safety + /// + /// The provided allocator must be the one this raw vector was created with. + pub unsafe fn shrink_to(&mut self, allocator: &A, min_capacity: usize) { + let min_capacity = min_capacity.max(self.len()); + if self.capacity() <= min_capacity { + return; + } + + self.try_realloc_with_capacity(allocator, min_capacity) + .unwrap(); + } + + /// Shrinks the capacity of the vector as much as possible. + /// + /// # Safety + /// + /// The provided allocator must be the one this raw vector was created with. + pub unsafe fn shrink_to_fit(&mut self, allocator: &A) { + self.shrink_to(allocator, self.len()) + } + + /// Removes the specified range from the vector in bulk, returning all + /// removed elements as an iterator. If the iterator is dropped before + /// being fully consumed, it drops the remaining removed elements. + /// + /// The returned iterator keeps a mutable borrow on the vector to optimize + /// its implementation. + /// + /// # Panics + /// + /// Panics if the starting point is greater than the end point or if + /// the end point is greater than the length of the vector. + /// + /// # Leaking + /// + /// If the returned iterator goes out of scope without being dropped (due to + /// [`mem::forget`], for example), the vector may have lost and leaked + /// elements arbitrarily, including elements outside the range. + /// + pub fn drain(&mut self, range: R) -> Drain<'_, T> + where + R: RangeBounds, + { + // Memory safety + // + // When the Drain is first created, it shortens the length of + // the source vector to make sure no uninitialized or moved-from elements + // are accessible at all if the Drain's destructor never gets to run. + // + // Drain will ptr::read out the values to remove. + // When finished, remaining tail of the vec is copied back to cover + // the hole, and the vector length is restored to the new length. + // + use core::ops::Bound::*; + let len = self.len(); + let end = match range.end_bound() { + Included(n) => *n + 1, + Excluded(n) => *n, + Unbounded => len, + }; + let start = match range.start_bound() { + Included(n) => *n, + Excluded(n) => *n + 1, + Unbounded => 0, + }; + assert!(end <= len); + assert!(start <= end); + + unsafe { + // Set self.vec length's to start, to be safe in case Drain is leaked + self.header.len = start as u32; + let range_slice = core::slice::from_raw_parts(self.as_ptr().add(start), end - start); + Drain { + tail_start: end, + tail_len: len - end, + iter: range_slice.iter(), + vec: NonNull::from(self), + } + } + } + + /// Creates a splicing iterator that replaces the specified range in the vector + /// with the given `replace_with` iterator and yields the removed items. + /// `replace_with` does not need to be the same length as `range`. + /// + /// `range` is removed even if the iterator is not consumed until the end. + /// + /// It is unspecified how many elements are removed from the vector + /// if the `Splice` value is leaked. + /// + /// The input iterator `replace_with` is only consumed when the `Splice` value is dropped. + /// + /// This is optimal if: + /// + /// * The tail (elements in the vector after `range`) is empty, + /// * or `replace_with` yields fewer or equal elements than `range`’s length + /// * or the lower bound of its `size_hint()` is exact. + /// + /// Otherwise, a temporary vector is allocated and the tail is moved twice. + /// + /// # Panics + /// + /// Panics if the starting point is greater than the end point or if + /// the end point is greater than the length of the vector. + /// + pub unsafe fn splice<'l, A, R, I>( + &'l mut self, + allocator: &'l A, + range: R, + replace_with: I, + ) -> Splice<'l, ::IntoIter, A> + where + A: Allocator, + R: RangeBounds, + I: IntoIterator, + { + Splice { + drain: self.drain(range), + replace_with: replace_with.into_iter(), + allocator, + } + } + + /// Retains only the elements specified by the predicate. + /// + /// In other words, remove all elements `e` for which `f(&e)` returns `false`. + /// This method operates in place, visiting each element exactly once in the + /// original order, and preserves the order of the retained elements. + pub fn retain(&mut self, mut f: F) + where + F: FnMut(&T) -> bool, + { + self.retain_mut(|elem| f(elem)); + } + + /// Retains only the elements specified by the predicate, passing a mutable reference to it. + /// + /// In other words, remove all elements `e` such that `f(&mut e)` returns `false`. + /// This method operates in place, visiting each element exactly once in the + /// original order, and preserves the order of the retained elements. + pub fn retain_mut(&mut self, mut f: F) + where + F: FnMut(&mut T) -> bool, + { + let original_len = self.len(); + // Avoid double drop if the drop guard is not executed, + // since we may make some holes during the process. + self.header.len = 0; + + // Vec: [Kept, Kept, Hole, Hole, Hole, Hole, Unchecked, Unchecked] + // |<- processed len ->| ^- next to check + // |<- deleted cnt ->| + // |<- original_len ->| + // Kept: Elements which predicate returns true on. + // Hole: Moved or dropped element slot. + // Unchecked: Unchecked valid elements. + // + // This drop guard will be invoked when predicate or `drop` of element panicked. + // It shifts unchecked elements to cover holes and `set_len` to the correct length. + // In cases when predicate and `drop` never panick, it will be optimized out. + struct BackshiftOnDrop<'a, T> { + v: &'a mut RawVector, + processed_len: usize, + deleted_cnt: usize, + original_len: usize, + } + + impl Drop for BackshiftOnDrop<'_, T> { + fn drop(&mut self) { + if self.deleted_cnt > 0 { + // SAFETY: Trailing unchecked items must be valid since we never touch them. + unsafe { + ptr::copy( + self.v.as_ptr().add(self.processed_len), + self.v + .as_mut_ptr() + .add(self.processed_len - self.deleted_cnt), + self.original_len - self.processed_len, + ); + } + } + // SAFETY: After filling holes, all items are in contiguous memory. + self.v.header.len = (self.original_len - self.deleted_cnt) as u32; + } + } + + let mut g = BackshiftOnDrop { + v: self, + processed_len: 0, + deleted_cnt: 0, + original_len, + }; + + fn process_loop( + original_len: usize, + f: &mut F, + g: &mut BackshiftOnDrop<'_, T>, + ) where + F: FnMut(&mut T) -> bool, + { + while g.processed_len != original_len { + // SAFETY: Unchecked element must be valid. + let cur = unsafe { &mut *g.v.as_mut_ptr().add(g.processed_len) }; + if !f(cur) { + // Advance early to avoid double drop if `drop_in_place` panicked. + g.processed_len += 1; + g.deleted_cnt += 1; + // SAFETY: We never touch this element again after dropped. + unsafe { ptr::drop_in_place(cur) }; + // We already advanced the counter. + if DELETED { + continue; + } else { + break; + } + } + if DELETED { + // SAFETY: `deleted_cnt` > 0, so the hole slot must not overlap with current element. + // We use copy for move, and never touch this element again. + unsafe { + let hole_slot = g.v.as_mut_ptr().add(g.processed_len - g.deleted_cnt); + ptr::copy_nonoverlapping(cur, hole_slot, 1); + } + } + g.processed_len += 1; + } + } + + // Stage 1: Nothing was deleted. + process_loop::(original_len, &mut f, &mut g); + + // Stage 2: Some elements were deleted. + process_loop::(original_len, &mut f, &mut g); + + // All item are processed. This can be optimized to `set_len` by LLVM. + drop(g); + } + + /// Transfers ownership of this raw vector's contents to the one that is returned, and leaves + /// this one empty and unallocated. + pub fn take(&mut self) -> Self { + mem::replace(self, RawVector::new()) + } +} + +impl> PartialEq> for RawVector { + fn eq(&self, other: &Self) -> bool { + self.as_slice() == other.as_slice() + } +} + +impl> PartialEq<&[T]> for RawVector { + fn eq(&self, other: &&[T]) -> bool { + self.as_slice() == *other + } +} + +impl Eq for RawVector {} + +impl AsRef<[T]> for RawVector { + fn as_ref(&self) -> &[T] { + self.as_slice() + } +} + +impl AsMut<[T]> for RawVector { + fn as_mut(&mut self) -> &mut [T] { + self.as_mut_slice() + } +} + +impl Default for RawVector { + fn default() -> Self { + Self::new() + } +} + +impl<'a, T> IntoIterator for &'a RawVector { + type Item = &'a T; + type IntoIter = core::slice::Iter<'a, T>; + fn into_iter(self) -> core::slice::Iter<'a, T> { + self.as_slice().iter() + } +} + +impl<'a, T> IntoIterator for &'a mut RawVector { + type Item = &'a mut T; + type IntoIter = core::slice::IterMut<'a, T>; + fn into_iter(self) -> core::slice::IterMut<'a, T> { + self.as_mut_slice().iter_mut() + } +} + +impl Index for RawVector +where + I: core::slice::SliceIndex<[T]>, +{ + type Output = >::Output; + fn index(&self, index: I) -> &Self::Output { + self.as_slice().index(index) + } +} + +impl IndexMut for RawVector +where + I: core::slice::SliceIndex<[T]>, +{ + fn index_mut(&mut self, index: I) -> &mut Self::Output { + self.as_mut_slice().index_mut(index) + } +} + +impl Deref for RawVector { + type Target = [T]; + fn deref(&self) -> &[T] { + self.as_slice() + } +} + +impl DerefMut for RawVector { + fn deref_mut(&mut self) -> &mut [T] { + self.as_mut_slice() + } +} + +impl Debug for RawVector { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> { + self.as_slice().fmt(f) + } +} + +impl core::hash::Hash for RawVector { + fn hash(&self, state: &mut H) + where + H: core::hash::Hasher, + { + self.as_slice().hash(state) + } +} + +/// A heap allocated, mutable contiguous buffer containing elements of type `T`. +/// +/// +/// +/// Similar in principle to a `Vec`. +/// It can be converted for free into a reference counted `SharedVector` or `AtomicSharedVector`. +/// +/// Unique and shared vectors expose similar functionality. `Vector` takes advantage of +/// the guaranteed uniqueness at the type level to provide overall faster operations than its +/// shared counterparts, while its memory layout makes it very cheap to convert to a shared vector +/// (involving no allocation or copy). +/// +/// # Internal representation +/// +/// `Vector` stores its length and capacity inline and points to the first element of the +/// allocated buffer. Room for a header is left uninitialized before the elements so that the +/// vector can be converted into a `SharedVector` or `AtomicSharedVector` without reallocating +/// the storage. +/// +/// Internally, `Vector` is built on top of `RawVector`. +pub struct Vector { + pub(crate) raw: RawVector, + pub(crate) allocator: A, +} + +impl Vector { + /// Creates an empty vector. + /// + /// This does not allocate memory. + pub fn new() -> Vector { + Vector { + raw: RawVector::new(), + allocator: Global, + } + } + + /// Creates an empty pre-allocated vector with a given storage capacity. + /// + /// Does not allocate memory if `cap` is zero. + pub fn with_capacity(cap: usize) -> Vector { + Self::try_with_capacity(cap).unwrap() + } + + /// Creates an empty pre-allocated vector with a given storage capacity. + /// + /// Does not allocate memory if `cap` is zero. + pub fn try_with_capacity(cap: usize) -> Result, AllocError> { + Vector::try_with_capacity_in(cap, Global) + } + + pub fn from_slice(data: &[T]) -> Self + where + T: Clone, + { + Vector { + raw: RawVector::try_from_slice(&Global, data).unwrap(), + allocator: Global, + } + } + + /// Creates a vector with `n` clones of `elem`. + pub fn from_elem(elem: T, n: usize) -> Vector + where + T: Clone, + { + Vector { + raw: RawVector::try_from_elem(&Global, elem, n).unwrap(), + allocator: Global, + } + } +} + +impl Vector { + /// Creates an empty vector without allocating memory. + pub fn new_in(allocator: A) -> Self { + Self::try_with_capacity_in(0, allocator).unwrap() + } + + /// Creates an empty pre-allocated vector with a given storage capacity. + /// + /// Does not allocate memory if `cap` is zero. + pub fn with_capacity_in(cap: usize, allocator: A) -> Self { + Self::try_with_capacity_in(cap, allocator).unwrap() + } + + /// Creates an empty pre-allocated vector with a given storage capacity. + /// + /// Does not allocate memory if `cap` is zero. + pub fn try_with_capacity_in(cap: usize, allocator: A) -> Result, AllocError> { + let raw = RawVector::try_with_capacity(&allocator, cap)?; + + Ok(Vector { raw, allocator }) + } + + #[inline(always)] + /// Returns `true` if the vector contains no elements. + pub fn is_empty(&self) -> bool { + self.raw.is_empty() + } + + #[inline(always)] + /// Returns the number of elements in the vector, also referred to as its ‘length’. + pub fn len(&self) -> usize { + self.raw.len() + } + + #[inline(always)] + /// Returns the total number of elements the vector can hold without reallocating. + pub fn capacity(&self) -> usize { + self.raw.capacity() + } + + /// Returns number of elements that can be added without reallocating. + #[inline(always)] + pub fn remaining_capacity(&self) -> usize { + self.raw.remaining_capacity() + } + + /// Returns a reference to the underlying allocator. + #[inline(always)] + pub fn allocator(&self) -> &A { + &self.allocator + } + + #[inline(always)] + pub fn as_slice(&self) -> &[T] { + self.raw.as_slice() + } + + #[inline(always)] + pub fn as_mut_slice(&mut self) -> &mut [T] { + self.raw.as_mut_slice() + } + + /// Clears the vector, removing all values. + pub fn clear(&mut self) { + unsafe { raw::clear(self.raw.data_ptr(), &mut self.raw.header) } + } + + unsafe fn into_header_buffer(mut self) -> HeaderBuffer + where + R: RefCount, + { + debug_assert!(self.raw.header.cap != 0); + unsafe { + let mut header = raw::header_from_data_ptr(self.raw.data); + + *header.as_mut() = raw::Header { + vec: VecHeader { + len: self.raw.header.len, + cap: self.raw.header.cap, + }, + ref_count: R::new(1), + allocator: ptr::read(&mut self.allocator), + }; + + mem::forget(self); + + HeaderBuffer::from_raw(header) + } + } + + /// Make this vector immutable. + /// + /// This operation is cheap, the underlying storage does not not need + /// to be reallocated. + #[inline] + pub fn into_shared(self) -> SharedVector + where + A: Allocator + Clone, + { + if self.raw.header.cap == 0 { + return SharedVector::try_with_capacity_in(0, self.allocator.clone()).unwrap(); + } + unsafe { + let inner = self.into_header_buffer::(); + SharedVector { inner } + } + } + + /// Make this vector immutable. + /// + /// This operation is cheap, the underlying storage does not not need + /// to be reallocated. + #[inline] + pub fn into_shared_atomic(self) -> AtomicSharedVector + where + A: Allocator + Clone, + { + if self.raw.header.cap == 0 { + return AtomicSharedVector::try_with_capacity_in(0, self.allocator.clone()).unwrap(); + } + unsafe { + let inner = self.into_header_buffer::(); + AtomicSharedVector { inner } + } + } + + /// Appends an element to the back of a collection. + /// + /// # Panics + /// + /// Panics if the new capacity exceeds `u32::MAX` bytes. + #[inline(always)] + pub fn push(&mut self, val: T) { + unsafe { + self.raw.push(&self.allocator, val); + } + } + + /// Appends an element if there is sufficient spare capacity, otherwise an error is returned + /// with the element. + /// + /// Unlike push this method will not reallocate when there’s insufficient capacity. + /// The caller should use reserve or try_reserve to ensure that there is enough capacity. + #[inline(always)] + pub fn push_within_capacity(&mut self, val: T) -> Result<(), T> { + self.raw.push_within_capacity(val) + } + + /// Removes the last element from the vector and returns it, or `None` if it is empty. + #[inline(always)] + pub fn pop(&mut self) -> Option { + self.raw.pop() + } + + /// Removes and returns the element at position `index` within the vector, + /// shifting all elements after it to the left. + /// + /// # Panics + /// + /// Panics if `index` is out of bounds. + /// + #[inline(always)] + pub fn remove(&mut self, index: usize) -> T { + self.raw.remove(index) + } + + /// Removes an element from the vector and returns it. + /// + /// The removed element is replaced by the last element of the vector. + /// + /// # Panics + /// + /// Panics if index is out of bounds. + #[inline(always)] + pub fn swap_remove(&mut self, idx: usize) -> T { + self.raw.swap_remove(idx) + } + + /// Inserts an element at position `index` within the vector, shifting all + /// elements after it to the right. + /// + /// # Panics + /// + /// Panics if `index > len`. + #[inline(always)] + pub fn insert(&mut self, index: usize, element: T) { + unsafe { self.raw.insert(&self.allocator, index, element) } + } + + /// Clones and appends the contents of the slice to the back of a collection. + #[inline(always)] + pub fn extend_from_slice(&mut self, data: &[T]) + where + T: Clone, + { + unsafe { self.raw.extend_from_slice(&self.allocator, data) } + } + + /// Moves all the elements of `other` into `self`, leaving `other` empty. + #[inline(always)] + pub fn append(&mut self, other: &mut Self) + where + T: Clone, + { + unsafe { self.raw.append(&self.allocator, &mut other.raw) } + } + + /// Appends the contents of an iterator to the back of a collection. + #[inline(always)] + pub fn extend(&mut self, data: impl IntoIterator) { + unsafe { self.raw.extend(&self.allocator, data) } + } + + /// Allocates a clone of this buffer. + #[inline(always)] + pub fn clone_buffer(&self) -> Self + where + T: Clone, + A: Clone, + { + Vector { + raw: self.raw.clone_buffer(&self.allocator), + allocator: self.allocator.clone(), + } + } + + /// Allocate a clone of this buffer with a different capacity + /// + /// The capacity must be at least as large as the buffer's length. + #[inline(always)] + pub fn clone_buffer_with_capacity(&self, cap: usize) -> Self + where + T: Clone, + A: Clone, + { + Vector { + raw: self.raw.clone_buffer_with_capacity(&self.allocator, cap), + allocator: self.allocator.clone(), + } + } + + #[inline(always)] + pub fn reserve(&mut self, additional: usize) { + unsafe { self.raw.try_reserve(&self.allocator, additional).unwrap() } + } + + #[inline(always)] + pub fn try_reserve(&mut self, additional: usize) -> Result<(), AllocError> { + unsafe { self.raw.try_reserve(&self.allocator, additional) } + } + + /// Reserves the minimum capacity for at least `additional` elements to be inserted in the given vector. + /// + /// Unlike `reserve`, this will not deliberately over-allocate to speculatively avoid frequent allocations. + /// After calling `try_reserve_exact`, capacity will be greater than or equal to `self.len() + additional` if + /// it returns `Ok(())`. + /// This will also allocate if the vector is not unique. + /// Does nothing if the capacity is already sufficient and the vector is unique. + /// + /// Note that the allocator may give the collection more space than it requests. Therefore, capacity can not + /// be relied upon to be precisely minimal. Prefer `try_reserve` if future insertions are expected. + pub fn reserve_exact(&mut self, additional: usize) + where + T: Clone, + { + self.try_reserve_exact(additional).unwrap(); + } + + /// Tries to reserve the minimum capacity for at least `additional` elements to be inserted in the given vector. + /// + /// Unlike `try_reserve`, this will not deliberately over-allocate to speculatively avoid frequent allocations. + /// After calling `reserve_exact`, capacity will be greater than or equal to `self.len() + additional`. + /// This will also allocate if the vector is not unique. + /// Does nothing if the capacity is already sufficient and the vector is unique. + /// + /// Note that the allocator may give the collection more space than it requests. Therefore, capacity can not + /// be relied upon to be precisely minimal. Prefer `try_reserve` if future insertions are expected. + pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), AllocError> { + unsafe { self.raw.try_reserve_exact(&self.allocator, additional) } + } + + /// Shrinks the capacity of the vector with a lower bound. + /// + /// The capacity will remain at least as large as both the length and the supplied value. + /// If the current capacity is less than the lower limit, this is a no-op. + #[inline(always)] + pub fn shrink_to(&mut self, min_capacity: usize) + where + T: Clone, + { + unsafe { self.raw.shrink_to(&self.allocator, min_capacity) } + } + + /// Shrinks the capacity of the vector as much as possible. + #[inline(always)] + pub fn shrink_to_fit(&mut self) + where + T: Clone, + { + unsafe { self.raw.shrink_to_fit(&self.allocator) } + } + + /// Removes the specified range from the vector in bulk, returning all + /// removed elements as an iterator. If the iterator is dropped before + /// being fully consumed, it drops the remaining removed elements. + /// + /// The returned iterator keeps a mutable borrow on the vector to optimize + /// its implementation. + /// + /// # Panics + /// + /// Panics if the starting point is greater than the end point or if + /// the end point is greater than the length of the vector. + /// + /// # Leaking + /// + /// If the returned iterator goes out of scope without being dropped (due to + /// [`mem::forget`], for example), the vector may have lost and leaked + /// elements arbitrarily, including elements outside the range. + /// + pub fn drain(&mut self, range: R) -> Drain<'_, T> + where + R: RangeBounds, + { + self.raw.drain(range) + } + + /// Creates a splicing iterator that replaces the specified range in the vector + /// with the given `replace_with` iterator and yields the removed items. + /// `replace_with` does not need to be the same length as `range`. + /// + /// `range` is removed even if the iterator is not consumed until the end. + /// + /// It is unspecified how many elements are removed from the vector + /// if the `Splice` value is leaked. + /// + /// The input iterator `replace_with` is only consumed when the `Splice` value is dropped. + /// + /// This is optimal if: + /// + /// * The tail (elements in the vector after `range`) is empty, + /// * or `replace_with` yields fewer or equal elements than `range`’s length + /// * or the lower bound of its `size_hint()` is exact. + /// + /// Otherwise, a temporary vector is allocated and the tail is moved twice. + /// + /// # Panics + /// + /// Panics if the starting point is greater than the end point or if + /// the end point is greater than the length of the vector. + /// + pub fn splice( + &mut self, + range: R, + replace_with: I, + ) -> Splice<'_, ::IntoIter, A> + where + R: RangeBounds, + I: IntoIterator, + { + unsafe { self.raw.splice(&self.allocator, range, replace_with) } + } + + /// Retains only the elements specified by the predicate. + /// + /// In other words, remove all elements `e` for which `f(&e)` returns `false`. + /// This method operates in place, visiting each element exactly once in the + /// original order, and preserves the order of the retained elements. + pub fn retain(&mut self, f: F) + where + F: FnMut(&T) -> bool, + { + self.raw.retain(f) + } + + /// Retains only the elements specified by the predicate, passing a mutable reference to it. + /// + /// In other words, remove all elements `e` such that `f(&mut e)` returns `false`. + /// This method operates in place, visiting each element exactly once in the + /// original order, and preserves the order of the retained elements. + pub fn retain_mut(&mut self, f: F) + where + F: FnMut(&mut T) -> bool, + { + self.raw.retain_mut(f) + } + + #[inline(always)] + pub fn take(&mut self) -> Self + where + A: Clone, + { + let other = Vector { + raw: self.raw.take(), + allocator: self.allocator.clone(), + }; + + other + } +} + +impl Drop for Vector { + fn drop(&mut self) { + unsafe { self.raw.deallocate(&self.allocator) } + } +} + +impl Clone for Vector { + fn clone(&self) -> Self { + self.clone_buffer() + } +} + +impl, A: Allocator> PartialEq> for Vector { + fn eq(&self, other: &Self) -> bool { + self.as_slice() == other.as_slice() + } +} + +impl, A: Allocator> PartialEq<&[T]> for Vector { + fn eq(&self, other: &&[T]) -> bool { + self.as_slice() == *other + } +} + +impl AsRef<[T]> for Vector { + fn as_ref(&self) -> &[T] { + self.as_slice() + } +} + +impl AsMut<[T]> for Vector { + fn as_mut(&mut self) -> &mut [T] { + self.as_mut_slice() + } +} + +impl Default for Vector { + fn default() -> Self { + Self::new() + } +} + +impl<'a, T, A: Allocator> IntoIterator for &'a Vector { + type Item = &'a T; + type IntoIter = core::slice::Iter<'a, T>; + fn into_iter(self) -> core::slice::Iter<'a, T> { + self.as_slice().iter() + } +} + +impl<'a, T, A: Allocator> IntoIterator for &'a mut Vector { + type Item = &'a mut T; + type IntoIter = core::slice::IterMut<'a, T>; + fn into_iter(self) -> core::slice::IterMut<'a, T> { + self.as_mut_slice().iter_mut() + } +} + +impl Index for Vector +where + I: core::slice::SliceIndex<[T]>, +{ + type Output = >::Output; + fn index(&self, index: I) -> &Self::Output { + self.as_slice().index(index) + } +} + +impl IndexMut for Vector +where + I: core::slice::SliceIndex<[T]>, +{ + fn index_mut(&mut self, index: I) -> &mut Self::Output { + self.as_mut_slice().index_mut(index) + } +} + +impl Deref for Vector { + type Target = [T]; + fn deref(&self) -> &[T] { + self.as_slice() + } +} + +impl DerefMut for Vector { + fn deref_mut(&mut self) -> &mut [T] { + self.as_mut_slice() + } +} + +impl Debug for Vector { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> { + self.as_slice().fmt(f) + } +} + +impl From> for Vector { + fn from(shared: SharedVector) -> Self { + shared.into_unique() + } +} + +impl From> for Vector { + fn from(shared: AtomicSharedVector) -> Self { + shared.into_unique() + } +} + +impl core::hash::Hash for Vector { + fn hash(&self, state: &mut H) + where + H: core::hash::Hasher, + { + self.as_slice().hash(state) + } +} + +#[test] +fn basic_unique() { + fn num(val: u32) -> Box { + Box::new(val) + } + + let mut a = Vector::with_capacity(256); + + a.push(num(0)); + a.push(num(1)); + a.push(num(2)); + + let a = a.into_shared(); + + assert_eq!(a.len(), 3); + + assert_eq!(a.as_slice(), &[num(0), num(1), num(2)]); + + assert!(a.is_unique()); + + let b = Vector::from_slice(&[num(0), num(1), num(2), num(3), num(4)]); + + assert_eq!(b.as_slice(), &[num(0), num(1), num(2), num(3), num(4)]); + + let c = a.clone_buffer(); + assert!(!c.ptr_eq(&a)); + + let a2 = a.new_ref(); + assert!(a2.ptr_eq(&a)); + assert!(!a.is_unique()); + assert!(!a2.is_unique()); + + mem::drop(a2); + + assert!(a.is_unique()); + + let _ = c.clone_buffer(); + let _ = b.clone_buffer(); + + let mut d = Vector::with_capacity(64); + d.extend_from_slice(&[num(0), num(1), num(2)]); + d.extend_from_slice(&[]); + d.extend_from_slice(&[num(3), num(4)]); + + assert_eq!(d.as_slice(), &[num(0), num(1), num(2), num(3), num(4)]); +} + +#[test] +fn shrink() { + let mut v: Vector = Vector::with_capacity(32); + v.shrink_to(8); +} + +#[test] +fn zst() { + let mut v = Vector::new(); + v.push(()); + v.push(()); + v.push(()); + v.push(()); + + assert_eq!(v.len(), 4); +} + +#[test] +fn dyn_allocator() { + let allocator: &dyn Allocator = &Global; + let mut v = crate::vector!([1u32, 2, 3] in allocator); + + v.push(4); + + assert_eq!(&v[..], &[1, 2, 3, 4]); +} + +#[test] +fn borrowd_dyn_alloc() { + struct DataStructure<'a> { + data: Vector, + } + + impl DataStructure<'static> { + fn new() -> DataStructure<'static> { + DataStructure { + data: Vector::new_in(&Global as &'static dyn Allocator), + } + } + } + + impl<'a> DataStructure<'a> { + fn new_in(allocator: &'a dyn Allocator) -> DataStructure<'a> { + DataStructure { + data: Vector::new_in(allocator), + } + } + + fn push(&mut self, val: u32) { + self.data.push(val); + } + } + + let mut ds1 = DataStructure::new(); + ds1.push(1); + + let alloc = Global; + let mut ds2 = DataStructure::new_in(&alloc); + ds2.push(2); +} + +#[test] +fn splice1() { + let mut vec = Vector::new(); + vec.splice(0..0, vec![Box::new(1); 5].into_iter()); + vec.splice(0..0, vec![Box::new(2); 5].into_iter()); +} + +#[test] +fn drain1() { + let mut vectors: [Vector>; 4] = + [Vector::new(), Vector::new(), Vector::new(), Vector::new()]; + vectors[0].shrink_to(3906369431118283232); + vectors[2].extend_from_slice(&[Box::new(1), Box::new(2), Box::new(3)]); + let vec = &mut vectors[2]; + let len = vec.len(); + let start = if len > 0 { + 16059518370053021184 % len + } else { + 0 + }; + let end = 16059518370053021185.min(len); + vectors[2].drain(start..end); +} diff --git a/src/unrolled.rs b/src/unrolled.rs index f430ff7..3a91d8b 100644 --- a/src/unrolled.rs +++ b/src/unrolled.rs @@ -1,6 +1,7 @@ #[cfg(test)] mod proptests; +use crate::shared_vector::{AtomicSharedVector, Vector}; use smallvec::SmallVec; use crate::shared::PointerFamily; @@ -9,84 +10,31 @@ use std::cmp::Ordering; use std::iter::{FlatMap, FromIterator, Rev}; use std::marker::PhantomData; -type ConsumingIter = FlatMap< +type ConsumingIter = FlatMap< NodeIter, // Rev>>, - MaybeCloned, - fn(UnrolledList) -> MaybeCloned, // Rev>>, + MaybeCloned, + fn(UnrolledList) -> MaybeCloned, // Rev>>, >; -enum MaybeCloned { - Owned(Rev>>), - Cloned(OwnedNodeIterator), -} - -impl Iterator - for MaybeCloned -{ - type Item = T; +type MaybeCloned = + Rev>>; - #[inline(always)] - fn next(&mut self) -> Option { - match self { - MaybeCloned::Owned(o) => o.next(), - MaybeCloned::Cloned(r) => r.next(), - } - } - - #[inline(always)] - fn size_hint(&self) -> (usize, Option) { - match self { - MaybeCloned::Owned(o) => o.size_hint(), - MaybeCloned::Cloned(r) => r.size_hint(), - } - } - - #[inline(always)] - fn fold(self, init: B, f: F) -> B - where - Self: Sized, - F: FnMut(B, Self::Item) -> B, - { - match self { - MaybeCloned::Owned(o) => o.fold(init, f), - MaybeCloned::Cloned(r) => r.fold(init, f), - } - } - - #[inline] - fn nth(&mut self, n: usize) -> Option { - match self { - MaybeCloned::Owned(o) => o.nth(n), - MaybeCloned::Cloned(r) => r.nth(n), - } - } - - #[inline] - fn find(&mut self, predicate: F) -> Option - where - F: FnMut(&Self::Item) -> bool, - { - match self { - MaybeCloned::Owned(o) => o.find(predicate), - MaybeCloned::Cloned(r) => r.find(predicate), - } - } -} - -type RefIter<'a, T, P, const N: usize, const G: usize> = FlatMap< +type RefIter<'a, T, P, const N: u32, const G: u32> = FlatMap< NodeIterRef<'a, T, P, N, G>, Rev>, fn(&'a UnrolledList) -> Rev>, >; -type DrainingConsumingIter = FlatMap< +type DrainingConsumingIter = FlatMap< DrainingNodeIter, - Rev>>, - fn(UnrolledList) -> Rev>>, + // Rev>>, + Rev>>, + // fn(UnrolledList) -> Rev>>, + fn(UnrolledList) -> Rev>>, >; -fn empty_list( +fn empty_list( ) -> P::Pointer> where P::Pointer>: 'static, @@ -104,20 +52,18 @@ where #[derive(Eq)] #[repr(transparent)] -pub struct UnrolledList( +pub struct UnrolledList( pub(crate) P::Pointer>, ); -impl Clone - for UnrolledList -{ +impl Clone for UnrolledList { fn clone(&self) -> Self { Self(P::clone(&self.0)) } } // Check if these lists are equivalent via the iterator -impl PartialEq +impl PartialEq for UnrolledList { fn eq(&self, other: &Self) -> bool { @@ -125,7 +71,7 @@ impl Par } } -impl PartialOrd +impl PartialOrd for UnrolledList { fn partial_cmp(&self, other: &Self) -> Option { @@ -133,15 +79,13 @@ impl Pa } } -impl Default - for UnrolledList -{ +impl Default for UnrolledList { fn default() -> Self { Self::new() } } -impl UnrolledList { +impl UnrolledList { pub fn new() -> Self { // UnrolledList(P::new(UnrolledCell::new())) @@ -167,7 +111,8 @@ impl UnrolledList bool { - P::ptr_eq(&self.0.elements, &other.0.elements) && self.0.index == other.0.index + // P::ptr_eq(&self.0.elements, &other.0.elements) && self.0.index == other.0.index + self.0.elements.ptr_eq(&other.0.elements) && self.0.index == other.0.index } pub fn as_ptr_usize(&self) -> usize { @@ -175,7 +120,8 @@ impl UnrolledList usize { - P::as_ptr(&self.0.elements) as usize + // P::as_ptr(&self.0.elements) as usize + self.0.elements.as_ptr() as usize } pub fn next_ptr_as_usize(&self) -> Option { @@ -183,7 +129,7 @@ impl UnrolledList impl Iterator { - self.0.elements.iter().take(self.index()).rev() + self.0.elements.iter().take(self.index() as _).rev() } pub fn draining_iterator(self) -> DrainingConsumingWrapper { @@ -192,11 +138,22 @@ impl UnrolledList UnrolledList usize { - self.node_iter().map(|node| node.index()).sum() + self.node_iter().map(|node| node.index() as usize).sum() } // [0 1 2 3 4 5] -> [6 7 8 9 10] @@ -219,25 +176,30 @@ impl UnrolledList UnrolledList self.0.index && self.0.next.is_none() { + if count > self.0.index() && self.0.next.is_none() { return self.clone(); } for mut node in self.clone().into_node_iter() { - if count < node.0.index { + if count < node.0.index() { let inner = P::make_mut(&mut node.0); // this is the new tail, point to the end inner.next = None; // We want to chop off whatever we need to - let elements_mut = P::make_mut(&mut inner.elements); + // let elements_mut = P::make_mut(&mut inner.elements); + + inner.elements.ensure_unique(); // Grab the end of the vector, this will be the new backing - let remaining = elements_mut.split_off(inner.index - count); - inner.index = count; - *elements_mut = remaining; + // let remaining = elements_mut.split_off(inner.index - count); + + let remaining = split_off(&mut inner.elements, inner.index as usize - count); + + inner.index = count as _; + + // *elements_mut = remaining; + + inner.elements = remaining; + nodes.push(node); break; } else { // Note: We might want to truncate the remaining // elements of the vector. - count -= node.0.index; + count -= node.0.index(); nodes.push(node); } } @@ -318,14 +289,14 @@ impl UnrolledList UnrolledList self.size() - 1 { + if self.elements().len() > self.size() as usize - 1 { // Always initialize a vec with half the capacity of the previous one - let mut vec = Vec::with_capacity(self.size() / 2); + let mut vec = Vector::with_capacity(self.size() as usize / 2); vec.push(value); // Make dummy node // return reference to this new node let mut default = UnrolledList(P::new(UnrolledCell { index: 1, - elements: P::new(vec), + elements: vec.into_shared_atomic(), next: Some(self.clone()), size: self.size() * UnrolledCell::::GROWTH_RATE, })); @@ -372,16 +345,23 @@ impl UnrolledList { - match P::get_mut(&mut inner.elements) { - Some(reference) => { - reference.push(value); - inner.index += 1; - } - // Just check if its bigger than half, point to it. - None => { - self.slow_path_new_node(value); - } + if inner.elements.is_unique() { + inner.elements.push(value); + inner.index += 1; + } else { + self.slow_path_new_node(value); } + + // match P::get_mut(&mut inner.elements) { + // Some(reference) => { + // reference.push(value); + // inner.index += 1; + // } + // // Just check if its bigger than half, point to it. + // None => { + // self.slow_path_new_node(value); + // } + // } } None => { self.slow_path_new_node(value); @@ -391,13 +371,13 @@ impl UnrolledList self.size() / 2 { - let mut vec = Vec::with_capacity(N); + if self.elements().len() > self.size() as usize / 2 { + let mut vec = Vector::with_capacity(N as _); vec.push(value); let mut default = UnrolledList(P::new(UnrolledCell { index: 1, - elements: P::new(vec), + elements: vec.into_shared_atomic(), next: Some(self.clone()), size: N, })); @@ -418,9 +398,9 @@ impl UnrolledList Option { let cell = P::make_mut(&mut self.0); - let elements = P::make_mut(&mut cell.elements); - - let ret = elements.pop(); + // let elements = P::make_mut(&mut cell.elements); + // let ret = elements.pop(); + let ret = cell.elements.pop(); if ret.is_some() { cell.index -= 1; @@ -484,13 +464,13 @@ impl UnrolledList usize { + fn size(&self) -> u32 { self.0.size } #[cfg(test)] fn does_node_satisfy_invariant(&self) -> bool { - self.elements().len() <= self.size() + self.elements().len() <= self.size() as usize } #[cfg(test)] @@ -523,7 +503,7 @@ impl UnrolledList impl Iterator { self.node_iter() - .flat_map(|x| x.elements()[0..x.index()].iter().rev()) + .flat_map(|x| x.elements()[0..x.index() as usize].iter().rev()) } // Every node must have either CAPACITY elements, or be marked as full @@ -534,19 +514,19 @@ impl UnrolledList Option<&T> { - if index < self.0.index { - self.0.elements.get(self.0.index - index - 1) + if index < self.0.index() { + self.0.elements.get(self.0.index() - index - 1) } else { let mut cur = self.0.next.as_ref(); - index -= self.0.index; + index -= self.0.index(); while let Some(node) = cur { - if index < node.0.index { + if index < node.0.index() { let node_cap = node.0.index; - return node.0.elements.get(node_cap - index - 1); + return node.0.elements.get(node_cap as usize - index - 1); } else { cur = node.0.next.as_ref(); - index -= node.0.index; + index -= node.0.index(); } } @@ -606,13 +586,13 @@ impl UnrolledList usize { + pub fn index(&self) -> u32 { self.0.index } } // Don't blow the stack -impl Drop +impl Drop for UnrolledCell { fn drop(&mut self) { @@ -630,27 +610,25 @@ impl Drop } #[repr(C)] -pub struct UnrolledCell { - index: usize, - pub(crate) elements: P::Pointer>, +pub struct UnrolledCell { + index: u32, + pub(crate) elements: AtomicSharedVector, pub(crate) next: Option>, - size: usize, + size: u32, } -impl Clone - for UnrolledCell -{ +impl Clone for UnrolledCell { fn clone(&self) -> Self { Self { index: self.index, - elements: P::clone(&self.elements), + elements: self.elements.clone(), next: self.next.clone(), size: self.size, } } } -impl std::fmt::Debug +impl std::fmt::Debug for UnrolledList { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -658,22 +636,26 @@ impl UnrolledCell { - const GROWTH_RATE: usize = if G == 0 { 1 } else { G }; +impl UnrolledCell { + const GROWTH_RATE: u32 = if G == 0 { 1 } else { G }; fn new() -> Self { UnrolledCell { index: 0, - elements: P::new(Vec::new()), + elements: AtomicSharedVector::new(), next: None, size: N, } } + fn index(&self) -> usize { + self.index as _ + } + fn new_with_capacity() -> Self { UnrolledCell { index: 0, - elements: P::new(Vec::with_capacity(N)), + elements: AtomicSharedVector::with_capacity(N as _), next: None, size: N, } @@ -684,7 +666,7 @@ impl UnrolledCell UnrolledCell Self { UnrolledCell { index: self.index - 1, - elements: P::clone(&self.elements), + elements: self.elements.clone(), next: self.next.clone(), size: self.size, } @@ -708,8 +690,9 @@ impl UnrolledCell UnrolledCell) -> UnrolledList { let size = cdr.size(); - if cdr.elements().len() > size - 1 { + if cdr.elements().len() > size as usize - 1 { UnrolledList(P::new(UnrolledCell { index: 1, - elements: P::new(vec![value]), + // elements: P::new(vec![value]), + elements: crate::arc_vector![value], next: Some(cdr), size: size * Self::GROWTH_RATE, })) } else { let inner = P::make_mut(&mut cdr.0); - let elements = P::make_mut(&mut inner.elements); + // let elements = P::make_mut(&mut inner.elements); inner.index += 1; - elements.push(value); + // elements.push(value); + + inner.elements.push(value); cdr } } } -impl Extend +impl Extend for UnrolledList { fn extend>(&mut self, iter: I) { @@ -744,17 +730,13 @@ impl Extend } } -pub(crate) struct DrainingNodeIter< - T: Clone + 'static, - P: PointerFamily, - const N: usize, - const G: usize, -> { +pub(crate) struct DrainingNodeIter +{ cur: Option>, _inner: PhantomData, } -impl Iterator +impl Iterator for DrainingNodeIter { type Item = UnrolledList; @@ -764,7 +746,8 @@ impl Iterator if let Some(mut _self) = std::mem::take(&mut self.cur) { if let Some(next) = _self.0.next.as_ref() { // If we can, drop these values! - if next.strong_count() == 1 && P::strong_count(&next.0.elements) == 1 { + // if next.strong_count() == 1 && P::strong_count(&next.0.elements) == 1 { + if next.strong_count() == 1 && next.0.elements.is_unique() { // self.cur = _self.0.next.clone(); self.cur = P::get_mut(&mut _self.0).and_then(|x| x.next.take()); } else { @@ -814,12 +797,12 @@ impl Iterator } } -pub(crate) struct NodeIter { +pub(crate) struct NodeIter { cur: Option>, _inner: PhantomData, } -impl Iterator for NodeIter { +impl Iterator for NodeIter { type Item = UnrolledList; fn next(&mut self) -> Option { if let Some(_self) = std::mem::take(&mut self.cur) { @@ -831,18 +814,13 @@ impl Iterator for No } } -pub(crate) struct NodeIterRef< - 'a, - T: Clone + 'static, - P: PointerFamily, - const N: usize, - const G: usize, -> { +pub(crate) struct NodeIterRef<'a, T: Clone + 'static, P: PointerFamily, const N: u32, const G: u32> +{ cur: Option<&'a UnrolledList>, _inner: PhantomData, } -impl<'a, T: Clone + 'static, P: PointerFamily, const N: usize, const G: usize> Iterator +impl<'a, T: Clone + 'static, P: PointerFamily, const N: u32, const G: u32> Iterator for NodeIterRef<'a, T, P, N, G> { type Item = &'a UnrolledList; @@ -860,11 +838,11 @@ impl<'a, T: Clone + 'static, P: PointerFamily, const N: usize, const G: usize> I pub struct DrainingConsumingWrapper< T: Clone + 'static, P: PointerFamily, - const N: usize, - const G: usize, + const N: u32, + const G: u32, >(DrainingConsumingIter); -impl Iterator +impl Iterator for DrainingConsumingWrapper { type Item = T; @@ -889,11 +867,11 @@ impl Iterator } } -pub struct ConsumingWrapper( +pub struct ConsumingWrapper( ConsumingIter, ); -impl Iterator +impl Iterator for ConsumingWrapper { type Item = T; @@ -918,27 +896,27 @@ impl Iterator } } -struct OwnedNodeIterator { - list: UnrolledCell, -} - -impl Iterator - for OwnedNodeIterator -{ - type Item = T; - - fn next(&mut self) -> Option { - let value = self.list.car().cloned(); - if self.list.index > 0 { - self.list.index -= 1; - value - } else { - None - } - } -} - -impl IntoIterator +// struct OwnedNodeIterator { +// list: UnrolledCell, +// } + +// impl Iterator +// for OwnedNodeIterator +// { +// type Item = T; + +// fn next(&mut self) -> Option { +// let value = self.list.car().cloned(); +// if self.list.index > 0 { +// self.list.index -= 1; +// value +// } else { +// None +// } +// } +// } + +impl IntoIterator for UnrolledList { type Item = T; @@ -956,6 +934,7 @@ impl IntoIterator // Otherwise, we end up with a worst of both worlds, since // we need to reallocate the whole vec _just_ to get owned // references. + /* let cell = P::make_mut(&mut x.0); match P::get_mut(&mut cell.elements) { @@ -967,6 +946,21 @@ impl IntoIterator list: cell.to_owned(), }), // None => cell.elements.iter().take(x.index()).rev(), } + */ + + let cell = P::make_mut(&mut x.0); + + let v = std::mem::take(&mut cell.elements); + + let unique = v.into_unique(); + + unique.into_iter().take(x.index() as _).rev() + + // cell.elements.into_unique() + + // MaybeCloned::Owned(v.into_unique().into_iter().cloned().take(x.index()).rev()) + + // todo!() // let vec = P::make_mut(&mut cell.elements); @@ -977,11 +971,11 @@ impl IntoIterator } // TODO have this also expose TryFold -pub struct IterWrapper<'a, T: Clone + 'static, P: PointerFamily, const N: usize, const G: usize>( +pub struct IterWrapper<'a, T: Clone + 'static, P: PointerFamily, const N: u32, const G: u32>( RefIter<'a, T, P, N, G>, ); -impl<'a, T: Clone, P: PointerFamily, const N: usize, const G: usize> Iterator +impl<'a, T: Clone, P: PointerFamily, const N: u32, const G: u32> Iterator for IterWrapper<'a, T, P, N, G> { type Item = &'a T; @@ -1006,7 +1000,7 @@ impl<'a, T: Clone, P: PointerFamily, const N: usize, const G: usize> Iterator } } -impl<'a, T: Clone, P: PointerFamily, const N: usize, const G: usize> IntoIterator +impl<'a, T: Clone, P: PointerFamily, const N: u32, const G: u32> IntoIterator for &'a UnrolledList { type Item = &'a T; @@ -1016,12 +1010,12 @@ impl<'a, T: Clone, P: PointerFamily, const N: usize, const G: usize> IntoIterato fn into_iter(self) -> Self::IntoIter { IterWrapper( self.node_iter() - .flat_map(|x| x.elements()[0..x.index()].iter().rev()), + .flat_map(|x| x.elements()[0..x.index() as _].iter().rev()), ) } } -struct ExponentialChunks +struct ExponentialChunks where I: Iterator, { @@ -1031,7 +1025,7 @@ where running_sum: usize, } -impl ExponentialChunks +impl ExponentialChunks where I: Iterator, { @@ -1039,7 +1033,7 @@ where let mut running_sum = size; while running_sum < length { - size *= G; + size *= G as usize; running_sum += size; } @@ -1052,11 +1046,11 @@ where } } -impl Iterator for ExponentialChunks +impl Iterator for ExponentialChunks where I: Iterator, { - type Item = (usize, Vec); + type Item = (usize, Vector); fn next(&mut self) -> Option { let chunk_size = if self.length > self.running_sum { @@ -1067,7 +1061,7 @@ where let iter = self.iter.by_ref().take(chunk_size); - let mut chunk = Vec::with_capacity(iter.size_hint().1.unwrap_or(0)); + let mut chunk = Vector::with_capacity(iter.size_hint().1.unwrap_or(0)); for item in iter { chunk.push(item); @@ -1080,29 +1074,29 @@ where let result = chunk; let size = self.size; - self.size /= G; + self.size /= G as usize; self.length -= result.len(); Some((size, result)) } } -fn from_vec( +fn from_vec( vec: Vec, ) -> UnrolledList { let length = vec.len(); let mut pairs: SmallVec<[UnrolledList<_, _, N, G>; 16]> = - ExponentialChunks::<_, N, G>::new(vec.into_iter(), length, N) + ExponentialChunks::<_, N, G>::new(vec.into_iter(), length, N as usize) .map(|(size, x)| { let mut elements = x; elements.reverse(); UnrolledList(P::new(UnrolledCell { - index: elements.len(), - elements: P::new(elements), + index: elements.len() as u32, + elements: elements.into_shared_atomic(), next: None, - size, + size: size as u32, })) }) .collect(); @@ -1127,7 +1121,7 @@ fn from_vec( // and we'll implement FromIterator // TODO specialize this for the into version? -impl FromIterator +impl FromIterator for UnrolledList { fn from_iter>(iter: I) -> Self { @@ -1136,8 +1130,8 @@ impl FromIterator } } -impl - FromIterator> for UnrolledList +impl FromIterator> + for UnrolledList { fn from_iter>>(iter: I) -> Self { // Links up the nodes @@ -1152,30 +1146,37 @@ impl if let Some(UnrolledList(cell)) = nodes.get_mut(i) { // Check if this node can fit entirely into the previous one - if cell.elements.len() + prev.0.elements.len() <= prev.0.size { + if cell.elements.len() + prev.0.elements.len() <= prev.0.size as usize { let left_inner = P::make_mut(cell); let right_inner = P::make_mut(&mut prev.0); - let left_vector = P::make_mut(&mut left_inner.elements); - let right_vector = P::make_mut(&mut right_inner.elements); + // let left_vector = P::make_mut(&mut left_inner.elements); + // let right_vector = P::make_mut(&mut right_inner.elements); // Drop the useless elements - if left_inner.index < left_vector.len() { - left_vector.truncate(left_inner.index); + if left_inner.index() < left_inner.elements.len() { + // left_vector.truncate(left_inner.index); + + truncate(&mut left_inner.elements, left_inner.index as _); } // TODO - if right_inner.index < right_vector.len() { - right_vector.truncate(right_inner.index); + if right_inner.index() < right_inner.elements.len() { + // right_vector.truncate(right_inner.index); + + truncate(&mut right_inner.elements, right_inner.index as _); } // Perform the actual move of the values - right_vector.append(left_vector); + // right_vector.append(left_vector); + + right_inner.elements.append(&mut left_inner.elements); // Swap the locations now after we've done the update - std::mem::swap(left_vector, right_vector); + std::mem::swap(&mut left_inner.elements, &mut right_inner.elements); + // Adjust the indices accordingly - left_inner.index = left_vector.len(); + left_inner.index = left_inner.elements.len() as u32; right_inner.index = 0; // Update this node to now point to the right nodes tail @@ -1192,7 +1193,7 @@ impl } } -impl<'a, T: 'a + Clone, P: 'a + PointerFamily, const N: usize, const G: usize> +impl<'a, T: 'a + Clone, P: 'a + PointerFamily, const N: u32, const G: u32> FromIterator<&'a UnrolledList> for UnrolledList { fn from_iter>>(iter: I) -> Self { @@ -1200,7 +1201,7 @@ impl<'a, T: 'a + Clone, P: 'a + PointerFamily, const N: usize, const G: usize> } } -impl From> +impl From> for UnrolledList { fn from(vec: Vec) -> Self { @@ -1208,7 +1209,7 @@ impl From> } } -impl From<&[T]> +impl From<&[T]> for UnrolledList { fn from(vec: &[T]) -> Self { @@ -1216,6 +1217,26 @@ impl From<&[T]> } } +fn truncate(value: &mut AtomicSharedVector, index: usize) { + for _ in 0..value.len() - index { + value.pop(); + } +} + +fn split_off(value: &mut AtomicSharedVector, index: usize) -> AtomicSharedVector { + let capacity = value.len() - index; + let mut new = Vector::with_capacity(index); + for i in index..value.len() { + let value = value.get_mut(i).unwrap(); + new.push(unsafe { std::ptr::read(value) }) + } + for _ in 0..capacity { + std::mem::forget(value.pop()); + } + + new.into_shared_atomic() +} + #[cfg(test)] mod tests { @@ -1455,7 +1476,6 @@ mod iterator_tests { fn take() { let list: RcList = (0..2 * CAPACITY).into_iter().collect(); let next = list.take(100); - assert!(Iterator::eq(0..100usize, next.into_iter())) }