refactor(transformer): move common stack functionality into StackCommon trait (#6114)

`Stack` and `NonEmptyStack` contained a lot of the same logic. Move shared logic into a `StackCommon` trait that they both implement.

Also split out core allocation logic into non-generic free functions for faster compile time.
This commit is contained in:
overlookmotel 2024-09-27 16:48:36 +00:00
parent 98390594d2
commit c50500ec42
4 changed files with 389 additions and 308 deletions

View file

@ -0,0 +1,281 @@
#![expect(clippy::unnecessary_safety_comment)]
use std::{
alloc::{self, Layout},
mem::{align_of, size_of},
ptr::{self, NonNull},
};
use assert_unchecked::assert_unchecked;
use super::StackCapacity;
pub trait StackCommon<T>: StackCapacity<T> {
// Getter setter methods defined by implementer
fn start(&self) -> NonNull<T>;
fn end(&self) -> NonNull<T>;
fn cursor(&self) -> NonNull<T>;
fn set_start(&mut self, start: NonNull<T>);
fn set_end(&mut self, end: NonNull<T>);
fn set_cursor(&mut self, cursor: NonNull<T>);
/// Make allocation of `capacity_bytes` bytes, aligned for `T`.
///
/// # Panics
/// Panics if out of memory (or may abort, depending on global allocator's behavior).
///
/// # SAFETY
/// * `capacity_bytes` must not be 0.
/// * `capacity_bytes` must be a multiple of `mem::size_of::<T>()`.
/// * `capacity_bytes` must not exceed `Self::MAX_CAPACITY_BYTES`.
#[inline]
unsafe fn allocate(capacity_bytes: usize) -> (NonNull<T>, NonNull<T>) {
// SAFETY: Caller guarantees `capacity_bytes` satisfies requirements
let layout = Self::layout_for(capacity_bytes);
let (start, end) = allocate(layout);
// SAFETY: `start` and `end` are `NonNull` - just casting them
let start = NonNull::new_unchecked(start.as_ptr().cast::<T>());
let end = NonNull::new_unchecked(end.as_ptr().cast::<T>());
(start, end)
}
/// Grow allocation.
///
/// `start` and `end` are set to the start and end of new allocation.
/// `current` is set so distance from `start` is old `capacity_bytes`.
/// This is where it should be if stack was previously full to capacity.
///
/// # Panics
/// Panics if stack is already at maximum capacity.
///
/// # SAFETY
/// Stack must have already allocated. i.e. `start` is not a dangling pointer.
#[inline]
unsafe fn grow(&mut self) {
// Grow allocation.
// SAFETY: Caller guarantees stack has allocated.
// `start` and `end` are boundaries of that allocation (`alloc` and `grow` ensure that).
// So `old_start_ptr` and `old_layout` accurately describe the current allocation.
// `grow` creates new allocation with byte size double what it currently is, or caps it
// at `MAX_CAPACITY_BYTES`.
// Old capacity in bytes was a multiple of `size_of::<T>()`, so double that must be too.
// `MAX_CAPACITY_BYTES` is also a multiple of `size_of::<T>()`.
// So new capacity in bytes must be a multiple of `size_of::<T>()`.
// `MAX_CAPACITY_BYTES <= isize::MAX`.
let old_start_ptr = NonNull::new_unchecked(self.start().as_ptr().cast::<u8>());
let old_layout = Self::layout_for(self.capacity_bytes());
let (start, end, current) = grow(old_start_ptr, old_layout, Self::MAX_CAPACITY_BYTES);
// Update pointers.
// SAFETY: `start`, `end`, and `current` are all `NonNull` - just casting them.
// All pointers returned from `grow` are aligned for `T`.
// Old capacity and new capacity in bytes are both multiples of `size_of::<T>()`,
// so distances `end - start` and `current - start` are both multiples of `size_of::<T>()`.
self.set_start(NonNull::new_unchecked(start.as_ptr().cast::<T>()));
self.set_end(NonNull::new_unchecked(end.as_ptr().cast::<T>()));
self.set_cursor(NonNull::new_unchecked(current.as_ptr().cast::<T>()));
}
/// Deallocate stack memory.
///
/// Note: Does *not* drop the contents of the stack (the `T`s), only the memory allocated
/// by `allocate` / `grow`. If stack is not empty, also call `drop_contents()` before calling this.
///
/// # SAFETY
/// Stack must have already allocated. i.e. `start` is not a dangling pointer.
#[inline]
unsafe fn deallocate(&self) {
// SAFETY: Caller guarantees stack is allocated.
// `start` and `end` are boundaries of that allocation (`allocate` and `grow` ensure that).
// So `start` and `layout` accurately describe the current allocation.
let layout = Self::layout_for(self.capacity_bytes());
alloc::dealloc(self.start().as_ptr().cast::<u8>(), layout);
}
/// Drop contents of stack.
///
/// This function will be optimized out if `T` is non-drop, as `drop_in_place` calls
/// `std::mem::needs_drop` internally and is a no-op if it returns true.
///
/// # SAFETY
/// * Stack must be allocated.
/// * Stack must contain `len` initialized entries, starting at `self.start()`.
#[inline]
unsafe fn drop_contents(&self, len: usize) {
// Drop contents. Next line copied from `std`'s `Vec`.
// SAFETY: Caller guarantees stack contains `len` initialized entries, starting at `start`.
ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.start().as_ptr(), len));
}
/// Get layout for allocation of `capacity_bytes` bytes.
///
/// # SAFETY
/// * `capacity_bytes` must not be 0.
/// * `capacity_bytes` must be a multiple of `mem::size_of::<T>()`.
/// * `capacity_bytes` must not exceed `Self::MAX_CAPACITY_BYTES`.
#[inline]
unsafe fn layout_for(capacity_bytes: usize) -> Layout {
// `capacity_bytes` must not be 0 because cannot make 0-size allocations.
debug_assert!(capacity_bytes > 0);
// `capacity_bytes` must be a multiple of `size_of::<T>()` so that `cursor == end`
// checks in `push` methods accurately detects when full to capacity
debug_assert!(capacity_bytes % size_of::<T>() == 0);
// `capacity_bytes` must not exceed `Self::MAX_CAPACITY_BYTES` to prevent creating an allocation
// of illegal size
debug_assert!(capacity_bytes <= Self::MAX_CAPACITY_BYTES);
// SAFETY: `align_of::<T>()` trivially satisfies alignment requirements.
// Caller guarantees `capacity_bytes <= MAX_CAPACITY_BYTES`.
// `MAX_CAPACITY_BYTES` takes into account the rounding-up by alignment requirement.
Layout::from_size_align_unchecked(capacity_bytes, align_of::<T>())
}
/// Get offset of `cursor` in number of `T`s.
///
/// # SAFETY
/// * `self.cursor()` and `self.start()` must be derived from same pointer.
/// * `self.cursor()` must be `>= self.start()`.
/// * Byte distance between `self.cursor()` and `self.start()` must be a multiple of `size_of::<T>()`.
unsafe fn cursor_offset(&self) -> usize {
// `offset_from` returns offset in units of `T`.
// SAFETY: Caller guarantees `cursor` and `start` are derived from same pointer.
// This implies that both pointers are always within bounds of a single allocation.
// Caller guarantees `cursor >= start`.
// Caller guarantees distance between pointers is a multiple of `size_of::<T>()`.
// `assert_unchecked!` is to help compiler to optimize.
// See: https://doc.rust-lang.org/std/primitive.pointer.html#method.sub_ptr
#[expect(clippy::cast_sign_loss)]
unsafe {
assert_unchecked!(self.cursor() >= self.start());
self.cursor().as_ptr().offset_from(self.start().as_ptr()) as usize
}
}
/// Get capacity.
#[inline]
fn capacity(&self) -> usize {
// SAFETY: `allocate` and `grow` both ensure:
// * `start` and `end` are both derived from same pointer
// * `start` and `end` are both within bounds of a single allocation.
// * `end` is always >= `start`.
// * Distance between `start` and `end` is always a multiple of `size_of::<T>()`.
// `assert_unchecked!` is to help compiler to optimize.
// See: https://doc.rust-lang.org/std/primitive.pointer.html#method.sub_ptr
#[expect(clippy::cast_sign_loss)]
unsafe {
assert_unchecked!(self.end() >= self.start());
self.end().as_ptr().offset_from(self.start().as_ptr()) as usize
}
}
/// Get capacity in bytes.
#[inline]
fn capacity_bytes(&self) -> usize {
// SAFETY: `allocate` and `grow` both ensure:
// * `start` and `end` are both derived from same pointer
// * `start` and `end` are both within bounds of a single allocation.
// * `end` is always >= `start`.
// * Distance between `start` and `end` is always a multiple of `size_of::<T>()`.
// `assert_unchecked!` is to help compiler to optimize.
// See: https://doc.rust-lang.org/std/primitive.pointer.html#method.sub_ptr
#[expect(clippy::cast_sign_loss)]
unsafe {
assert_unchecked!(self.end() >= self.start());
self.end().as_ptr().byte_offset_from(self.start().as_ptr()) as usize
}
}
}
/// Make allocation of with provided layout.
///
/// This is a separate non-generic function to improve compile time
/// (same pattern as `std::vec::Vec` uses).
///
/// # Panics
/// Panics if out of memory (or may abort, depending on global allocator's behavior).
///
/// # SAFETY
/// `layout` must have non-zero size.
#[inline]
unsafe fn allocate(layout: Layout) -> (/* start */ NonNull<u8>, /* end */ NonNull<u8>) {
// SAFETY: Caller guarantees `layout` has non-zero-size
let ptr = alloc::alloc(layout);
if ptr.is_null() {
alloc::handle_alloc_error(layout);
}
// SAFETY: We checked `ptr` is non-null
let start = NonNull::new_unchecked(ptr);
// SAFETY: We allocated `layout.size()` bytes, so `end` is end of allocation
let end = NonNull::new_unchecked(ptr.add(layout.size()));
(start, end)
}
/// Grow existing allocation.
///
/// Grow by doubling size, with high bound of `max_capacity_bytes`.
///
/// # SAFETY
/// * `old_start` and `old_layout` must describe an existing allocation.
/// * `max_capacity_bytes` must be `>= old_layout.size()`.
/// * `max_capacity_bytes` must be `<= isize::MAX`.
unsafe fn grow(
old_start: NonNull<u8>,
old_layout: Layout,
max_capacity_bytes: usize,
) -> (/* start */ NonNull<u8>, /* end */ NonNull<u8>, /* current */ NonNull<u8>) {
// Get new capacity
// Capacity in bytes cannot be larger than `isize::MAX`, so `* 2` cannot overflow
let old_capacity_bytes = old_layout.size();
let mut new_capacity_bytes = old_capacity_bytes * 2;
if new_capacity_bytes > max_capacity_bytes {
assert!(old_capacity_bytes < max_capacity_bytes, "Cannot grow beyond `Self::MAX_CAPACITY`");
new_capacity_bytes = max_capacity_bytes;
}
debug_assert!(new_capacity_bytes > old_capacity_bytes);
// Reallocate.
// SAFETY: Caller guarantees `old_start` and `old_layout` describe an existing allocation.
// Caller guarantees that `max_capacity_bytes <= isize::MAX`.
// `new_capacity_bytes` is capped above at `max_capacity_bytes`, so is a legal allocation size.
// `start` and `end` are boundaries of that allocation (`alloc` and `grow` ensure that).
// So `start` and `old_layout` accurately describe the current allocation.
// `old_capacity_bytes` was a multiple of `size_of::<T>()`, so double that must be too.
// `MAX_CAPACITY_BYTES` is also a multiple of `size_of::<T>()`.
// So `new_capacity_bytes` must be a multiple of `size_of::<T>()`.
// `new_capacity_bytes` is `<= MAX_CAPACITY_BYTES`, so is a legal allocation size.
// `layout_for` produces a layout with `T`'s alignment, so `new_ptr` is aligned for `T`.
let new_ptr = unsafe {
let old_ptr = old_start.as_ptr();
let new_ptr = alloc::realloc(old_ptr, old_layout, new_capacity_bytes);
if new_ptr.is_null() {
let new_layout =
Layout::from_size_align_unchecked(old_capacity_bytes, old_layout.align());
alloc::handle_alloc_error(new_layout);
}
new_ptr
};
// Update pointers.
//
// Stack was full to capacity, so new last index after push is the old capacity.
// i.e. `new_cursor - new_start == old_end - old_start`.
// Note: All pointers need to be updated even if allocation grew in place.
// From docs for `GlobalAlloc::realloc`:
// "Any access to the old `ptr` is Undefined Behavior, even if the allocation remained in-place."
// <https://doc.rust-lang.org/std/alloc/trait.GlobalAlloc.html#method.realloc>
// `end` changes whatever happens, so always need to be updated.
// `cursor` needs to be derived from `start` to make `offset_from` valid, so also needs updating.
//
// SAFETY: We checked that `new_ptr` is non-null.
// `old_capacity_bytes < new_capacity_bytes` (ensured above), so `new_cursor` must be in bounds.
let new_start = NonNull::new_unchecked(new_ptr);
let new_end = NonNull::new_unchecked(new_ptr.add(new_capacity_bytes));
let new_cursor = NonNull::new_unchecked(new_ptr.add(old_capacity_bytes));
(new_start, new_end, new_cursor)
}

View file

@ -1,9 +1,11 @@
mod capacity;
mod common;
mod non_empty;
mod sparse;
mod standard;
use capacity::StackCapacity;
use common::StackCommon;
pub use non_empty::NonEmptyStack;
pub use sparse::SparseStack;
pub use standard::Stack;

View file

@ -1,14 +1,8 @@
#![expect(clippy::unnecessary_safety_comment)]
use std::{
alloc::{self, Layout},
mem::{align_of, size_of},
ptr::{self, NonNull},
};
use std::{mem::size_of, ptr::NonNull};
use assert_unchecked::assert_unchecked;
use super::StackCapacity;
use super::{StackCapacity, StackCommon};
/// A stack which can never be empty.
///
@ -62,6 +56,38 @@ pub struct NonEmptyStack<T> {
impl<T> StackCapacity<T> for NonEmptyStack<T> {}
impl<T> StackCommon<T> for NonEmptyStack<T> {
#[inline]
fn start(&self) -> NonNull<T> {
self.start
}
#[inline]
fn end(&self) -> NonNull<T> {
self.end
}
#[inline]
fn cursor(&self) -> NonNull<T> {
self.cursor
}
#[inline]
fn set_start(&mut self, start: NonNull<T>) {
self.start = start;
}
#[inline]
fn set_end(&mut self, end: NonNull<T>) {
self.end = end;
}
#[inline]
fn set_cursor(&mut self, cursor: NonNull<T>) {
self.cursor = cursor;
}
}
impl<T> NonEmptyStack<T> {
/// Maximum capacity.
///
@ -136,18 +162,7 @@ impl<T> NonEmptyStack<T> {
assert!(size_of::<T>() > 0, "Zero sized types are not supported");
// SAFETY: Caller guarantees `capacity_bytes` satisfies requirements
let layout = Self::layout_for(capacity_bytes);
let ptr = alloc::alloc(layout);
if ptr.is_null() {
alloc::handle_alloc_error(layout);
}
// `layout_for` produces a layout with `T`'s alignment, so `ptr` is aligned for `T`
let ptr = ptr.cast::<T>();
// SAFETY: We checked `ptr` is non-null
let start = NonNull::new_unchecked(ptr);
// SAFETY: We allocated `capacity_bytes` bytes, so `end` is end of allocation
let end = NonNull::new_unchecked(ptr.byte_add(capacity_bytes));
let (start, end) = Self::allocate(capacity_bytes);
// Write initial value to start of allocation.
// SAFETY: Allocation was created with alignment of `T`, and with capacity for at least 1 entry,
@ -158,29 +173,6 @@ impl<T> NonEmptyStack<T> {
Self { cursor: start, start, end }
}
/// Get layout for allocation of `capacity_bytes` bytes.
///
/// # SAFETY
/// * `capacity_bytes` must not be 0.
/// * `capacity_bytes` must be a multiple of `mem::size_of::<T>()`.
/// * `capacity_bytes` must not exceed [`Self::MAX_CAPACITY_BYTES`].
#[inline]
unsafe fn layout_for(capacity_bytes: usize) -> Layout {
// `capacity_bytes` must not be 0 because stack can never be empty.
debug_assert!(capacity_bytes > 0);
// `capacity_bytes` must be a multiple of `size_of::<T>()` so that `new_cursor == self.end`
// check in `push` accurately detects when full to capacity
debug_assert!(capacity_bytes % size_of::<T>() == 0);
// `capacity_bytes` must not exceed `Self::MAX_CAPACITY_BYTES` to prevent creating an allocation
// of illegal size
debug_assert!(capacity_bytes <= Self::MAX_CAPACITY_BYTES);
// SAFETY: `align_of::<T>()` trivially satisfies alignment requirements.
// Caller guarantees `capacity_bytes <= MAX_CAPACITY_BYTES`.
// `MAX_CAPACITY_BYTES` takes into account the rounding-up by alignment requirement.
Layout::from_size_align_unchecked(capacity_bytes, align_of::<T>())
}
/// Get reference to last value on stack.
#[inline]
pub fn last(&self) -> &T {
@ -236,58 +228,9 @@ impl<T> NonEmptyStack<T> {
#[cold]
#[inline(never)]
unsafe fn push_slow(&mut self, value: T) {
// Get new capacity
let old_capacity_bytes = self.capacity_bytes();
// Capacity in bytes cannot be larger than `isize::MAX`, so `* 2` cannot overflow
let mut new_capacity_bytes = old_capacity_bytes * 2;
if new_capacity_bytes > Self::MAX_CAPACITY_BYTES {
assert!(
old_capacity_bytes < Self::MAX_CAPACITY_BYTES,
"Cannot grow beyond `Self::MAX_CAPACITY`"
);
new_capacity_bytes = Self::MAX_CAPACITY_BYTES;
}
debug_assert!(new_capacity_bytes > old_capacity_bytes);
// Reallocate.
// SAFETY:
// Stack is always allocated, and `self.start` and `self.end` are boundaries of that allocation.
// So `self.start` and `old_layout` accurately describe the current allocation.
// `old_capacity_bytes` was a multiple of `size_of::<T>()`, so double that must be too.
// `MAX_CAPACITY_BYTES` is also a multiple of `size_of::<T>()`.
// So `new_capacity_bytes` must be a multiple of `size_of::<T>()`.
// `new_capacity_bytes` is `<= MAX_CAPACITY_BYTES`, so is a legal allocation size.
// `layout_for` produces a layout with `T`'s alignment, so `new_ptr` is aligned for `T`.
let new_ptr = unsafe {
let old_ptr = self.start.as_ptr().cast::<u8>();
let old_layout = Self::layout_for(old_capacity_bytes);
let new_ptr = alloc::realloc(old_ptr, old_layout, new_capacity_bytes);
if new_ptr.is_null() {
let new_layout = Self::layout_for(new_capacity_bytes);
alloc::handle_alloc_error(new_layout);
}
new_ptr.cast::<T>()
};
// Update pointers.
// Stack was full to capacity, so new last index after push is the old capacity.
// i.e. `self.cursor - self.start == old_end - old_start`.
// Note: All pointers need to be updated even if allocation grew in place.
// From docs for `GlobalAlloc::realloc`:
// "Any access to the old `ptr` is Undefined Behavior, even if the allocation remained in-place."
// <https://doc.rust-lang.org/std/alloc/trait.GlobalAlloc.html#method.realloc>
// `end` changes whatever happens, so always need to be updated.
// `cursor` needs to be derived from `start` to make `offset_from` valid, so also needs updating.
// SAFETY: We checked that `new_ptr` is non-null.
// `old_capacity_bytes` and `new_capacity_bytes` are both multiples of `size_of::<T>()`.
// `size_of::<T>()` is always a multiple of `T`'s alignment, and `new_ptr` is aligned for `T`,
// so new `self.cursor` and `self.end` are aligned for `T`.
// `old_capacity_bytes` is always `< new_capacity_bytes`, so new `self.cursor` must be in bounds.
unsafe {
self.start = NonNull::new_unchecked(new_ptr);
self.end = NonNull::new_unchecked(new_ptr.byte_add(new_capacity_bytes));
self.cursor = NonNull::new_unchecked(new_ptr.byte_add(old_capacity_bytes));
}
// Grow allocation.
// SAFETY: Stack is always allocated.
self.grow();
// Write value.
// SAFETY: We just allocated additional capacity, so `self.cursor` is in bounds.
@ -330,75 +273,34 @@ impl<T> NonEmptyStack<T> {
/// Number of entries is always at least 1. Stack is never empty.
#[inline]
pub fn len(&self) -> usize {
// `offset_from` returns offset in units of `T`.
// When stack has 1 entry, `start - cursor == 0`, so add 1 to get number of entries.
// SAFETY: `self.start` and `self.cursor` are both derived from same pointer
// (in `new_with_capacity_bytes_unchecked` and `push_slow`).
// Both pointers are always within bounds of a single allocation.
// Distance between pointers is always a multiple of `size_of::<T>()`.
// Byte size of allocation cannot exceed `isize::MAX`, so `+ 1` cannot wrap around.
// SAFETY: `self.start` and `self.cursor` are both derived from same pointer.
// `self.cursor` is always >= `self.start`.
// `assert_unchecked!` is to help compiler to optimize.
// See: https://doc.rust-lang.org/std/primitive.pointer.html#method.sub_ptr
#[expect(clippy::cast_sign_loss)]
unsafe {
assert_unchecked!(self.cursor >= self.start);
self.cursor.as_ptr().offset_from(self.start.as_ptr()) as usize + 1
}
// Distance between pointers is always a multiple of `size_of::<T>()`.
let offset = unsafe { self.cursor_offset() };
// When stack has 1 entry, `start - cursor == 0`, so add 1 to get number of entries.
// SAFETY: Capacity cannot exceed `Self::MAX_CAPACITY`, which is `<= isize::MAX`,
// and offset can't exceed capacity, so `+ 1` cannot wrap around.
offset + 1
}
/// Get capacity.
#[inline]
pub fn capacity(&self) -> usize {
// SAFETY: `self.start` and `self.end` are both derived from same pointer
// (in `new_with_capacity_bytes_unchecked` and `push_slow`).
// Both pointers are always within bounds of single allocation.
// Distance between pointers is always a multiple of `size_of::<T>()`.
// `self.end` is always > `self.start`, because stack is never empty.
// `assert_unchecked!` is to help compiler to optimize.
// See: https://doc.rust-lang.org/std/primitive.pointer.html#method.sub_ptr
#[expect(clippy::cast_sign_loss)]
unsafe {
assert_unchecked!(self.end > self.start);
self.end.as_ptr().offset_from(self.start.as_ptr()) as usize
}
}
/// Get capacity in bytes.
#[inline]
fn capacity_bytes(&self) -> usize {
// SAFETY: `self.start` and `self.end` are both derived from same pointer
// (in `new_with_capacity_bytes_unchecked` and `push_slow`).
// Both pointers are always within bounds of single allocation.
// Distance between pointers is always a multiple of `size_of::<T>()`.
// `self.end` is always > `self.start`, because stack is never empty.
// `assert_unchecked!` is to help compiler to optimize.
// See: https://doc.rust-lang.org/std/primitive.pointer.html#method.sub_ptr
#[expect(clippy::cast_sign_loss)]
unsafe {
assert_unchecked!(self.end > self.start);
self.end.as_ptr().byte_offset_from(self.start.as_ptr()) as usize
}
<Self as StackCommon<T>>::capacity(self)
}
}
impl<T> Drop for NonEmptyStack<T> {
fn drop(&mut self) {
// Drop contents. This block copied from `std`'s `Vec`.
// Will be optimized out if `T` is non-drop, as `drop_in_place` calls `std::mem::needs_drop`.
// SAFETY: Stack contains `self.len()` initialized entries, starting at `self.start`.
unsafe {
ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.start.as_ptr(), self.len()));
}
// Drop contents.
// SAFETY: Stack is always allocated, and contains `self.len()` initialized entries,
// starting at `self.start`.
unsafe { self.drop_contents(self.len()) };
// Drop the memory
// SAFETY:
// Stack is always allocated, and `self.start` and `self.end` are boundaries of that allocation.
// So `self.start` and `layout` accurately describe the current allocation.
unsafe {
let layout = Self::layout_for(self.capacity_bytes());
alloc::dealloc(self.start.as_ptr().cast::<u8>(), layout);
}
// SAFETY: Stack is always allocated.
unsafe { self.deallocate() };
}
}

View file

@ -1,14 +1,8 @@
#![expect(clippy::unnecessary_safety_comment)]
use std::{
alloc::{self, Layout},
mem::{align_of, size_of, ManuallyDrop},
ptr::{self, NonNull},
};
use std::{mem::size_of, ptr::NonNull};
use assert_unchecked::assert_unchecked;
use super::StackCapacity;
use super::{StackCapacity, StackCommon};
/// A simple stack.
///
@ -44,6 +38,38 @@ pub struct Stack<T> {
impl<T> StackCapacity<T> for Stack<T> {}
impl<T> StackCommon<T> for Stack<T> {
#[inline]
fn start(&self) -> NonNull<T> {
self.start
}
#[inline]
fn end(&self) -> NonNull<T> {
self.end
}
#[inline]
fn cursor(&self) -> NonNull<T> {
self.cursor
}
#[inline]
fn set_start(&mut self, start: NonNull<T>) {
self.start = start;
}
#[inline]
fn set_end(&mut self, end: NonNull<T>) {
self.end = end;
}
#[inline]
fn set_cursor(&mut self, cursor: NonNull<T>) {
self.cursor = cursor;
}
}
impl<T> Stack<T> {
/// Maximum capacity.
///
@ -120,46 +146,12 @@ impl<T> Stack<T> {
assert!(size_of::<T>() > 0, "Zero sized types are not supported");
// SAFETY: Caller guarantees `capacity_bytes` satisfies requirements
let layout = Self::layout_for(capacity_bytes);
let ptr = alloc::alloc(layout);
if ptr.is_null() {
alloc::handle_alloc_error(layout);
}
// `layout_for` produces a layout with `T`'s alignment, so `ptr` is aligned for `T`
let ptr = ptr.cast::<T>();
// SAFETY: We checked `ptr` is non-null
let start = NonNull::new_unchecked(ptr);
// SAFETY: We allocated `capacity_bytes` bytes, so `end` is end of allocation
let end = NonNull::new_unchecked(ptr.byte_add(capacity_bytes));
let (start, end) = Self::allocate(capacity_bytes);
// `cursor` is positioned at start
Self { cursor: start, start, end }
}
/// Get layout for allocation of `capacity_bytes` bytes.
///
/// # SAFETY
/// * `capacity_bytes` must not be 0.
/// * `capacity_bytes` must be a multiple of `mem::size_of::<T>()`.
/// * `capacity_bytes` must not exceed [`Self::MAX_CAPACITY_BYTES`].
#[inline]
unsafe fn layout_for(capacity_bytes: usize) -> Layout {
// `capacity_bytes` must not be 0 because cannot make 0-size allocations.
debug_assert!(capacity_bytes > 0);
// `capacity_bytes` must be a multiple of `size_of::<T>()` so that `new_cursor == self.end`
// check in `push` accurately detects when full to capacity
debug_assert!(capacity_bytes % size_of::<T>() == 0);
// `capacity_bytes` must not exceed `Self::MAX_CAPACITY_BYTES` to prevent creating an allocation
// of illegal size
debug_assert!(capacity_bytes <= Self::MAX_CAPACITY_BYTES);
// SAFETY: `align_of::<T>()` trivially satisfies alignment requirements.
// Caller guarantees `capacity_bytes <= MAX_CAPACITY_BYTES`.
// `MAX_CAPACITY_BYTES` takes into account the rounding-up by alignment requirement.
Layout::from_size_align_unchecked(capacity_bytes, align_of::<T>())
}
/// Get reference to last value on stack.
#[inline]
#[cfg_attr(not(test), expect(dead_code))]
@ -249,69 +241,18 @@ impl<T> Stack<T> {
#[cold]
#[inline(never)]
unsafe fn push_slow(&mut self, value: T) {
if self.end == self.start {
#[expect(clippy::if_not_else)]
if self.end != self.start {
// Stack was already allocated. Grow capacity.
// SAFETY: Checked above that is already allocated.
self.grow();
} else {
// Stack was not allocated yet.
// SAFETY: `DEFAULT_CAPACITY_BYTES` satisfies requirements.
let new = ManuallyDrop::new(Self::new_with_capacity_bytes_unchecked(
Self::DEFAULT_CAPACITY_BYTES,
));
self.start = new.start;
self.cursor = new.start;
self.end = new.end;
} else {
// Stack was already allocated. Grow capacity.
// Get new capacity
let old_capacity_bytes = self.capacity_bytes();
// Capacity in bytes cannot be larger than `isize::MAX`, so `* 2` cannot overflow.
let mut new_capacity_bytes = old_capacity_bytes * 2;
if new_capacity_bytes > Self::MAX_CAPACITY_BYTES {
assert!(
old_capacity_bytes < Self::MAX_CAPACITY_BYTES,
"Cannot grow beyond `Self::MAX_CAPACITY`"
);
new_capacity_bytes = Self::MAX_CAPACITY_BYTES;
}
debug_assert!(new_capacity_bytes > old_capacity_bytes);
// Reallocate.
// SAFETY:
// Stack is allocated, and `self.start` and `self.end` are boundaries of that allocation.
// So `self.start` and `old_layout` accurately describe the current allocation.
// `old_capacity_bytes` was a multiple of `size_of::<T>()`, so double that must be too.
// `MAX_CAPACITY_BYTES` is also a multiple of `size_of::<T>()`.
// So `new_capacity_bytes` must be a multiple of `size_of::<T>()`.
// `new_capacity_bytes` is `<= MAX_CAPACITY_BYTES`, so is a legal allocation size.
// `layout_for` produces a layout with `T`'s alignment, so `new_ptr` is aligned for `T`.
let new_ptr = unsafe {
let old_ptr = self.start.as_ptr().cast::<u8>();
let old_layout = Self::layout_for(old_capacity_bytes);
let new_ptr = alloc::realloc(old_ptr, old_layout, new_capacity_bytes);
if new_ptr.is_null() {
let new_layout = Self::layout_for(new_capacity_bytes);
alloc::handle_alloc_error(new_layout);
}
new_ptr.cast::<T>()
};
// Update pointers.
// Stack was full to capacity, so new last index after push is the old capacity.
// i.e. `self.cursor - self.start == old_end - old_start`.
// Note: All pointers need to be updated even if allocation grew in place.
// From docs for `GlobalAlloc::realloc`:
// "Any access to the old `ptr` is Undefined Behavior, even if the allocation remained in-place."
// <https://doc.rust-lang.org/std/alloc/trait.GlobalAlloc.html#method.realloc>
// `end` changes whatever happens, so always need to be updated.
// `cursor` needs to be derived from `start` to make `offset_from` valid, so also needs updating.
// SAFETY: We checked that `new_ptr` is non-null.
// `old_capacity_bytes` and `new_capacity_bytes` are both multiples of `size_of::<T>()`.
// `size_of::<T>()` is always a multiple of `T`'s alignment, and `new_ptr` is aligned for `T`,
// so new `self.cursor` and `self.end` are aligned for `T`.
// `old_capacity_bytes` is always `< new_capacity_bytes`, so new `self.cursor` must be in bounds.
unsafe {
self.start = NonNull::new_unchecked(new_ptr);
self.end = NonNull::new_unchecked(new_ptr.byte_add(new_capacity_bytes));
self.cursor = NonNull::new_unchecked(new_ptr.byte_add(old_capacity_bytes));
}
let (start, end) = Self::allocate(Self::DEFAULT_CAPACITY_BYTES);
self.start = start;
self.cursor = start;
self.end = end;
}
// Write value + increment cursor.
@ -355,19 +296,10 @@ impl<T> Stack<T> {
/// Get number of entries on stack.
#[inline]
pub fn len(&self) -> usize {
// `offset_from` returns offset in units of `T`.
// SAFETY: `self.start` and `self.cursor` are both derived from same pointer
// (in `new`, `new_with_capacity_bytes_unchecked` and `push_slow`).
// Both pointers are always within bounds of a single allocation.
// Distance between pointers is always a multiple of `size_of::<T>()`.
// SAFETY: `self.start` and `self.cursor` are both derived from same pointer.
// `self.cursor` is always >= `self.start`.
// `assert_unchecked!` is to help compiler to optimize.
// See: https://doc.rust-lang.org/std/primitive.pointer.html#method.sub_ptr
#[expect(clippy::cast_sign_loss)]
unsafe {
assert_unchecked!(self.cursor >= self.start);
self.cursor.as_ptr().offset_from(self.start.as_ptr()) as usize
}
// Distance between pointers is always a multiple of `size_of::<T>()`.
unsafe { self.cursor_offset() }
}
/// Get if stack is empty.
@ -379,35 +311,7 @@ impl<T> Stack<T> {
/// Get capacity.
#[inline]
pub fn capacity(&self) -> usize {
// SAFETY: `self.start` and `self.end` are both derived from same pointer
// (in `new`, `new_with_capacity_bytes_unchecked` and `push_slow`).
// Both pointers are always within bounds of single allocation.
// Distance between pointers is always a multiple of `size_of::<T>()`.
// `self.end` is always >= `self.start`.
// `assert_unchecked!` is to help compiler to optimize.
// See: https://doc.rust-lang.org/std/primitive.pointer.html#method.sub_ptr
#[expect(clippy::cast_sign_loss)]
unsafe {
assert_unchecked!(self.end >= self.start);
self.end.as_ptr().offset_from(self.start.as_ptr()) as usize
}
}
/// Get capacity in bytes.
#[inline]
fn capacity_bytes(&self) -> usize {
// SAFETY: `self.start` and `self.end` are both derived from same pointer
// (in `new`, `new_with_capacity_bytes_unchecked` and `push_slow`).
// Both pointers are always within bounds of single allocation.
// Distance between pointers is always a multiple of `size_of::<T>()`.
// `self.end` is always >= `self.start`.
// `assert_unchecked!` is to help compiler to optimize.
// See: https://doc.rust-lang.org/std/primitive.pointer.html#method.sub_ptr
#[expect(clippy::cast_sign_loss)]
unsafe {
assert_unchecked!(self.end >= self.start);
self.end.as_ptr().byte_offset_from(self.start.as_ptr()) as usize
}
<Self as StackCommon<T>>::capacity(self)
}
}
@ -419,22 +323,14 @@ impl<T> Drop for Stack<T> {
}
if !self.is_empty() {
// Drop contents. This block copied from `std`'s `Vec`.
// Will be optimized out if `T` is non-drop, as `drop_in_place` calls `std::mem::needs_drop`.
// SAFETY: Stack contains `self.len()` initialized entries, starting at `self.start`.
unsafe {
ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.start.as_ptr(), self.len()));
}
// SAFETY: Checked above that stack is allocated.
// Stack contains `self.len()` initialized entries, starting at `self.start`
unsafe { self.drop_contents(self.len()) };
}
// Drop the memory
// SAFETY: Checked above that stack is allocated.
// `self.start` and `self.end` are boundaries of that allocation.
// So `self.start` and `layout` accurately describe the current allocation.
unsafe {
let layout = Self::layout_for(self.capacity_bytes());
alloc::dealloc(self.start.as_ptr().cast::<u8>(), layout);
}
unsafe { self.deallocate() };
}
}