#![cfg_attr(not(all(test, feature = "float")), allow(dead_code, unused_macros))]
#[macro_use]
#[path = "gen/utils.rs"]
mod gen;
use core::sync::atomic::Ordering;
macro_rules! static_assert {
($cond:expr $(,)?) => {{
let [] = [(); true as usize - $crate::utils::_assert_is_bool($cond) as usize];
}};
}
pub(crate) const fn _assert_is_bool(v: bool) -> bool {
v
}
macro_rules! static_assert_layout {
($atomic_type:ty, $value_type:ty) => {
static_assert!(
core::mem::align_of::<$atomic_type>() == core::mem::size_of::<$atomic_type>()
);
static_assert!(core::mem::size_of::<$atomic_type>() == core::mem::size_of::<$value_type>());
};
}
macro_rules! doc_comment {
($doc:expr, $($tt:tt)*) => {
#[doc = $doc]
$($tt)*
};
}
#[allow(unused_macros)]
#[cfg(not(portable_atomic_no_outline_atomics))]
#[cfg(any(
target_arch = "aarch64",
target_arch = "arm",
target_arch = "arm64ec",
target_arch = "powerpc64",
target_arch = "riscv32",
target_arch = "riscv64",
all(target_arch = "x86_64", not(any(target_env = "sgx", miri))),
))]
macro_rules! ifunc {
(unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)? { $($detect_body:tt)* }) => {{
type FnTy = unsafe fn($($arg_ty),*) $(-> $ret_ty)?;
static FUNC: core::sync::atomic::AtomicPtr<()>
= core::sync::atomic::AtomicPtr::new(detect as *mut ());
#[cold]
unsafe fn detect($($arg_pat: $arg_ty),*) $(-> $ret_ty)? {
let func: FnTy = { $($detect_body)* };
FUNC.store(func as *mut (), core::sync::atomic::Ordering::Relaxed);
unsafe { func($($arg_pat),*) }
}
let func = {
core::mem::transmute::<*mut (), FnTy>(FUNC.load(core::sync::atomic::Ordering::Relaxed))
};
func($($arg_pat),*)
}};
}
#[allow(unused_macros)]
#[cfg(not(portable_atomic_no_outline_atomics))]
#[cfg(any(
target_arch = "aarch64",
target_arch = "arm",
target_arch = "arm64ec",
target_arch = "powerpc64",
target_arch = "riscv32",
target_arch = "riscv64",
all(target_arch = "x86_64", not(any(target_env = "sgx", miri))),
))]
macro_rules! fn_alias {
(
$(#[$($fn_attr:tt)*])*
$vis:vis unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)?;
$(#[$($alias_attr:tt)*])*
$new:ident = $from:ident($($last_args:tt)*);
$($rest:tt)*
) => {
$(#[$($fn_attr)*])*
$(#[$($alias_attr)*])*
$vis unsafe fn $new($($arg_pat: $arg_ty),*) $(-> $ret_ty)? {
unsafe { $from($($arg_pat,)* $($last_args)*) }
}
fn_alias! {
$(#[$($fn_attr)*])*
$vis unsafe fn($($arg_pat: $arg_ty),*) $(-> $ret_ty)?;
$($rest)*
}
};
(
$(#[$($attr:tt)*])*
$vis:vis unsafe fn($($arg_pat:ident: $arg_ty:ty),*) $(-> $ret_ty:ty)?;
) => {}
}
macro_rules! const_fn {
(
const_if: #[cfg($($cfg:tt)+)];
$(#[$($attr:tt)*])*
$vis:vis const fn $($rest:tt)*
) => {
#[cfg($($cfg)+)]
$(#[$($attr)*])*
$vis const fn $($rest)*
#[cfg(not($($cfg)+))]
$(#[$($attr)*])*
$vis fn $($rest)*
};
}
macro_rules! impl_debug_and_serde {
($atomic_type:ident) => {
impl fmt::Debug for $atomic_type {
#[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Debug::fmt(&self.load(Ordering::Relaxed), f)
}
}
#[cfg(feature = "serde")]
#[cfg_attr(docsrs, doc(cfg(feature = "serde")))]
impl serde::ser::Serialize for $atomic_type {
#[allow(clippy::missing_inline_in_public_items)] fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::ser::Serializer,
{
self.load(Ordering::Relaxed).serialize(serializer)
}
}
#[cfg(feature = "serde")]
#[cfg_attr(docsrs, doc(cfg(feature = "serde")))]
impl<'de> serde::de::Deserialize<'de> for $atomic_type {
#[allow(clippy::missing_inline_in_public_items)] fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::de::Deserializer<'de>,
{
serde::de::Deserialize::deserialize(deserializer).map(Self::new)
}
}
};
}
macro_rules! impl_default_no_fetch_ops {
($atomic_type:ident, bool) => {
impl $atomic_type {
#[inline]
#[cfg_attr(miri, track_caller)] pub(crate) fn and(&self, val: bool, order: Ordering) {
self.fetch_and(val, order);
}
#[inline]
#[cfg_attr(miri, track_caller)] pub(crate) fn or(&self, val: bool, order: Ordering) {
self.fetch_or(val, order);
}
#[inline]
#[cfg_attr(miri, track_caller)] pub(crate) fn xor(&self, val: bool, order: Ordering) {
self.fetch_xor(val, order);
}
}
};
($atomic_type:ident, $int_type:ty) => {
impl $atomic_type {
#[inline]
#[cfg_attr(miri, track_caller)] pub(crate) fn add(&self, val: $int_type, order: Ordering) {
self.fetch_add(val, order);
}
#[inline]
#[cfg_attr(miri, track_caller)] pub(crate) fn sub(&self, val: $int_type, order: Ordering) {
self.fetch_sub(val, order);
}
#[inline]
#[cfg_attr(miri, track_caller)] pub(crate) fn and(&self, val: $int_type, order: Ordering) {
self.fetch_and(val, order);
}
#[inline]
#[cfg_attr(miri, track_caller)] pub(crate) fn or(&self, val: $int_type, order: Ordering) {
self.fetch_or(val, order);
}
#[inline]
#[cfg_attr(miri, track_caller)] pub(crate) fn xor(&self, val: $int_type, order: Ordering) {
self.fetch_xor(val, order);
}
}
};
}
macro_rules! impl_default_bit_opts {
($atomic_type:ident, $int_type:ty) => {
impl $atomic_type {
#[inline]
#[cfg_attr(miri, track_caller)] pub(crate) fn bit_set(&self, bit: u32, order: Ordering) -> bool {
let mask = <$int_type>::wrapping_shl(1, bit);
self.fetch_or(mask, order) & mask != 0
}
#[inline]
#[cfg_attr(miri, track_caller)] pub(crate) fn bit_clear(&self, bit: u32, order: Ordering) -> bool {
let mask = <$int_type>::wrapping_shl(1, bit);
self.fetch_and(!mask, order) & mask != 0
}
#[inline]
#[cfg_attr(miri, track_caller)] pub(crate) fn bit_toggle(&self, bit: u32, order: Ordering) -> bool {
let mask = <$int_type>::wrapping_shl(1, bit);
self.fetch_xor(mask, order) & mask != 0
}
}
};
}
macro_rules! items {
($($tt:tt)*) => {
$($tt)*
};
}
#[allow(dead_code)]
#[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
#[inline(always)]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub(crate) unsafe fn assert_unchecked(cond: bool) {
if !cond {
if cfg!(debug_assertions) {
unreachable!()
} else {
unsafe { core::hint::unreachable_unchecked() }
}
}
}
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub(crate) fn assert_load_ordering(order: Ordering) {
match order {
Ordering::Acquire | Ordering::Relaxed | Ordering::SeqCst => {}
Ordering::Release => panic!("there is no such thing as a release load"),
Ordering::AcqRel => panic!("there is no such thing as an acquire-release load"),
_ => unreachable!(),
}
}
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub(crate) fn assert_store_ordering(order: Ordering) {
match order {
Ordering::Release | Ordering::Relaxed | Ordering::SeqCst => {}
Ordering::Acquire => panic!("there is no such thing as an acquire store"),
Ordering::AcqRel => panic!("there is no such thing as an acquire-release store"),
_ => unreachable!(),
}
}
#[inline]
#[cfg_attr(all(debug_assertions, not(portable_atomic_no_track_caller)), track_caller)]
pub(crate) fn assert_compare_exchange_ordering(success: Ordering, failure: Ordering) {
match success {
Ordering::AcqRel
| Ordering::Acquire
| Ordering::Relaxed
| Ordering::Release
| Ordering::SeqCst => {}
_ => unreachable!(),
}
match failure {
Ordering::Acquire | Ordering::Relaxed | Ordering::SeqCst => {}
Ordering::Release => panic!("there is no such thing as a release failure ordering"),
Ordering::AcqRel => panic!("there is no such thing as an acquire-release failure ordering"),
_ => unreachable!(),
}
}
#[allow(dead_code)]
#[inline]
pub(crate) fn upgrade_success_ordering(success: Ordering, failure: Ordering) -> Ordering {
match (success, failure) {
(Ordering::Relaxed, Ordering::Acquire) => Ordering::Acquire,
(Ordering::Release, Ordering::Acquire) => Ordering::AcqRel,
(_, Ordering::SeqCst) => Ordering::SeqCst,
_ => success,
}
}
#[cfg(not(portable_atomic_no_asm_maybe_uninit))]
#[cfg(target_pointer_width = "32")]
#[allow(dead_code)]
#[inline]
pub(crate) fn zero_extend64_ptr(v: *mut ()) -> core::mem::MaybeUninit<u64> {
#[repr(C)]
struct ZeroExtended {
#[cfg(target_endian = "big")]
pad: *mut (),
v: *mut (),
#[cfg(target_endian = "little")]
pad: *mut (),
}
unsafe { core::mem::transmute(ZeroExtended { v, pad: core::ptr::null_mut() }) }
}
#[allow(dead_code)]
#[cfg(any(
target_arch = "aarch64",
target_arch = "arm64ec",
target_arch = "powerpc64",
target_arch = "riscv64",
target_arch = "s390x",
target_arch = "x86_64",
))]
#[derive(Clone, Copy)]
#[repr(C)]
pub(crate) union U128 {
pub(crate) whole: u128,
pub(crate) pair: Pair<u64>,
}
#[allow(dead_code)]
#[cfg(any(target_arch = "arm", target_arch = "riscv32"))]
#[derive(Clone, Copy)]
#[repr(C)]
pub(crate) union U64 {
pub(crate) whole: u64,
pub(crate) pair: Pair<u32>,
}
#[allow(dead_code)]
#[derive(Clone, Copy)]
#[repr(C)]
pub(crate) struct Pair<T: Copy> {
#[cfg(any(
target_endian = "little",
target_arch = "aarch64",
target_arch = "arm",
target_arch = "arm64ec",
))]
pub(crate) lo: T,
pub(crate) hi: T,
#[cfg(not(any(
target_endian = "little",
target_arch = "aarch64",
target_arch = "arm",
target_arch = "arm64ec",
)))]
pub(crate) lo: T,
}
#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
type MinWord = u32;
#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
type RetInt = u32;
#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
#[allow(dead_code)]
#[inline]
pub(crate) fn create_sub_word_mask_values<T>(ptr: *mut T) -> (*mut MinWord, RetInt, RetInt) {
use core::mem;
const SHIFT_MASK: bool = !cfg!(any(
target_arch = "riscv32",
target_arch = "riscv64",
target_arch = "loongarch64",
target_arch = "s390x",
));
let ptr_mask = mem::size_of::<MinWord>() - 1;
let aligned_ptr = strict::with_addr(ptr, ptr as usize & !ptr_mask) as *mut MinWord;
let ptr_lsb = if SHIFT_MASK {
ptr as usize & ptr_mask
} else {
ptr as usize
};
let shift = if cfg!(any(target_endian = "little", target_arch = "s390x")) {
ptr_lsb.wrapping_mul(8)
} else {
(ptr_lsb ^ (mem::size_of::<MinWord>() - mem::size_of::<T>())).wrapping_mul(8)
};
let mut mask: RetInt = (1 << (mem::size_of::<T>() * 8)) - 1; if SHIFT_MASK {
mask <<= shift;
}
(aligned_ptr, shift as RetInt, mask)
}
#[cfg(any(miri, target_arch = "riscv32", target_arch = "riscv64"))]
#[allow(dead_code)]
pub(crate) mod strict {
#[inline]
#[must_use]
pub(crate) fn with_addr<T>(ptr: *mut T, addr: usize) -> *mut T {
let offset = addr.wrapping_sub(ptr as usize);
(ptr as *mut u8).wrapping_add(offset) as *mut T
}
#[inline]
#[must_use]
pub(crate) fn map_addr<T>(ptr: *mut T, f: impl FnOnce(usize) -> usize) -> *mut T {
with_addr(ptr, f(ptr as usize))
}
}