portable_atomic/imp/
mod.rs

1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3// -----------------------------------------------------------------------------
4// Lock-free implementations
5
6#[cfg(not(any(
7    all(
8        portable_atomic_no_atomic_load_store,
9        not(all(target_arch = "bpf", not(feature = "critical-section"))),
10    ),
11    target_arch = "avr",
12    target_arch = "msp430",
13)))]
14#[cfg_attr(
15    portable_atomic_no_cfg_target_has_atomic,
16    cfg(not(all(
17        any(
18            target_arch = "riscv32",
19            target_arch = "riscv64",
20            feature = "critical-section",
21            portable_atomic_unsafe_assume_single_core,
22        ),
23        portable_atomic_no_atomic_cas,
24    )))
25)]
26#[cfg_attr(
27    not(portable_atomic_no_cfg_target_has_atomic),
28    cfg(not(all(
29        any(
30            target_arch = "riscv32",
31            target_arch = "riscv64",
32            feature = "critical-section",
33            portable_atomic_unsafe_assume_single_core,
34        ),
35        not(target_has_atomic = "ptr"),
36    )))
37)]
38mod core_atomic;
39
40// MSP430
41#[cfg(target_arch = "msp430")]
42pub(crate) mod msp430;
43
44// RISC-V without A-extension
45#[cfg(any(test, not(feature = "critical-section")))]
46#[cfg_attr(
47    portable_atomic_no_cfg_target_has_atomic,
48    cfg(any(
49        all(test, not(any(miri, portable_atomic_sanitize_thread))),
50        portable_atomic_no_atomic_cas,
51    ))
52)]
53#[cfg_attr(
54    not(portable_atomic_no_cfg_target_has_atomic),
55    cfg(any(
56        all(test, not(any(miri, portable_atomic_sanitize_thread))),
57        not(target_has_atomic = "ptr"),
58    ))
59)]
60#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
61mod riscv;
62
63// x86-specific optimizations
64// Miri and Sanitizer do not support inline assembly.
65#[cfg(all(
66    any(target_arch = "x86", target_arch = "x86_64"),
67    not(any(miri, portable_atomic_sanitize_thread)),
68    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
69))]
70mod x86;
71
72// 64-bit atomic implementations on 32-bit architectures
73#[cfg(any(target_arch = "arm", target_arch = "riscv32"))]
74mod atomic64;
75
76// 128-bit atomic implementations on 64-bit architectures
77#[cfg(any(
78    target_arch = "aarch64",
79    target_arch = "arm64ec",
80    target_arch = "powerpc64",
81    target_arch = "riscv64",
82    target_arch = "s390x",
83    target_arch = "x86_64",
84))]
85mod atomic128;
86
87// -----------------------------------------------------------------------------
88// Lock-based fallback implementations
89
90#[cfg(feature = "fallback")]
91#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
92#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
93#[cfg(any(
94    test,
95    not(any(
96        all(
97            target_arch = "aarch64",
98            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
99        ),
100        all(target_arch = "arm64ec", portable_atomic_unstable_asm_experimental_arch),
101        all(
102            target_arch = "x86_64",
103            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
104            any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
105        ),
106        all(
107            target_arch = "riscv64",
108            not(portable_atomic_no_asm),
109            any(
110                target_feature = "experimental-zacas",
111                portable_atomic_target_feature = "experimental-zacas",
112            ),
113        ),
114        all(
115            target_arch = "powerpc64",
116            portable_atomic_unstable_asm_experimental_arch,
117            any(
118                target_feature = "quadword-atomics",
119                portable_atomic_target_feature = "quadword-atomics",
120            ),
121        ),
122        all(target_arch = "s390x", portable_atomic_unstable_asm_experimental_arch),
123    ))
124))]
125mod fallback;
126
127// -----------------------------------------------------------------------------
128// Critical section based fallback implementations
129
130// On AVR, we always use critical section based fallback implementation.
131// AVR can be safely assumed to be single-core, so this is sound.
132// https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/Target/AVR/AVRExpandPseudoInsts.cpp#L1074
133// MSP430 as well.
134#[cfg(any(
135    all(test, target_os = "none"),
136    portable_atomic_unsafe_assume_single_core,
137    feature = "critical-section",
138    target_arch = "avr",
139    target_arch = "msp430",
140))]
141#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(any(test, portable_atomic_no_atomic_cas)))]
142#[cfg_attr(
143    not(portable_atomic_no_cfg_target_has_atomic),
144    cfg(any(test, not(target_has_atomic = "ptr")))
145)]
146#[cfg(any(
147    target_arch = "arm",
148    target_arch = "avr",
149    target_arch = "msp430",
150    target_arch = "riscv32",
151    target_arch = "riscv64",
152    target_arch = "xtensa",
153    feature = "critical-section",
154))]
155mod interrupt;
156
157// -----------------------------------------------------------------------------
158// Atomic float implementations
159
160#[cfg(feature = "float")]
161#[allow(clippy::float_arithmetic)]
162pub(crate) mod float;
163
164// -----------------------------------------------------------------------------
165
166#[cfg(not(any(
167    portable_atomic_no_atomic_load_store,
168    target_arch = "avr",
169    target_arch = "msp430",
170)))]
171#[cfg_attr(
172    portable_atomic_no_cfg_target_has_atomic,
173    cfg(not(all(
174        any(
175            target_arch = "riscv32",
176            target_arch = "riscv64",
177            feature = "critical-section",
178            portable_atomic_unsafe_assume_single_core,
179        ),
180        portable_atomic_no_atomic_cas,
181    )))
182)]
183#[cfg_attr(
184    not(portable_atomic_no_cfg_target_has_atomic),
185    cfg(not(all(
186        any(
187            target_arch = "riscv32",
188            target_arch = "riscv64",
189            feature = "critical-section",
190            portable_atomic_unsafe_assume_single_core,
191        ),
192        not(target_has_atomic = "ptr"),
193    )))
194)]
195items! {
196    pub(crate) use self::core_atomic::{
197        AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, AtomicU8,
198        AtomicUsize,
199    };
200    #[cfg_attr(
201        portable_atomic_no_cfg_target_has_atomic,
202        cfg(any(
203            not(portable_atomic_no_atomic_64),
204            not(any(target_pointer_width = "16", target_pointer_width = "32")),
205        ))
206    )]
207    #[cfg_attr(
208        not(portable_atomic_no_cfg_target_has_atomic),
209        cfg(any(
210            target_has_atomic = "64",
211            not(any(target_pointer_width = "16", target_pointer_width = "32")),
212        ))
213    )]
214    pub(crate) use self::core_atomic::{AtomicI64, AtomicU64};
215}
216// bpf
217#[cfg(all(
218    target_arch = "bpf",
219    portable_atomic_no_atomic_load_store,
220    not(feature = "critical-section"),
221))]
222pub(crate) use self::core_atomic::{AtomicI64, AtomicIsize, AtomicPtr, AtomicU64, AtomicUsize};
223
224// RISC-V without A-extension & !(assume single core | critical section)
225#[cfg(not(any(portable_atomic_unsafe_assume_single_core, feature = "critical-section")))]
226#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_cas))]
227#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "ptr")))]
228#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
229items! {
230    pub(crate) use self::riscv::{
231        AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, AtomicU8,
232        AtomicUsize,
233    };
234    #[cfg(target_arch = "riscv64")]
235    pub(crate) use self::riscv::{AtomicI64, AtomicU64};
236}
237
238// no core atomic CAS & (assume single core | critical section) => critical section based fallback
239#[cfg(any(
240    portable_atomic_unsafe_assume_single_core,
241    feature = "critical-section",
242    target_arch = "avr",
243    target_arch = "msp430",
244))]
245#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_cas))]
246#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "ptr")))]
247items! {
248    pub(crate) use self::interrupt::{
249        AtomicI16, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU8, AtomicUsize,
250    };
251    #[cfg(any(not(target_pointer_width = "16"), feature = "fallback"))]
252    pub(crate) use self::interrupt::{AtomicI32, AtomicU32};
253    #[cfg(any(
254        not(any(target_pointer_width = "16", target_pointer_width = "32")),
255        feature = "fallback",
256    ))]
257    pub(crate) use self::interrupt::{AtomicI64, AtomicU64};
258    #[cfg(feature = "fallback")]
259    pub(crate) use self::interrupt::{AtomicI128, AtomicU128};
260}
261
262// no core (64-bit | 128-bit) atomic & has CAS => use lock-base fallback
263#[cfg(feature = "fallback")]
264#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
265#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
266items! {
267    #[cfg(not(any(
268        all(
269            target_arch = "arm",
270            not(any(miri, portable_atomic_sanitize_thread)),
271            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
272            any(target_os = "linux", target_os = "android"),
273            not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
274            not(portable_atomic_no_outline_atomics),
275        ),
276        all(
277            target_arch = "riscv32",
278            not(any(miri, portable_atomic_sanitize_thread)),
279            not(portable_atomic_no_asm),
280            not(portable_atomic_pre_llvm_19),
281            any(
282                target_feature = "experimental-zacas",
283                portable_atomic_target_feature = "experimental-zacas",
284                all(
285                    feature = "fallback",
286                    not(portable_atomic_no_outline_atomics),
287                    any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
288                    any(target_os = "linux", target_os = "android"),
289                ),
290            ),
291        ),
292    )))]
293    #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))]
294    #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))]
295    pub(crate) use self::fallback::{AtomicI64, AtomicU64};
296    #[cfg(not(any(
297        all(
298            target_arch = "aarch64",
299            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
300        ),
301        all(target_arch = "arm64ec", portable_atomic_unstable_asm_experimental_arch),
302        all(
303            target_arch = "x86_64",
304            not(all(
305                any(miri, portable_atomic_sanitize_thread),
306                portable_atomic_no_cmpxchg16b_intrinsic,
307            )),
308            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
309            any(
310                target_feature = "cmpxchg16b",
311                portable_atomic_target_feature = "cmpxchg16b",
312                all(
313                    feature = "fallback",
314                    not(portable_atomic_no_outline_atomics),
315                    not(any(target_env = "sgx", miri)),
316                ),
317            ),
318        ),
319        all(
320            target_arch = "riscv64",
321            not(portable_atomic_no_asm),
322            not(portable_atomic_pre_llvm_19),
323            any(
324                target_feature = "experimental-zacas",
325                portable_atomic_target_feature = "experimental-zacas",
326                all(
327                    feature = "fallback",
328                    not(portable_atomic_no_outline_atomics),
329                    any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
330                    any(target_os = "linux", target_os = "android"),
331                    not(any(miri, portable_atomic_sanitize_thread)),
332                ),
333            ),
334        ),
335        all(
336            target_arch = "powerpc64",
337            portable_atomic_unstable_asm_experimental_arch,
338            any(
339                target_feature = "quadword-atomics",
340                portable_atomic_target_feature = "quadword-atomics",
341                all(
342                    feature = "fallback",
343                    not(portable_atomic_no_outline_atomics),
344                    portable_atomic_outline_atomics, // TODO(powerpc64): currently disabled by default
345                    any(
346                        all(
347                            target_os = "linux",
348                            any(
349                                target_env = "gnu",
350                                all(
351                                    any(target_env = "musl", target_env = "ohos"),
352                                    not(target_feature = "crt-static"),
353                                ),
354                                portable_atomic_outline_atomics,
355                            ),
356                        ),
357                        target_os = "android",
358                        target_os = "freebsd",
359                        all(target_os = "openbsd", portable_atomic_outline_atomics),
360                    ),
361                    not(any(miri, portable_atomic_sanitize_thread)),
362                ),
363            ),
364        ),
365        all(target_arch = "s390x", portable_atomic_unstable_asm_experimental_arch),
366    )))]
367    pub(crate) use self::fallback::{AtomicI128, AtomicU128};
368}
369
370// 64-bit atomics (platform-specific)
371// pre-v6 Arm Linux
372#[cfg(feature = "fallback")]
373#[cfg(all(
374    target_arch = "arm",
375    not(any(miri, portable_atomic_sanitize_thread)),
376    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
377    any(target_os = "linux", target_os = "android"),
378    not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
379    not(portable_atomic_no_outline_atomics),
380))]
381#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))]
382#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))]
383pub(crate) use self::atomic64::arm_linux::{AtomicI64, AtomicU64};
384#[cfg(all(
385    target_arch = "riscv32",
386    not(any(miri, portable_atomic_sanitize_thread)),
387    not(portable_atomic_no_asm),
388    not(portable_atomic_pre_llvm_19),
389    any(
390        target_feature = "experimental-zacas",
391        portable_atomic_target_feature = "experimental-zacas",
392        all(
393            feature = "fallback",
394            not(portable_atomic_no_outline_atomics),
395            any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
396            any(target_os = "linux", target_os = "android"),
397        ),
398    ),
399))]
400pub(crate) use self::atomic64::riscv32::{AtomicI64, AtomicU64};
401
402// 128-bit atomics (platform-specific)
403// AArch64
404#[cfg(any(
405    all(target_arch = "aarch64", any(not(portable_atomic_no_asm), portable_atomic_unstable_asm)),
406    all(target_arch = "arm64ec", portable_atomic_unstable_asm_experimental_arch)
407))]
408pub(crate) use self::atomic128::aarch64::{AtomicI128, AtomicU128};
409// x86_64 & (cmpxchg16b | outline-atomics)
410#[cfg(all(
411    target_arch = "x86_64",
412    not(all(any(miri, portable_atomic_sanitize_thread), portable_atomic_no_cmpxchg16b_intrinsic)),
413    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
414    any(
415        target_feature = "cmpxchg16b",
416        portable_atomic_target_feature = "cmpxchg16b",
417        all(
418            feature = "fallback",
419            not(portable_atomic_no_outline_atomics),
420            not(any(target_env = "sgx", miri)),
421        ),
422    ),
423))]
424pub(crate) use self::atomic128::x86_64::{AtomicI128, AtomicU128};
425// riscv64 & zacas
426#[cfg(all(
427    target_arch = "riscv64",
428    not(portable_atomic_no_asm),
429    not(portable_atomic_pre_llvm_19),
430    any(
431        target_feature = "experimental-zacas",
432        portable_atomic_target_feature = "experimental-zacas",
433        all(
434            feature = "fallback",
435            not(portable_atomic_no_outline_atomics),
436            any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
437            any(target_os = "linux", target_os = "android"),
438            not(any(miri, portable_atomic_sanitize_thread)),
439        ),
440    ),
441))]
442pub(crate) use self::atomic128::riscv64::{AtomicI128, AtomicU128};
443// powerpc64 & (pwr8 | outline-atomics)
444#[cfg(all(
445    target_arch = "powerpc64",
446    portable_atomic_unstable_asm_experimental_arch,
447    any(
448        target_feature = "quadword-atomics",
449        portable_atomic_target_feature = "quadword-atomics",
450        all(
451            feature = "fallback",
452            not(portable_atomic_no_outline_atomics),
453            portable_atomic_outline_atomics, // TODO(powerpc64): currently disabled by default
454            any(
455                all(
456                    target_os = "linux",
457                    any(
458                        target_env = "gnu",
459                        all(
460                            any(target_env = "musl", target_env = "ohos"),
461                            not(target_feature = "crt-static"),
462                        ),
463                        portable_atomic_outline_atomics,
464                    ),
465                ),
466                target_os = "android",
467                target_os = "freebsd",
468                all(target_os = "openbsd", portable_atomic_outline_atomics),
469            ),
470            not(any(miri, portable_atomic_sanitize_thread)),
471        ),
472    ),
473))]
474pub(crate) use self::atomic128::powerpc64::{AtomicI128, AtomicU128};
475// s390x
476#[cfg(all(target_arch = "s390x", portable_atomic_unstable_asm_experimental_arch))]
477pub(crate) use self::atomic128::s390x::{AtomicI128, AtomicU128};
portable_atomic/imp/mod.rs

portable_atomic/imp/
mod.rs
