1use crate::phase_locker::{MutPhaseLocker, PhaseGuard};
2use crate::phase_locker::{SyncPhaseLocker, UnSyncPhaseLocker};
3use crate::{GeneratorTolerance, Phase, Sequential};
4use core::marker::PhantomData;
5
6pub(crate) type SyncSequentializer<G> = generic::LazySequentializer<SyncPhaseLocker, G>;
7
8pub(crate) type UnSyncSequentializer<G> = generic::LazySequentializer<UnSyncPhaseLocker, G>;
9
10#[inline]
11#[cold]
12fn lazy_initialization_only<'a, T: 'a, P: PhaseGuard<'a, T>>(
13 mut phase_guard: P,
14 init: impl FnOnce(&'a T),
15) -> P {
16 let cur = Phase::empty();
17
18 let initialized = cur | Phase::INITIALIZED;
19
20 let initialization_panic = cur | Phase::INITIALIZATION_PANICKED | Phase::INITIALIZATION_SKIPED;
21
22 phase_guard.transition(init, initialized, initialization_panic);
23
24 phase_guard
25}
26
27#[inline]
28#[cold]
29fn mut_lazy_initialization_only<P: MutPhaseLocker>(locker: &mut P, init: impl FnOnce()) {
30 let cur = Phase::empty();
31
32 let initialized = cur | Phase::INITIALIZED;
33
34 let initialization_panic = cur | Phase::INITIALIZATION_PANICKED | Phase::INITIALIZATION_SKIPED;
35
36 locker.transition(init, initialized, initialization_panic);
37}
38
39struct Unit<T>(PhantomData<T>);
40
41impl<T> Unit<T> {
42 const fn new() -> Self {
43 Self(PhantomData)
44 }
45}
46
47#[inline]
48#[cold]
49fn lazy_initialization<'a, P: PhaseGuard<'a, S>, S: Sequential + 'a, Tol: GeneratorTolerance>(
50 mut phase_guard: P,
51 init: impl FnOnce(&'a <S as Sequential>::Data),
52 reg: impl FnOnce(&'a S) -> bool,
53 _: Unit<Tol>,
54) -> P
55where
56 <S as Sequential>::Data: 'a,
57{
58 let cur = phase_guard.phase();
59
60 debug_assert!(!cur.intersects(Phase::FINALIZED | Phase::FINALIZATION_PANICKED));
61
62 debug_assert!(!cur.intersects(Phase::INITIALIZED));
63
64 let registration_finished;
65
66 if !Tol::INIT_FAILURE || cur.is_empty() {
67 debug_assert!(cur.is_empty());
68
69 let registration_failed = Phase::REGISTRATION_PANICKED | Phase::INITIALIZATION_SKIPED;
70
71 if phase_guard.transition(reg, cur, registration_failed) {
72 registration_finished = Phase::REGISTERED;
73 } else {
74 registration_finished = Phase::REGISTRATION_REFUSED;
75 }
76 } else {
77 registration_finished = cur;
78 }
79
80 if registration_finished.intersects(Phase::REGISTERED) {
81 let before_init = if Tol::INIT_FAILURE {
82 registration_finished
83 & !(Phase::INITIALIZED
84 | Phase::INITIALIZATION_PANICKED
85 | Phase::INITIALIZATION_SKIPED)
86 } else {
87 registration_finished
88 };
89
90 let initialized = before_init | Phase::INITIALIZED;
91
92 let initialization_panic =
93 before_init | Phase::INITIALIZATION_PANICKED | Phase::INITIALIZATION_SKIPED;
94
95 phase_guard.transition(
96 |s| init(Sequential::data(s)),
97 initialized,
98 initialization_panic,
99 )
100 } else if Tol::FINAL_REGISTRATION_FAILURE {
101 let before_init = if Tol::INIT_FAILURE {
102 registration_finished
103 & !(Phase::INITIALIZED
104 | Phase::INITIALIZATION_PANICKED
105 | Phase::INITIALIZATION_SKIPED)
106 } else {
107 registration_finished
108 };
109
110 let initialized = before_init | Phase::INITIALIZED;
111
112 let initialization_panic =
113 before_init | Phase::INITIALIZATION_PANICKED | Phase::INITIALIZATION_SKIPED;
114
115 phase_guard.transition(
116 |s| init(Sequential::data(s)),
117 initialized,
118 initialization_panic,
119 )
120 } else {
121 let no_init = registration_finished | Phase::INITIALIZATION_SKIPED;
122
123 phase_guard.set_phase(no_init);
124 }
125 phase_guard
126}
127
128fn lazy_finalization<'a, T: 'a, P: PhaseGuard<'a, T>>(mut phase_guard: P, f: impl FnOnce(&'a T)) {
129 let cur = phase_guard.phase();
130
131 let finalizing_success = cur | Phase::FINALIZED;
132
133 let finalizing_failed = cur | Phase::FINALIZATION_PANICKED;
134
135 phase_guard.transition(f, finalizing_success, finalizing_failed);
136}
137
138mod generic {
139 use super::{
140 lazy_finalization, lazy_initialization, lazy_initialization_only,
141 mut_lazy_initialization_only, Unit,
142 };
143 use crate::phase_locker::{
144 LockNature, LockResult, Mappable, MutPhaseLocker, PhaseGuard, PhaseLocker,
145 };
146 use crate::{
147 FinalizableLazySequentializer, GeneratorTolerance,
148 LazySequentializer as LazySequentializerTrait, Phase, Phased, Sequential, Sequentializer,
149 UniqueLazySequentializer,
150 };
151
152 #[cfg(debug_mode)]
153 use crate::CyclicPanic;
154 use core::hint::unreachable_unchecked;
155 use core::marker::PhantomData;
156 #[cfg(debug_mode)]
157 use core::sync::atomic::{AtomicUsize, Ordering};
158
159 pub struct LazySequentializer<Locker, G>(
223 Locker,
224 PhantomData<G>,
225 #[cfg(debug_mode)] AtomicUsize,
226 );
227
228 impl<L, G> Phased for LazySequentializer<L, G>
229 where
230 L: Phased,
231 {
232 #[inline(always)]
233 fn phase(this: &Self) -> Phase {
234 Phased::phase(&this.0)
235 }
236 }
237 impl<L, G> LazySequentializer<L, G> {
238 #[inline(always)]
239 pub const fn new(locker: L) -> Self {
240 Self(
241 locker,
242 PhantomData,
243 #[cfg(debug_mode)]
244 AtomicUsize::new(0),
245 )
246 }
247 }
248
249 unsafe impl<'a, T: Sequential + 'a, L: 'a, G: 'a + GeneratorTolerance> Sequentializer<'a, T>
251 for LazySequentializer<L, G>
252 where
253 T::Sequentializer: AsRef<LazySequentializer<L, G>>,
254 T::Sequentializer: AsMut<LazySequentializer<L, G>>,
255 L: PhaseLocker<'a, T::Data>,
256 L: Phased,
257 {
258 type ReadGuard = L::ReadGuard;
259 type WriteGuard = L::WriteGuard;
260
261 #[inline(always)]
262 fn lock(
263 s: &'a T,
264 lock_nature: impl Fn(Phase) -> LockNature,
265 hint: Phase,
266 ) -> LockResult<Self::ReadGuard, Self::WriteGuard> {
267 let this = Sequential::sequentializer(s).as_ref();
268
269 let data = Sequential::data(s);
270
271 this.0.lock(data, &lock_nature, &lock_nature, hint)
272 }
273
274 #[inline(always)]
275 fn try_lock(
276 s: &'a T,
277 lock_nature: impl Fn(Phase) -> LockNature,
278 hint: Phase,
279 ) -> Option<LockResult<Self::ReadGuard, Self::WriteGuard>> {
280 let this = Sequential::sequentializer(s).as_ref();
281
282 let data = Sequential::data(s);
283
284 this.0.try_lock(data, &lock_nature, hint)
285 }
286
287 #[inline(always)]
288 fn lock_mut(s: &'a mut T) -> Self::WriteGuard {
289 let (that, data) = Sequential::sequentializer_data_mut(s);
290
291 that.as_mut().0.lock_mut(data)
292 }
293 }
294
295 #[inline(always)]
296 fn whole_lock<'a, T: Sequential + 'a, L: 'a, G: 'a>(
297 s: &'a T,
298 lock_nature: impl Fn(Phase) -> LockNature,
299 hint: Phase,
300 ) -> LockResult<L::ReadGuard, L::WriteGuard>
301 where
302 T::Sequentializer: AsRef<LazySequentializer<L, G>>,
303 T::Sequentializer: AsMut<LazySequentializer<L, G>>,
304 L: PhaseLocker<'a, T>,
305 {
306 let this = Sequential::sequentializer(s).as_ref();
307
308 this.0.lock(s, &lock_nature, &lock_nature, hint)
309 }
310
311 #[inline(always)]
312 fn try_whole_lock<'a, T: Sequential + 'a, L: 'a, G: 'a>(
313 s: &'a T,
314 lock_nature: impl Fn(Phase) -> LockNature,
315 hint: Phase,
316 ) -> Option<LockResult<L::ReadGuard, L::WriteGuard>>
317 where
318 T::Sequentializer: AsRef<LazySequentializer<L, G>>,
319 T::Sequentializer: AsMut<LazySequentializer<L, G>>,
320 L: PhaseLocker<'a, T>,
321 {
322 let this = Sequential::sequentializer(s).as_ref();
323
324 this.0.try_lock(s, &lock_nature, hint)
325 }
326
327 #[cfg(not(debug_mode))]
328 fn debug_save_thread<T>(_: &T) {}
329
330 #[cfg(debug_mode)]
331 fn debug_save_thread<T: Sequential, L, G>(s: &T) -> DebugGuard<'_, T, L, G>
332 where
333 T::Sequentializer: AsRef<LazySequentializer<L, G>>,
334 {
335 let this = Sequential::sequentializer(s).as_ref();
336 use parking_lot::lock_api::GetThreadId;
337 this.2.store(
338 parking_lot::RawThreadId.nonzero_thread_id().into(),
339 Ordering::Relaxed,
340 );
341 DebugGuard(s, PhantomData)
342 }
343
344 #[must_use]
345 #[cfg(debug_mode)]
346 struct DebugGuard<'a, T: Sequential, L, G>(&'a T, PhantomData<(L, G)>)
347 where
348 T::Sequentializer: AsRef<LazySequentializer<L, G>>;
349
350 #[cfg(debug_mode)]
351 impl<'a, T: Sequential, L, G> Drop for DebugGuard<'a, T, L, G>
352 where
353 T::Sequentializer: AsRef<LazySequentializer<L, G>>,
354 {
355 fn drop(&mut self) {
356 let this = Sequential::sequentializer(self.0).as_ref();
357 this.2.store(0, Ordering::Relaxed);
358 }
359 }
360
361 #[inline(always)]
362 fn debug_test<T: Sequential, L, G>(_s: &T)
363 where
364 T::Sequentializer: AsRef<LazySequentializer<L, G>>,
365 {
366 #[cfg(debug_mode)]
367 {
368 let this = Sequential::sequentializer(_s).as_ref();
369 let id = this.2.load(Ordering::Relaxed);
370 if id != 0 {
371 use parking_lot::lock_api::GetThreadId;
372 if id == parking_lot::RawThreadId.nonzero_thread_id().into() {
373 std::panic::panic_any(CyclicPanic);
374 }
375 }
376 }
377 }
378 unsafe impl<'a, T: Sequential + 'a, L: 'a, G: 'a> FinalizableLazySequentializer<'a, T>
380 for LazySequentializer<L, G>
381 where
382 T::Sequentializer: AsRef<LazySequentializer<L, G>>,
383 T::Sequentializer: AsMut<LazySequentializer<L, G>>,
384 L: PhaseLocker<'a, T>,
385 L: PhaseLocker<'a, T::Data>,
386 L: Phased,
387 <L as PhaseLocker<'a, T>>::ReadGuard:
388 Mappable<T, T::Data, <L as PhaseLocker<'a, T::Data>>::ReadGuard>,
389 <L as PhaseLocker<'a, T>>::WriteGuard:
390 Mappable<T, T::Data, <L as PhaseLocker<'a, T::Data>>::WriteGuard>,
391 <L as PhaseLocker<'a, T::Data>>::ReadGuard:
392 From<<L as PhaseLocker<'a, T::Data>>::WriteGuard>,
393 G: GeneratorTolerance,
394 {
395 #[inline(always)]
396 fn init(
397 s: &'a T,
398 shall_init: impl Fn(Phase) -> bool,
399 init: impl FnOnce(&'a <T as Sequential>::Data),
400 reg: impl FnOnce(&'a T) -> bool,
401 ) -> Phase {
402 let this = Sequential::sequentializer(s).as_ref();
403
404 let phase_guard = match this.0.lock(
405 s,
406 |p| {
407 if shall_init(p) {
408 debug_test(s);
409 LockNature::Write
410 } else {
411 LockNature::None
412 }
413 },
414 |_| LockNature::Read,
415 Phase::INITIALIZED | Phase::REGISTERED,
416 ) {
417 LockResult::None(p) => return p,
418 LockResult::Write(l) => l,
419 LockResult::Read(l) => return Phased::phase(&l),
420 };
421
422 let _g = debug_save_thread(s);
423 let ph = lazy_initialization(phase_guard, init, reg, Unit::<G>::new());
424 ph.phase()
425 }
426
427 #[inline(always)]
428 fn init_then_read_guard(
429 s: &'a T,
430 shall_init: impl Fn(Phase) -> bool,
431 init: impl FnOnce(&'a <T as Sequential>::Data),
432 reg: impl FnOnce(&'a T) -> bool,
433 ) -> Self::ReadGuard {
434 let this = Sequential::sequentializer(s).as_ref();
435
436 match this.0.lock(
437 s,
438 |p| {
439 if shall_init(p) {
440 debug_test(s);
441 LockNature::Write
442 } else {
443 LockNature::Read
444 }
445 },
446 |_| LockNature::Read,
447 Phase::INITIALIZED | Phase::REGISTERED,
448 ) {
449 LockResult::Read(l) => l.map(|s| Sequential::data(s)),
450 LockResult::Write(l) => {
451 let _g = debug_save_thread(s);
452 let l = lazy_initialization(l, init, reg, Unit::<G>::new());
453 l.map(|s| Sequential::data(s)).into()
454 }
455 LockResult::None(_) => unsafe { unreachable_unchecked() },
456 }
457 }
458 #[inline(always)]
459 fn init_then_write_guard(
460 s: &'a T,
461 shall_init: impl Fn(Phase) -> bool,
462 init: impl FnOnce(&'a <T as Sequential>::Data),
463 reg: impl FnOnce(&'a T) -> bool,
464 ) -> Self::WriteGuard {
465 match whole_lock(s, |_| LockNature::Write, Phase::INITIALIZED) {
466 LockResult::Write(l) => {
467 if shall_init(l.phase()) {
468 debug_test(s);
469 let _g = debug_save_thread(s);
470 let l = lazy_initialization(l, init, reg, Unit::<G>::new());
471 l.map(|s| Sequential::data(s))
472 } else {
473 l.map(|s| Sequential::data(s))
474 }
475 }
476 LockResult::Read(_) => unsafe { unreachable_unchecked() },
477 LockResult::None(_) => unsafe { unreachable_unchecked() },
478 }
479 }
480
481 #[inline(always)]
482 fn try_init_then_read_guard(
483 s: &'a T,
484 shall_init: impl Fn(Phase) -> bool,
485 init: impl FnOnce(&'a <T as Sequential>::Data),
486 reg: impl FnOnce(&'a T) -> bool,
487 ) -> Option<Self::ReadGuard> {
488 let this = Sequential::sequentializer(s).as_ref();
489
490 this.0
491 .try_lock(
492 s,
493 |p| {
494 if shall_init(p) {
495 debug_test(s);
496 LockNature::Write
497 } else {
498 LockNature::Read
499 }
500 },
501 Phase::INITIALIZED | Phase::REGISTERED,
502 )
503 .map(|l| match l {
504 LockResult::Read(l) => l.map(|s| Sequential::data(s)),
505 LockResult::Write(l) => {
506 let _g = debug_save_thread(s);
507 let l = lazy_initialization(l, init, reg, Unit::<G>::new());
508 l.map(|s| Sequential::data(s)).into()
509 }
510 LockResult::None(_) => unsafe { unreachable_unchecked() },
511 })
512 }
513 #[inline(always)]
514 fn try_init_then_write_guard(
515 s: &'a T,
516 shall_init: impl Fn(Phase) -> bool,
517 init: impl FnOnce(&'a <T as Sequential>::Data),
518 reg: impl FnOnce(&'a T) -> bool,
519 ) -> Option<Self::WriteGuard> {
520 try_whole_lock(
521 s,
522 |_| LockNature::Write,
523 Phase::INITIALIZED | Phase::REGISTERED,
524 )
525 .map(|l| match l {
526 LockResult::Write(l) => {
527 if shall_init(l.phase()) {
528 debug_test(s);
529 let _g = debug_save_thread(s);
530 let l = lazy_initialization(l, init, reg, Unit::<G>::new());
531 l.map(|s| Sequential::data(s))
532 } else {
533 l.map(|s| Sequential::data(s))
534 }
535 }
536 LockResult::Read(_) => unsafe { unreachable_unchecked() },
537 LockResult::None(_) => unsafe { unreachable_unchecked() },
538 })
539 }
540 #[inline(always)]
541
542 fn finalize_callback(s: &'a T, f: impl FnOnce(&'a T::Data)) {
543 let this = Sequential::sequentializer(s).as_ref();
544
545 let how = |p: Phase| {
546 if p.intersects(Phase::INITIALIZED) {
547 LockNature::Write
548 } else {
549 LockNature::None
550 }
551 };
552
553 let phase_guard = match this.0.lock(
554 Sequential::data(s),
555 how,
556 how,
557 Phase::INITIALIZED | Phase::REGISTERED,
558 ) {
559 LockResult::None(_) => return,
560 LockResult::Write(l) => l,
561 LockResult::Read(_) => unsafe { unreachable_unchecked() },
562 };
563
564 debug_assert!((phase_guard.phase()
565 & (Phase::FINALIZED | Phase::FINALIZATION_PANICKED))
566 .is_empty());
567
568 lazy_finalization(phase_guard, f);
569 }
570 }
571
572 impl<L, G> AsRef<LazySequentializer<L, G>> for LazySequentializer<L, G> {
573 #[inline(always)]
574 fn as_ref(&self) -> &Self {
575 self
576 }
577 }
578 impl<L, G> AsMut<LazySequentializer<L, G>> for LazySequentializer<L, G> {
579 #[inline(always)]
580 fn as_mut(&mut self) -> &mut Self {
581 self
582 }
583 }
584
585 unsafe impl<'a, T: Sequential + 'a, L: 'a, G: 'a> LazySequentializerTrait<'a, T>
587 for LazySequentializer<L, G>
588 where
589 T::Sequentializer: AsRef<LazySequentializer<L, G>>,
590 T::Sequentializer: AsMut<LazySequentializer<L, G>>,
591 L: PhaseLocker<'a, T>,
592 L: PhaseLocker<'a, T::Data>,
593 L: Phased,
594 <L as PhaseLocker<'a, T>>::ReadGuard:
595 Mappable<T, T::Data, <L as PhaseLocker<'a, T::Data>>::ReadGuard>,
596 <L as PhaseLocker<'a, T>>::WriteGuard:
597 Mappable<T, T::Data, <L as PhaseLocker<'a, T::Data>>::WriteGuard>,
598 <L as PhaseLocker<'a, T::Data>>::ReadGuard:
599 From<<L as PhaseLocker<'a, T::Data>>::WriteGuard>,
600 G: GeneratorTolerance,
601 {
602 const INITIALIZED_HINT: Phase = Phase::INITIALIZED;
603 #[inline(always)]
604 fn init(
605 s: &'a T,
606 shall_init: impl Fn(Phase) -> bool,
607 init: impl FnOnce(&'a <T as Sequential>::Data),
608 ) -> Phase {
609 let this = Sequential::sequentializer(s).as_ref();
610
611 let phase_guard = match this.0.lock(
612 Sequential::data(s),
613 |p| {
614 if shall_init(p) {
615 debug_test(s);
616 LockNature::Write
617 } else {
618 LockNature::None
619 }
620 },
621 |_| LockNature::Read,
622 Phase::INITIALIZED,
623 ) {
624 LockResult::None(p) => return p,
625 LockResult::Write(l) => l,
626 LockResult::Read(l) => return Phased::phase(&l),
627 };
628
629 let _g = debug_save_thread(s);
630 let ph = lazy_initialization_only(phase_guard, init);
631 ph.phase()
632 }
633
634 #[inline(always)]
635 fn init_then_read_guard(
636 s: &'a T,
637 shall_init: impl Fn(Phase) -> bool,
638 init: impl FnOnce(&'a <T as Sequential>::Data),
639 ) -> Self::ReadGuard {
640 let this = Sequential::sequentializer(s).as_ref();
641
642 match this.0.lock(
643 Sequential::data(s),
644 |p| {
645 if shall_init(p) {
646 debug_test(s);
647 LockNature::Write
648 } else {
649 LockNature::Read
650 }
651 },
652 |_| LockNature::Read,
653 Phase::INITIALIZED,
654 ) {
655 LockResult::Read(l) => l,
656 LockResult::Write(l) => {
657 let _g = debug_save_thread(s);
658 let l = lazy_initialization_only(l, init);
659 l.into()
660 }
661 LockResult::None(_) => unsafe { unreachable_unchecked() },
662 }
663 }
664 #[inline(always)]
665 fn init_then_write_guard(
666 s: &'a T,
667 shall_init: impl Fn(Phase) -> bool,
668 init: impl FnOnce(&'a <T as Sequential>::Data),
669 ) -> Self::WriteGuard {
670 match <Self as Sequentializer<'a, T>>::lock(
671 s,
672 |_| LockNature::Write,
673 Phase::INITIALIZED,
674 ) {
675 LockResult::Write(l) => {
676 if shall_init(l.phase()) {
677 debug_test(s);
678 let _g = debug_save_thread(s);
679 lazy_initialization_only(l, init)
680 } else {
681 l
682 }
683 }
684 LockResult::Read(_) => unsafe { unreachable_unchecked() },
685 LockResult::None(_) => unsafe { unreachable_unchecked() },
686 }
687 }
688
689 #[inline(always)]
690 fn try_init_then_read_guard(
691 s: &'a T,
692 shall_init: impl Fn(Phase) -> bool,
693 init: impl FnOnce(&'a <T as Sequential>::Data),
694 ) -> Option<Self::ReadGuard> {
695 let this = Sequential::sequentializer(s).as_ref();
696
697 this.0
698 .try_lock(
699 Sequential::data(s),
700 |p| {
701 if shall_init(p) {
702 debug_test(s);
703 LockNature::Write
704 } else {
705 LockNature::Read
706 }
707 },
708 Phase::INITIALIZED,
709 )
710 .map(|l| match l {
711 LockResult::Read(l) => l,
712 LockResult::Write(l) => {
713 let _g = debug_save_thread(s);
714 let l = lazy_initialization_only(l, init);
715 l.into()
716 }
717 LockResult::None(_) => unsafe { unreachable_unchecked() },
718 })
719 }
720 #[inline(always)]
721 fn try_init_then_write_guard(
722 s: &'a T,
723 shall_init: impl Fn(Phase) -> bool,
724 init: impl FnOnce(&'a <T as Sequential>::Data),
725 ) -> Option<Self::WriteGuard> {
726 <Self as Sequentializer<'a, T>>::try_lock(s, |_| LockNature::Write, Phase::INITIALIZED)
727 .map(|l| match l {
728 LockResult::Write(l) => {
729 if shall_init(l.phase()) {
730 debug_test(s);
731 let _g = debug_save_thread(s);
732 lazy_initialization_only(l, init)
733 } else {
734 l
735 }
736 }
737 LockResult::Read(_) => unsafe { unreachable_unchecked() },
738 LockResult::None(_) => unsafe { unreachable_unchecked() },
739 })
740 }
741 }
742
743 impl<L, T: Sequential<Sequentializer = Self>, G: GeneratorTolerance> UniqueLazySequentializer<T>
744 for LazySequentializer<L, G>
745 where
746 L: MutPhaseLocker,
747 {
748 fn init_unique(
749 target: &mut T,
750 shall_init: impl Fn(Phase) -> bool,
751 init: impl FnOnce(&mut <T as Sequential>::Data),
752 ) -> Phase {
753 let (that, data) = Sequential::sequentializer_data_mut(target);
754
755 if shall_init(that.0.get_phase_unique()) {
758 mut_lazy_initialization_only(&mut that.0, || init(data));
759 }
760 that.0.get_phase_unique()
761 }
762 }
763}