cranelift_codegen/isa/x64/inst/
args.rs

1//! Instruction operand sub-components (aka "parts"): definitions and printing.
2
3use super::regs::{self};
4use super::EmitState;
5use crate::ir::condcodes::{FloatCC, IntCC};
6use crate::ir::{MemFlags, Type};
7use crate::isa::x64::inst::regs::pretty_print_reg;
8use crate::isa::x64::inst::Inst;
9use crate::machinst::*;
10use regalloc2::VReg;
11use smallvec::{smallvec, SmallVec};
12use std::fmt;
13use std::string::String;
14
15pub use crate::isa::x64::lower::isle::generated_code::DivSignedness;
16
17/// An extenstion trait for converting `Writable{Xmm,Gpr}` to `Writable<Reg>`.
18pub trait ToWritableReg {
19    /// Convert `Writable{Xmm,Gpr}` to `Writable<Reg>`.
20    fn to_writable_reg(&self) -> Writable<Reg>;
21}
22
23/// An extension trait for converting `Writable<Reg>` to `Writable{Xmm,Gpr}`.
24pub trait FromWritableReg: Sized {
25    /// Convert `Writable<Reg>` to `Writable{Xmm,Gpr}`.
26    fn from_writable_reg(w: Writable<Reg>) -> Option<Self>;
27}
28
29/// A macro for defining a newtype of `Reg` that enforces some invariant about
30/// the wrapped `Reg` (such as that it is of a particular register class).
31macro_rules! newtype_of_reg {
32    (
33        $newtype_reg:ident,
34        $newtype_writable_reg:ident,
35        $newtype_option_writable_reg:ident,
36        reg_mem: ($($newtype_reg_mem:ident $(aligned:$aligned:ident)?),*),
37        reg_mem_imm: ($($newtype_reg_mem_imm:ident $(aligned:$aligned_imm:ident)?),*),
38        $newtype_imm8_reg:ident,
39        |$check_reg:ident| $check:expr
40    ) => {
41        /// A newtype wrapper around `Reg`.
42        #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
43        pub struct $newtype_reg(Reg);
44
45        impl PartialEq<Reg> for $newtype_reg {
46            fn eq(&self, other: &Reg) -> bool {
47                self.0 == *other
48            }
49        }
50
51        impl From<$newtype_reg> for Reg {
52            fn from(r: $newtype_reg) -> Self {
53                r.0
54            }
55        }
56
57        impl $newtype_reg {
58            /// Create this newtype from the given register, or return `None` if the register
59            /// is not a valid instance of this newtype.
60            pub fn new($check_reg: Reg) -> Option<Self> {
61                if $check {
62                    Some(Self($check_reg))
63                } else {
64                    None
65                }
66            }
67
68            /// Get this newtype's underlying `Reg`.
69            pub fn to_reg(self) -> Reg {
70                self.0
71            }
72        }
73
74        // Convenience impl so that people working with this newtype can use it
75        // "just like" a plain `Reg`.
76        //
77        // NB: We cannot implement `DerefMut` because that would let people do
78        // nasty stuff like `*my_gpr.deref_mut() = some_xmm_reg`, breaking the
79        // invariants that `Gpr` provides.
80        impl std::ops::Deref for $newtype_reg {
81            type Target = Reg;
82
83            fn deref(&self) -> &Reg {
84                &self.0
85            }
86        }
87
88        /// Writable Gpr.
89        pub type $newtype_writable_reg = Writable<$newtype_reg>;
90
91        #[allow(dead_code)] // Used by some newtypes and not others.
92        /// Optional writable Gpr.
93        pub type $newtype_option_writable_reg = Option<Writable<$newtype_reg>>;
94
95        impl ToWritableReg for $newtype_writable_reg {
96            fn to_writable_reg(&self) -> Writable<Reg> {
97                Writable::from_reg(self.to_reg().to_reg())
98            }
99        }
100
101        impl FromWritableReg for $newtype_writable_reg {
102            fn from_writable_reg(w: Writable<Reg>) -> Option<Self> {
103                Some(Writable::from_reg($newtype_reg::new(w.to_reg())?))
104            }
105        }
106
107        $(
108            /// A newtype wrapper around `RegMem` for general-purpose registers.
109            #[derive(Clone, Debug)]
110            pub struct $newtype_reg_mem(RegMem);
111
112            impl From<$newtype_reg_mem> for RegMem {
113                fn from(rm: $newtype_reg_mem) -> Self {
114                    rm.0
115                }
116            }
117
118            impl From<$newtype_reg> for $newtype_reg_mem {
119                fn from(r: $newtype_reg) -> Self {
120                    $newtype_reg_mem(RegMem::reg(r.into()))
121                }
122            }
123
124            impl $newtype_reg_mem {
125                /// Construct a `RegMem` newtype from the given `RegMem`, or return
126                /// `None` if the `RegMem` is not a valid instance of this `RegMem`
127                /// newtype.
128                pub fn new(rm: RegMem) -> Option<Self> {
129                    match rm {
130                        RegMem::Mem { addr } => {
131                            let mut _allow = true;
132                            $(
133                                if $aligned {
134                                    _allow = addr.aligned();
135                                }
136                            )?
137                            if _allow {
138                                Some(Self(RegMem::Mem { addr }))
139                            } else {
140                                None
141                            }
142                        }
143                        RegMem::Reg { reg: $check_reg } if $check => Some(Self(rm)),
144                        RegMem::Reg { reg: _ } => None,
145                    }
146                }
147
148                /// Convert this newtype into its underlying `RegMem`.
149                pub fn to_reg_mem(self) -> RegMem {
150                    self.0
151                }
152
153                #[allow(dead_code)] // Used by some newtypes and not others.
154                pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
155                    &self,
156                    collector: &mut OperandCollector<'_, F>,
157                ) {
158                    self.0.get_operands(collector);
159                }
160            }
161            impl PrettyPrint for $newtype_reg_mem {
162                fn pretty_print(&self, size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
163                    self.0.pretty_print(size, allocs)
164                }
165            }
166        )*
167
168        $(
169            /// A newtype wrapper around `RegMemImm`.
170            #[derive(Clone, Debug)]
171            pub struct $newtype_reg_mem_imm(RegMemImm);
172
173            impl From<$newtype_reg_mem_imm> for RegMemImm {
174                fn from(rmi: $newtype_reg_mem_imm) -> RegMemImm {
175                    rmi.0
176                }
177            }
178
179            impl From<$newtype_reg> for $newtype_reg_mem_imm {
180                fn from(r: $newtype_reg) -> Self {
181                    $newtype_reg_mem_imm(RegMemImm::reg(r.into()))
182                }
183            }
184
185            impl $newtype_reg_mem_imm {
186                /// Construct this newtype from the given `RegMemImm`, or return
187                /// `None` if the `RegMemImm` is not a valid instance of this
188                /// newtype.
189                pub fn new(rmi: RegMemImm) -> Option<Self> {
190                    match rmi {
191                        RegMemImm::Imm { .. } => Some(Self(rmi)),
192                        RegMemImm::Mem { addr } => {
193                            let mut _allow = true;
194                            $(
195                                if $aligned_imm {
196                                    _allow = addr.aligned();
197                                }
198                            )?
199                            if _allow {
200                                Some(Self(RegMemImm::Mem { addr }))
201                            } else {
202                                None
203                            }
204                        }
205                        RegMemImm::Reg { reg: $check_reg } if $check => Some(Self(rmi)),
206                        RegMemImm::Reg { reg: _ } => None,
207                    }
208                }
209
210                /// Convert this newtype into its underlying `RegMemImm`.
211                #[allow(dead_code)] // Used by some newtypes and not others.
212                pub fn to_reg_mem_imm(self) -> RegMemImm {
213                    self.0
214                }
215
216                #[allow(dead_code)] // Used by some newtypes and not others.
217                pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
218                    &self,
219                    collector: &mut OperandCollector<'_, F>,
220                ) {
221                    self.0.get_operands(collector);
222                }
223            }
224
225            impl PrettyPrint for $newtype_reg_mem_imm {
226                fn pretty_print(&self, size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
227                    self.0.pretty_print(size, allocs)
228                }
229            }
230        )*
231
232        /// A newtype wrapper around `Imm8Reg`.
233        #[derive(Clone, Debug)]
234        #[allow(dead_code)] // Used by some newtypes and not others.
235        pub struct $newtype_imm8_reg(Imm8Reg);
236
237        impl From<$newtype_reg> for $newtype_imm8_reg {
238            fn from(r: $newtype_reg) -> Self {
239                Self(Imm8Reg::Reg { reg: r.to_reg() })
240            }
241        }
242
243        impl $newtype_imm8_reg {
244            /// Construct this newtype from the given `Imm8Reg`, or return
245            /// `None` if the `Imm8Reg` is not a valid instance of this newtype.
246            #[allow(dead_code)] // Used by some newtypes and not others.
247            pub fn new(imm8_reg: Imm8Reg) -> Option<Self> {
248                match imm8_reg {
249                    Imm8Reg::Imm8 { .. } => Some(Self(imm8_reg)),
250                    Imm8Reg::Reg { reg: $check_reg } if $check => Some(Self(imm8_reg)),
251                    Imm8Reg::Reg { reg: _ } => None,
252                }
253            }
254
255            /// Convert this newtype into its underlying `Imm8Reg`.
256            #[allow(dead_code)] // Used by some newtypes and not others.
257            pub fn to_imm8_reg(self) -> Imm8Reg {
258                self.0
259            }
260        }
261    };
262}
263
264// Define a newtype of `Reg` for general-purpose registers.
265newtype_of_reg!(
266    Gpr,
267    WritableGpr,
268    OptionWritableGpr,
269    reg_mem: (GprMem),
270    reg_mem_imm: (GprMemImm),
271    Imm8Gpr,
272    |reg| reg.class() == RegClass::Int
273);
274
275// Define a newtype of `Reg` for XMM registers.
276newtype_of_reg!(
277    Xmm,
278    WritableXmm,
279    OptionWritableXmm,
280    reg_mem: (XmmMem, XmmMemAligned aligned:true),
281    reg_mem_imm: (XmmMemImm, XmmMemAlignedImm aligned:true),
282    Imm8Xmm,
283    |reg| reg.class() == RegClass::Float
284);
285
286// N.B.: `Amode` is defined in `inst.isle`. We add some convenience
287// constructors here.
288
289// Re-export the type from the ISLE generated code.
290pub use crate::isa::x64::lower::isle::generated_code::Amode;
291
292impl Amode {
293    /// Create an immediate sign-extended and register addressing mode.
294    pub fn imm_reg(simm32: u32, base: Reg) -> Self {
295        debug_assert!(base.class() == RegClass::Int);
296        Self::ImmReg {
297            simm32,
298            base,
299            flags: MemFlags::trusted(),
300        }
301    }
302
303    /// Create a sign-extended-32-to-64 with register and shift addressing mode.
304    pub fn imm_reg_reg_shift(simm32: u32, base: Gpr, index: Gpr, shift: u8) -> Self {
305        debug_assert!(base.class() == RegClass::Int);
306        debug_assert!(index.class() == RegClass::Int);
307        debug_assert!(shift <= 3);
308        Self::ImmRegRegShift {
309            simm32,
310            base,
311            index,
312            shift,
313            flags: MemFlags::trusted(),
314        }
315    }
316
317    pub(crate) fn rip_relative(target: MachLabel) -> Self {
318        Self::RipRelative { target }
319    }
320
321    pub(crate) fn with_flags(&self, flags: MemFlags) -> Self {
322        match self {
323            &Self::ImmReg { simm32, base, .. } => Self::ImmReg {
324                simm32,
325                base,
326                flags,
327            },
328            &Self::ImmRegRegShift {
329                simm32,
330                base,
331                index,
332                shift,
333                ..
334            } => Self::ImmRegRegShift {
335                simm32,
336                base,
337                index,
338                shift,
339                flags,
340            },
341            _ => panic!("Amode {:?} cannot take memflags", self),
342        }
343    }
344
345    /// Add the registers mentioned by `self` to `collector`.
346    pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
347        &self,
348        collector: &mut OperandCollector<'_, F>,
349    ) {
350        match self {
351            Amode::ImmReg { base, .. } => {
352                if *base != regs::rbp() && *base != regs::rsp() {
353                    collector.reg_use(*base);
354                }
355            }
356            Amode::ImmRegRegShift { base, index, .. } => {
357                debug_assert_ne!(base.to_reg(), regs::rbp());
358                debug_assert_ne!(base.to_reg(), regs::rsp());
359                collector.reg_use(base.to_reg());
360                debug_assert_ne!(index.to_reg(), regs::rbp());
361                debug_assert_ne!(index.to_reg(), regs::rsp());
362                collector.reg_use(index.to_reg());
363            }
364            Amode::RipRelative { .. } => {
365                // RIP isn't involved in regalloc.
366            }
367        }
368    }
369
370    /// Same as `get_operands`, but add the registers in the "late" phase.
371    pub(crate) fn get_operands_late<F: Fn(VReg) -> VReg>(
372        &self,
373        collector: &mut OperandCollector<'_, F>,
374    ) {
375        match self {
376            Amode::ImmReg { base, .. } => {
377                collector.reg_late_use(*base);
378            }
379            Amode::ImmRegRegShift { base, index, .. } => {
380                collector.reg_late_use(base.to_reg());
381                collector.reg_late_use(index.to_reg());
382            }
383            Amode::RipRelative { .. } => {
384                // RIP isn't involved in regalloc.
385            }
386        }
387    }
388
389    pub(crate) fn get_flags(&self) -> MemFlags {
390        match self {
391            Amode::ImmReg { flags, .. } | Amode::ImmRegRegShift { flags, .. } => *flags,
392            Amode::RipRelative { .. } => MemFlags::trusted(),
393        }
394    }
395
396    pub(crate) fn can_trap(&self) -> bool {
397        !self.get_flags().notrap()
398    }
399
400    pub(crate) fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {
401        // The order in which we consume allocs here must match the
402        // order in which we produce operands in get_operands() above.
403        match self {
404            &Amode::ImmReg {
405                simm32,
406                base,
407                flags,
408            } => {
409                let base = if base == regs::rsp() || base == regs::rbp() {
410                    base
411                } else {
412                    allocs.next(base)
413                };
414                Amode::ImmReg {
415                    simm32,
416                    flags,
417                    base,
418                }
419            }
420            &Amode::ImmRegRegShift {
421                simm32,
422                base,
423                index,
424                shift,
425                flags,
426            } => Amode::ImmRegRegShift {
427                simm32,
428                shift,
429                flags,
430                base: Gpr::new(allocs.next(*base)).unwrap(),
431                index: Gpr::new(allocs.next(*index)).unwrap(),
432            },
433            &Amode::RipRelative { target } => Amode::RipRelative { target },
434        }
435    }
436
437    /// Offset the amode by a fixed offset.
438    pub(crate) fn offset(&self, offset: u32) -> Self {
439        let mut ret = self.clone();
440        match &mut ret {
441            &mut Amode::ImmReg { ref mut simm32, .. } => *simm32 += offset,
442            &mut Amode::ImmRegRegShift { ref mut simm32, .. } => *simm32 += offset,
443            _ => panic!("Cannot offset amode: {:?}", self),
444        }
445        ret
446    }
447
448    pub(crate) fn aligned(&self) -> bool {
449        self.get_flags().aligned()
450    }
451}
452
453impl PrettyPrint for Amode {
454    fn pretty_print(&self, _size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
455        match self {
456            Amode::ImmReg { simm32, base, .. } => {
457                // Note: size is always 8; the address is 64 bits,
458                // even if the addressed operand is smaller.
459                format!("{}({})", *simm32 as i32, pretty_print_reg(*base, 8, allocs))
460            }
461            Amode::ImmRegRegShift {
462                simm32,
463                base,
464                index,
465                shift,
466                ..
467            } => format!(
468                "{}({},{},{})",
469                *simm32 as i32,
470                pretty_print_reg(base.to_reg(), 8, allocs),
471                pretty_print_reg(index.to_reg(), 8, allocs),
472                1 << shift
473            ),
474            Amode::RipRelative { ref target } => format!("label{}(%rip)", target.get()),
475        }
476    }
477}
478
479/// A Memory Address. These denote a 64-bit value only.
480/// Used for usual addressing modes as well as addressing modes used during compilation, when the
481/// moving SP offset is not known.
482#[derive(Clone, Debug)]
483pub enum SyntheticAmode {
484    /// A real amode.
485    Real(Amode),
486
487    /// A (virtual) offset to the "nominal SP" value, which will be recomputed as we push and pop
488    /// within the function.
489    NominalSPOffset {
490        /// The nominal stack pointer value.
491        simm32: u32,
492    },
493
494    /// A virtual offset to a constant that will be emitted in the constant section of the buffer.
495    ConstantOffset(VCodeConstant),
496}
497
498impl SyntheticAmode {
499    /// Create a real addressing mode.
500    pub fn real(amode: Amode) -> Self {
501        Self::Real(amode)
502    }
503
504    pub(crate) fn nominal_sp_offset(simm32: u32) -> Self {
505        SyntheticAmode::NominalSPOffset { simm32 }
506    }
507
508    /// Add the registers mentioned by `self` to `collector`.
509    pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
510        &self,
511        collector: &mut OperandCollector<'_, F>,
512    ) {
513        match self {
514            SyntheticAmode::Real(addr) => addr.get_operands(collector),
515            SyntheticAmode::NominalSPOffset { .. } => {
516                // Nothing to do; the base is SP and isn't involved in regalloc.
517            }
518            SyntheticAmode::ConstantOffset(_) => {}
519        }
520    }
521
522    /// Same as `get_operands`, but add the register in the "late" phase.
523    pub(crate) fn get_operands_late<F: Fn(VReg) -> VReg>(
524        &self,
525        collector: &mut OperandCollector<'_, F>,
526    ) {
527        match self {
528            SyntheticAmode::Real(addr) => addr.get_operands_late(collector),
529            SyntheticAmode::NominalSPOffset { .. } => {
530                // Nothing to do; the base is SP and isn't involved in regalloc.
531            }
532            SyntheticAmode::ConstantOffset(_) => {}
533        }
534    }
535
536    pub(crate) fn finalize(&self, state: &mut EmitState, buffer: &MachBuffer<Inst>) -> Amode {
537        match self {
538            SyntheticAmode::Real(addr) => addr.clone(),
539            SyntheticAmode::NominalSPOffset { simm32 } => {
540                let off = *simm32 as i64 + state.virtual_sp_offset;
541                // TODO will require a sequence of add etc.
542                assert!(
543                    off <= u32::max_value() as i64,
544                    "amode finalize: add sequence NYI"
545                );
546                Amode::imm_reg(off as u32, regs::rsp())
547            }
548            SyntheticAmode::ConstantOffset(c) => {
549                Amode::rip_relative(buffer.get_label_for_constant(*c))
550            }
551        }
552    }
553
554    pub(crate) fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {
555        match self {
556            SyntheticAmode::Real(addr) => SyntheticAmode::Real(addr.with_allocs(allocs)),
557            &SyntheticAmode::NominalSPOffset { .. } | &SyntheticAmode::ConstantOffset { .. } => {
558                self.clone()
559            }
560        }
561    }
562
563    pub(crate) fn aligned(&self) -> bool {
564        match self {
565            SyntheticAmode::Real(addr) => addr.aligned(),
566            SyntheticAmode::NominalSPOffset { .. } | SyntheticAmode::ConstantOffset { .. } => true,
567        }
568    }
569}
570
571impl Into<SyntheticAmode> for Amode {
572    fn into(self) -> SyntheticAmode {
573        SyntheticAmode::Real(self)
574    }
575}
576
577impl Into<SyntheticAmode> for VCodeConstant {
578    fn into(self) -> SyntheticAmode {
579        SyntheticAmode::ConstantOffset(self)
580    }
581}
582
583impl PrettyPrint for SyntheticAmode {
584    fn pretty_print(&self, _size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
585        match self {
586            // See note in `Amode` regarding constant size of `8`.
587            SyntheticAmode::Real(addr) => addr.pretty_print(8, allocs),
588            SyntheticAmode::NominalSPOffset { simm32 } => {
589                format!("rsp({} + virtual offset)", *simm32 as i32)
590            }
591            SyntheticAmode::ConstantOffset(c) => format!("const({})", c.as_u32()),
592        }
593    }
594}
595
596/// An operand which is either an integer Register, a value in Memory or an Immediate.  This can
597/// denote an 8, 16, 32 or 64 bit value.  For the Immediate form, in the 8- and 16-bit case, only
598/// the lower 8 or 16 bits of `simm32` is relevant.  In the 64-bit case, the value denoted by
599/// `simm32` is its sign-extension out to 64 bits.
600#[derive(Clone, Debug)]
601pub enum RegMemImm {
602    /// A register operand.
603    Reg {
604        /// The underlying register.
605        reg: Reg,
606    },
607    /// A memory operand.
608    Mem {
609        /// The memory address.
610        addr: SyntheticAmode,
611    },
612    /// An immediate operand.
613    Imm {
614        /// The immediate value.
615        simm32: u32,
616    },
617}
618
619impl RegMemImm {
620    /// Create a register operand.
621    pub fn reg(reg: Reg) -> Self {
622        debug_assert!(reg.class() == RegClass::Int || reg.class() == RegClass::Float);
623        Self::Reg { reg }
624    }
625
626    /// Create a memory operand.
627    pub fn mem(addr: impl Into<SyntheticAmode>) -> Self {
628        Self::Mem { addr: addr.into() }
629    }
630
631    /// Create an immediate operand.
632    pub fn imm(simm32: u32) -> Self {
633        Self::Imm { simm32 }
634    }
635
636    /// Asserts that in register mode, the reg class is the one that's expected.
637    pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) {
638        if let Self::Reg { reg } = self {
639            debug_assert_eq!(reg.class(), expected_reg_class);
640        }
641    }
642
643    /// Add the regs mentioned by `self` to `collector`.
644    pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
645        &self,
646        collector: &mut OperandCollector<'_, F>,
647    ) {
648        match self {
649            Self::Reg { reg } => collector.reg_use(*reg),
650            Self::Mem { addr } => addr.get_operands(collector),
651            Self::Imm { .. } => {}
652        }
653    }
654
655    pub(crate) fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {
656        match self {
657            Self::Reg { reg } => Self::Reg {
658                reg: allocs.next(*reg),
659            },
660            Self::Mem { addr } => Self::Mem {
661                addr: addr.with_allocs(allocs),
662            },
663            Self::Imm { .. } => self.clone(),
664        }
665    }
666}
667
668impl From<RegMem> for RegMemImm {
669    fn from(rm: RegMem) -> RegMemImm {
670        match rm {
671            RegMem::Reg { reg } => RegMemImm::Reg { reg },
672            RegMem::Mem { addr } => RegMemImm::Mem { addr },
673        }
674    }
675}
676
677impl PrettyPrint for RegMemImm {
678    fn pretty_print(&self, size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
679        match self {
680            Self::Reg { reg } => pretty_print_reg(*reg, size, allocs),
681            Self::Mem { addr } => addr.pretty_print(size, allocs),
682            Self::Imm { simm32 } => format!("${}", *simm32 as i32),
683        }
684    }
685}
686
687/// An operand which is either an 8-bit integer immediate or a register.
688#[derive(Clone, Debug)]
689pub enum Imm8Reg {
690    /// 8-bit immediate operand.
691    Imm8 {
692        /// The 8-bit immediate value.
693        imm: u8,
694    },
695    /// A register operand.
696    Reg {
697        /// The underlying register.
698        reg: Reg,
699    },
700}
701
702impl From<u8> for Imm8Reg {
703    fn from(imm: u8) -> Self {
704        Self::Imm8 { imm }
705    }
706}
707
708impl From<Reg> for Imm8Reg {
709    fn from(reg: Reg) -> Self {
710        Self::Reg { reg }
711    }
712}
713
714/// An operand which is either an integer Register or a value in Memory.  This can denote an 8, 16,
715/// 32, 64, or 128 bit value.
716#[derive(Clone, Debug)]
717pub enum RegMem {
718    /// A register operand.
719    Reg {
720        /// The underlying register.
721        reg: Reg,
722    },
723    /// A memory operand.
724    Mem {
725        /// The memory address.
726        addr: SyntheticAmode,
727    },
728}
729
730impl RegMem {
731    /// Create a register operand.
732    pub fn reg(reg: Reg) -> Self {
733        debug_assert!(reg.class() == RegClass::Int || reg.class() == RegClass::Float);
734        Self::Reg { reg }
735    }
736
737    /// Create a memory operand.
738    pub fn mem(addr: impl Into<SyntheticAmode>) -> Self {
739        Self::Mem { addr: addr.into() }
740    }
741    /// Asserts that in register mode, the reg class is the one that's expected.
742    pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) {
743        if let Self::Reg { reg } = self {
744            debug_assert_eq!(reg.class(), expected_reg_class);
745        }
746    }
747    /// Add the regs mentioned by `self` to `collector`.
748    pub(crate) fn get_operands<F: Fn(VReg) -> VReg>(
749        &self,
750        collector: &mut OperandCollector<'_, F>,
751    ) {
752        match self {
753            RegMem::Reg { reg } => collector.reg_use(*reg),
754            RegMem::Mem { addr, .. } => addr.get_operands(collector),
755        }
756    }
757
758    pub(crate) fn with_allocs(&self, allocs: &mut AllocationConsumer<'_>) -> Self {
759        match self {
760            RegMem::Reg { reg } => RegMem::Reg {
761                reg: allocs.next(*reg),
762            },
763            RegMem::Mem { addr } => RegMem::Mem {
764                addr: addr.with_allocs(allocs),
765            },
766        }
767    }
768}
769
770impl From<Reg> for RegMem {
771    fn from(reg: Reg) -> RegMem {
772        RegMem::Reg { reg }
773    }
774}
775
776impl From<Writable<Reg>> for RegMem {
777    fn from(r: Writable<Reg>) -> Self {
778        RegMem::reg(r.to_reg())
779    }
780}
781
782impl PrettyPrint for RegMem {
783    fn pretty_print(&self, size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
784        match self {
785            RegMem::Reg { reg } => pretty_print_reg(*reg, size, allocs),
786            RegMem::Mem { addr, .. } => addr.pretty_print(size, allocs),
787        }
788    }
789}
790
791/// Some basic ALU operations.
792#[derive(Copy, Clone, PartialEq)]
793pub enum AluRmiROpcode {
794    /// Add operation.
795    Add,
796    /// Add with carry.
797    Adc,
798    /// Integer subtraction.
799    Sub,
800    /// Integer subtraction with borrow.
801    Sbb,
802    /// Bitwise AND operation.
803    And,
804    /// Bitwise inclusive OR.
805    Or,
806    /// Bitwise exclusive OR.
807    Xor,
808    /// The signless, non-extending (N x N -> N, for N in {32,64}) variant.
809    Mul,
810}
811
812impl fmt::Debug for AluRmiROpcode {
813    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
814        let name = match self {
815            AluRmiROpcode::Add => "add",
816            AluRmiROpcode::Adc => "adc",
817            AluRmiROpcode::Sub => "sub",
818            AluRmiROpcode::Sbb => "sbb",
819            AluRmiROpcode::And => "and",
820            AluRmiROpcode::Or => "or",
821            AluRmiROpcode::Xor => "xor",
822            AluRmiROpcode::Mul => "imul",
823        };
824        write!(fmt, "{}", name)
825    }
826}
827
828impl fmt::Display for AluRmiROpcode {
829    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
830        fmt::Debug::fmt(self, f)
831    }
832}
833
834/// ALU operations that don't accept intermediates.
835#[derive(Copy, Clone, PartialEq)]
836pub enum AluRmROpcode {
837    /// And with negated second operand.
838    Andn,
839}
840
841impl AluRmROpcode {
842    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
843        match self {
844            AluRmROpcode::Andn => smallvec![InstructionSet::BMI1],
845        }
846    }
847}
848
849impl fmt::Debug for AluRmROpcode {
850    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
851        let name = match self {
852            AluRmROpcode::Andn => "andn",
853        };
854        write!(fmt, "{}", name)
855    }
856}
857
858impl fmt::Display for AluRmROpcode {
859    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
860        fmt::Debug::fmt(self, f)
861    }
862}
863
864#[derive(Clone, PartialEq)]
865/// Unary operations requiring register or memory and register operands.
866pub enum UnaryRmROpcode {
867    /// Bit-scan reverse.
868    Bsr,
869    /// Bit-scan forward.
870    Bsf,
871    /// Counts leading zeroes (Leading Zero CouNT).
872    Lzcnt,
873    /// Counts trailing zeroes (Trailing Zero CouNT).
874    Tzcnt,
875    /// Counts the number of ones (POPulation CouNT).
876    Popcnt,
877}
878
879impl UnaryRmROpcode {
880    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
881        match self {
882            UnaryRmROpcode::Bsr | UnaryRmROpcode::Bsf => smallvec![],
883            UnaryRmROpcode::Lzcnt => smallvec![InstructionSet::Lzcnt],
884            UnaryRmROpcode::Tzcnt => smallvec![InstructionSet::BMI1],
885            UnaryRmROpcode::Popcnt => smallvec![InstructionSet::Popcnt],
886        }
887    }
888}
889
890impl fmt::Debug for UnaryRmROpcode {
891    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
892        match self {
893            UnaryRmROpcode::Bsr => write!(fmt, "bsr"),
894            UnaryRmROpcode::Bsf => write!(fmt, "bsf"),
895            UnaryRmROpcode::Lzcnt => write!(fmt, "lzcnt"),
896            UnaryRmROpcode::Tzcnt => write!(fmt, "tzcnt"),
897            UnaryRmROpcode::Popcnt => write!(fmt, "popcnt"),
898        }
899    }
900}
901
902impl fmt::Display for UnaryRmROpcode {
903    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
904        fmt::Debug::fmt(self, f)
905    }
906}
907
908#[derive(Clone, Copy, PartialEq)]
909/// Comparison operations.
910pub enum CmpOpcode {
911    /// CMP instruction: compute `a - b` and set flags from result.
912    Cmp,
913    /// TEST instruction: compute `a & b` and set flags from result.
914    Test,
915}
916
917#[derive(Debug)]
918pub(crate) enum InstructionSet {
919    SSE,
920    SSE2,
921    SSSE3,
922    SSE41,
923    SSE42,
924    Popcnt,
925    Lzcnt,
926    BMI1,
927    #[allow(dead_code)] // never constructed (yet).
928    BMI2,
929    FMA,
930    AVX,
931    AVX2,
932    AVX512BITALG,
933    AVX512DQ,
934    AVX512F,
935    AVX512VBMI,
936    AVX512VL,
937}
938
939/// Some SSE operations requiring 2 operands r/m and r.
940#[derive(Clone, Copy, PartialEq)]
941#[allow(dead_code)] // some variants here aren't used just yet
942#[allow(missing_docs)]
943pub enum SseOpcode {
944    Addps,
945    Addpd,
946    Addss,
947    Addsd,
948    Andps,
949    Andpd,
950    Andnps,
951    Andnpd,
952    Blendvpd,
953    Blendvps,
954    Comiss,
955    Comisd,
956    Cmpps,
957    Cmppd,
958    Cmpss,
959    Cmpsd,
960    Cvtdq2ps,
961    Cvtdq2pd,
962    Cvtpd2ps,
963    Cvtps2pd,
964    Cvtsd2ss,
965    Cvtsd2si,
966    Cvtsi2ss,
967    Cvtsi2sd,
968    Cvtss2si,
969    Cvtss2sd,
970    Cvttpd2dq,
971    Cvttps2dq,
972    Cvttss2si,
973    Cvttsd2si,
974    Divps,
975    Divpd,
976    Divss,
977    Divsd,
978    Insertps,
979    Maxps,
980    Maxpd,
981    Maxss,
982    Maxsd,
983    Minps,
984    Minpd,
985    Minss,
986    Minsd,
987    Movaps,
988    Movapd,
989    Movd,
990    Movdqa,
991    Movdqu,
992    Movlhps,
993    Movmskps,
994    Movmskpd,
995    Movq,
996    Movss,
997    Movsd,
998    Movups,
999    Movupd,
1000    Mulps,
1001    Mulpd,
1002    Mulss,
1003    Mulsd,
1004    Orps,
1005    Orpd,
1006    Pabsb,
1007    Pabsw,
1008    Pabsd,
1009    Packssdw,
1010    Packsswb,
1011    Packusdw,
1012    Packuswb,
1013    Paddb,
1014    Paddd,
1015    Paddq,
1016    Paddw,
1017    Paddsb,
1018    Paddsw,
1019    Paddusb,
1020    Paddusw,
1021    Palignr,
1022    Pand,
1023    Pandn,
1024    Pavgb,
1025    Pavgw,
1026    Pblendvb,
1027    Pcmpeqb,
1028    Pcmpeqw,
1029    Pcmpeqd,
1030    Pcmpeqq,
1031    Pcmpgtb,
1032    Pcmpgtw,
1033    Pcmpgtd,
1034    Pcmpgtq,
1035    Pextrb,
1036    Pextrw,
1037    Pextrd,
1038    Pextrq,
1039    Pinsrb,
1040    Pinsrw,
1041    Pinsrd,
1042    Pmaddubsw,
1043    Pmaddwd,
1044    Pmaxsb,
1045    Pmaxsw,
1046    Pmaxsd,
1047    Pmaxub,
1048    Pmaxuw,
1049    Pmaxud,
1050    Pminsb,
1051    Pminsw,
1052    Pminsd,
1053    Pminub,
1054    Pminuw,
1055    Pminud,
1056    Pmovmskb,
1057    Pmovsxbd,
1058    Pmovsxbw,
1059    Pmovsxbq,
1060    Pmovsxwd,
1061    Pmovsxwq,
1062    Pmovsxdq,
1063    Pmovzxbd,
1064    Pmovzxbw,
1065    Pmovzxbq,
1066    Pmovzxwd,
1067    Pmovzxwq,
1068    Pmovzxdq,
1069    Pmuldq,
1070    Pmulhw,
1071    Pmulhuw,
1072    Pmulhrsw,
1073    Pmulld,
1074    Pmullw,
1075    Pmuludq,
1076    Por,
1077    Pshufb,
1078    Pshufd,
1079    Psllw,
1080    Pslld,
1081    Psllq,
1082    Psraw,
1083    Psrad,
1084    Psrlw,
1085    Psrld,
1086    Psrlq,
1087    Psubb,
1088    Psubd,
1089    Psubq,
1090    Psubw,
1091    Psubsb,
1092    Psubsw,
1093    Psubusb,
1094    Psubusw,
1095    Ptest,
1096    Punpckhbw,
1097    Punpckhwd,
1098    Punpcklbw,
1099    Punpcklwd,
1100    Pxor,
1101    Rcpss,
1102    Roundps,
1103    Roundpd,
1104    Roundss,
1105    Roundsd,
1106    Rsqrtss,
1107    Shufps,
1108    Sqrtps,
1109    Sqrtpd,
1110    Sqrtss,
1111    Sqrtsd,
1112    Subps,
1113    Subpd,
1114    Subss,
1115    Subsd,
1116    Ucomiss,
1117    Ucomisd,
1118    Unpcklps,
1119    Xorps,
1120    Xorpd,
1121    Phaddw,
1122    Phaddd,
1123    Punpckhdq,
1124    Punpckldq,
1125    Punpckhqdq,
1126    Punpcklqdq,
1127    Pshuflw,
1128    Pshufhw,
1129    Pblendw,
1130    Movddup,
1131}
1132
1133impl SseOpcode {
1134    /// Which `InstructionSet` is the first supporting this opcode?
1135    pub(crate) fn available_from(&self) -> InstructionSet {
1136        use InstructionSet::*;
1137        match self {
1138            SseOpcode::Addps
1139            | SseOpcode::Addss
1140            | SseOpcode::Andps
1141            | SseOpcode::Andnps
1142            | SseOpcode::Comiss
1143            | SseOpcode::Cmpps
1144            | SseOpcode::Cmpss
1145            | SseOpcode::Cvtsi2ss
1146            | SseOpcode::Cvtss2si
1147            | SseOpcode::Cvttss2si
1148            | SseOpcode::Divps
1149            | SseOpcode::Divss
1150            | SseOpcode::Maxps
1151            | SseOpcode::Maxss
1152            | SseOpcode::Minps
1153            | SseOpcode::Minss
1154            | SseOpcode::Movaps
1155            | SseOpcode::Movlhps
1156            | SseOpcode::Movmskps
1157            | SseOpcode::Movss
1158            | SseOpcode::Movups
1159            | SseOpcode::Mulps
1160            | SseOpcode::Mulss
1161            | SseOpcode::Orps
1162            | SseOpcode::Rcpss
1163            | SseOpcode::Rsqrtss
1164            | SseOpcode::Shufps
1165            | SseOpcode::Sqrtps
1166            | SseOpcode::Sqrtss
1167            | SseOpcode::Subps
1168            | SseOpcode::Subss
1169            | SseOpcode::Ucomiss
1170            | SseOpcode::Unpcklps
1171            | SseOpcode::Xorps => SSE,
1172
1173            SseOpcode::Addpd
1174            | SseOpcode::Addsd
1175            | SseOpcode::Andpd
1176            | SseOpcode::Andnpd
1177            | SseOpcode::Cmppd
1178            | SseOpcode::Cmpsd
1179            | SseOpcode::Comisd
1180            | SseOpcode::Cvtdq2ps
1181            | SseOpcode::Cvtdq2pd
1182            | SseOpcode::Cvtpd2ps
1183            | SseOpcode::Cvtps2pd
1184            | SseOpcode::Cvtsd2ss
1185            | SseOpcode::Cvtsd2si
1186            | SseOpcode::Cvtsi2sd
1187            | SseOpcode::Cvtss2sd
1188            | SseOpcode::Cvttpd2dq
1189            | SseOpcode::Cvttps2dq
1190            | SseOpcode::Cvttsd2si
1191            | SseOpcode::Divpd
1192            | SseOpcode::Divsd
1193            | SseOpcode::Maxpd
1194            | SseOpcode::Maxsd
1195            | SseOpcode::Minpd
1196            | SseOpcode::Minsd
1197            | SseOpcode::Movapd
1198            | SseOpcode::Movd
1199            | SseOpcode::Movmskpd
1200            | SseOpcode::Movq
1201            | SseOpcode::Movsd
1202            | SseOpcode::Movupd
1203            | SseOpcode::Movdqa
1204            | SseOpcode::Movdqu
1205            | SseOpcode::Mulpd
1206            | SseOpcode::Mulsd
1207            | SseOpcode::Orpd
1208            | SseOpcode::Packssdw
1209            | SseOpcode::Packsswb
1210            | SseOpcode::Packuswb
1211            | SseOpcode::Paddb
1212            | SseOpcode::Paddd
1213            | SseOpcode::Paddq
1214            | SseOpcode::Paddw
1215            | SseOpcode::Paddsb
1216            | SseOpcode::Paddsw
1217            | SseOpcode::Paddusb
1218            | SseOpcode::Paddusw
1219            | SseOpcode::Pand
1220            | SseOpcode::Pandn
1221            | SseOpcode::Pavgb
1222            | SseOpcode::Pavgw
1223            | SseOpcode::Pcmpeqb
1224            | SseOpcode::Pcmpeqw
1225            | SseOpcode::Pcmpeqd
1226            | SseOpcode::Pcmpgtb
1227            | SseOpcode::Pcmpgtw
1228            | SseOpcode::Pcmpgtd
1229            | SseOpcode::Pextrw
1230            | SseOpcode::Pinsrw
1231            | SseOpcode::Pmaddubsw
1232            | SseOpcode::Pmaddwd
1233            | SseOpcode::Pmaxsw
1234            | SseOpcode::Pmaxub
1235            | SseOpcode::Pminsw
1236            | SseOpcode::Pminub
1237            | SseOpcode::Pmovmskb
1238            | SseOpcode::Pmulhw
1239            | SseOpcode::Pmulhuw
1240            | SseOpcode::Pmullw
1241            | SseOpcode::Pmuludq
1242            | SseOpcode::Por
1243            | SseOpcode::Pshufd
1244            | SseOpcode::Psllw
1245            | SseOpcode::Pslld
1246            | SseOpcode::Psllq
1247            | SseOpcode::Psraw
1248            | SseOpcode::Psrad
1249            | SseOpcode::Psrlw
1250            | SseOpcode::Psrld
1251            | SseOpcode::Psrlq
1252            | SseOpcode::Psubb
1253            | SseOpcode::Psubd
1254            | SseOpcode::Psubq
1255            | SseOpcode::Psubw
1256            | SseOpcode::Psubsb
1257            | SseOpcode::Psubsw
1258            | SseOpcode::Psubusb
1259            | SseOpcode::Psubusw
1260            | SseOpcode::Punpckhbw
1261            | SseOpcode::Punpckhwd
1262            | SseOpcode::Punpcklbw
1263            | SseOpcode::Punpcklwd
1264            | SseOpcode::Pxor
1265            | SseOpcode::Sqrtpd
1266            | SseOpcode::Sqrtsd
1267            | SseOpcode::Subpd
1268            | SseOpcode::Subsd
1269            | SseOpcode::Ucomisd
1270            | SseOpcode::Xorpd
1271            | SseOpcode::Punpckldq
1272            | SseOpcode::Punpckhdq
1273            | SseOpcode::Punpcklqdq
1274            | SseOpcode::Punpckhqdq
1275            | SseOpcode::Pshuflw
1276            | SseOpcode::Pshufhw => SSE2,
1277
1278            SseOpcode::Pabsb
1279            | SseOpcode::Pabsw
1280            | SseOpcode::Pabsd
1281            | SseOpcode::Palignr
1282            | SseOpcode::Pmulhrsw
1283            | SseOpcode::Pshufb
1284            | SseOpcode::Phaddw
1285            | SseOpcode::Phaddd
1286            | SseOpcode::Movddup => SSSE3,
1287
1288            SseOpcode::Blendvpd
1289            | SseOpcode::Blendvps
1290            | SseOpcode::Insertps
1291            | SseOpcode::Packusdw
1292            | SseOpcode::Pblendvb
1293            | SseOpcode::Pcmpeqq
1294            | SseOpcode::Pextrb
1295            | SseOpcode::Pextrd
1296            | SseOpcode::Pextrq
1297            | SseOpcode::Pinsrb
1298            | SseOpcode::Pinsrd
1299            | SseOpcode::Pmaxsb
1300            | SseOpcode::Pmaxsd
1301            | SseOpcode::Pmaxuw
1302            | SseOpcode::Pmaxud
1303            | SseOpcode::Pminsb
1304            | SseOpcode::Pminsd
1305            | SseOpcode::Pminuw
1306            | SseOpcode::Pminud
1307            | SseOpcode::Pmovsxbd
1308            | SseOpcode::Pmovsxbw
1309            | SseOpcode::Pmovsxbq
1310            | SseOpcode::Pmovsxwd
1311            | SseOpcode::Pmovsxwq
1312            | SseOpcode::Pmovsxdq
1313            | SseOpcode::Pmovzxbd
1314            | SseOpcode::Pmovzxbw
1315            | SseOpcode::Pmovzxbq
1316            | SseOpcode::Pmovzxwd
1317            | SseOpcode::Pmovzxwq
1318            | SseOpcode::Pmovzxdq
1319            | SseOpcode::Pmuldq
1320            | SseOpcode::Pmulld
1321            | SseOpcode::Ptest
1322            | SseOpcode::Roundps
1323            | SseOpcode::Roundpd
1324            | SseOpcode::Roundss
1325            | SseOpcode::Roundsd
1326            | SseOpcode::Pblendw => SSE41,
1327
1328            SseOpcode::Pcmpgtq => SSE42,
1329        }
1330    }
1331
1332    /// Returns the src operand size for an instruction.
1333    pub(crate) fn src_size(&self) -> u8 {
1334        match self {
1335            SseOpcode::Movd => 4,
1336            _ => 8,
1337        }
1338    }
1339}
1340
1341impl fmt::Debug for SseOpcode {
1342    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1343        let name = match self {
1344            SseOpcode::Addps => "addps",
1345            SseOpcode::Addpd => "addpd",
1346            SseOpcode::Addss => "addss",
1347            SseOpcode::Addsd => "addsd",
1348            SseOpcode::Andpd => "andpd",
1349            SseOpcode::Andps => "andps",
1350            SseOpcode::Andnps => "andnps",
1351            SseOpcode::Andnpd => "andnpd",
1352            SseOpcode::Blendvpd => "blendvpd",
1353            SseOpcode::Blendvps => "blendvps",
1354            SseOpcode::Cmpps => "cmpps",
1355            SseOpcode::Cmppd => "cmppd",
1356            SseOpcode::Cmpss => "cmpss",
1357            SseOpcode::Cmpsd => "cmpsd",
1358            SseOpcode::Comiss => "comiss",
1359            SseOpcode::Comisd => "comisd",
1360            SseOpcode::Cvtdq2ps => "cvtdq2ps",
1361            SseOpcode::Cvtdq2pd => "cvtdq2pd",
1362            SseOpcode::Cvtpd2ps => "cvtpd2ps",
1363            SseOpcode::Cvtps2pd => "cvtps2pd",
1364            SseOpcode::Cvtsd2ss => "cvtsd2ss",
1365            SseOpcode::Cvtsd2si => "cvtsd2si",
1366            SseOpcode::Cvtsi2ss => "cvtsi2ss",
1367            SseOpcode::Cvtsi2sd => "cvtsi2sd",
1368            SseOpcode::Cvtss2si => "cvtss2si",
1369            SseOpcode::Cvtss2sd => "cvtss2sd",
1370            SseOpcode::Cvttpd2dq => "cvttpd2dq",
1371            SseOpcode::Cvttps2dq => "cvttps2dq",
1372            SseOpcode::Cvttss2si => "cvttss2si",
1373            SseOpcode::Cvttsd2si => "cvttsd2si",
1374            SseOpcode::Divps => "divps",
1375            SseOpcode::Divpd => "divpd",
1376            SseOpcode::Divss => "divss",
1377            SseOpcode::Divsd => "divsd",
1378            SseOpcode::Insertps => "insertps",
1379            SseOpcode::Maxps => "maxps",
1380            SseOpcode::Maxpd => "maxpd",
1381            SseOpcode::Maxss => "maxss",
1382            SseOpcode::Maxsd => "maxsd",
1383            SseOpcode::Minps => "minps",
1384            SseOpcode::Minpd => "minpd",
1385            SseOpcode::Minss => "minss",
1386            SseOpcode::Minsd => "minsd",
1387            SseOpcode::Movaps => "movaps",
1388            SseOpcode::Movapd => "movapd",
1389            SseOpcode::Movd => "movd",
1390            SseOpcode::Movdqa => "movdqa",
1391            SseOpcode::Movdqu => "movdqu",
1392            SseOpcode::Movlhps => "movlhps",
1393            SseOpcode::Movmskps => "movmskps",
1394            SseOpcode::Movmskpd => "movmskpd",
1395            SseOpcode::Movq => "movq",
1396            SseOpcode::Movss => "movss",
1397            SseOpcode::Movsd => "movsd",
1398            SseOpcode::Movups => "movups",
1399            SseOpcode::Movupd => "movupd",
1400            SseOpcode::Mulps => "mulps",
1401            SseOpcode::Mulpd => "mulpd",
1402            SseOpcode::Mulss => "mulss",
1403            SseOpcode::Mulsd => "mulsd",
1404            SseOpcode::Orpd => "orpd",
1405            SseOpcode::Orps => "orps",
1406            SseOpcode::Pabsb => "pabsb",
1407            SseOpcode::Pabsw => "pabsw",
1408            SseOpcode::Pabsd => "pabsd",
1409            SseOpcode::Packssdw => "packssdw",
1410            SseOpcode::Packsswb => "packsswb",
1411            SseOpcode::Packusdw => "packusdw",
1412            SseOpcode::Packuswb => "packuswb",
1413            SseOpcode::Paddb => "paddb",
1414            SseOpcode::Paddd => "paddd",
1415            SseOpcode::Paddq => "paddq",
1416            SseOpcode::Paddw => "paddw",
1417            SseOpcode::Paddsb => "paddsb",
1418            SseOpcode::Paddsw => "paddsw",
1419            SseOpcode::Paddusb => "paddusb",
1420            SseOpcode::Paddusw => "paddusw",
1421            SseOpcode::Palignr => "palignr",
1422            SseOpcode::Pand => "pand",
1423            SseOpcode::Pandn => "pandn",
1424            SseOpcode::Pavgb => "pavgb",
1425            SseOpcode::Pavgw => "pavgw",
1426            SseOpcode::Pblendvb => "pblendvb",
1427            SseOpcode::Pcmpeqb => "pcmpeqb",
1428            SseOpcode::Pcmpeqw => "pcmpeqw",
1429            SseOpcode::Pcmpeqd => "pcmpeqd",
1430            SseOpcode::Pcmpeqq => "pcmpeqq",
1431            SseOpcode::Pcmpgtb => "pcmpgtb",
1432            SseOpcode::Pcmpgtw => "pcmpgtw",
1433            SseOpcode::Pcmpgtd => "pcmpgtd",
1434            SseOpcode::Pcmpgtq => "pcmpgtq",
1435            SseOpcode::Pextrb => "pextrb",
1436            SseOpcode::Pextrw => "pextrw",
1437            SseOpcode::Pextrd => "pextrd",
1438            SseOpcode::Pextrq => "pextrq",
1439            SseOpcode::Pinsrb => "pinsrb",
1440            SseOpcode::Pinsrw => "pinsrw",
1441            SseOpcode::Pinsrd => "pinsrd",
1442            SseOpcode::Pmaddubsw => "pmaddubsw",
1443            SseOpcode::Pmaddwd => "pmaddwd",
1444            SseOpcode::Pmaxsb => "pmaxsb",
1445            SseOpcode::Pmaxsw => "pmaxsw",
1446            SseOpcode::Pmaxsd => "pmaxsd",
1447            SseOpcode::Pmaxub => "pmaxub",
1448            SseOpcode::Pmaxuw => "pmaxuw",
1449            SseOpcode::Pmaxud => "pmaxud",
1450            SseOpcode::Pminsb => "pminsb",
1451            SseOpcode::Pminsw => "pminsw",
1452            SseOpcode::Pminsd => "pminsd",
1453            SseOpcode::Pminub => "pminub",
1454            SseOpcode::Pminuw => "pminuw",
1455            SseOpcode::Pminud => "pminud",
1456            SseOpcode::Pmovmskb => "pmovmskb",
1457            SseOpcode::Pmovsxbd => "pmovsxbd",
1458            SseOpcode::Pmovsxbw => "pmovsxbw",
1459            SseOpcode::Pmovsxbq => "pmovsxbq",
1460            SseOpcode::Pmovsxwd => "pmovsxwd",
1461            SseOpcode::Pmovsxwq => "pmovsxwq",
1462            SseOpcode::Pmovsxdq => "pmovsxdq",
1463            SseOpcode::Pmovzxbd => "pmovzxbd",
1464            SseOpcode::Pmovzxbw => "pmovzxbw",
1465            SseOpcode::Pmovzxbq => "pmovzxbq",
1466            SseOpcode::Pmovzxwd => "pmovzxwd",
1467            SseOpcode::Pmovzxwq => "pmovzxwq",
1468            SseOpcode::Pmovzxdq => "pmovzxdq",
1469            SseOpcode::Pmuldq => "pmuldq",
1470            SseOpcode::Pmulhw => "pmulhw",
1471            SseOpcode::Pmulhuw => "pmulhuw",
1472            SseOpcode::Pmulhrsw => "pmulhrsw",
1473            SseOpcode::Pmulld => "pmulld",
1474            SseOpcode::Pmullw => "pmullw",
1475            SseOpcode::Pmuludq => "pmuludq",
1476            SseOpcode::Por => "por",
1477            SseOpcode::Pshufb => "pshufb",
1478            SseOpcode::Pshufd => "pshufd",
1479            SseOpcode::Psllw => "psllw",
1480            SseOpcode::Pslld => "pslld",
1481            SseOpcode::Psllq => "psllq",
1482            SseOpcode::Psraw => "psraw",
1483            SseOpcode::Psrad => "psrad",
1484            SseOpcode::Psrlw => "psrlw",
1485            SseOpcode::Psrld => "psrld",
1486            SseOpcode::Psrlq => "psrlq",
1487            SseOpcode::Psubb => "psubb",
1488            SseOpcode::Psubd => "psubd",
1489            SseOpcode::Psubq => "psubq",
1490            SseOpcode::Psubw => "psubw",
1491            SseOpcode::Psubsb => "psubsb",
1492            SseOpcode::Psubsw => "psubsw",
1493            SseOpcode::Psubusb => "psubusb",
1494            SseOpcode::Psubusw => "psubusw",
1495            SseOpcode::Ptest => "ptest",
1496            SseOpcode::Punpckhbw => "punpckhbw",
1497            SseOpcode::Punpckhwd => "punpckhwd",
1498            SseOpcode::Punpcklbw => "punpcklbw",
1499            SseOpcode::Punpcklwd => "punpcklwd",
1500            SseOpcode::Pxor => "pxor",
1501            SseOpcode::Rcpss => "rcpss",
1502            SseOpcode::Roundps => "roundps",
1503            SseOpcode::Roundpd => "roundpd",
1504            SseOpcode::Roundss => "roundss",
1505            SseOpcode::Roundsd => "roundsd",
1506            SseOpcode::Rsqrtss => "rsqrtss",
1507            SseOpcode::Shufps => "shufps",
1508            SseOpcode::Sqrtps => "sqrtps",
1509            SseOpcode::Sqrtpd => "sqrtpd",
1510            SseOpcode::Sqrtss => "sqrtss",
1511            SseOpcode::Sqrtsd => "sqrtsd",
1512            SseOpcode::Subps => "subps",
1513            SseOpcode::Subpd => "subpd",
1514            SseOpcode::Subss => "subss",
1515            SseOpcode::Subsd => "subsd",
1516            SseOpcode::Ucomiss => "ucomiss",
1517            SseOpcode::Ucomisd => "ucomisd",
1518            SseOpcode::Unpcklps => "unpcklps",
1519            SseOpcode::Xorps => "xorps",
1520            SseOpcode::Xorpd => "xorpd",
1521            SseOpcode::Phaddw => "phaddw",
1522            SseOpcode::Phaddd => "phaddd",
1523            SseOpcode::Punpckldq => "punpckldq",
1524            SseOpcode::Punpckhdq => "punpckhdq",
1525            SseOpcode::Punpcklqdq => "punpcklqdq",
1526            SseOpcode::Punpckhqdq => "punpckhqdq",
1527            SseOpcode::Pshuflw => "pshuflw",
1528            SseOpcode::Pshufhw => "pshufhw",
1529            SseOpcode::Pblendw => "pblendw",
1530            SseOpcode::Movddup => "movddup",
1531        };
1532        write!(fmt, "{}", name)
1533    }
1534}
1535
1536impl fmt::Display for SseOpcode {
1537    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1538        fmt::Debug::fmt(self, f)
1539    }
1540}
1541
1542pub use crate::isa::x64::lower::isle::generated_code::AvxOpcode;
1543
1544impl AvxOpcode {
1545    /// Which `InstructionSet`s support the opcode?
1546    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
1547        match self {
1548            AvxOpcode::Vfmadd213ss
1549            | AvxOpcode::Vfmadd213sd
1550            | AvxOpcode::Vfmadd213ps
1551            | AvxOpcode::Vfmadd213pd
1552            | AvxOpcode::Vfmadd132ss
1553            | AvxOpcode::Vfmadd132sd
1554            | AvxOpcode::Vfmadd132ps
1555            | AvxOpcode::Vfmadd132pd
1556            | AvxOpcode::Vfnmadd213ss
1557            | AvxOpcode::Vfnmadd213sd
1558            | AvxOpcode::Vfnmadd213ps
1559            | AvxOpcode::Vfnmadd213pd
1560            | AvxOpcode::Vfnmadd132ss
1561            | AvxOpcode::Vfnmadd132sd
1562            | AvxOpcode::Vfnmadd132ps
1563            | AvxOpcode::Vfnmadd132pd => smallvec![InstructionSet::FMA],
1564            AvxOpcode::Vminps
1565            | AvxOpcode::Vminpd
1566            | AvxOpcode::Vmaxps
1567            | AvxOpcode::Vmaxpd
1568            | AvxOpcode::Vandnps
1569            | AvxOpcode::Vandnpd
1570            | AvxOpcode::Vpandn
1571            | AvxOpcode::Vcmpps
1572            | AvxOpcode::Vcmppd
1573            | AvxOpcode::Vpsrlw
1574            | AvxOpcode::Vpsrld
1575            | AvxOpcode::Vpsrlq
1576            | AvxOpcode::Vpaddb
1577            | AvxOpcode::Vpaddw
1578            | AvxOpcode::Vpaddd
1579            | AvxOpcode::Vpaddq
1580            | AvxOpcode::Vpaddsb
1581            | AvxOpcode::Vpaddsw
1582            | AvxOpcode::Vpaddusb
1583            | AvxOpcode::Vpaddusw
1584            | AvxOpcode::Vpsubb
1585            | AvxOpcode::Vpsubw
1586            | AvxOpcode::Vpsubd
1587            | AvxOpcode::Vpsubq
1588            | AvxOpcode::Vpsubsb
1589            | AvxOpcode::Vpsubsw
1590            | AvxOpcode::Vpsubusb
1591            | AvxOpcode::Vpsubusw
1592            | AvxOpcode::Vpavgb
1593            | AvxOpcode::Vpavgw
1594            | AvxOpcode::Vpand
1595            | AvxOpcode::Vandps
1596            | AvxOpcode::Vandpd
1597            | AvxOpcode::Vpor
1598            | AvxOpcode::Vorps
1599            | AvxOpcode::Vorpd
1600            | AvxOpcode::Vpxor
1601            | AvxOpcode::Vxorps
1602            | AvxOpcode::Vxorpd
1603            | AvxOpcode::Vpmullw
1604            | AvxOpcode::Vpmulld
1605            | AvxOpcode::Vpmulhw
1606            | AvxOpcode::Vpmulhd
1607            | AvxOpcode::Vpmulhrsw
1608            | AvxOpcode::Vpmulhuw
1609            | AvxOpcode::Vpmuldq
1610            | AvxOpcode::Vpmuludq
1611            | AvxOpcode::Vpunpckhwd
1612            | AvxOpcode::Vpunpcklwd
1613            | AvxOpcode::Vunpcklps
1614            | AvxOpcode::Vaddps
1615            | AvxOpcode::Vaddpd
1616            | AvxOpcode::Vsubps
1617            | AvxOpcode::Vsubpd
1618            | AvxOpcode::Vmulps
1619            | AvxOpcode::Vmulpd
1620            | AvxOpcode::Vdivps
1621            | AvxOpcode::Vdivpd
1622            | AvxOpcode::Vpcmpeqb
1623            | AvxOpcode::Vpcmpeqw
1624            | AvxOpcode::Vpcmpeqd
1625            | AvxOpcode::Vpcmpeqq
1626            | AvxOpcode::Vpcmpgtb
1627            | AvxOpcode::Vpcmpgtw
1628            | AvxOpcode::Vpcmpgtd
1629            | AvxOpcode::Vpcmpgtq
1630            | AvxOpcode::Vblendvps
1631            | AvxOpcode::Vblendvpd
1632            | AvxOpcode::Vpblendvb
1633            | AvxOpcode::Vmovlhps
1634            | AvxOpcode::Vpminsb
1635            | AvxOpcode::Vpminsw
1636            | AvxOpcode::Vpminsd
1637            | AvxOpcode::Vpminub
1638            | AvxOpcode::Vpminuw
1639            | AvxOpcode::Vpminud
1640            | AvxOpcode::Vpmaxsb
1641            | AvxOpcode::Vpmaxsw
1642            | AvxOpcode::Vpmaxsd
1643            | AvxOpcode::Vpmaxub
1644            | AvxOpcode::Vpmaxuw
1645            | AvxOpcode::Vpmaxud
1646            | AvxOpcode::Vpunpcklbw
1647            | AvxOpcode::Vpunpckhbw
1648            | AvxOpcode::Vpacksswb
1649            | AvxOpcode::Vpackssdw
1650            | AvxOpcode::Vpackuswb
1651            | AvxOpcode::Vpackusdw
1652            | AvxOpcode::Vpalignr
1653            | AvxOpcode::Vpinsrb
1654            | AvxOpcode::Vpinsrw
1655            | AvxOpcode::Vpinsrd
1656            | AvxOpcode::Vpinsrq
1657            | AvxOpcode::Vpmaddwd
1658            | AvxOpcode::Vpmaddubsw
1659            | AvxOpcode::Vinsertps
1660            | AvxOpcode::Vpshufb
1661            | AvxOpcode::Vshufps
1662            | AvxOpcode::Vpsllw
1663            | AvxOpcode::Vpslld
1664            | AvxOpcode::Vpsllq
1665            | AvxOpcode::Vpsraw
1666            | AvxOpcode::Vpsrad
1667            | AvxOpcode::Vpmovsxbw
1668            | AvxOpcode::Vpmovzxbw
1669            | AvxOpcode::Vpmovsxwd
1670            | AvxOpcode::Vpmovzxwd
1671            | AvxOpcode::Vpmovsxdq
1672            | AvxOpcode::Vpmovzxdq
1673            | AvxOpcode::Vaddss
1674            | AvxOpcode::Vaddsd
1675            | AvxOpcode::Vmulss
1676            | AvxOpcode::Vmulsd
1677            | AvxOpcode::Vsubss
1678            | AvxOpcode::Vsubsd
1679            | AvxOpcode::Vdivss
1680            | AvxOpcode::Vdivsd
1681            | AvxOpcode::Vpabsb
1682            | AvxOpcode::Vpabsw
1683            | AvxOpcode::Vpabsd
1684            | AvxOpcode::Vminss
1685            | AvxOpcode::Vminsd
1686            | AvxOpcode::Vmaxss
1687            | AvxOpcode::Vmaxsd
1688            | AvxOpcode::Vsqrtps
1689            | AvxOpcode::Vsqrtpd
1690            | AvxOpcode::Vroundpd
1691            | AvxOpcode::Vroundps
1692            | AvxOpcode::Vcvtdq2pd
1693            | AvxOpcode::Vcvtdq2ps
1694            | AvxOpcode::Vcvtpd2ps
1695            | AvxOpcode::Vcvtps2pd
1696            | AvxOpcode::Vcvttpd2dq
1697            | AvxOpcode::Vcvttps2dq
1698            | AvxOpcode::Vphaddw
1699            | AvxOpcode::Vphaddd
1700            | AvxOpcode::Vpunpckldq
1701            | AvxOpcode::Vpunpckhdq
1702            | AvxOpcode::Vpunpcklqdq
1703            | AvxOpcode::Vpunpckhqdq
1704            | AvxOpcode::Vpshuflw
1705            | AvxOpcode::Vpshufhw
1706            | AvxOpcode::Vpshufd
1707            | AvxOpcode::Vmovss
1708            | AvxOpcode::Vmovsd
1709            | AvxOpcode::Vmovups
1710            | AvxOpcode::Vmovupd
1711            | AvxOpcode::Vmovdqu
1712            | AvxOpcode::Vpextrb
1713            | AvxOpcode::Vpextrw
1714            | AvxOpcode::Vpextrd
1715            | AvxOpcode::Vpextrq
1716            | AvxOpcode::Vpblendw
1717            | AvxOpcode::Vmovddup
1718            | AvxOpcode::Vbroadcastss
1719            | AvxOpcode::Vmovd
1720            | AvxOpcode::Vmovq
1721            | AvxOpcode::Vmovmskps
1722            | AvxOpcode::Vmovmskpd
1723            | AvxOpcode::Vpmovmskb
1724            | AvxOpcode::Vcvtsi2ss
1725            | AvxOpcode::Vcvtsi2sd
1726            | AvxOpcode::Vcvtss2sd
1727            | AvxOpcode::Vcvtsd2ss
1728            | AvxOpcode::Vsqrtss
1729            | AvxOpcode::Vsqrtsd
1730            | AvxOpcode::Vroundss
1731            | AvxOpcode::Vroundsd => {
1732                smallvec![InstructionSet::AVX]
1733            }
1734
1735            AvxOpcode::Vpbroadcastb | AvxOpcode::Vpbroadcastw | AvxOpcode::Vpbroadcastd => {
1736                smallvec![InstructionSet::AVX2]
1737            }
1738        }
1739    }
1740}
1741
1742impl fmt::Display for AvxOpcode {
1743    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1744        format!("{self:?}").to_lowercase().fmt(f)
1745    }
1746}
1747
1748#[derive(Clone, PartialEq)]
1749#[allow(missing_docs)]
1750pub enum Avx512Opcode {
1751    Vcvtudq2ps,
1752    Vpabsq,
1753    Vpermi2b,
1754    Vpmullq,
1755    Vpopcntb,
1756}
1757
1758impl Avx512Opcode {
1759    /// Which `InstructionSet`s support the opcode?
1760    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
1761        match self {
1762            Avx512Opcode::Vcvtudq2ps => {
1763                smallvec![InstructionSet::AVX512F, InstructionSet::AVX512VL]
1764            }
1765            Avx512Opcode::Vpabsq => smallvec![InstructionSet::AVX512F, InstructionSet::AVX512VL],
1766            Avx512Opcode::Vpermi2b => {
1767                smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512VBMI]
1768            }
1769            Avx512Opcode::Vpmullq => smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512DQ],
1770            Avx512Opcode::Vpopcntb => {
1771                smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512BITALG]
1772            }
1773        }
1774    }
1775}
1776
1777impl fmt::Debug for Avx512Opcode {
1778    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1779        let name = match self {
1780            Avx512Opcode::Vcvtudq2ps => "vcvtudq2ps",
1781            Avx512Opcode::Vpabsq => "vpabsq",
1782            Avx512Opcode::Vpermi2b => "vpermi2b",
1783            Avx512Opcode::Vpmullq => "vpmullq",
1784            Avx512Opcode::Vpopcntb => "vpopcntb",
1785        };
1786        write!(fmt, "{}", name)
1787    }
1788}
1789
1790impl fmt::Display for Avx512Opcode {
1791    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1792        fmt::Debug::fmt(self, f)
1793    }
1794}
1795
1796/// This defines the ways a value can be extended: either signed- or zero-extension, or none for
1797/// types that are not extended. Contrast with [ExtMode], which defines the widths from and to which
1798/// values can be extended.
1799#[allow(dead_code)]
1800#[derive(Clone, PartialEq)]
1801pub enum ExtKind {
1802    /// No extension.
1803    None,
1804    /// Sign-extend.
1805    SignExtend,
1806    /// Zero-extend.
1807    ZeroExtend,
1808}
1809
1810/// These indicate ways of extending (widening) a value, using the Intel
1811/// naming: B(yte) = u8, W(ord) = u16, L(ong)word = u32, Q(uad)word = u64
1812#[derive(Clone, PartialEq)]
1813pub enum ExtMode {
1814    /// Byte -> Longword.
1815    BL,
1816    /// Byte -> Quadword.
1817    BQ,
1818    /// Word -> Longword.
1819    WL,
1820    /// Word -> Quadword.
1821    WQ,
1822    /// Longword -> Quadword.
1823    LQ,
1824}
1825
1826impl ExtMode {
1827    /// Calculate the `ExtMode` from passed bit lengths of the from/to types.
1828    pub(crate) fn new(from_bits: u16, to_bits: u16) -> Option<ExtMode> {
1829        match (from_bits, to_bits) {
1830            (1, 8) | (1, 16) | (1, 32) | (8, 16) | (8, 32) => Some(ExtMode::BL),
1831            (1, 64) | (8, 64) => Some(ExtMode::BQ),
1832            (16, 32) => Some(ExtMode::WL),
1833            (16, 64) => Some(ExtMode::WQ),
1834            (32, 64) => Some(ExtMode::LQ),
1835            _ => None,
1836        }
1837    }
1838
1839    /// Return the source register size in bytes.
1840    pub(crate) fn src_size(&self) -> u8 {
1841        match self {
1842            ExtMode::BL | ExtMode::BQ => 1,
1843            ExtMode::WL | ExtMode::WQ => 2,
1844            ExtMode::LQ => 4,
1845        }
1846    }
1847
1848    /// Return the destination register size in bytes.
1849    pub(crate) fn dst_size(&self) -> u8 {
1850        match self {
1851            ExtMode::BL | ExtMode::WL => 4,
1852            ExtMode::BQ | ExtMode::WQ | ExtMode::LQ => 8,
1853        }
1854    }
1855}
1856
1857impl fmt::Debug for ExtMode {
1858    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1859        let name = match self {
1860            ExtMode::BL => "bl",
1861            ExtMode::BQ => "bq",
1862            ExtMode::WL => "wl",
1863            ExtMode::WQ => "wq",
1864            ExtMode::LQ => "lq",
1865        };
1866        write!(fmt, "{}", name)
1867    }
1868}
1869
1870impl fmt::Display for ExtMode {
1871    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1872        fmt::Debug::fmt(self, f)
1873    }
1874}
1875
1876/// These indicate the form of a scalar shift/rotate: left, signed right, unsigned right.
1877#[derive(Clone, Copy)]
1878pub enum ShiftKind {
1879    /// Left shift.
1880    ShiftLeft,
1881    /// Inserts zeros in the most significant bits.
1882    ShiftRightLogical,
1883    /// Replicates the sign bit in the most significant bits.
1884    ShiftRightArithmetic,
1885    /// Left rotation.
1886    RotateLeft,
1887    /// Right rotation.
1888    RotateRight,
1889}
1890
1891impl fmt::Debug for ShiftKind {
1892    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1893        let name = match self {
1894            ShiftKind::ShiftLeft => "shl",
1895            ShiftKind::ShiftRightLogical => "shr",
1896            ShiftKind::ShiftRightArithmetic => "sar",
1897            ShiftKind::RotateLeft => "rol",
1898            ShiftKind::RotateRight => "ror",
1899        };
1900        write!(fmt, "{}", name)
1901    }
1902}
1903
1904impl fmt::Display for ShiftKind {
1905    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1906        fmt::Debug::fmt(self, f)
1907    }
1908}
1909
1910/// These indicate condition code tests.  Not all are represented since not all are useful in
1911/// compiler-generated code.
1912#[derive(Copy, Clone)]
1913#[repr(u8)]
1914pub enum CC {
1915    ///  overflow
1916    O = 0,
1917    /// no overflow
1918    NO = 1,
1919
1920    /// < unsigned
1921    B = 2,
1922    /// >= unsigned
1923    NB = 3,
1924
1925    /// zero
1926    Z = 4,
1927    /// not-zero
1928    NZ = 5,
1929
1930    /// <= unsigned
1931    BE = 6,
1932    /// > unsigned
1933    NBE = 7,
1934
1935    /// negative
1936    S = 8,
1937    /// not-negative
1938    NS = 9,
1939
1940    /// < signed
1941    L = 12,
1942    /// >= signed
1943    NL = 13,
1944
1945    /// <= signed
1946    LE = 14,
1947    /// > signed
1948    NLE = 15,
1949
1950    /// parity
1951    P = 10,
1952
1953    /// not parity
1954    NP = 11,
1955}
1956
1957impl CC {
1958    pub(crate) fn from_intcc(intcc: IntCC) -> Self {
1959        match intcc {
1960            IntCC::Equal => CC::Z,
1961            IntCC::NotEqual => CC::NZ,
1962            IntCC::SignedGreaterThanOrEqual => CC::NL,
1963            IntCC::SignedGreaterThan => CC::NLE,
1964            IntCC::SignedLessThanOrEqual => CC::LE,
1965            IntCC::SignedLessThan => CC::L,
1966            IntCC::UnsignedGreaterThanOrEqual => CC::NB,
1967            IntCC::UnsignedGreaterThan => CC::NBE,
1968            IntCC::UnsignedLessThanOrEqual => CC::BE,
1969            IntCC::UnsignedLessThan => CC::B,
1970        }
1971    }
1972
1973    pub(crate) fn invert(&self) -> Self {
1974        match self {
1975            CC::O => CC::NO,
1976            CC::NO => CC::O,
1977
1978            CC::B => CC::NB,
1979            CC::NB => CC::B,
1980
1981            CC::Z => CC::NZ,
1982            CC::NZ => CC::Z,
1983
1984            CC::BE => CC::NBE,
1985            CC::NBE => CC::BE,
1986
1987            CC::S => CC::NS,
1988            CC::NS => CC::S,
1989
1990            CC::L => CC::NL,
1991            CC::NL => CC::L,
1992
1993            CC::LE => CC::NLE,
1994            CC::NLE => CC::LE,
1995
1996            CC::P => CC::NP,
1997            CC::NP => CC::P,
1998        }
1999    }
2000
2001    pub(crate) fn get_enc(self) -> u8 {
2002        self as u8
2003    }
2004}
2005
2006impl fmt::Debug for CC {
2007    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
2008        let name = match self {
2009            CC::O => "o",
2010            CC::NO => "no",
2011            CC::B => "b",
2012            CC::NB => "nb",
2013            CC::Z => "z",
2014            CC::NZ => "nz",
2015            CC::BE => "be",
2016            CC::NBE => "nbe",
2017            CC::S => "s",
2018            CC::NS => "ns",
2019            CC::L => "l",
2020            CC::NL => "nl",
2021            CC::LE => "le",
2022            CC::NLE => "nle",
2023            CC::P => "p",
2024            CC::NP => "np",
2025        };
2026        write!(fmt, "{}", name)
2027    }
2028}
2029
2030impl fmt::Display for CC {
2031    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2032        fmt::Debug::fmt(self, f)
2033    }
2034}
2035
2036/// Encode the ways that floats can be compared. This is used in float comparisons such as `cmpps`,
2037/// e.g.; it is distinguished from other float comparisons (e.g. `ucomiss`) in that those use EFLAGS
2038/// whereas [FcmpImm] is used as an immediate.
2039#[derive(Clone, Copy)]
2040pub enum FcmpImm {
2041    /// Equal comparison.
2042    Equal = 0x00,
2043    /// Less than comparison.
2044    LessThan = 0x01,
2045    /// Less than or equal comparison.
2046    LessThanOrEqual = 0x02,
2047    /// Unordered.
2048    Unordered = 0x03,
2049    /// Not equal comparison.
2050    NotEqual = 0x04,
2051    /// Unordered of greater than or equal comparison.
2052    UnorderedOrGreaterThanOrEqual = 0x05,
2053    /// Unordered or greater than comparison.
2054    UnorderedOrGreaterThan = 0x06,
2055    /// Ordered.
2056    Ordered = 0x07,
2057}
2058
2059impl FcmpImm {
2060    pub(crate) fn encode(self) -> u8 {
2061        self as u8
2062    }
2063}
2064
2065impl From<FloatCC> for FcmpImm {
2066    fn from(cond: FloatCC) -> Self {
2067        match cond {
2068            FloatCC::Equal => FcmpImm::Equal,
2069            FloatCC::LessThan => FcmpImm::LessThan,
2070            FloatCC::LessThanOrEqual => FcmpImm::LessThanOrEqual,
2071            FloatCC::Unordered => FcmpImm::Unordered,
2072            FloatCC::NotEqual => FcmpImm::NotEqual,
2073            FloatCC::UnorderedOrGreaterThanOrEqual => FcmpImm::UnorderedOrGreaterThanOrEqual,
2074            FloatCC::UnorderedOrGreaterThan => FcmpImm::UnorderedOrGreaterThan,
2075            FloatCC::Ordered => FcmpImm::Ordered,
2076            _ => panic!("unable to create comparison predicate for {}", cond),
2077        }
2078    }
2079}
2080
2081/// Encode the rounding modes used as part of the Rounding Control field.
2082/// Note, these rounding immediates only consider the rounding control field
2083/// (i.e. the rounding mode) which only take up the first two bits when encoded.
2084/// However the rounding immediate which this field helps make up, also includes
2085/// bits 3 and 4 which define the rounding select and precision mask respectively.
2086/// These two bits are not defined here and are implictly set to zero when encoded.
2087#[derive(Clone, Copy)]
2088pub enum RoundImm {
2089    /// Round to nearest mode.
2090    RoundNearest = 0x00,
2091    /// Round down mode.
2092    RoundDown = 0x01,
2093    /// Round up mode.
2094    RoundUp = 0x02,
2095    /// Round to zero mode.
2096    RoundZero = 0x03,
2097}
2098
2099impl RoundImm {
2100    pub(crate) fn encode(self) -> u8 {
2101        self as u8
2102    }
2103}
2104
2105/// An operand's size in bits.
2106#[derive(Clone, Copy, PartialEq)]
2107pub enum OperandSize {
2108    /// 8-bit.
2109    Size8,
2110    /// 16-bit.
2111    Size16,
2112    /// 32-bit.
2113    Size32,
2114    /// 64-bit.
2115    Size64,
2116}
2117
2118impl OperandSize {
2119    pub(crate) fn from_bytes(num_bytes: u32) -> Self {
2120        match num_bytes {
2121            1 => OperandSize::Size8,
2122            2 => OperandSize::Size16,
2123            4 => OperandSize::Size32,
2124            8 => OperandSize::Size64,
2125            _ => unreachable!("Invalid OperandSize: {}", num_bytes),
2126        }
2127    }
2128
2129    // Computes the OperandSize for a given type.
2130    // For vectors, the OperandSize of the lanes is returned.
2131    pub(crate) fn from_ty(ty: Type) -> Self {
2132        Self::from_bytes(ty.lane_type().bytes())
2133    }
2134
2135    // Check that the value of self is one of the allowed sizes.
2136    pub(crate) fn is_one_of(&self, sizes: &[Self]) -> bool {
2137        sizes.iter().any(|val| *self == *val)
2138    }
2139
2140    pub(crate) fn to_bytes(&self) -> u8 {
2141        match self {
2142            Self::Size8 => 1,
2143            Self::Size16 => 2,
2144            Self::Size32 => 4,
2145            Self::Size64 => 8,
2146        }
2147    }
2148
2149    pub(crate) fn to_bits(&self) -> u8 {
2150        self.to_bytes() * 8
2151    }
2152}
2153
2154/// An x64 memory fence kind.
2155#[derive(Clone)]
2156#[allow(dead_code)]
2157pub enum FenceKind {
2158    /// `mfence` instruction ("Memory Fence")
2159    MFence,
2160    /// `lfence` instruction ("Load Fence")
2161    LFence,
2162    /// `sfence` instruction ("Store Fence")
2163    SFence,
2164}
cranelift_codegen/isa/x64/inst/args.rs

cranelift_codegen/isa/x64/inst/
args.rs
