1use crate::binemit::{Addend, CodeOffset, Reloc, StackMap};
4use crate::ir::{types, ExternalName, LibCall, Opcode, RelSourceLoc, TrapCode, Type};
5use crate::isa::x64::abi::X64ABIMachineSpec;
6use crate::isa::x64::inst::regs::{pretty_print_reg, show_ireg_sized};
7use crate::isa::x64::settings as x64_settings;
8use crate::isa::CallConv;
9use crate::{machinst::*, trace};
10use crate::{settings, CodegenError, CodegenResult};
11use alloc::boxed::Box;
12use alloc::vec::Vec;
13use regalloc2::{Allocation, PRegSet, VReg};
14use smallvec::{smallvec, SmallVec};
15use std::fmt;
16use std::string::{String, ToString};
17
18pub mod args;
19mod emit;
20#[cfg(test)]
21mod emit_tests;
22pub mod regs;
23pub mod unwind;
24
25use args::*;
26
27pub use super::lower::isle::generated_code::MInst as Inst;
32
33#[derive(Clone, Debug)]
35pub struct CallInfo {
36 pub uses: CallArgList,
38 pub defs: CallRetList,
40 pub clobbers: PRegSet,
42 pub opcode: Opcode,
44}
45
46#[test]
47#[cfg(target_pointer_width = "64")]
48fn inst_size_test() {
49 assert_eq!(40, std::mem::size_of::<Inst>());
52}
53
54pub(crate) fn low32_will_sign_extend_to_64(x: u64) -> bool {
55 let xs = x as i64;
56 xs == ((xs << 32) >> 32)
57}
58
59impl Inst {
60 fn available_in_any_isa(&self) -> SmallVec<[InstructionSet; 2]> {
65 match self {
66 Inst::AluRmiR { .. }
69 | Inst::AluRM { .. }
70 | Inst::AtomicRmwSeq { .. }
71 | Inst::Bswap { .. }
72 | Inst::CallKnown { .. }
73 | Inst::CallUnknown { .. }
74 | Inst::CheckedSRemSeq { .. }
75 | Inst::CheckedSRemSeq8 { .. }
76 | Inst::Cmove { .. }
77 | Inst::CmpRmiR { .. }
78 | Inst::CvtFloatToSintSeq { .. }
79 | Inst::CvtFloatToUintSeq { .. }
80 | Inst::CvtUint64ToFloatSeq { .. }
81 | Inst::Div { .. }
82 | Inst::Div8 { .. }
83 | Inst::Fence { .. }
84 | Inst::Hlt
85 | Inst::Imm { .. }
86 | Inst::JmpCond { .. }
87 | Inst::JmpIf { .. }
88 | Inst::JmpKnown { .. }
89 | Inst::JmpTableSeq { .. }
90 | Inst::JmpUnknown { .. }
91 | Inst::LoadEffectiveAddress { .. }
92 | Inst::LoadExtName { .. }
93 | Inst::LockCmpxchg { .. }
94 | Inst::Mov64MR { .. }
95 | Inst::MovImmM { .. }
96 | Inst::MovRM { .. }
97 | Inst::MovRR { .. }
98 | Inst::MovFromPReg { .. }
99 | Inst::MovToPReg { .. }
100 | Inst::MovsxRmR { .. }
101 | Inst::MovzxRmR { .. }
102 | Inst::MulHi { .. }
103 | Inst::Neg { .. }
104 | Inst::Not { .. }
105 | Inst::Nop { .. }
106 | Inst::Pop64 { .. }
107 | Inst::Push64 { .. }
108 | Inst::StackProbeLoop { .. }
109 | Inst::Args { .. }
110 | Inst::Ret { .. }
111 | Inst::Setcc { .. }
112 | Inst::ShiftR { .. }
113 | Inst::SignExtendData { .. }
114 | Inst::TrapIf { .. }
115 | Inst::TrapIfAnd { .. }
116 | Inst::TrapIfOr { .. }
117 | Inst::Ud2 { .. }
118 | Inst::VirtualSPOffsetAdj { .. }
119 | Inst::XmmCmove { .. }
120 | Inst::XmmCmpRmR { .. }
121 | Inst::XmmMinMaxSeq { .. }
122 | Inst::XmmUninitializedValue { .. }
123 | Inst::ElfTlsGetAddr { .. }
124 | Inst::MachOTlsGetAddr { .. }
125 | Inst::CoffTlsGetAddr { .. }
126 | Inst::Unwind { .. }
127 | Inst::DummyUse { .. }
128 | Inst::AluConstOp { .. } => smallvec![],
129
130 Inst::AluRmRVex { op, .. } => op.available_from(),
131 Inst::UnaryRmR { op, .. } => op.available_from(),
132
133 Inst::GprToXmm { op, .. }
135 | Inst::XmmMovRM { op, .. }
136 | Inst::XmmMovRMImm { op, .. }
137 | Inst::XmmRmiReg { opcode: op, .. }
138 | Inst::XmmRmR { op, .. }
139 | Inst::XmmRmRUnaligned { op, .. }
140 | Inst::XmmRmRBlend { op, .. }
141 | Inst::XmmRmRImm { op, .. }
142 | Inst::XmmToGpr { op, .. }
143 | Inst::XmmToGprImm { op, .. }
144 | Inst::XmmUnaryRmRImm { op, .. }
145 | Inst::XmmUnaryRmRUnaligned { op, .. }
146 | Inst::XmmUnaryRmR { op, .. } => smallvec![op.available_from()],
147
148 Inst::XmmUnaryRmREvex { op, .. }
149 | Inst::XmmRmREvex { op, .. }
150 | Inst::XmmRmREvex3 { op, .. } => op.available_from(),
151
152 Inst::XmmRmiRVex { op, .. }
153 | Inst::XmmRmRVex3 { op, .. }
154 | Inst::XmmRmRImmVex { op, .. }
155 | Inst::XmmRmRBlendVex { op, .. }
156 | Inst::XmmVexPinsr { op, .. }
157 | Inst::XmmUnaryRmRVex { op, .. }
158 | Inst::XmmUnaryRmRImmVex { op, .. }
159 | Inst::XmmMovRMVex { op, .. }
160 | Inst::XmmMovRMImmVex { op, .. }
161 | Inst::XmmToGprImmVex { op, .. }
162 | Inst::XmmToGprVex { op, .. }
163 | Inst::GprToXmmVex { op, .. } => op.available_from(),
164 }
165 }
166}
167
168impl Inst {
171 pub(crate) fn nop(len: u8) -> Self {
172 debug_assert!(len <= 15);
173 Self::Nop { len }
174 }
175
176 pub(crate) fn alu_rmi_r(
177 size: OperandSize,
178 op: AluRmiROpcode,
179 src: RegMemImm,
180 dst: Writable<Reg>,
181 ) -> Self {
182 debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
183 src.assert_regclass_is(RegClass::Int);
184 debug_assert!(dst.to_reg().class() == RegClass::Int);
185 Self::AluRmiR {
186 size,
187 op,
188 src1: Gpr::new(dst.to_reg()).unwrap(),
189 src2: GprMemImm::new(src).unwrap(),
190 dst: WritableGpr::from_writable_reg(dst).unwrap(),
191 }
192 }
193
194 #[allow(dead_code)]
195 pub(crate) fn unary_rm_r(
196 size: OperandSize,
197 op: UnaryRmROpcode,
198 src: RegMem,
199 dst: Writable<Reg>,
200 ) -> Self {
201 src.assert_regclass_is(RegClass::Int);
202 debug_assert!(dst.to_reg().class() == RegClass::Int);
203 debug_assert!(size.is_one_of(&[
204 OperandSize::Size16,
205 OperandSize::Size32,
206 OperandSize::Size64
207 ]));
208 Self::UnaryRmR {
209 size,
210 op,
211 src: GprMem::new(src).unwrap(),
212 dst: WritableGpr::from_writable_reg(dst).unwrap(),
213 }
214 }
215
216 pub(crate) fn not(size: OperandSize, src: Writable<Reg>) -> Inst {
217 debug_assert_eq!(src.to_reg().class(), RegClass::Int);
218 Inst::Not {
219 size,
220 src: Gpr::new(src.to_reg()).unwrap(),
221 dst: WritableGpr::from_writable_reg(src).unwrap(),
222 }
223 }
224
225 pub(crate) fn div(
226 size: OperandSize,
227 sign: DivSignedness,
228 trap: TrapCode,
229 divisor: RegMem,
230 dividend_lo: Gpr,
231 dividend_hi: Gpr,
232 dst_quotient: WritableGpr,
233 dst_remainder: WritableGpr,
234 ) -> Inst {
235 divisor.assert_regclass_is(RegClass::Int);
236 Inst::Div {
237 size,
238 sign,
239 trap,
240 divisor: GprMem::new(divisor).unwrap(),
241 dividend_lo,
242 dividend_hi,
243 dst_quotient,
244 dst_remainder,
245 }
246 }
247
248 pub(crate) fn div8(
249 sign: DivSignedness,
250 trap: TrapCode,
251 divisor: RegMem,
252 dividend: Gpr,
253 dst: WritableGpr,
254 ) -> Inst {
255 divisor.assert_regclass_is(RegClass::Int);
256 Inst::Div8 {
257 sign,
258 trap,
259 divisor: GprMem::new(divisor).unwrap(),
260 dividend,
261 dst,
262 }
263 }
264
265 pub(crate) fn imm(dst_size: OperandSize, simm64: u64, dst: Writable<Reg>) -> Inst {
266 debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
267 debug_assert!(dst.to_reg().class() == RegClass::Int);
268 let dst_size = match dst_size {
271 OperandSize::Size64 if simm64 > u32::max_value() as u64 => OperandSize::Size64,
272 _ => OperandSize::Size32,
273 };
274 Inst::Imm {
275 dst_size,
276 simm64,
277 dst: WritableGpr::from_writable_reg(dst).unwrap(),
278 }
279 }
280
281 pub(crate) fn mov_r_r(size: OperandSize, src: Reg, dst: Writable<Reg>) -> Inst {
282 debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
283 debug_assert!(src.class() == RegClass::Int);
284 debug_assert!(dst.to_reg().class() == RegClass::Int);
285 let src = Gpr::new(src).unwrap();
286 let dst = WritableGpr::from_writable_reg(dst).unwrap();
287 Inst::MovRR { size, src, dst }
288 }
289
290 pub(crate) fn xmm_unary_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Inst {
292 src.assert_regclass_is(RegClass::Float);
293 debug_assert!(dst.to_reg().class() == RegClass::Float);
294 Inst::XmmUnaryRmR {
295 op,
296 src: XmmMemAligned::new(src).unwrap(),
297 dst: WritableXmm::from_writable_reg(dst).unwrap(),
298 }
299 }
300
301 pub(crate) fn xmm_rm_r(op: SseOpcode, src: RegMem, dst: Writable<Reg>) -> Self {
302 src.assert_regclass_is(RegClass::Float);
303 debug_assert!(dst.to_reg().class() == RegClass::Float);
304 Inst::XmmRmR {
305 op,
306 src1: Xmm::new(dst.to_reg()).unwrap(),
307 src2: XmmMemAligned::new(src).unwrap(),
308 dst: WritableXmm::from_writable_reg(dst).unwrap(),
309 }
310 }
311
312 #[cfg(test)]
313 pub(crate) fn xmm_rmr_vex3(op: AvxOpcode, src3: RegMem, src2: Reg, dst: Writable<Reg>) -> Self {
314 src3.assert_regclass_is(RegClass::Float);
315 debug_assert!(src2.class() == RegClass::Float);
316 debug_assert!(dst.to_reg().class() == RegClass::Float);
317 Inst::XmmRmRVex3 {
318 op,
319 src3: XmmMem::new(src3).unwrap(),
320 src2: Xmm::new(src2).unwrap(),
321 src1: Xmm::new(dst.to_reg()).unwrap(),
322 dst: WritableXmm::from_writable_reg(dst).unwrap(),
323 }
324 }
325
326 pub(crate) fn xmm_mov_r_m(op: SseOpcode, src: Reg, dst: impl Into<SyntheticAmode>) -> Inst {
327 debug_assert!(src.class() == RegClass::Float);
328 Inst::XmmMovRM {
329 op,
330 src: Xmm::new(src).unwrap(),
331 dst: dst.into(),
332 }
333 }
334
335 pub(crate) fn xmm_to_gpr(
336 op: SseOpcode,
337 src: Reg,
338 dst: Writable<Reg>,
339 dst_size: OperandSize,
340 ) -> Inst {
341 debug_assert!(src.class() == RegClass::Float);
342 debug_assert!(dst.to_reg().class() == RegClass::Int);
343 debug_assert!(dst_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
344 Inst::XmmToGpr {
345 op,
346 src: Xmm::new(src).unwrap(),
347 dst: WritableGpr::from_writable_reg(dst).unwrap(),
348 dst_size,
349 }
350 }
351
352 pub(crate) fn gpr_to_xmm(
353 op: SseOpcode,
354 src: RegMem,
355 src_size: OperandSize,
356 dst: Writable<Reg>,
357 ) -> Inst {
358 src.assert_regclass_is(RegClass::Int);
359 debug_assert!(src_size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
360 debug_assert!(dst.to_reg().class() == RegClass::Float);
361 Inst::GprToXmm {
362 op,
363 src: GprMem::new(src).unwrap(),
364 dst: WritableXmm::from_writable_reg(dst).unwrap(),
365 src_size,
366 }
367 }
368
369 pub(crate) fn xmm_cmp_rm_r(op: SseOpcode, src: RegMem, dst: Reg) -> Inst {
370 src.assert_regclass_is(RegClass::Float);
371 debug_assert!(dst.class() == RegClass::Float);
372 let src = XmmMemAligned::new(src).unwrap();
373 let dst = Xmm::new(dst).unwrap();
374 Inst::XmmCmpRmR { op, src, dst }
375 }
376
377 #[allow(dead_code)]
378 pub(crate) fn xmm_min_max_seq(
379 size: OperandSize,
380 is_min: bool,
381 lhs: Reg,
382 rhs: Reg,
383 dst: Writable<Reg>,
384 ) -> Inst {
385 debug_assert!(size.is_one_of(&[OperandSize::Size32, OperandSize::Size64]));
386 debug_assert_eq!(lhs.class(), RegClass::Float);
387 debug_assert_eq!(rhs.class(), RegClass::Float);
388 debug_assert_eq!(dst.to_reg().class(), RegClass::Float);
389 Inst::XmmMinMaxSeq {
390 size,
391 is_min,
392 lhs: Xmm::new(lhs).unwrap(),
393 rhs: Xmm::new(rhs).unwrap(),
394 dst: WritableXmm::from_writable_reg(dst).unwrap(),
395 }
396 }
397
398 pub(crate) fn movzx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
399 src.assert_regclass_is(RegClass::Int);
400 debug_assert!(dst.to_reg().class() == RegClass::Int);
401 let src = GprMem::new(src).unwrap();
402 let dst = WritableGpr::from_writable_reg(dst).unwrap();
403 Inst::MovzxRmR { ext_mode, src, dst }
404 }
405
406 pub(crate) fn movsx_rm_r(ext_mode: ExtMode, src: RegMem, dst: Writable<Reg>) -> Inst {
407 src.assert_regclass_is(RegClass::Int);
408 debug_assert!(dst.to_reg().class() == RegClass::Int);
409 let src = GprMem::new(src).unwrap();
410 let dst = WritableGpr::from_writable_reg(dst).unwrap();
411 Inst::MovsxRmR { ext_mode, src, dst }
412 }
413
414 pub(crate) fn mov64_m_r(src: impl Into<SyntheticAmode>, dst: Writable<Reg>) -> Inst {
415 debug_assert!(dst.to_reg().class() == RegClass::Int);
416 Inst::Mov64MR {
417 src: src.into(),
418 dst: WritableGpr::from_writable_reg(dst).unwrap(),
419 }
420 }
421
422 pub(crate) fn mov_r_m(size: OperandSize, src: Reg, dst: impl Into<SyntheticAmode>) -> Inst {
423 debug_assert!(src.class() == RegClass::Int);
424 Inst::MovRM {
425 size,
426 src: Gpr::new(src).unwrap(),
427 dst: dst.into(),
428 }
429 }
430
431 pub(crate) fn lea(addr: impl Into<SyntheticAmode>, dst: Writable<Reg>) -> Inst {
432 debug_assert!(dst.to_reg().class() == RegClass::Int);
433 Inst::LoadEffectiveAddress {
434 addr: addr.into(),
435 dst: WritableGpr::from_writable_reg(dst).unwrap(),
436 size: OperandSize::Size64,
437 }
438 }
439
440 pub(crate) fn shift_r(
441 size: OperandSize,
442 kind: ShiftKind,
443 num_bits: Imm8Gpr,
444 src: Reg,
445 dst: Writable<Reg>,
446 ) -> Inst {
447 if let Imm8Reg::Imm8 { imm: num_bits } = num_bits.clone().to_imm8_reg() {
448 debug_assert!(num_bits < size.to_bits());
449 }
450 debug_assert!(dst.to_reg().class() == RegClass::Int);
451 Inst::ShiftR {
452 size,
453 kind,
454 src: Gpr::new(src).unwrap(),
455 num_bits,
456 dst: WritableGpr::from_writable_reg(dst).unwrap(),
457 }
458 }
459
460 pub(crate) fn cmp_rmi_r(size: OperandSize, src: RegMemImm, dst: Reg) -> Inst {
463 src.assert_regclass_is(RegClass::Int);
464 debug_assert_eq!(dst.class(), RegClass::Int);
465 Inst::CmpRmiR {
466 size,
467 src: GprMemImm::new(src).unwrap(),
468 dst: Gpr::new(dst).unwrap(),
469 opcode: CmpOpcode::Cmp,
470 }
471 }
472
473 pub(crate) fn trap(trap_code: TrapCode) -> Inst {
474 Inst::Ud2 { trap_code }
475 }
476
477 pub(crate) fn trap_if(cc: CC, trap_code: TrapCode) -> Inst {
478 Inst::TrapIf { cc, trap_code }
479 }
480
481 pub(crate) fn cmove(size: OperandSize, cc: CC, src: RegMem, dst: Writable<Reg>) -> Inst {
482 debug_assert!(size.is_one_of(&[
483 OperandSize::Size16,
484 OperandSize::Size32,
485 OperandSize::Size64
486 ]));
487 debug_assert!(dst.to_reg().class() == RegClass::Int);
488 Inst::Cmove {
489 size,
490 cc,
491 consequent: GprMem::new(src).unwrap(),
492 alternative: Gpr::new(dst.to_reg()).unwrap(),
493 dst: WritableGpr::from_writable_reg(dst).unwrap(),
494 }
495 }
496
497 pub(crate) fn push64(src: RegMemImm) -> Inst {
498 src.assert_regclass_is(RegClass::Int);
499 let src = GprMemImm::new(src).unwrap();
500 Inst::Push64 { src }
501 }
502
503 pub(crate) fn pop64(dst: Writable<Reg>) -> Inst {
504 debug_assert!(dst.to_reg().class() == RegClass::Int);
505 let dst = WritableGpr::from_writable_reg(dst).unwrap();
506 Inst::Pop64 { dst }
507 }
508
509 pub(crate) fn call_known(
510 dest: ExternalName,
511 uses: CallArgList,
512 defs: CallRetList,
513 clobbers: PRegSet,
514 opcode: Opcode,
515 ) -> Inst {
516 Inst::CallKnown {
517 dest,
518 info: Box::new(CallInfo {
519 uses,
520 defs,
521 clobbers,
522 opcode,
523 }),
524 }
525 }
526
527 pub(crate) fn call_unknown(
528 dest: RegMem,
529 uses: CallArgList,
530 defs: CallRetList,
531 clobbers: PRegSet,
532 opcode: Opcode,
533 ) -> Inst {
534 dest.assert_regclass_is(RegClass::Int);
535 Inst::CallUnknown {
536 dest,
537 info: Box::new(CallInfo {
538 uses,
539 defs,
540 clobbers,
541 opcode,
542 }),
543 }
544 }
545
546 pub(crate) fn ret(rets: Vec<RetPair>) -> Inst {
547 Inst::Ret { rets }
548 }
549
550 pub(crate) fn jmp_known(dst: MachLabel) -> Inst {
551 Inst::JmpKnown { dst }
552 }
553
554 pub(crate) fn jmp_unknown(target: RegMem) -> Inst {
555 target.assert_regclass_is(RegClass::Int);
556 Inst::JmpUnknown { target }
557 }
558
559 pub(crate) fn load(
563 ty: Type,
564 from_addr: impl Into<SyntheticAmode>,
565 to_reg: Writable<Reg>,
566 ext_kind: ExtKind,
567 ) -> Inst {
568 let rc = to_reg.to_reg().class();
569 match rc {
570 RegClass::Int => {
571 let ext_mode = match ty.bytes() {
572 1 => Some(ExtMode::BQ),
573 2 => Some(ExtMode::WQ),
574 4 => Some(ExtMode::LQ),
575 8 => None,
576 _ => unreachable!("the type should never use a scalar load: {}", ty),
577 };
578 if let Some(ext_mode) = ext_mode {
579 match ext_kind {
581 ExtKind::SignExtend => {
582 Inst::movsx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg)
583 }
584 ExtKind::ZeroExtend => {
585 Inst::movzx_rm_r(ext_mode, RegMem::mem(from_addr), to_reg)
586 }
587 ExtKind::None => panic!(
588 "expected an extension kind for extension mode: {:?}",
589 ext_mode
590 ),
591 }
592 } else {
593 Inst::mov64_m_r(from_addr, to_reg)
595 }
596 }
597 RegClass::Float => {
598 let opcode = match ty {
599 types::F32 => SseOpcode::Movss,
600 types::F64 => SseOpcode::Movsd,
601 types::F32X4 => SseOpcode::Movups,
602 types::F64X2 => SseOpcode::Movupd,
603 _ if ty.is_vector() && ty.bits() == 128 => SseOpcode::Movdqu,
604 _ => unimplemented!("unable to load type: {}", ty),
605 };
606 Inst::xmm_unary_rm_r(opcode, RegMem::mem(from_addr), to_reg)
607 }
608 }
609 }
610
611 pub(crate) fn store(ty: Type, from_reg: Reg, to_addr: impl Into<SyntheticAmode>) -> Inst {
613 let rc = from_reg.class();
614 match rc {
615 RegClass::Int => Inst::mov_r_m(OperandSize::from_ty(ty), from_reg, to_addr),
616 RegClass::Float => {
617 let opcode = match ty {
618 types::F32 => SseOpcode::Movss,
619 types::F64 => SseOpcode::Movsd,
620 types::F32X4 => SseOpcode::Movups,
621 types::F64X2 => SseOpcode::Movupd,
622 _ if ty.is_vector() && ty.bits() == 128 => SseOpcode::Movdqu,
623 _ => unimplemented!("unable to store type: {}", ty),
624 };
625 Inst::xmm_mov_r_m(opcode, from_reg, to_addr)
626 }
627 }
628 }
629}
630
631impl PrettyPrint for Inst {
635 fn pretty_print(&self, _size: u8, allocs: &mut AllocationConsumer<'_>) -> String {
636 fn ljustify(s: String) -> String {
637 let w = 7;
638 if s.len() >= w {
639 s
640 } else {
641 let need = usize::min(w, w - s.len());
642 s + &format!("{nil: <width$}", nil = "", width = need)
643 }
644 }
645
646 fn ljustify2(s1: String, s2: String) -> String {
647 ljustify(s1 + &s2)
648 }
649
650 fn suffix_lq(size: OperandSize) -> String {
651 match size {
652 OperandSize::Size32 => "l",
653 OperandSize::Size64 => "q",
654 _ => unreachable!(),
655 }
656 .to_string()
657 }
658
659 fn suffix_lqb(size: OperandSize) -> String {
660 match size {
661 OperandSize::Size32 => "l",
662 OperandSize::Size64 => "q",
663 _ => unreachable!(),
664 }
665 .to_string()
666 }
667
668 fn suffix_bwlq(size: OperandSize) -> String {
669 match size {
670 OperandSize::Size8 => "b".to_string(),
671 OperandSize::Size16 => "w".to_string(),
672 OperandSize::Size32 => "l".to_string(),
673 OperandSize::Size64 => "q".to_string(),
674 }
675 }
676
677 match self {
678 Inst::Nop { len } => format!("{} len={}", ljustify("nop".to_string()), len),
679
680 Inst::AluRmiR {
681 size,
682 op,
683 src1,
684 src2,
685 dst,
686 } => {
687 let size_bytes = size.to_bytes();
688 let src1 = pretty_print_reg(src1.to_reg(), size_bytes, allocs);
689 let dst = pretty_print_reg(dst.to_reg().to_reg(), size_bytes, allocs);
690 let src2 = src2.pretty_print(size_bytes, allocs);
691 format!(
692 "{} {}, {}, {}",
693 ljustify2(op.to_string(), suffix_lqb(*size)),
694 src1,
695 src2,
696 dst
697 )
698 }
699 Inst::AluConstOp { op, dst, size } => {
700 let size_bytes = size.to_bytes();
701 let dst = pretty_print_reg(dst.to_reg().to_reg(), size_bytes, allocs);
702 format!(
703 "{} {dst}, {dst}, {dst}",
704 ljustify2(op.to_string(), suffix_lqb(*size)),
705 )
706 }
707 Inst::AluRM {
708 size,
709 op,
710 src1_dst,
711 src2,
712 } => {
713 let size_bytes = size.to_bytes();
714 let src2 = pretty_print_reg(src2.to_reg(), size_bytes, allocs);
715 let src1_dst = src1_dst.pretty_print(size_bytes, allocs);
716 format!(
717 "{} {}, {}",
718 ljustify2(op.to_string(), suffix_lqb(*size)),
719 src2,
720 src1_dst,
721 )
722 }
723 Inst::AluRmRVex {
724 size,
725 op,
726 src1,
727 src2,
728 dst,
729 } => {
730 let size_bytes = size.to_bytes();
731 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes(), allocs);
732 let src1 = pretty_print_reg(src1.to_reg(), size_bytes, allocs);
733 let src2 = pretty_print_reg(src2.to_reg(), size_bytes, allocs);
734 format!(
735 "{} {}, {}, {}",
736 ljustify2(op.to_string(), String::new()),
737 src2,
738 src1,
739 dst,
740 )
741 }
742 Inst::UnaryRmR { src, dst, op, size } => {
743 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes(), allocs);
744 let src = src.pretty_print(size.to_bytes(), allocs);
745 format!(
746 "{} {}, {}",
747 ljustify2(op.to_string(), suffix_bwlq(*size)),
748 src,
749 dst,
750 )
751 }
752
753 Inst::Not { size, src, dst } => {
754 let src = pretty_print_reg(src.to_reg(), size.to_bytes(), allocs);
755 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes(), allocs);
756 format!(
757 "{} {}, {}",
758 ljustify2("not".to_string(), suffix_bwlq(*size)),
759 src,
760 dst,
761 )
762 }
763
764 Inst::Neg { size, src, dst } => {
765 let src = pretty_print_reg(src.to_reg(), size.to_bytes(), allocs);
766 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes(), allocs);
767 format!(
768 "{} {}, {}",
769 ljustify2("neg".to_string(), suffix_bwlq(*size)),
770 src,
771 dst,
772 )
773 }
774
775 Inst::Div {
776 size,
777 sign,
778 trap,
779 divisor,
780 dividend_lo,
781 dividend_hi,
782 dst_quotient,
783 dst_remainder,
784 } => {
785 let divisor = divisor.pretty_print(size.to_bytes(), allocs);
786 let dividend_lo = pretty_print_reg(dividend_lo.to_reg(), size.to_bytes(), allocs);
787 let dividend_hi = pretty_print_reg(dividend_hi.to_reg(), size.to_bytes(), allocs);
788 let dst_quotient =
789 pretty_print_reg(dst_quotient.to_reg().to_reg(), size.to_bytes(), allocs);
790 let dst_remainder =
791 pretty_print_reg(dst_remainder.to_reg().to_reg(), size.to_bytes(), allocs);
792 format!(
793 "{} {}, {}, {}, {}, {} ; trap={trap}",
794 ljustify(match sign {
795 DivSignedness::Signed => "idiv".to_string(),
796 DivSignedness::Unsigned => "div".to_string(),
797 }),
798 dividend_lo,
799 dividend_hi,
800 divisor,
801 dst_quotient,
802 dst_remainder,
803 )
804 }
805
806 Inst::Div8 {
807 sign,
808 trap,
809 divisor,
810 dividend,
811 dst,
812 } => {
813 let divisor = divisor.pretty_print(1, allocs);
814 let dividend = pretty_print_reg(dividend.to_reg(), 1, allocs);
815 let dst = pretty_print_reg(dst.to_reg().to_reg(), 1, allocs);
816 format!(
817 "{} {dividend}, {divisor}, {dst} ; trap={trap}",
818 ljustify(match sign {
819 DivSignedness::Signed => "idiv".to_string(),
820 DivSignedness::Unsigned => "div".to_string(),
821 }),
822 )
823 }
824
825 Inst::MulHi {
826 size,
827 signed,
828 src1,
829 src2,
830 dst_lo,
831 dst_hi,
832 } => {
833 let src1 = pretty_print_reg(src1.to_reg(), size.to_bytes(), allocs);
834 let dst_lo = pretty_print_reg(dst_lo.to_reg().to_reg(), size.to_bytes(), allocs);
835 let dst_hi = pretty_print_reg(dst_hi.to_reg().to_reg(), size.to_bytes(), allocs);
836 let src2 = src2.pretty_print(size.to_bytes(), allocs);
837 format!(
838 "{} {}, {}, {}, {}",
839 ljustify(if *signed {
840 "imul".to_string()
841 } else {
842 "mul".to_string()
843 }),
844 src1,
845 src2,
846 dst_lo,
847 dst_hi,
848 )
849 }
850
851 Inst::CheckedSRemSeq {
852 size,
853 divisor,
854 dividend_lo,
855 dividend_hi,
856 dst_quotient,
857 dst_remainder,
858 } => {
859 let divisor = pretty_print_reg(divisor.to_reg(), size.to_bytes(), allocs);
860 let dividend_lo = pretty_print_reg(dividend_lo.to_reg(), size.to_bytes(), allocs);
861 let dividend_hi = pretty_print_reg(dividend_hi.to_reg(), size.to_bytes(), allocs);
862 let dst_quotient =
863 pretty_print_reg(dst_quotient.to_reg().to_reg(), size.to_bytes(), allocs);
864 let dst_remainder =
865 pretty_print_reg(dst_remainder.to_reg().to_reg(), size.to_bytes(), allocs);
866 format!(
867 "checked_srem_seq {dividend_lo}, {dividend_hi}, \
868 {divisor}, {dst_quotient}, {dst_remainder}",
869 )
870 }
871
872 Inst::CheckedSRemSeq8 {
873 divisor,
874 dividend,
875 dst,
876 } => {
877 let divisor = pretty_print_reg(divisor.to_reg(), 1, allocs);
878 let dividend = pretty_print_reg(dividend.to_reg(), 1, allocs);
879 let dst = pretty_print_reg(dst.to_reg().to_reg(), 1, allocs);
880 format!("checked_srem_seq {dividend}, {divisor}, {dst}")
881 }
882
883 Inst::SignExtendData { size, src, dst } => {
884 let src = pretty_print_reg(src.to_reg(), size.to_bytes(), allocs);
885 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes(), allocs);
886 format!(
887 "{} {}, {}",
888 match size {
889 OperandSize::Size8 => "cbw",
890 OperandSize::Size16 => "cwd",
891 OperandSize::Size32 => "cdq",
892 OperandSize::Size64 => "cqo",
893 },
894 src,
895 dst,
896 )
897 }
898
899 Inst::XmmUnaryRmR { op, src, dst, .. } => {
900 let dst = pretty_print_reg(dst.to_reg().to_reg(), op.src_size(), allocs);
901 let src = src.pretty_print(op.src_size(), allocs);
902 format!("{} {}, {}", ljustify(op.to_string()), src, dst)
903 }
904
905 Inst::XmmUnaryRmRUnaligned { op, src, dst, .. } => {
906 let dst = pretty_print_reg(dst.to_reg().to_reg(), op.src_size(), allocs);
907 let src = src.pretty_print(op.src_size(), allocs);
908 format!("{} {}, {}", ljustify(op.to_string()), src, dst)
909 }
910
911 Inst::XmmUnaryRmRImm {
912 op, src, dst, imm, ..
913 } => {
914 let dst = pretty_print_reg(dst.to_reg().to_reg(), op.src_size(), allocs);
915 let src = src.pretty_print(op.src_size(), allocs);
916 format!("{} ${}, {}, {}", ljustify(op.to_string()), imm, src, dst)
917 }
918
919 Inst::XmmUnaryRmRVex { op, src, dst, .. } => {
920 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
921 let src = src.pretty_print(8, allocs);
922 format!("{} {}, {}", ljustify(op.to_string()), src, dst)
923 }
924
925 Inst::XmmUnaryRmRImmVex {
926 op, src, dst, imm, ..
927 } => {
928 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
929 let src = src.pretty_print(8, allocs);
930 format!("{} ${imm}, {}, {}", ljustify(op.to_string()), src, dst)
931 }
932
933 Inst::XmmUnaryRmREvex { op, src, dst, .. } => {
934 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
935 let src = src.pretty_print(8, allocs);
936 format!("{} {}, {}", ljustify(op.to_string()), src, dst)
937 }
938
939 Inst::XmmMovRM { op, src, dst, .. } => {
940 let src = pretty_print_reg(src.to_reg(), 8, allocs);
941 let dst = dst.pretty_print(8, allocs);
942 format!("{} {}, {}", ljustify(op.to_string()), src, dst)
943 }
944
945 Inst::XmmMovRMVex { op, src, dst, .. } => {
946 let src = pretty_print_reg(src.to_reg(), 8, allocs);
947 let dst = dst.pretty_print(8, allocs);
948 format!("{} {}, {}", ljustify(op.to_string()), src, dst)
949 }
950
951 Inst::XmmMovRMImm {
952 op, src, dst, imm, ..
953 } => {
954 let src = pretty_print_reg(src.to_reg(), 8, allocs);
955 let dst = dst.pretty_print(8, allocs);
956 format!("{} ${imm}, {}, {}", ljustify(op.to_string()), src, dst)
957 }
958
959 Inst::XmmMovRMImmVex {
960 op, src, dst, imm, ..
961 } => {
962 let src = pretty_print_reg(src.to_reg(), 8, allocs);
963 let dst = dst.pretty_print(8, allocs);
964 format!("{} ${imm}, {}, {}", ljustify(op.to_string()), src, dst)
965 }
966
967 Inst::XmmRmR {
968 op,
969 src1,
970 src2,
971 dst,
972 ..
973 } => {
974 let src1 = pretty_print_reg(src1.to_reg(), 8, allocs);
975 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
976 let src2 = src2.pretty_print(8, allocs);
977 format!("{} {}, {}, {}", ljustify(op.to_string()), src1, src2, dst)
978 }
979
980 Inst::XmmRmRUnaligned {
981 op,
982 src1,
983 src2,
984 dst,
985 ..
986 } => {
987 let src1 = pretty_print_reg(src1.to_reg(), 8, allocs);
988 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
989 let src2 = src2.pretty_print(8, allocs);
990 format!("{} {}, {}, {}", ljustify(op.to_string()), src1, src2, dst)
991 }
992
993 Inst::XmmRmRBlend {
994 op,
995 src1,
996 src2,
997 mask,
998 dst,
999 } => {
1000 let src1 = pretty_print_reg(src1.to_reg(), 8, allocs);
1001 let mask = allocs.next(mask.to_reg());
1002 let mask = if mask.is_virtual() {
1003 format!(" <{}>", show_ireg_sized(mask, 8))
1004 } else {
1005 debug_assert_eq!(mask, regs::xmm0());
1006 String::new()
1007 };
1008 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1009 let src2 = src2.pretty_print(8, allocs);
1010 format!(
1011 "{} {}, {}, {}{}",
1012 ljustify(op.to_string()),
1013 src1,
1014 src2,
1015 dst,
1016 mask
1017 )
1018 }
1019
1020 Inst::XmmRmiRVex {
1021 op,
1022 src1,
1023 src2,
1024 dst,
1025 ..
1026 } => {
1027 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1028 let src1 = pretty_print_reg(src1.to_reg(), 8, allocs);
1029 let src2 = src2.pretty_print(8, allocs);
1030
1031 format!("{} {}, {}, {}", ljustify(op.to_string()), src1, src2, dst)
1032 }
1033
1034 Inst::XmmRmRImmVex {
1035 op,
1036 src1,
1037 src2,
1038 dst,
1039 imm,
1040 ..
1041 } => {
1042 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1043 let src1 = pretty_print_reg(src1.to_reg(), 8, allocs);
1044 let src2 = src2.pretty_print(8, allocs);
1045
1046 format!("{} ${imm}, {src1}, {src2}, {dst}", ljustify(op.to_string()))
1047 }
1048
1049 Inst::XmmVexPinsr {
1050 op,
1051 src1,
1052 src2,
1053 dst,
1054 imm,
1055 ..
1056 } => {
1057 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1058 let src1 = pretty_print_reg(src1.to_reg(), 8, allocs);
1059 let src2 = src2.pretty_print(8, allocs);
1060
1061 format!("{} ${imm}, {src1}, {src2}, {dst}", ljustify(op.to_string()))
1062 }
1063
1064 Inst::XmmRmRVex3 {
1065 op,
1066 src1,
1067 src2,
1068 src3,
1069 dst,
1070 ..
1071 } => {
1072 let src1 = pretty_print_reg(src1.to_reg(), 8, allocs);
1073 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1074 let src2 = pretty_print_reg(src2.to_reg(), 8, allocs);
1075 let src3 = src3.pretty_print(8, allocs);
1076
1077 format!(
1078 "{} {}, {}, {}, {}",
1079 ljustify(op.to_string()),
1080 src1,
1081 src2,
1082 src3,
1083 dst
1084 )
1085 }
1086
1087 Inst::XmmRmRBlendVex {
1088 op,
1089 src1,
1090 src2,
1091 mask,
1092 dst,
1093 ..
1094 } => {
1095 let src1 = pretty_print_reg(src1.to_reg(), 8, allocs);
1096 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1097 let src2 = src2.pretty_print(8, allocs);
1098 let mask = pretty_print_reg(mask.to_reg(), 8, allocs);
1099
1100 format!("{} {src1}, {src2}, {mask}, {dst}", ljustify(op.to_string()))
1101 }
1102
1103 Inst::XmmRmREvex {
1104 op,
1105 src1,
1106 src2,
1107 dst,
1108 ..
1109 } => {
1110 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1111 let src2 = pretty_print_reg(src2.to_reg(), 8, allocs);
1112 let src1 = src1.pretty_print(8, allocs);
1113 format!("{} {}, {}, {}", ljustify(op.to_string()), src1, src2, dst)
1114 }
1115
1116 Inst::XmmRmREvex3 {
1117 op,
1118 src1,
1119 src2,
1120 src3,
1121 dst,
1122 ..
1123 } => {
1124 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1125 let src2 = pretty_print_reg(src2.to_reg(), 8, allocs);
1126 let src3 = pretty_print_reg(src3.to_reg(), 8, allocs);
1127 let src1 = src1.pretty_print(8, allocs);
1128 format!(
1129 "{} {}, {}, {}, {}",
1130 ljustify(op.to_string()),
1131 src1,
1132 src2,
1133 src3,
1134 dst
1135 )
1136 }
1137
1138 Inst::XmmMinMaxSeq {
1139 lhs,
1140 rhs,
1141 dst,
1142 is_min,
1143 size,
1144 } => {
1145 let rhs = pretty_print_reg(rhs.to_reg(), 8, allocs);
1146 let lhs = pretty_print_reg(lhs.to_reg(), 8, allocs);
1147 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1148 format!(
1149 "{} {}, {}, {}",
1150 ljustify2(
1151 if *is_min {
1152 "xmm min seq ".to_string()
1153 } else {
1154 "xmm max seq ".to_string()
1155 },
1156 format!("f{}", size.to_bits())
1157 ),
1158 lhs,
1159 rhs,
1160 dst
1161 )
1162 }
1163
1164 Inst::XmmRmRImm {
1165 op,
1166 src1,
1167 src2,
1168 dst,
1169 imm,
1170 size,
1171 ..
1172 } => {
1173 let src1 = pretty_print_reg(*src1, 8, allocs);
1174 let dst = pretty_print_reg(dst.to_reg(), 8, allocs);
1175 let src2 = src2.pretty_print(8, allocs);
1176 format!(
1177 "{} ${imm}, {src1}, {src2}, {dst}",
1178 ljustify(format!(
1179 "{}{}",
1180 op.to_string(),
1181 if *size == OperandSize::Size64 {
1182 ".w"
1183 } else {
1184 ""
1185 }
1186 )),
1187 )
1188 }
1189
1190 Inst::XmmUninitializedValue { dst } => {
1191 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1192 format!("{} {}", ljustify("uninit".into()), dst)
1193 }
1194
1195 Inst::XmmToGpr {
1196 op,
1197 src,
1198 dst,
1199 dst_size,
1200 } => {
1201 let dst_size = dst_size.to_bytes();
1202 let src = pretty_print_reg(src.to_reg(), 8, allocs);
1203 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size, allocs);
1204 format!("{} {}, {}", ljustify(op.to_string()), src, dst)
1205 }
1206
1207 Inst::XmmToGprVex {
1208 op,
1209 src,
1210 dst,
1211 dst_size,
1212 } => {
1213 let dst_size = dst_size.to_bytes();
1214 let src = pretty_print_reg(src.to_reg(), 8, allocs);
1215 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size, allocs);
1216 format!("{} {src}, {dst}", ljustify(op.to_string()))
1217 }
1218
1219 Inst::XmmToGprImm { op, src, dst, imm } => {
1220 let src = pretty_print_reg(src.to_reg(), 8, allocs);
1221 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1222 format!("{} ${imm}, {}, {}", ljustify(op.to_string()), src, dst)
1223 }
1224
1225 Inst::XmmToGprImmVex { op, src, dst, imm } => {
1226 let src = pretty_print_reg(src.to_reg(), 8, allocs);
1227 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1228 format!("{} ${imm}, {}, {}", ljustify(op.to_string()), src, dst)
1229 }
1230
1231 Inst::GprToXmm {
1232 op,
1233 src,
1234 src_size,
1235 dst,
1236 } => {
1237 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1238 let src = src.pretty_print(src_size.to_bytes(), allocs);
1239 format!("{} {}, {}", ljustify(op.to_string()), src, dst)
1240 }
1241
1242 Inst::GprToXmmVex {
1243 op,
1244 src,
1245 src_size,
1246 dst,
1247 } => {
1248 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1249 let src = src.pretty_print(src_size.to_bytes(), allocs);
1250 format!("{} {src}, {dst}", ljustify(op.to_string()))
1251 }
1252
1253 Inst::XmmCmpRmR { op, src, dst } => {
1254 let dst = pretty_print_reg(dst.to_reg(), 8, allocs);
1255 let src = src.pretty_print(8, allocs);
1256 format!("{} {}, {}", ljustify(op.to_string()), src, dst)
1257 }
1258
1259 Inst::CvtUint64ToFloatSeq {
1260 src,
1261 dst,
1262 dst_size,
1263 tmp_gpr1,
1264 tmp_gpr2,
1265 ..
1266 } => {
1267 let src = pretty_print_reg(src.to_reg(), 8, allocs);
1268 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes(), allocs);
1269 let tmp_gpr1 = pretty_print_reg(tmp_gpr1.to_reg().to_reg(), 8, allocs);
1270 let tmp_gpr2 = pretty_print_reg(tmp_gpr2.to_reg().to_reg(), 8, allocs);
1271 format!(
1272 "{} {}, {}, {}, {}",
1273 ljustify(format!(
1274 "u64_to_{}_seq",
1275 if *dst_size == OperandSize::Size64 {
1276 "f64"
1277 } else {
1278 "f32"
1279 }
1280 )),
1281 src,
1282 dst,
1283 tmp_gpr1,
1284 tmp_gpr2
1285 )
1286 }
1287
1288 Inst::CvtFloatToSintSeq {
1289 src,
1290 dst,
1291 src_size,
1292 dst_size,
1293 tmp_xmm,
1294 tmp_gpr,
1295 is_saturating,
1296 } => {
1297 let src = pretty_print_reg(src.to_reg(), src_size.to_bytes(), allocs);
1298 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes(), allocs);
1299 let tmp_gpr = pretty_print_reg(tmp_gpr.to_reg().to_reg(), 8, allocs);
1300 let tmp_xmm = pretty_print_reg(tmp_xmm.to_reg().to_reg(), 8, allocs);
1301 format!(
1302 "{} {}, {}, {}, {}",
1303 ljustify(format!(
1304 "cvt_float{}_to_sint{}{}_seq",
1305 src_size.to_bits(),
1306 dst_size.to_bits(),
1307 if *is_saturating { "_sat" } else { "" },
1308 )),
1309 src,
1310 dst,
1311 tmp_gpr,
1312 tmp_xmm,
1313 )
1314 }
1315
1316 Inst::CvtFloatToUintSeq {
1317 src,
1318 dst,
1319 src_size,
1320 dst_size,
1321 tmp_gpr,
1322 tmp_xmm,
1323 tmp_xmm2,
1324 is_saturating,
1325 } => {
1326 let src = pretty_print_reg(src.to_reg(), src_size.to_bytes(), allocs);
1327 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes(), allocs);
1328 let tmp_gpr = pretty_print_reg(tmp_gpr.to_reg().to_reg(), 8, allocs);
1329 let tmp_xmm = pretty_print_reg(tmp_xmm.to_reg().to_reg(), 8, allocs);
1330 let tmp_xmm2 = pretty_print_reg(tmp_xmm2.to_reg().to_reg(), 8, allocs);
1331 format!(
1332 "{} {}, {}, {}, {}, {}",
1333 ljustify(format!(
1334 "cvt_float{}_to_uint{}{}_seq",
1335 src_size.to_bits(),
1336 dst_size.to_bits(),
1337 if *is_saturating { "_sat" } else { "" },
1338 )),
1339 src,
1340 dst,
1341 tmp_gpr,
1342 tmp_xmm,
1343 tmp_xmm2,
1344 )
1345 }
1346
1347 Inst::Imm {
1348 dst_size,
1349 simm64,
1350 dst,
1351 } => {
1352 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size.to_bytes(), allocs);
1353 if *dst_size == OperandSize::Size64 {
1354 format!(
1355 "{} ${}, {}",
1356 ljustify("movabsq".to_string()),
1357 *simm64 as i64,
1358 dst,
1359 )
1360 } else {
1361 format!(
1362 "{} ${}, {}",
1363 ljustify("movl".to_string()),
1364 (*simm64 as u32) as i32,
1365 dst,
1366 )
1367 }
1368 }
1369
1370 Inst::MovImmM { size, simm64, dst } => {
1371 let dst = dst.pretty_print(size.to_bytes(), allocs);
1372 let suffix = suffix_bwlq(*size);
1373 let instruction = ljustify2("mov".to_string(), suffix);
1374
1375 match *size {
1376 OperandSize::Size8 => {
1377 format!("{} ${}, {}", instruction, (*simm64 as u8) as i8, dst)
1378 }
1379 OperandSize::Size16 => {
1380 format!("{} ${}, {}", instruction, (*simm64 as u16) as i16, dst)
1381 }
1382 OperandSize::Size32 => {
1383 format!("{} ${}, {}", instruction, (*simm64 as u32) as i32, dst)
1384 }
1385 OperandSize::Size64 => format!("{} ${}, {}", instruction, *simm64 as i64, dst),
1386 }
1387 }
1388
1389 Inst::MovRR { size, src, dst } => {
1390 let src = pretty_print_reg(src.to_reg(), size.to_bytes(), allocs);
1391 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes(), allocs);
1392 format!(
1393 "{} {}, {}",
1394 ljustify2("mov".to_string(), suffix_lq(*size)),
1395 src,
1396 dst
1397 )
1398 }
1399
1400 Inst::MovFromPReg { src, dst } => {
1401 allocs.next_fixed_nonallocatable(*src);
1402 let src: Reg = (*src).into();
1403 let src = regs::show_ireg_sized(src, 8);
1404 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1405 format!("{} {}, {}", ljustify("movq".to_string()), src, dst)
1406 }
1407
1408 Inst::MovToPReg { src, dst } => {
1409 let src = pretty_print_reg(src.to_reg(), 8, allocs);
1410 allocs.next_fixed_nonallocatable(*dst);
1411 let dst: Reg = (*dst).into();
1412 let dst = regs::show_ireg_sized(dst, 8);
1413 format!("{} {}, {}", ljustify("movq".to_string()), src, dst)
1414 }
1415
1416 Inst::MovzxRmR {
1417 ext_mode, src, dst, ..
1418 } => {
1419 let dst_size = if *ext_mode == ExtMode::LQ {
1420 4
1421 } else {
1422 ext_mode.dst_size()
1423 };
1424 let dst = pretty_print_reg(dst.to_reg().to_reg(), dst_size, allocs);
1425 let src = src.pretty_print(ext_mode.src_size(), allocs);
1426 if *ext_mode == ExtMode::LQ {
1427 format!("{} {}, {}", ljustify("movl".to_string()), src, dst)
1428 } else {
1429 format!(
1430 "{} {}, {}",
1431 ljustify2("movz".to_string(), ext_mode.to_string()),
1432 src,
1433 dst,
1434 )
1435 }
1436 }
1437
1438 Inst::Mov64MR { src, dst, .. } => {
1439 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1440 let src = src.pretty_print(8, allocs);
1441 format!("{} {}, {}", ljustify("movq".to_string()), src, dst)
1442 }
1443
1444 Inst::LoadEffectiveAddress { addr, dst, size } => {
1445 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes(), allocs);
1446 let addr = addr.pretty_print(8, allocs);
1447 format!("{} {}, {}", ljustify("lea".to_string()), addr, dst)
1448 }
1449
1450 Inst::MovsxRmR {
1451 ext_mode, src, dst, ..
1452 } => {
1453 let dst = pretty_print_reg(dst.to_reg().to_reg(), ext_mode.dst_size(), allocs);
1454 let src = src.pretty_print(ext_mode.src_size(), allocs);
1455 format!(
1456 "{} {}, {}",
1457 ljustify2("movs".to_string(), ext_mode.to_string()),
1458 src,
1459 dst
1460 )
1461 }
1462
1463 Inst::MovRM { size, src, dst, .. } => {
1464 let src = pretty_print_reg(src.to_reg(), size.to_bytes(), allocs);
1465 let dst = dst.pretty_print(size.to_bytes(), allocs);
1466 format!(
1467 "{} {}, {}",
1468 ljustify2("mov".to_string(), suffix_bwlq(*size)),
1469 src,
1470 dst
1471 )
1472 }
1473
1474 Inst::ShiftR {
1475 size,
1476 kind,
1477 num_bits,
1478 src,
1479 dst,
1480 ..
1481 } => {
1482 let src = pretty_print_reg(src.to_reg(), size.to_bytes(), allocs);
1483 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes(), allocs);
1484 match num_bits.clone().to_imm8_reg() {
1485 Imm8Reg::Reg { reg } => {
1486 let reg = pretty_print_reg(reg, 1, allocs);
1487 format!(
1488 "{} {}, {}, {}",
1489 ljustify2(kind.to_string(), suffix_bwlq(*size)),
1490 reg,
1491 src,
1492 dst,
1493 )
1494 }
1495
1496 Imm8Reg::Imm8 { imm: num_bits } => format!(
1497 "{} ${}, {}, {}",
1498 ljustify2(kind.to_string(), suffix_bwlq(*size)),
1499 num_bits,
1500 src,
1501 dst,
1502 ),
1503 }
1504 }
1505
1506 Inst::XmmRmiReg {
1507 opcode,
1508 src1,
1509 src2,
1510 dst,
1511 ..
1512 } => {
1513 let src1 = pretty_print_reg(src1.to_reg(), 8, allocs);
1514 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1515 let src2 = src2.pretty_print(8, allocs);
1516 format!(
1517 "{} {}, {}, {}",
1518 ljustify(opcode.to_string()),
1519 src1,
1520 src2,
1521 dst,
1522 )
1523 }
1524
1525 Inst::CmpRmiR {
1526 size,
1527 src,
1528 dst,
1529 opcode,
1530 } => {
1531 let dst = pretty_print_reg(dst.to_reg(), size.to_bytes(), allocs);
1532 let src = src.pretty_print(size.to_bytes(), allocs);
1533 let op = match opcode {
1534 CmpOpcode::Cmp => "cmp",
1535 CmpOpcode::Test => "test",
1536 };
1537 format!(
1538 "{} {}, {}",
1539 ljustify2(op.to_string(), suffix_bwlq(*size)),
1540 src,
1541 dst,
1542 )
1543 }
1544
1545 Inst::Setcc { cc, dst } => {
1546 let dst = pretty_print_reg(dst.to_reg().to_reg(), 1, allocs);
1547 format!("{} {}", ljustify2("set".to_string(), cc.to_string()), dst)
1548 }
1549
1550 Inst::Bswap { size, src, dst } => {
1551 let src = pretty_print_reg(src.to_reg(), size.to_bytes(), allocs);
1552 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes(), allocs);
1553 format!(
1554 "{} {}, {}",
1555 ljustify2("bswap".to_string(), suffix_bwlq(*size)),
1556 src,
1557 dst
1558 )
1559 }
1560
1561 Inst::Cmove {
1562 size,
1563 cc,
1564 consequent,
1565 alternative,
1566 dst,
1567 } => {
1568 let alternative = pretty_print_reg(alternative.to_reg(), size.to_bytes(), allocs);
1569 let dst = pretty_print_reg(dst.to_reg().to_reg(), size.to_bytes(), allocs);
1570 let consequent = consequent.pretty_print(size.to_bytes(), allocs);
1571 format!(
1572 "{} {}, {}, {}",
1573 ljustify(format!("cmov{}{}", cc.to_string(), suffix_bwlq(*size))),
1574 consequent,
1575 alternative,
1576 dst,
1577 )
1578 }
1579
1580 Inst::XmmCmove {
1581 ty,
1582 cc,
1583 consequent,
1584 alternative,
1585 dst,
1586 ..
1587 } => {
1588 let size = u8::try_from(ty.bytes()).unwrap();
1589 let alternative = pretty_print_reg(alternative.to_reg(), size, allocs);
1590 let dst = pretty_print_reg(dst.to_reg().to_reg(), size, allocs);
1591 let consequent = consequent.pretty_print(size, allocs);
1592 let suffix = match *ty {
1593 types::F64 => "sd",
1594 types::F32 => "ss",
1595 types::F32X4 => "aps",
1596 types::F64X2 => "apd",
1597 _ => "dqa",
1598 };
1599 format!(
1600 "mov{suffix} {alternative}, {dst}; \
1601 j{} $next; \
1602 mov{suffix} {consequent}, {dst}; \
1603 $next:",
1604 cc.invert().to_string(),
1605 )
1606 }
1607
1608 Inst::Push64 { src } => {
1609 let src = src.pretty_print(8, allocs);
1610 format!("{} {}", ljustify("pushq".to_string()), src)
1611 }
1612
1613 Inst::StackProbeLoop {
1614 tmp,
1615 frame_size,
1616 guard_size,
1617 } => {
1618 let tmp = pretty_print_reg(tmp.to_reg(), 8, allocs);
1619 format!(
1620 "{} {}, frame_size={}, guard_size={}",
1621 ljustify("stack_probe_loop".to_string()),
1622 tmp,
1623 frame_size,
1624 guard_size
1625 )
1626 }
1627
1628 Inst::Pop64 { dst } => {
1629 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1630 format!("{} {}", ljustify("popq".to_string()), dst)
1631 }
1632
1633 Inst::CallKnown { dest, .. } => {
1634 format!("{} {:?}", ljustify("call".to_string()), dest)
1635 }
1636
1637 Inst::CallUnknown { dest, .. } => {
1638 let dest = dest.pretty_print(8, allocs);
1639 format!("{} *{}", ljustify("call".to_string()), dest)
1640 }
1641
1642 Inst::Args { args } => {
1643 let mut s = "args".to_string();
1644 for arg in args {
1645 use std::fmt::Write;
1646 let preg = regs::show_reg(arg.preg);
1647 let def = pretty_print_reg(arg.vreg.to_reg(), 8, allocs);
1648 write!(&mut s, " {}={}", def, preg).unwrap();
1649 }
1650 s
1651 }
1652
1653 Inst::Ret { rets } => {
1654 let mut s = "ret".to_string();
1655 for ret in rets {
1656 use std::fmt::Write;
1657 let preg = regs::show_reg(ret.preg);
1658 let vreg = pretty_print_reg(ret.vreg, 8, allocs);
1659 write!(&mut s, " {}={}", vreg, preg).unwrap();
1660 }
1661 s
1662 }
1663
1664 Inst::JmpKnown { dst } => {
1665 format!("{} {}", ljustify("jmp".to_string()), dst.to_string())
1666 }
1667
1668 Inst::JmpIf { cc, taken } => format!(
1669 "{} {}",
1670 ljustify2("j".to_string(), cc.to_string()),
1671 taken.to_string(),
1672 ),
1673
1674 Inst::JmpCond {
1675 cc,
1676 taken,
1677 not_taken,
1678 } => format!(
1679 "{} {}; j {}",
1680 ljustify2("j".to_string(), cc.to_string()),
1681 taken.to_string(),
1682 not_taken.to_string()
1683 ),
1684
1685 Inst::JmpTableSeq {
1686 idx, tmp1, tmp2, ..
1687 } => {
1688 let idx = pretty_print_reg(*idx, 8, allocs);
1689 let tmp1 = pretty_print_reg(tmp1.to_reg(), 8, allocs);
1690 let tmp2 = pretty_print_reg(tmp2.to_reg(), 8, allocs);
1691 format!(
1692 "{} {}, {}, {}",
1693 ljustify("br_table".into()),
1694 idx,
1695 tmp1,
1696 tmp2
1697 )
1698 }
1699
1700 Inst::JmpUnknown { target } => {
1701 let target = target.pretty_print(8, allocs);
1702 format!("{} *{}", ljustify("jmp".to_string()), target)
1703 }
1704
1705 Inst::TrapIf { cc, trap_code, .. } => {
1706 format!("j{cc} #trap={trap_code}")
1707 }
1708
1709 Inst::TrapIfAnd {
1710 cc1,
1711 cc2,
1712 trap_code,
1713 ..
1714 } => {
1715 format!(
1716 "trap_if_and {}, {}, {}",
1717 cc1.invert().to_string(),
1718 cc2.invert().to_string(),
1719 trap_code
1720 )
1721 }
1722
1723 Inst::TrapIfOr {
1724 cc1,
1725 cc2,
1726 trap_code,
1727 ..
1728 } => {
1729 format!(
1730 "trap_if_or {}, {}, {}",
1731 cc1.to_string(),
1732 cc2.invert().to_string(),
1733 trap_code
1734 )
1735 }
1736
1737 Inst::LoadExtName {
1738 dst, name, offset, ..
1739 } => {
1740 let dst = pretty_print_reg(dst.to_reg(), 8, allocs);
1741 format!(
1742 "{} {}+{}, {}",
1743 ljustify("load_ext_name".into()),
1744 name.display(None),
1745 offset,
1746 dst,
1747 )
1748 }
1749
1750 Inst::LockCmpxchg {
1751 ty,
1752 replacement,
1753 expected,
1754 mem,
1755 dst_old,
1756 ..
1757 } => {
1758 let size = ty.bytes() as u8;
1759 let replacement = pretty_print_reg(*replacement, size, allocs);
1760 let expected = pretty_print_reg(*expected, size, allocs);
1761 let dst_old = pretty_print_reg(dst_old.to_reg(), size, allocs);
1762 let mem = mem.pretty_print(size, allocs);
1763 format!(
1764 "lock cmpxchg{} {}, {}, expected={}, dst_old={}",
1765 suffix_bwlq(OperandSize::from_bytes(size as u32)),
1766 replacement,
1767 mem,
1768 expected,
1769 dst_old,
1770 )
1771 }
1772
1773 Inst::AtomicRmwSeq { ty, op, .. } => {
1774 format!(
1775 "atomically {{ {}_bits_at_[%r9]) {:?}= %r10; %rax = old_value_at_[%r9]; %r11, %rflags = trash }}",
1776 ty.bits(), op)
1777 }
1778
1779 Inst::Fence { kind } => match kind {
1780 FenceKind::MFence => "mfence".to_string(),
1781 FenceKind::LFence => "lfence".to_string(),
1782 FenceKind::SFence => "sfence".to_string(),
1783 },
1784
1785 Inst::VirtualSPOffsetAdj { offset } => format!("virtual_sp_offset_adjust {}", offset),
1786
1787 Inst::Hlt => "hlt".into(),
1788
1789 Inst::Ud2 { trap_code } => format!("ud2 {}", trap_code),
1790
1791 Inst::ElfTlsGetAddr { ref symbol, dst } => {
1792 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1793 format!("{} = elf_tls_get_addr {:?}", dst, symbol)
1794 }
1795
1796 Inst::MachOTlsGetAddr { ref symbol, dst } => {
1797 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1798 format!("{} = macho_tls_get_addr {:?}", dst, symbol)
1799 }
1800
1801 Inst::CoffTlsGetAddr {
1802 ref symbol,
1803 dst,
1804 tmp,
1805 } => {
1806 use std::fmt::Write;
1807
1808 let dst = pretty_print_reg(dst.to_reg().to_reg(), 8, allocs);
1809 let tmp = allocs.next(tmp.to_reg().to_reg());
1810
1811 let mut s = format!("{} = coff_tls_get_addr {:?}", dst, symbol);
1812 if tmp.is_virtual() {
1813 write!(&mut s, ", {}", show_ireg_sized(tmp, 8)).unwrap();
1814 };
1815
1816 s
1817 }
1818
1819 Inst::Unwind { inst } => {
1820 format!("unwind {:?}", inst)
1821 }
1822
1823 Inst::DummyUse { reg } => {
1824 let reg = pretty_print_reg(*reg, 8, allocs);
1825 format!("dummy_use {}", reg)
1826 }
1827 }
1828 }
1829}
1830
1831impl fmt::Debug for Inst {
1832 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1833 write!(
1834 fmt,
1835 "{}",
1836 self.pretty_print_inst(&[], &mut Default::default())
1837 )
1838 }
1839}
1840
1841fn x64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut OperandCollector<'_, F>) {
1842 match inst {
1855 Inst::AluRmiR {
1856 src1, src2, dst, ..
1857 } => {
1858 collector.reg_use(src1.to_reg());
1859 collector.reg_reuse_def(dst.to_writable_reg(), 0);
1860 src2.get_operands(collector);
1861 }
1862 Inst::AluConstOp { dst, .. } => collector.reg_def(dst.to_writable_reg()),
1863 Inst::AluRM { src1_dst, src2, .. } => {
1864 collector.reg_use(src2.to_reg());
1865 src1_dst.get_operands(collector);
1866 }
1867 Inst::AluRmRVex {
1868 src1, src2, dst, ..
1869 } => {
1870 collector.reg_def(dst.to_writable_reg());
1871 collector.reg_use(src1.to_reg());
1872 collector.reg_use(src2.to_reg());
1873 }
1874 Inst::Not { src, dst, .. } => {
1875 collector.reg_use(src.to_reg());
1876 collector.reg_reuse_def(dst.to_writable_reg(), 0);
1877 }
1878 Inst::Neg { src, dst, .. } => {
1879 collector.reg_use(src.to_reg());
1880 collector.reg_reuse_def(dst.to_writable_reg(), 0);
1881 }
1882 Inst::Div {
1883 dividend_lo,
1884 dividend_hi,
1885 dst_quotient,
1886 dst_remainder,
1887 ..
1888 }
1889 | Inst::CheckedSRemSeq {
1890 dividend_lo,
1891 dividend_hi,
1892 dst_quotient,
1893 dst_remainder,
1894 ..
1895 } => {
1896 match inst {
1897 Inst::Div { divisor, .. } => divisor.get_operands(collector),
1898 Inst::CheckedSRemSeq { divisor, .. } => collector.reg_use(divisor.to_reg()),
1899 _ => {}
1900 }
1901 collector.reg_fixed_use(dividend_lo.to_reg(), regs::rax());
1902 collector.reg_fixed_use(dividend_hi.to_reg(), regs::rdx());
1903 collector.reg_fixed_def(dst_quotient.to_writable_reg(), regs::rax());
1904 collector.reg_fixed_def(dst_remainder.to_writable_reg(), regs::rdx());
1905 }
1906 Inst::Div8 { dividend, dst, .. } | Inst::CheckedSRemSeq8 { dividend, dst, .. } => {
1907 match inst {
1908 Inst::Div8 { divisor, .. } => divisor.get_operands(collector),
1909 Inst::CheckedSRemSeq8 { divisor, .. } => collector.reg_use(divisor.to_reg()),
1910 _ => {}
1911 }
1912 collector.reg_fixed_use(dividend.to_reg(), regs::rax());
1913 collector.reg_fixed_def(dst.to_writable_reg(), regs::rax());
1914 }
1915 Inst::MulHi {
1916 src1,
1917 src2,
1918 dst_lo,
1919 dst_hi,
1920 ..
1921 } => {
1922 collector.reg_fixed_use(src1.to_reg(), regs::rax());
1923 collector.reg_fixed_def(dst_lo.to_writable_reg(), regs::rax());
1924 collector.reg_fixed_def(dst_hi.to_writable_reg(), regs::rdx());
1925 src2.get_operands(collector);
1926 }
1927 Inst::SignExtendData { size, src, dst } => {
1928 match size {
1929 OperandSize::Size8 => {
1930 collector.reg_fixed_use(src.to_reg(), regs::rax());
1933 collector.reg_fixed_def(dst.to_writable_reg(), regs::rax());
1934 }
1935 _ => {
1936 collector.reg_fixed_use(src.to_reg(), regs::rax());
1939 collector.reg_fixed_def(dst.to_writable_reg(), regs::rdx());
1940 }
1941 }
1942 }
1943 Inst::UnaryRmR { src, dst, .. } => {
1944 collector.reg_def(dst.to_writable_reg());
1945 src.get_operands(collector);
1946 }
1947 Inst::XmmUnaryRmR { src, dst, .. } | Inst::XmmUnaryRmRImm { src, dst, .. } => {
1948 collector.reg_def(dst.to_writable_reg());
1949 src.get_operands(collector);
1950 }
1951 Inst::XmmUnaryRmREvex { src, dst, .. }
1952 | Inst::XmmUnaryRmRUnaligned { src, dst, .. }
1953 | Inst::XmmUnaryRmRVex { src, dst, .. }
1954 | Inst::XmmUnaryRmRImmVex { src, dst, .. } => {
1955 collector.reg_def(dst.to_writable_reg());
1956 src.get_operands(collector);
1957 }
1958 Inst::XmmRmR {
1959 src1, src2, dst, ..
1960 } => {
1961 collector.reg_use(src1.to_reg());
1962 collector.reg_reuse_def(dst.to_writable_reg(), 0);
1963 src2.get_operands(collector);
1964 }
1965 Inst::XmmRmRUnaligned {
1966 src1, src2, dst, ..
1967 } => {
1968 collector.reg_use(src1.to_reg());
1969 collector.reg_reuse_def(dst.to_writable_reg(), 0);
1970 src2.get_operands(collector);
1971 }
1972 Inst::XmmRmRBlend {
1973 src1,
1974 src2,
1975 mask,
1976 dst,
1977 op,
1978 } => {
1979 assert!(
1980 *op == SseOpcode::Blendvpd
1981 || *op == SseOpcode::Blendvps
1982 || *op == SseOpcode::Pblendvb
1983 );
1984 collector.reg_use(src1.to_reg());
1985 collector.reg_fixed_use(mask.to_reg(), regs::xmm0());
1986 collector.reg_reuse_def(dst.to_writable_reg(), 0);
1987 src2.get_operands(collector);
1988 }
1989 Inst::XmmRmiRVex {
1990 src1, src2, dst, ..
1991 } => {
1992 collector.reg_def(dst.to_writable_reg());
1993 collector.reg_use(src1.to_reg());
1994 src2.get_operands(collector);
1995 }
1996 Inst::XmmRmRImmVex {
1997 src1, src2, dst, ..
1998 } => {
1999 collector.reg_def(dst.to_writable_reg());
2000 collector.reg_use(src1.to_reg());
2001 src2.get_operands(collector);
2002 }
2003 Inst::XmmVexPinsr {
2004 src1, src2, dst, ..
2005 } => {
2006 collector.reg_def(dst.to_writable_reg());
2007 collector.reg_use(src1.to_reg());
2008 src2.get_operands(collector);
2009 }
2010 Inst::XmmRmRVex3 {
2011 src1,
2012 src2,
2013 src3,
2014 dst,
2015 ..
2016 } => {
2017 collector.reg_use(src1.to_reg());
2018 collector.reg_reuse_def(dst.to_writable_reg(), 0);
2019 collector.reg_use(src2.to_reg());
2020 src3.get_operands(collector);
2021 }
2022 Inst::XmmRmRBlendVex {
2023 src1,
2024 src2,
2025 mask,
2026 dst,
2027 ..
2028 } => {
2029 collector.reg_def(dst.to_writable_reg());
2030 collector.reg_use(src1.to_reg());
2031 src2.get_operands(collector);
2032 collector.reg_use(mask.to_reg());
2033 }
2034 Inst::XmmRmREvex {
2035 op,
2036 src1,
2037 src2,
2038 dst,
2039 ..
2040 } => {
2041 assert_ne!(*op, Avx512Opcode::Vpermi2b);
2042 collector.reg_def(dst.to_writable_reg());
2043 collector.reg_use(src2.to_reg());
2044 src1.get_operands(collector);
2045 }
2046 Inst::XmmRmREvex3 {
2047 op,
2048 src1,
2049 src2,
2050 src3,
2051 dst,
2052 ..
2053 } => {
2054 assert_eq!(*op, Avx512Opcode::Vpermi2b);
2055 collector.reg_reuse_def(dst.to_writable_reg(), 2); collector.reg_use(src2.to_reg());
2057 collector.reg_use(src3.to_reg());
2058 src1.get_operands(collector);
2059 }
2060 Inst::XmmRmRImm {
2061 src1, src2, dst, ..
2062 } => {
2063 collector.reg_use(*src1);
2064 collector.reg_reuse_def(*dst, 0);
2065 src2.get_operands(collector);
2066 }
2067 Inst::XmmUninitializedValue { dst } => collector.reg_def(dst.to_writable_reg()),
2068 Inst::XmmMinMaxSeq { lhs, rhs, dst, .. } => {
2069 collector.reg_use(rhs.to_reg());
2070 collector.reg_use(lhs.to_reg());
2071 collector.reg_reuse_def(dst.to_writable_reg(), 0); }
2073 Inst::XmmRmiReg {
2074 src1, src2, dst, ..
2075 } => {
2076 collector.reg_use(src1.to_reg());
2077 collector.reg_reuse_def(dst.to_writable_reg(), 0); src2.get_operands(collector);
2079 }
2080 Inst::XmmMovRM { src, dst, .. }
2081 | Inst::XmmMovRMVex { src, dst, .. }
2082 | Inst::XmmMovRMImm { src, dst, .. }
2083 | Inst::XmmMovRMImmVex { src, dst, .. } => {
2084 collector.reg_use(src.to_reg());
2085 dst.get_operands(collector);
2086 }
2087 Inst::XmmCmpRmR { src, dst, .. } => {
2088 collector.reg_use(dst.to_reg());
2089 src.get_operands(collector);
2090 }
2091 Inst::Imm { dst, .. } => {
2092 collector.reg_def(dst.to_writable_reg());
2093 }
2094 Inst::MovRR { src, dst, .. } => {
2095 collector.reg_use(src.to_reg());
2096 collector.reg_def(dst.to_writable_reg());
2097 }
2098 Inst::MovFromPReg { dst, src } => {
2099 debug_assert!(dst.to_reg().to_reg().is_virtual());
2100 collector.reg_fixed_nonallocatable(*src);
2101 collector.reg_def(dst.to_writable_reg());
2102 }
2103 Inst::MovToPReg { dst, src } => {
2104 debug_assert!(src.to_reg().is_virtual());
2105 collector.reg_use(src.to_reg());
2106 collector.reg_fixed_nonallocatable(*dst);
2107 }
2108 Inst::XmmToGpr { src, dst, .. }
2109 | Inst::XmmToGprVex { src, dst, .. }
2110 | Inst::XmmToGprImm { src, dst, .. }
2111 | Inst::XmmToGprImmVex { src, dst, .. } => {
2112 collector.reg_use(src.to_reg());
2113 collector.reg_def(dst.to_writable_reg());
2114 }
2115 Inst::GprToXmm { src, dst, .. } | Inst::GprToXmmVex { src, dst, .. } => {
2116 collector.reg_def(dst.to_writable_reg());
2117 src.get_operands(collector);
2118 }
2119 Inst::CvtUint64ToFloatSeq {
2120 src,
2121 dst,
2122 tmp_gpr1,
2123 tmp_gpr2,
2124 ..
2125 } => {
2126 collector.reg_use(src.to_reg());
2127 collector.reg_early_def(dst.to_writable_reg());
2128 collector.reg_early_def(tmp_gpr1.to_writable_reg());
2129 collector.reg_early_def(tmp_gpr2.to_writable_reg());
2130 }
2131 Inst::CvtFloatToSintSeq {
2132 src,
2133 dst,
2134 tmp_xmm,
2135 tmp_gpr,
2136 ..
2137 } => {
2138 collector.reg_use(src.to_reg());
2139 collector.reg_early_def(dst.to_writable_reg());
2140 collector.reg_early_def(tmp_gpr.to_writable_reg());
2141 collector.reg_early_def(tmp_xmm.to_writable_reg());
2142 }
2143 Inst::CvtFloatToUintSeq {
2144 src,
2145 dst,
2146 tmp_gpr,
2147 tmp_xmm,
2148 tmp_xmm2,
2149 ..
2150 } => {
2151 collector.reg_use(src.to_reg());
2152 collector.reg_early_def(dst.to_writable_reg());
2153 collector.reg_early_def(tmp_gpr.to_writable_reg());
2154 collector.reg_early_def(tmp_xmm.to_writable_reg());
2155 collector.reg_early_def(tmp_xmm2.to_writable_reg());
2156 }
2157
2158 Inst::MovImmM { dst, .. } => {
2159 dst.get_operands(collector);
2160 }
2161
2162 Inst::MovzxRmR { src, dst, .. } => {
2163 collector.reg_def(dst.to_writable_reg());
2164 src.get_operands(collector);
2165 }
2166 Inst::Mov64MR { src, dst, .. } => {
2167 collector.reg_def(dst.to_writable_reg());
2168 src.get_operands(collector);
2169 }
2170 Inst::LoadEffectiveAddress { addr: src, dst, .. } => {
2171 collector.reg_def(dst.to_writable_reg());
2172 src.get_operands(collector);
2173 }
2174 Inst::MovsxRmR { src, dst, .. } => {
2175 collector.reg_def(dst.to_writable_reg());
2176 src.get_operands(collector);
2177 }
2178 Inst::MovRM { src, dst, .. } => {
2179 collector.reg_use(src.to_reg());
2180 dst.get_operands(collector);
2181 }
2182 Inst::ShiftR {
2183 num_bits, src, dst, ..
2184 } => {
2185 collector.reg_use(src.to_reg());
2186 collector.reg_reuse_def(dst.to_writable_reg(), 0);
2187 if let Imm8Reg::Reg { reg } = num_bits.clone().to_imm8_reg() {
2188 collector.reg_fixed_use(reg, regs::rcx());
2189 }
2190 }
2191 Inst::CmpRmiR { src, dst, .. } => {
2192 collector.reg_use(dst.to_reg());
2194 src.get_operands(collector);
2195 }
2196 Inst::Setcc { dst, .. } => {
2197 collector.reg_def(dst.to_writable_reg());
2198 }
2199 Inst::Bswap { src, dst, .. } => {
2200 collector.reg_use(src.to_reg());
2201 collector.reg_reuse_def(dst.to_writable_reg(), 0);
2202 }
2203 Inst::Cmove {
2204 consequent,
2205 alternative,
2206 dst,
2207 ..
2208 } => {
2209 collector.reg_use(alternative.to_reg());
2210 collector.reg_reuse_def(dst.to_writable_reg(), 0);
2211 consequent.get_operands(collector);
2212 }
2213 Inst::XmmCmove {
2214 consequent,
2215 alternative,
2216 dst,
2217 ..
2218 } => {
2219 collector.reg_use(alternative.to_reg());
2220 collector.reg_reuse_def(dst.to_writable_reg(), 0);
2221 consequent.get_operands(collector);
2222 }
2223 Inst::Push64 { src } => {
2224 src.get_operands(collector);
2225 }
2226 Inst::Pop64 { dst } => {
2227 collector.reg_def(dst.to_writable_reg());
2228 }
2229 Inst::StackProbeLoop { tmp, .. } => {
2230 collector.reg_early_def(*tmp);
2231 }
2232
2233 Inst::CallKnown { dest, ref info, .. } => {
2234 debug_assert_ne!(*dest, ExternalName::LibCall(LibCall::Probestack));
2239 for u in &info.uses {
2240 collector.reg_fixed_use(u.vreg, u.preg);
2241 }
2242 for d in &info.defs {
2243 collector.reg_fixed_def(d.vreg, d.preg);
2244 }
2245 collector.reg_clobbers(info.clobbers);
2246 }
2247
2248 Inst::CallUnknown { ref info, dest, .. } => {
2249 dest.get_operands(collector);
2250 for u in &info.uses {
2251 collector.reg_fixed_use(u.vreg, u.preg);
2252 }
2253 for d in &info.defs {
2254 collector.reg_fixed_def(d.vreg, d.preg);
2255 }
2256 collector.reg_clobbers(info.clobbers);
2257 }
2258
2259 Inst::JmpTableSeq {
2260 ref idx,
2261 ref tmp1,
2262 ref tmp2,
2263 ..
2264 } => {
2265 collector.reg_use(*idx);
2266 collector.reg_early_def(*tmp1);
2267 collector.reg_def(*tmp2);
2271 }
2272
2273 Inst::JmpUnknown { target } => {
2274 target.get_operands(collector);
2275 }
2276
2277 Inst::LoadExtName { dst, .. } => {
2278 collector.reg_def(*dst);
2279 }
2280
2281 Inst::LockCmpxchg {
2282 replacement,
2283 expected,
2284 mem,
2285 dst_old,
2286 ..
2287 } => {
2288 collector.reg_use(*replacement);
2289 collector.reg_fixed_use(*expected, regs::rax());
2290 collector.reg_fixed_def(*dst_old, regs::rax());
2291 mem.get_operands(collector);
2292 }
2293
2294 Inst::AtomicRmwSeq {
2295 operand,
2296 temp,
2297 dst_old,
2298 mem,
2299 ..
2300 } => {
2301 collector.reg_late_use(*operand);
2302 collector.reg_early_def(*temp);
2303 collector.reg_fixed_def(*dst_old, regs::rax());
2306 mem.get_operands_late(collector)
2307 }
2308
2309 Inst::Args { args } => {
2310 for arg in args {
2311 collector.reg_fixed_def(arg.vreg, arg.preg);
2312 }
2313 }
2314
2315 Inst::Ret { rets } => {
2316 for ret in rets.iter() {
2319 collector.reg_fixed_use(ret.vreg, ret.preg);
2320 }
2321 }
2322
2323 Inst::JmpKnown { .. }
2324 | Inst::JmpIf { .. }
2325 | Inst::JmpCond { .. }
2326 | Inst::Nop { .. }
2327 | Inst::TrapIf { .. }
2328 | Inst::TrapIfAnd { .. }
2329 | Inst::TrapIfOr { .. }
2330 | Inst::VirtualSPOffsetAdj { .. }
2331 | Inst::Hlt
2332 | Inst::Ud2 { .. }
2333 | Inst::Fence { .. } => {
2334 }
2336
2337 Inst::ElfTlsGetAddr { dst, .. } | Inst::MachOTlsGetAddr { dst, .. } => {
2338 collector.reg_fixed_def(dst.to_writable_reg(), regs::rax());
2339 let mut clobbers = X64ABIMachineSpec::get_regs_clobbered_by_call(CallConv::SystemV);
2346 clobbers.remove(regs::gpr_preg(regs::ENC_RAX));
2347 collector.reg_clobbers(clobbers);
2348 }
2349
2350 Inst::CoffTlsGetAddr { dst, tmp, .. } => {
2351 collector.reg_fixed_def(dst.to_writable_reg(), regs::rax());
2356
2357 collector.reg_fixed_def(tmp.to_writable_reg(), regs::rcx());
2359 }
2360
2361 Inst::Unwind { .. } => {}
2362
2363 Inst::DummyUse { reg } => {
2364 collector.reg_use(*reg);
2365 }
2366 }
2367}
2368
2369impl MachInst for Inst {
2373 type ABIMachineSpec = X64ABIMachineSpec;
2374
2375 fn get_operands<F: Fn(VReg) -> VReg>(&self, collector: &mut OperandCollector<'_, F>) {
2376 x64_get_operands(&self, collector)
2377 }
2378
2379 fn is_move(&self) -> Option<(Writable<Reg>, Reg)> {
2380 match self {
2381 Self::MovRR { size, src, dst, .. } if *size == OperandSize::Size64 => {
2386 Some((dst.to_writable_reg(), src.to_reg()))
2387 }
2388 Self::XmmUnaryRmR { op, src, dst, .. }
2393 if *op == SseOpcode::Movss
2394 || *op == SseOpcode::Movsd
2395 || *op == SseOpcode::Movaps
2396 || *op == SseOpcode::Movapd
2397 || *op == SseOpcode::Movups
2398 || *op == SseOpcode::Movupd
2399 || *op == SseOpcode::Movdqa
2400 || *op == SseOpcode::Movdqu =>
2401 {
2402 if let RegMem::Reg { reg } = src.clone().to_reg_mem() {
2403 Some((dst.to_writable_reg(), reg))
2404 } else {
2405 None
2406 }
2407 }
2408 _ => None,
2409 }
2410 }
2411
2412 fn is_included_in_clobbers(&self) -> bool {
2413 match self {
2414 &Inst::Args { .. } => false,
2415 _ => true,
2416 }
2417 }
2418
2419 fn is_trap(&self) -> bool {
2420 match self {
2421 Self::Ud2 { .. } => true,
2422 _ => false,
2423 }
2424 }
2425
2426 fn is_args(&self) -> bool {
2427 match self {
2428 Self::Args { .. } => true,
2429 _ => false,
2430 }
2431 }
2432
2433 fn is_term(&self) -> MachTerminator {
2434 match self {
2435 &Self::Ret { .. } => MachTerminator::Ret,
2437 &Self::JmpKnown { .. } => MachTerminator::Uncond,
2438 &Self::JmpCond { .. } => MachTerminator::Cond,
2439 &Self::JmpTableSeq { .. } => MachTerminator::Indirect,
2440 _ => MachTerminator::None,
2442 }
2443 }
2444
2445 fn gen_move(dst_reg: Writable<Reg>, src_reg: Reg, ty: Type) -> Inst {
2446 trace!(
2447 "Inst::gen_move {:?} -> {:?} (type: {:?})",
2448 src_reg,
2449 dst_reg.to_reg(),
2450 ty
2451 );
2452 let rc_dst = dst_reg.to_reg().class();
2453 let rc_src = src_reg.class();
2454 debug_assert!(rc_dst == rc_src);
2456 match rc_dst {
2457 RegClass::Int => Inst::mov_r_r(OperandSize::Size64, src_reg, dst_reg),
2458 RegClass::Float => {
2459 let opcode = match ty {
2464 types::F32 | types::F64 | types::F32X4 => SseOpcode::Movaps,
2465 types::F64X2 => SseOpcode::Movapd,
2466 _ if ty.is_vector() && ty.bits() == 128 => SseOpcode::Movdqa,
2467 _ => unimplemented!("unable to move type: {}", ty),
2468 };
2469 Inst::xmm_unary_rm_r(opcode, RegMem::reg(src_reg), dst_reg)
2470 }
2471 }
2472 }
2473
2474 fn gen_nop(preferred_size: usize) -> Inst {
2475 Inst::nop(std::cmp::min(preferred_size, 15) as u8)
2476 }
2477
2478 fn rc_for_type(ty: Type) -> CodegenResult<(&'static [RegClass], &'static [Type])> {
2479 match ty {
2480 types::I8 => Ok((&[RegClass::Int], &[types::I8])),
2481 types::I16 => Ok((&[RegClass::Int], &[types::I16])),
2482 types::I32 => Ok((&[RegClass::Int], &[types::I32])),
2483 types::I64 => Ok((&[RegClass::Int], &[types::I64])),
2484 types::R32 => panic!("32-bit reftype pointer should never be seen on x86-64"),
2485 types::R64 => Ok((&[RegClass::Int], &[types::R64])),
2486 types::F32 => Ok((&[RegClass::Float], &[types::F32])),
2487 types::F64 => Ok((&[RegClass::Float], &[types::F64])),
2488 types::I128 => Ok((&[RegClass::Int, RegClass::Int], &[types::I64, types::I64])),
2489 _ if ty.is_vector() => {
2490 assert!(ty.bits() <= 128);
2491 Ok((&[RegClass::Float], &[types::I8X16]))
2492 }
2493 _ => Err(CodegenError::Unsupported(format!(
2494 "Unexpected SSA-value type: {}",
2495 ty
2496 ))),
2497 }
2498 }
2499
2500 fn canonical_type_for_rc(rc: RegClass) -> Type {
2501 match rc {
2502 RegClass::Float => types::I8X16,
2503 RegClass::Int => types::I64,
2504 }
2505 }
2506
2507 fn gen_jump(label: MachLabel) -> Inst {
2508 Inst::jmp_known(label)
2509 }
2510
2511 fn gen_dummy_use(reg: Reg) -> Self {
2512 Inst::DummyUse { reg }
2513 }
2514
2515 fn worst_case_size() -> CodeOffset {
2516 15
2517 }
2518
2519 fn ref_type_regclass(_: &settings::Flags) -> RegClass {
2520 RegClass::Int
2521 }
2522
2523 fn is_safepoint(&self) -> bool {
2524 match self {
2525 Inst::CallKnown { .. }
2526 | Inst::CallUnknown { .. }
2527 | Inst::TrapIf { .. }
2528 | Inst::Ud2 { .. } => true,
2529 _ => false,
2530 }
2531 }
2532
2533 type LabelUse = LabelUse;
2534
2535 const TRAP_OPCODE: &'static [u8] = &[0x0f, 0x0b];
2536}
2537
2538#[derive(Default, Clone, Debug)]
2540pub struct EmitState {
2541 pub(crate) virtual_sp_offset: i64,
2544 pub(crate) nominal_sp_to_fp: i64,
2546 stack_map: Option<StackMap>,
2548 cur_srcloc: RelSourceLoc,
2550}
2551
2552pub struct EmitInfo {
2554 pub(super) flags: settings::Flags,
2555 isa_flags: x64_settings::Flags,
2556}
2557
2558impl EmitInfo {
2559 pub fn new(flags: settings::Flags, isa_flags: x64_settings::Flags) -> Self {
2561 Self { flags, isa_flags }
2562 }
2563}
2564
2565impl MachInstEmit for Inst {
2566 type State = EmitState;
2567 type Info = EmitInfo;
2568
2569 fn emit(
2570 &self,
2571 allocs: &[Allocation],
2572 sink: &mut MachBuffer<Inst>,
2573 info: &Self::Info,
2574 state: &mut Self::State,
2575 ) {
2576 let mut allocs = AllocationConsumer::new(allocs);
2577 emit::emit(self, &mut allocs, sink, info, state);
2578 }
2579
2580 fn pretty_print_inst(&self, allocs: &[Allocation], _: &mut Self::State) -> String {
2581 PrettyPrint::pretty_print(self, 0, &mut AllocationConsumer::new(allocs))
2582 }
2583}
2584
2585impl MachInstEmitState<Inst> for EmitState {
2586 fn new(abi: &Callee<X64ABIMachineSpec>) -> Self {
2587 EmitState {
2588 virtual_sp_offset: 0,
2589 nominal_sp_to_fp: abi.frame_size() as i64,
2590 stack_map: None,
2591 cur_srcloc: Default::default(),
2592 }
2593 }
2594
2595 fn pre_safepoint(&mut self, stack_map: StackMap) {
2596 self.stack_map = Some(stack_map);
2597 }
2598
2599 fn pre_sourceloc(&mut self, srcloc: RelSourceLoc) {
2600 self.cur_srcloc = srcloc;
2601 }
2602}
2603
2604impl EmitState {
2605 fn take_stack_map(&mut self) -> Option<StackMap> {
2606 self.stack_map.take()
2607 }
2608
2609 fn clear_post_insn(&mut self) {
2610 self.stack_map = None;
2611 }
2612}
2613
2614#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2616pub enum LabelUse {
2617 JmpRel32,
2621
2622 PCRel32,
2625}
2626
2627impl MachInstLabelUse for LabelUse {
2628 const ALIGN: CodeOffset = 1;
2629
2630 fn max_pos_range(self) -> CodeOffset {
2631 match self {
2632 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0x7fff_ffff,
2633 }
2634 }
2635
2636 fn max_neg_range(self) -> CodeOffset {
2637 match self {
2638 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0x8000_0000,
2639 }
2640 }
2641
2642 fn patch_size(self) -> CodeOffset {
2643 match self {
2644 LabelUse::JmpRel32 | LabelUse::PCRel32 => 4,
2645 }
2646 }
2647
2648 fn patch(self, buffer: &mut [u8], use_offset: CodeOffset, label_offset: CodeOffset) {
2649 let pc_rel = (label_offset as i64) - (use_offset as i64);
2650 debug_assert!(pc_rel <= self.max_pos_range() as i64);
2651 debug_assert!(pc_rel >= -(self.max_neg_range() as i64));
2652 let pc_rel = pc_rel as u32;
2653 match self {
2654 LabelUse::JmpRel32 => {
2655 let addend = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2656 let value = pc_rel.wrapping_add(addend).wrapping_sub(4);
2657 buffer.copy_from_slice(&value.to_le_bytes()[..]);
2658 }
2659 LabelUse::PCRel32 => {
2660 let addend = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
2661 let value = pc_rel.wrapping_add(addend);
2662 buffer.copy_from_slice(&value.to_le_bytes()[..]);
2663 }
2664 }
2665 }
2666
2667 fn supports_veneer(self) -> bool {
2668 match self {
2669 LabelUse::JmpRel32 | LabelUse::PCRel32 => false,
2670 }
2671 }
2672
2673 fn veneer_size(self) -> CodeOffset {
2674 match self {
2675 LabelUse::JmpRel32 | LabelUse::PCRel32 => 0,
2676 }
2677 }
2678
2679 fn generate_veneer(self, _: &mut [u8], _: CodeOffset) -> (CodeOffset, LabelUse) {
2680 match self {
2681 LabelUse::JmpRel32 | LabelUse::PCRel32 => {
2682 panic!("Veneer not supported for JumpRel32 label-use.");
2683 }
2684 }
2685 }
2686
2687 fn from_reloc(reloc: Reloc, addend: Addend) -> Option<Self> {
2688 match (reloc, addend) {
2689 (Reloc::X86CallPCRel4, -4) => Some(LabelUse::JmpRel32),
2690 _ => None,
2691 }
2692 }
2693}