From 375e336aaea92ba2485fbdeec00b6105053f9518 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 7 Jan 2022 10:58:41 -0800 Subject: [PATCH] cranelift: port `sshr` to ISLE on x64 --- .../lower/isle/generated_code.manifest | 2 +- .../isa/aarch64/lower/isle/generated_code.rs | 17 +- cranelift/codegen/src/isa/x64/inst.isle | 88 ++ cranelift/codegen/src/isa/x64/lower.isle | 117 +++ cranelift/codegen/src/isa/x64/lower.rs | 183 +--- cranelift/codegen/src/isa/x64/lower/isle.rs | 46 + .../x64/lower/isle/generated_code.manifest | 6 +- .../src/isa/x64/lower/isle/generated_code.rs | 845 +++++++++++++----- cranelift/codegen/src/machinst/isle.rs | 12 + cranelift/codegen/src/prelude.isle | 10 + .../filetests/filetests/isa/x64/i128.clif | 29 +- .../isa/x64/simd-bitwise-compile.clif | 31 +- cranelift/filetests/src/runner.rs | 2 +- 13 files changed, 942 insertions(+), 446 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index f17a07ac3199..9a4fec314329 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle f176ef3bba99365 -src/prelude.isle babc931e5dc5b4cf +src/prelude.isle e5e437b1aed2a1e1 src/isa/aarch64/inst.isle 36d057f98a944e4 src/isa/aarch64/lower.isle 43467df9d06b00ac diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index 4f63278523e4..3fc210a96b4b 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -20,6 +20,8 @@ pub trait Context { fn pack_value_array_2(&mut self, arg0: Value, arg1: Value) -> ValueArray2; fn unpack_value_array_3(&mut self, arg0: &ValueArray3) -> (Value, Value, Value); fn pack_value_array_3(&mut self, arg0: Value, arg1: Value, arg2: Value) -> ValueArray3; + fn u32_add(&mut self, arg0: u32, arg1: u32) -> u32; + fn u8_and(&mut self, arg0: u8, arg1: u8) -> u8; fn value_reg(&mut self, arg0: Reg) -> ValueRegs; fn value_regs(&mut self, arg0: Reg, arg1: Reg) -> ValueRegs; fn temp_writable_reg(&mut self, arg0: Type) -> WritableReg; @@ -32,6 +34,7 @@ pub trait Context { fn u32_as_u64(&mut self, arg0: u32) -> u64; fn ty_bits(&mut self, arg0: Type) -> u8; fn ty_bits_u16(&mut self, arg0: Type) -> u16; + fn lane_type(&mut self, arg0: Type) -> Type; fn fits_in_16(&mut self, arg0: Type) -> Option; fn fits_in_32(&mut self, arg0: Type) -> Option; fn fits_in_64(&mut self, arg0: Type) -> Option; @@ -89,13 +92,13 @@ pub trait Context { fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift; } -/// Internal type ProducesFlags: defined at src/prelude.isle line 263. +/// Internal type ProducesFlags: defined at src/prelude.isle line 273. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 266. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 276. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, @@ -975,7 +978,7 @@ pub enum AtomicRMWOp { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 60. + // Rule at src/prelude.isle line 66. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -984,7 +987,7 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 95. + // Rule at src/prelude.isle line 101. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -1009,7 +1012,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 276. + // Rule at src/prelude.isle line 286. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1037,7 +1040,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 284. + // Rule at src/prelude.isle line 294. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -1071,7 +1074,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 294. + // Rule at src/prelude.isle line 304. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern5_0); let expr2_0 = C::emit(ctx, &pattern3_0); diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index 8f94248d764c..d8ca3d58ce4b 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -35,6 +35,7 @@ (dst WritableReg) (imm u8) (size OperandSize)) + (XmmUninitializedValue (dst WritableReg)) (CmpRmiR (size OperandSize) (opcode CmpOpcode) (src RegMemImm) @@ -292,6 +293,15 @@ (Mem (addr SyntheticAmode)) (Imm (simm32 u32)))) +;; Put the given clif value into a `RegMemImm` operand. +;; +;; Asserts that the value fits into a single register, and doesn't require +;; multiple registers for its representation (like `i128` for example). +;; +;; As a side effect, this marks the value as used. +(decl put_in_reg_mem_imm (Value) RegMemImm) +(extern constructor put_in_reg_mem_imm put_in_reg_mem_imm) + (type RegMem extern (enum (Reg (reg Reg)) @@ -319,6 +329,15 @@ (enum (Imm8 (imm u8)) (Reg (reg Reg)))) +;; Put the given clif value into a `Imm8Reg` operand. +;; +;; Asserts that the value fits into a single register, and doesn't require +;; multiple registers for its representation (like `i128` for example). +;; +;; As a side effect, this marks the value as used. +(decl put_in_imm8_reg (Value) Imm8Reg) +(extern constructor put_in_imm8_reg put_in_imm8_reg) + (type CC extern (enum O NO @@ -494,6 +513,37 @@ wr)))) r)) +;; Helper for creating an SSE register holding an `i64x2` from two `i64` values. +(decl make_i64x2_from_lanes (RegMem RegMem) Reg) +(rule (make_i64x2_from_lanes lo hi) + (let ((dst_w WritableReg (temp_writable_reg $I64X2)) + (dst_r Reg (writable_reg_to_reg dst_w)) + (_0 Unit (emit (MInst.XmmUninitializedValue dst_w))) + (_1 Unit (emit (MInst.XmmRmRImm (SseOpcode.Pinsrd) + dst_r + lo + dst_w + 0 + (OperandSize.Size64)))) + (_2 Unit (emit (MInst.XmmRmRImm (SseOpcode.Pinsrd) + dst_r + hi + dst_w + 1 + (OperandSize.Size64))))) + dst_r)) + +;; Move a `RegMemImm.Reg` operand to an XMM register, if necessary. +(decl reg_mem_imm_to_xmm (RegMemImm) RegMemImm) +(rule (reg_mem_imm_to_xmm rmi @ (RegMemImm.Mem _)) rmi) +(rule (reg_mem_imm_to_xmm rmi @ (RegMemImm.Imm _)) rmi) +(rule (reg_mem_imm_to_xmm (RegMemImm.Reg r)) + (RegMemImm.Reg (gpr_to_xmm $I8X16 + (SseOpcode.Movd) + (RegMem.Reg r) + (OperandSize.Size32)))) + + ;;;; Instruction Constructors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; ;; These constructors create SSA-style `MInst`s. It is their responsibility to @@ -1058,6 +1108,21 @@ (rule (pminud src1 src2) (xmm_rm_r $I8X16 (SseOpcode.Pminud) src1 src2)) +;; Helper for creating `punpcklbw` instructions. +(decl punpcklbw (Reg RegMem) Reg) +(rule (punpcklbw src1 src2) + (xmm_rm_r $I8X16 (SseOpcode.Punpcklbw) src1 src2)) + +;; Helper for creating `punpckhbw` instructions. +(decl punpckhbw (Reg RegMem) Reg) +(rule (punpckhbw src1 src2) + (xmm_rm_r $I8X16 (SseOpcode.Punpckhbw) src1 src2)) + +;; Helper for creating `packsswb` instructions. +(decl packsswb (Reg RegMem) Reg) +(rule (packsswb src1 src2) + (xmm_rm_r $I8X16 (SseOpcode.Packsswb) src1 src2)) + ;; Helper for creating `MInst.XmmRmRImm` instructions. (decl xmm_rm_r_imm (SseOpcode Reg RegMem u8 OperandSize) Reg) (rule (xmm_rm_r_imm op src1 src2 imm size) @@ -1180,6 +1245,16 @@ (rule (psrlq src1 src2) (xmm_rmi_reg (SseOpcode.Psrlq) src1 src2)) +;; Helper for creating `psraw` instructions. +(decl psraw (Reg RegMemImm) Reg) +(rule (psraw src1 src2) + (xmm_rmi_reg (SseOpcode.Psraw) src1 src2)) + +;; Helper for creating `psrad` instructions. +(decl psrad (Reg RegMemImm) Reg) +(rule (psrad src1 src2) + (xmm_rmi_reg (SseOpcode.Psrad) src1 src2)) + ;; Helper for creating `MInst.MulHi` instructions. ;; ;; Returns the (lo, hi) register halves of the multiplication. @@ -1252,6 +1327,19 @@ (rule (insertps src1 src2 lane) (xmm_rm_r_imm (SseOpcode.Insertps) src1 src2 lane (OperandSize.Size32))) +;; Helper for creating `pextrd` instructions. +(decl pextrd (Type Reg u8) Reg) +(rule (pextrd ty src lane) + (let ((w_dst WritableReg (temp_writable_reg ty)) + (r_dst Reg (writable_reg_to_reg w_dst)) + (_ Unit (emit (MInst.XmmRmRImm (SseOpcode.Pextrd) + r_dst + (RegMem.Reg src) + w_dst + lane + (operand_size_of_type_32_64 (lane_type ty)))))) + r_dst)) + ;; Helper for creating `not` instructions. (decl not (Type Reg) Reg) (rule (not ty src) diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 5647ef06dd6c..2bd9d8b942b8 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -623,6 +623,123 @@ (let ((amt_ Reg (lo_reg amt))) (shr_i128 (put_in_regs src) amt_))) +;;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; `i64` and smaller. + +(rule (lower (has_type (fits_in_64 ty) (sshr src amt))) + (let ((src_ Reg (extend_to_reg src ty (ExtendKind.Sign))) + ;; NB: Only the low bits of `amt` matter since we logically mask the + ;; shift amount to the value's bit width. + (amt_ Reg (lo_reg amt))) + (value_reg (sar ty src_ (Imm8Reg.Reg amt_))))) + +(rule (lower (has_type (fits_in_64 ty) (sshr src (imm8_from_value amt)))) + (let ((src_ Reg (extend_to_reg src ty (ExtendKind.Sign)))) + (value_reg (sar ty src_ amt)))) + +;; `i128`. + +(decl sar_i128 (ValueRegs Reg) ValueRegs) +(rule (sar_i128 src amt) + ;; Unpack the low/high halves of `src`. + (let ((src_lo Reg (value_regs_get src 0)) + (src_hi Reg (value_regs_get src 1)) + ;; Do a shift of each half. NB: the low half uses an unsigned shift + ;; because its MSB is not a sign bit. + (lo_shifted Reg (shr $I64 src_lo (Imm8Reg.Reg amt))) + (hi_shifted Reg (sar $I64 src_hi (Imm8Reg.Reg amt))) + ;; `src_hi << (64 - amt)` are the bits to carry over from the low + ;; half to the high half. + (carry Reg (shl $I64 src_hi (Imm8Reg.Reg (sub $I64 (imm $I64 64) (RegMemImm.Reg amt))))) + ;; Nullify the carry if we are shifting by a multiple of 128. + (carry_ Reg (with_flags_1 (test (OperandSize.Size64) (RegMemImm.Imm 127) amt) + (cmove $I64 (CC.Z) (RegMem.Reg (imm $I64 0)) carry))) + ;; Add the carry into the low half. + (lo_shifted_ Reg (or $I64 lo_shifted (RegMemImm.Reg carry_))) + ;; Get all sign bits. + (sign_bits Reg (sar $I64 src_hi (Imm8Reg.Imm8 63)))) + ;; Combine the two shifted halves. However, if we are shifting by >= 64 + ;; (modulo 128), then the hi bits are all sign bits and the lo bits are + ;; what would otherwise be our hi bits. + (with_flags_2 (test (OperandSize.Size64) (RegMemImm.Imm 64) amt) + (cmove $I64 (CC.Z) (RegMem.Reg lo_shifted_) hi_shifted) + (cmove $I64 (CC.Z) (RegMem.Reg hi_shifted) sign_bits)))) + +(rule (lower (has_type $I128 (sshr src amt))) + ;; NB: Only the low bits of `amt` matter since we logically mask the shift + ;; amount to the value's bit width. + (let ((amt_ Reg (lo_reg amt))) + (sar_i128 (put_in_regs src) amt_))) + +;; SSE. + +;; Since the x86 instruction set does not have an 8x16 shift instruction and the +;; approach used for `ishl` and `ushr` cannot be easily used (the masks do not +;; preserve the sign), we use a different approach here: separate the low and +;; high lanes, shift them separately, and merge them into the final result. +;; +;; Visually, this looks like the following, where `src.i8x16 = [s0, s1, ..., +;; s15]: +;; +;; lo.i16x8 = [(s0, s0), (s1, s1), ..., (s7, s7)] +;; shifted_lo.i16x8 = shift each lane of `low` +;; hi.i16x8 = [(s8, s8), (s9, s9), ..., (s15, s15)] +;; shifted_hi.i16x8 = shift each lane of `high` +;; result = [s0'', s1'', ..., s15''] +(rule (lower (has_type $I8X16 (sshr src amt @ (value_type amt_ty)))) + (let ((src_ Reg (put_in_reg src)) + ;; In order for `packsswb` later to only use the high byte of each + ;; 16x8 lane, we shift right an extra 8 bits, relying on `psraw` to + ;; fill in the upper bits appropriately. + (amt_ RegMemImm (sshr_i8x16_bigger_shift amt_ty (put_in_reg_mem_imm amt))) + (lo Reg (punpcklbw src_ (RegMem.Reg src_))) + (shifted_lo Reg (psraw lo amt_)) + (hi Reg (punpckhbw src_ (RegMem.Reg src_))) + (shifted_hi Reg (psraw hi amt_))) + (value_reg (packsswb shifted_lo (RegMem.Reg shifted_hi))))) + +(decl sshr_i8x16_bigger_shift (Type RegMemImm) RegMemImm) +(rule (sshr_i8x16_bigger_shift _ty (RegMemImm.Imm i)) + (RegMemImm.Imm (u32_add i 8))) +(rule (sshr_i8x16_bigger_shift ty (RegMemImm.Reg r)) + (reg_mem_imm_to_xmm (RegMemImm.Reg (add ty r (RegMemImm.Imm 8))))) +(rule (sshr_i8x16_bigger_shift ty rmi @ (RegMemImm.Mem _m)) + (reg_mem_imm_to_xmm (RegMemImm.Reg (add ty (imm ty 8) rmi)))) + +;; `sshr.{i16x8,i32x4}` can be a simple `psra{w,d}`, we just have to make sure +;; that if the shift amount is in a register, it is in an XMM register. +(rule (lower (has_type $I16X8 (sshr src amt))) + (value_reg (psraw (put_in_reg src) + (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) +(rule (lower (has_type $I32X4 (sshr src amt))) + (value_reg (psrad (put_in_reg src) + (reg_mem_imm_to_xmm (put_in_reg_mem_imm amt))))) + +;; The `sshr.i64x2` CLIF instruction has no single x86 instruction in the older +;; feature sets. Newer ones like AVX512VL + AVX512F include `vpsraq`, a 128-bit +;; instruction that would fit here, but this backend does not currently have +;; support for EVEX encodings. To remedy this, we extract each 64-bit lane to a +;; GPR, shift each using a scalar instruction, and insert the shifted values +;; back in the `dst` XMM register. +;; +;; (TODO: when EVEX support is available, add an alternate lowering here). +(rule (lower (has_type $I64X2 (sshr src amt))) + (let ((src_ Reg (put_in_reg src)) + (lo Reg (pextrd $I64 src_ 0)) + (hi Reg (pextrd $I64 src_ 1)) + (amt_ Imm8Reg (sshr_i64x2_shift_amount (put_in_imm8_reg amt))) + (shifted_lo Reg (sar $I64 lo amt_)) + (shifted_hi Reg (sar $I64 hi amt_))) + (value_reg (make_i64x2_from_lanes (RegMem.Reg shifted_lo) + (RegMem.Reg shifted_hi))))) + +;; Mask the shift amount according to Cranelift's semantics. +(decl sshr_i64x2_shift_amount (Imm8Reg) Imm8Reg) +(rule (sshr_i64x2_shift_amount reg @ (Imm8Reg.Reg _)) reg) +(rule (sshr_i64x2_shift_amount (Imm8Reg.Imm8 imm)) + (Imm8Reg.Imm8 (u8_and imm 63))) + ;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; `i16` and `i8`: we need to extend the shift amount, or mask the diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 5b4ed7fbbd79..6fd6a995b912 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -1538,13 +1538,18 @@ fn lower_insn_to_regs>( | Opcode::Umin | Opcode::Bnot | Opcode::Bitselect - | Opcode::Vselect => implemented_in_isle(ctx), + | Opcode::Vselect + | Opcode::Sshr => implemented_in_isle(ctx), - Opcode::Ishl | Opcode::Ushr | Opcode::Sshr | Opcode::Rotl | Opcode::Rotr => { + Opcode::Ishl | Opcode::Ushr | Opcode::Rotl | Opcode::Rotr => { let dst_ty = ctx.output_ty(insn, 0); debug_assert_eq!(ctx.input_ty(insn, 0), dst_ty); if !dst_ty.is_vector() && dst_ty.bits() <= 64 { + if op != Opcode::Rotr { + implemented_in_isle(ctx); + } + // Scalar shifts on x86 have various encodings: // - shift by one bit, e.g. `SAL r/m8, 1` (not used here) // - shift by an immediate amount, e.g. `SAL r/m8, imm8` @@ -1557,10 +1562,6 @@ fn lower_insn_to_regs>( OperandSize::Size32, extend_input_to_reg(ctx, inputs[0], ExtSpec::ZeroExtendTo32), ), - Opcode::Sshr => ( - OperandSize::Size32, - extend_input_to_reg(ctx, inputs[0], ExtSpec::SignExtendTo32), - ), Opcode::Rotl | Opcode::Rotr => ( OperandSize::from_ty(dst_ty), put_input_in_reg(ctx, inputs[0]), @@ -1590,7 +1591,6 @@ fn lower_insn_to_regs>( let shift_kind = match op { Opcode::Ishl => ShiftKind::ShiftLeft, Opcode::Ushr => ShiftKind::ShiftRightLogical, - Opcode::Sshr => ShiftKind::ShiftRightArithmetic, Opcode::Rotl => ShiftKind::RotateLeft, Opcode::Rotr => ShiftKind::RotateRight, _ => unreachable!(), @@ -1608,50 +1608,8 @@ fn lower_insn_to_regs>( let dst = get_output_reg(ctx, outputs[0]); match op { - Opcode::Ishl => { - emit_shl_i128(ctx, src, dst, amt_src); - } - Opcode::Ushr => { - emit_shr_i128(ctx, src, dst, amt_src, /* is_signed = */ false); - } - Opcode::Sshr => { - emit_shr_i128(ctx, src, dst, amt_src, /* is_signed = */ true); - } - Opcode::Rotl => { - // (mov tmp, src) - // (shl.i128 tmp, amt) - // (mov dst, src) - // (ushr.i128 dst, 128-amt) - // (or dst, tmp) - let tmp = ctx.alloc_tmp(types::I128); - emit_shl_i128(ctx, src, tmp, amt_src); - let inv_amt = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - ctx.emit(Inst::imm(OperandSize::Size64, 128, inv_amt)); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Sub, - RegMemImm::reg(amt_src), - inv_amt, - )); - emit_shr_i128( - ctx, - src, - dst, - inv_amt.to_reg(), - /* is_signed = */ false, - ); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Or, - RegMemImm::reg(tmp.regs()[0].to_reg()), - dst.regs()[0], - )); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Or, - RegMemImm::reg(tmp.regs()[1].to_reg()), - dst.regs()[1], - )); + Opcode::Ishl | Opcode::Ushr | Opcode::Rotl => { + implemented_in_isle(ctx); } Opcode::Rotr => { // (mov tmp, src) @@ -1808,127 +1766,6 @@ fn lower_insn_to_regs>( _ => SseOpcode::Pand, }; ctx.emit(Inst::xmm_rm_r(sse_op, RegMem::from(mask_value), dst)); - } else if dst_ty == types::I8X16 && op == Opcode::Sshr { - // Since the x86 instruction set does not have an 8x16 shift instruction and the approach used for - // `ishl` and `ushr` cannot be easily used (the masks do not preserve the sign), we use a different - // approach here: separate the low and high lanes, shift them separately, and merge them into the final - // result. Visually, this looks like the following, where `src.i8x16 = [s0, s1, ..., s15]: - // low.i16x8 = [(s0, s0), (s1, s1), ..., (s7, s7)] - // shifted_low.i16x8 = shift each lane of `low` - // high.i16x8 = [(s8, s8), (s9, s9), ..., (s15, s15)] - // shifted_high.i16x8 = shift each lane of `high` - // dst.i8x16 = [s0'', s1'', ..., s15''] - let src = put_input_in_reg(ctx, inputs[0]); - let shift_by = input_to_reg_mem_imm(ctx, inputs[1]); - let shift_by_ty = ctx.input_ty(insn, 1); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - - // In order for PACKSSWB later to only use the high byte of each 16x8 lane, we shift right an extra 8 - // bits, relying on PSRAW to fill in the upper bits appropriately. - let bigger_shift_by = match shift_by { - // When we know the shift amount at compile time, we add the extra shift amount statically. - RegMemImm::Imm { simm32 } => RegMemImm::imm(simm32 + 8), - // Otherwise we add instructions to add the extra shift amount and move the value into an XMM - // register. - RegMemImm::Reg { reg } => { - let bigger_shift_by_gpr = ctx.alloc_tmp(shift_by_ty).only_reg().unwrap(); - ctx.emit(Inst::mov_r_r(OperandSize::Size64, reg, bigger_shift_by_gpr)); - - let size = if shift_by_ty == types::I64 { - OperandSize::Size64 - } else { - OperandSize::Size32 - }; - let imm = RegMemImm::imm(8); - ctx.emit(Inst::alu_rmi_r( - size, - AluRmiROpcode::Add, - imm, - bigger_shift_by_gpr, - )); - - let bigger_shift_by_xmm = ctx.alloc_tmp(dst_ty).only_reg().unwrap(); - ctx.emit(Inst::gpr_to_xmm( - SseOpcode::Movd, - RegMem::from(bigger_shift_by_gpr), - OperandSize::Size32, - bigger_shift_by_xmm, - )); - RegMemImm::reg(bigger_shift_by_xmm.to_reg()) - } - RegMemImm::Mem { .. } => unimplemented!("load shift amount to XMM register"), - }; - - // Unpack and shift the lower lanes of `src` into the `dst` register. - ctx.emit(Inst::gen_move(dst, src, dst_ty)); - ctx.emit(Inst::xmm_rm_r(SseOpcode::Punpcklbw, RegMem::from(dst), dst)); - ctx.emit(Inst::xmm_rmi_reg( - SseOpcode::Psraw, - bigger_shift_by.clone(), - dst, - )); - - // Unpack and shift the upper lanes of `src` into a temporary register, `upper_lanes`. - let upper_lanes = ctx.alloc_tmp(dst_ty).only_reg().unwrap(); - ctx.emit(Inst::gen_move(upper_lanes, src, dst_ty)); - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Punpckhbw, - RegMem::from(upper_lanes), - upper_lanes, - )); - ctx.emit(Inst::xmm_rmi_reg( - SseOpcode::Psraw, - bigger_shift_by, - upper_lanes, - )); - - // Merge the upper and lower shifted lanes into `dst`. - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Packsswb, - RegMem::from(upper_lanes), - dst, - )); - } else if dst_ty == types::I64X2 && op == Opcode::Sshr { - // The `sshr.i8x16` CLIF instruction has no single x86 instruction in the older feature sets; newer ones - // like AVX512VL + AVX512F include VPSRAQ, a 128-bit instruction that would fit here, but this backend - // does not currently have support for EVEX encodings (TODO when EVEX support is available, add an - // alternate lowering here). To remedy this, we extract each 64-bit lane to a GPR, shift each using a - // scalar instruction, and insert the shifted values back in the `dst` XMM register. - let src = put_input_in_reg(ctx, inputs[0]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - ctx.emit(Inst::gen_move(dst, src, dst_ty)); - - // Extract the upper and lower lanes into temporary GPRs. - let lower_lane = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - emit_extract_lane(ctx, src, lower_lane, 0, types::I64); - let upper_lane = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - emit_extract_lane(ctx, src, upper_lane, 1, types::I64); - - // Shift each value. - let mut shift = |reg: Writable| { - let kind = ShiftKind::ShiftRightArithmetic; - if let Some(shift_by) = ctx.get_input_as_source_or_const(insn, 1).constant { - // Mask the shift amount according to Cranelift's semantics. - let shift_by = (shift_by as u8) & (types::I64.bits() as u8 - 1); - ctx.emit(Inst::shift_r( - OperandSize::Size64, - kind, - Some(shift_by), - reg, - )); - } else { - let dynamic_shift_by = put_input_in_reg(ctx, inputs[1]); - let w_rcx = Writable::from_reg(regs::rcx()); - ctx.emit(Inst::mov_r_r(OperandSize::Size64, dynamic_shift_by, w_rcx)); - ctx.emit(Inst::shift_r(OperandSize::Size64, kind, None, reg)); - }; - }; - shift(lower_lane); - shift(upper_lane); - - // Insert the scalar values back into the `dst` vector. - emit_insert_lane(ctx, RegMem::from(lower_lane), dst, 0, types::I64); - emit_insert_lane(ctx, RegMem::from(upper_lane), dst, 1, types::I64); } else { // For the remaining packed shifts not covered above, x86 has implementations that can either: // - shift using an immediate @@ -1940,13 +1777,11 @@ fn lower_insn_to_regs>( types::I16X8 => match op { Opcode::Ishl => SseOpcode::Psllw, Opcode::Ushr => SseOpcode::Psrlw, - Opcode::Sshr => SseOpcode::Psraw, _ => unimplemented!("{} is not implemented for type {}", op, dst_ty), }, types::I32X4 => match op { Opcode::Ishl => SseOpcode::Pslld, Opcode::Ushr => SseOpcode::Psrld, - Opcode::Sshr => SseOpcode::Psrad, _ => unimplemented!("{} is not implemented for type {}", op, dst_ty), }, types::I64X2 => match op { diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index bebedc324028..6898419aa00c 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -69,6 +69,31 @@ where OperandSize::from_ty(ty) } + fn put_in_reg_mem_imm(&mut self, val: Value) -> RegMemImm { + let inputs = self.lower_ctx.get_value_as_source_or_const(val); + + if let Some(c) = inputs.constant { + if let Some(imm) = to_simm32(c as i64) { + return imm; + } + + // Generate constants fresh at each use to minimize long-range + // register pressure. + let ty = self.value_type(val); + return RegMemImm::reg(generated_code::constructor_imm(self, ty, c).unwrap()); + } + + if let Some((src_insn, 0)) = inputs.inst { + if let Some((addr_input, offset)) = is_mergeable_load(self.lower_ctx, src_insn) { + self.lower_ctx.sink_inst(src_insn); + let amode = lower_to_amode(self.lower_ctx, addr_input, offset); + return RegMemImm::mem(amode); + } + } + + RegMemImm::reg(self.put_in_reg(val)) + } + fn put_in_reg_mem(&mut self, val: Value) -> RegMem { let inputs = self.lower_ctx.get_value_as_source_or_const(val); @@ -90,6 +115,27 @@ where RegMem::reg(self.put_in_reg(val)) } + fn put_in_imm8_reg(&mut self, val: Value) -> Imm8Reg { + let inputs = self.lower_ctx.get_value_as_source_or_const(val); + + if let Some(c) = inputs.constant { + if let Ok(imm) = u8::try_from(c) { + return Imm8Reg::Imm8 { imm }; + } + + // Generate constants fresh at each use to minimize long-range + // register pressure. + let ty = self.value_type(val); + return Imm8Reg::Reg { + reg: generated_code::constructor_imm(self, ty, c).unwrap(), + }; + } + + Imm8Reg::Reg { + reg: self.put_in_reg(val), + } + } + #[inline] fn encode_fcmp_imm(&mut self, imm: &FcmpImm) -> u8 { imm.encode() diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index e3cdbcd263f1..f8f54370d863 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle f176ef3bba99365 -src/prelude.isle babc931e5dc5b4cf -src/isa/x64/inst.isle bc5fc626492752c8 -src/isa/x64/lower.isle 33e94300f4c08455 +src/prelude.isle e5e437b1aed2a1e1 +src/isa/x64/inst.isle 128d6b7090756799 +src/isa/x64/lower.isle d66912b78779ce9d diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index f2f8dd0ca2ca..1cc560d03875 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -20,6 +20,8 @@ pub trait Context { fn pack_value_array_2(&mut self, arg0: Value, arg1: Value) -> ValueArray2; fn unpack_value_array_3(&mut self, arg0: &ValueArray3) -> (Value, Value, Value); fn pack_value_array_3(&mut self, arg0: Value, arg1: Value, arg2: Value) -> ValueArray3; + fn u32_add(&mut self, arg0: u32, arg1: u32) -> u32; + fn u8_and(&mut self, arg0: u8, arg1: u8) -> u8; fn value_reg(&mut self, arg0: Reg) -> ValueRegs; fn value_regs(&mut self, arg0: Reg, arg1: Reg) -> ValueRegs; fn temp_writable_reg(&mut self, arg0: Type) -> WritableReg; @@ -32,6 +34,7 @@ pub trait Context { fn u32_as_u64(&mut self, arg0: u32) -> u64; fn ty_bits(&mut self, arg0: Type) -> u8; fn ty_bits_u16(&mut self, arg0: Type) -> u16; + fn lane_type(&mut self, arg0: Type) -> Type; fn fits_in_16(&mut self, arg0: Type) -> Option; fn fits_in_32(&mut self, arg0: Type) -> Option; fn fits_in_64(&mut self, arg0: Type) -> Option; @@ -59,7 +62,9 @@ pub trait Context { fn trap_code_integer_overflow(&mut self) -> TrapCode; fn operand_size_of_type_32_64(&mut self, arg0: Type) -> OperandSize; fn raw_operand_size_of_type(&mut self, arg0: Type) -> OperandSize; + fn put_in_reg_mem_imm(&mut self, arg0: Value) -> RegMemImm; fn put_in_reg_mem(&mut self, arg0: Value) -> RegMem; + fn put_in_imm8_reg(&mut self, arg0: Value) -> Imm8Reg; fn encode_fcmp_imm(&mut self, arg0: &FcmpImm) -> u8; fn xmm0(&mut self) -> WritableReg; fn avx512vl_enabled(&mut self, arg0: Type) -> Option<()>; @@ -77,19 +82,19 @@ pub trait Context { fn sse_insertps_lane_imm(&mut self, arg0: u8) -> u8; } -/// Internal type ProducesFlags: defined at src/prelude.isle line 263. +/// Internal type ProducesFlags: defined at src/prelude.isle line 273. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlags { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 266. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 276. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlags { inst: MInst, result: Reg }, } -/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 416. +/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 435. #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum ExtendKind { Sign, @@ -99,7 +104,7 @@ pub enum ExtendKind { // Generated as internal constructor for term temp_reg. pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 60. + // Rule at src/prelude.isle line 66. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -108,7 +113,7 @@ pub fn constructor_temp_reg(ctx: &mut C, arg0: Type) -> Option // Generated as internal constructor for term lo_reg. pub fn constructor_lo_reg(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/prelude.isle line 95. + // Rule at src/prelude.isle line 101. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -133,7 +138,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 276. + // Rule at src/prelude.isle line 286. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -161,7 +166,7 @@ pub fn constructor_with_flags_1( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 284. + // Rule at src/prelude.isle line 294. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern3_0); return Some(pattern3_1); @@ -195,7 +200,7 @@ pub fn constructor_with_flags_2( result: pattern5_1, } = pattern4_0 { - // Rule at src/prelude.isle line 294. + // Rule at src/prelude.isle line 304. let expr0_0 = C::emit(ctx, &pattern1_0); let expr1_0 = C::emit(ctx, &pattern5_0); let expr2_0 = C::emit(ctx, &pattern3_0); @@ -212,22 +217,22 @@ pub fn constructor_operand_size_bits(ctx: &mut C, arg0: &OperandSize let pattern0_0 = arg0; match pattern0_0 { &OperandSize::Size8 => { - // Rule at src/isa/x64/inst.isle line 90. + // Rule at src/isa/x64/inst.isle line 91. let expr0_0: u16 = 8; return Some(expr0_0); } &OperandSize::Size16 => { - // Rule at src/isa/x64/inst.isle line 91. + // Rule at src/isa/x64/inst.isle line 92. let expr0_0: u16 = 16; return Some(expr0_0); } &OperandSize::Size32 => { - // Rule at src/isa/x64/inst.isle line 92. + // Rule at src/isa/x64/inst.isle line 93. let expr0_0: u16 = 32; return Some(expr0_0); } &OperandSize::Size64 => { - // Rule at src/isa/x64/inst.isle line 93. + // Rule at src/isa/x64/inst.isle line 94. let expr0_0: u16 = 64; return Some(expr0_0); } @@ -248,12 +253,12 @@ pub fn constructor_extend_to_reg( let pattern2_0 = arg1; if pattern2_0 == pattern1_0 { let pattern4_0 = arg2; - // Rule at src/isa/x64/inst.isle line 428. + // Rule at src/isa/x64/inst.isle line 447. let expr0_0 = C::put_in_reg(ctx, pattern0_0); return Some(expr0_0); } let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 431. + // Rule at src/isa/x64/inst.isle line 450. let expr0_0 = C::ty_bits_u16(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern2_0); let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?; @@ -277,7 +282,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 451. + // Rule at src/isa/x64/inst.isle line 470. let expr0_0 = constructor_movsx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -285,7 +290,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 447. + // Rule at src/isa/x64/inst.isle line 466. let expr0_0 = constructor_movzx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -298,17 +303,17 @@ pub fn constructor_extend( pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 458. + // Rule at src/isa/x64/inst.isle line 477. let expr0_0 = SseOpcode::Xorps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 459. + // Rule at src/isa/x64/inst.isle line 478. let expr0_0 = SseOpcode::Xorpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 460. + // Rule at src/isa/x64/inst.isle line 479. let expr0_0 = SseOpcode::Pxor; return Some(expr0_0); } @@ -325,7 +330,7 @@ pub fn constructor_sse_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 464. + // Rule at src/isa/x64/inst.isle line 483. let expr0_0 = constructor_sse_xor_op(ctx, pattern0_0)?; let expr1_0 = constructor_xmm_rm_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -335,40 +340,40 @@ pub fn constructor_sse_xor( pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 473. + // Rule at src/isa/x64/inst.isle line 492. let expr0_0 = SseOpcode::Cmpps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 474. + // Rule at src/isa/x64/inst.isle line 493. let expr0_0 = SseOpcode::Cmppd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { if pattern1_0 == 8 { if pattern1_1 == 16 { - // Rule at src/isa/x64/inst.isle line 469. + // Rule at src/isa/x64/inst.isle line 488. let expr0_0 = SseOpcode::Pcmpeqb; return Some(expr0_0); } } if pattern1_0 == 16 { if pattern1_1 == 8 { - // Rule at src/isa/x64/inst.isle line 470. + // Rule at src/isa/x64/inst.isle line 489. let expr0_0 = SseOpcode::Pcmpeqw; return Some(expr0_0); } } if pattern1_0 == 32 { if pattern1_1 == 4 { - // Rule at src/isa/x64/inst.isle line 471. + // Rule at src/isa/x64/inst.isle line 490. let expr0_0 = SseOpcode::Pcmpeqd; return Some(expr0_0); } } if pattern1_0 == 64 { if pattern1_1 == 2 { - // Rule at src/isa/x64/inst.isle line 472. + // Rule at src/isa/x64/inst.isle line 491. let expr0_0 = SseOpcode::Pcmpeqq; return Some(expr0_0); } @@ -380,7 +385,7 @@ pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 488. + // Rule at src/isa/x64/inst.isle line 507. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0: Type = I32X4; @@ -396,6 +401,79 @@ pub fn constructor_vector_all_ones(ctx: &mut C, arg0: Type) -> Optio return Some(expr1_0); } +// Generated as internal constructor for term make_i64x2_from_lanes. +pub fn constructor_make_i64x2_from_lanes( + ctx: &mut C, + arg0: &RegMem, + arg1: &RegMem, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 518. + let expr0_0: Type = I64X2; + let expr1_0 = C::temp_writable_reg(ctx, expr0_0); + let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); + let expr3_0 = MInst::XmmUninitializedValue { dst: expr1_0 }; + let expr4_0 = C::emit(ctx, &expr3_0); + let expr5_0 = SseOpcode::Pinsrd; + let expr6_0: u8 = 0; + let expr7_0 = OperandSize::Size64; + let expr8_0 = MInst::XmmRmRImm { + op: expr5_0, + src1: expr2_0, + src2: pattern0_0.clone(), + dst: expr1_0, + imm: expr6_0, + size: expr7_0, + }; + let expr9_0 = C::emit(ctx, &expr8_0); + let expr10_0 = SseOpcode::Pinsrd; + let expr11_0: u8 = 1; + let expr12_0 = OperandSize::Size64; + let expr13_0 = MInst::XmmRmRImm { + op: expr10_0, + src1: expr2_0, + src2: pattern1_0.clone(), + dst: expr1_0, + imm: expr11_0, + size: expr12_0, + }; + let expr14_0 = C::emit(ctx, &expr13_0); + return Some(expr2_0); +} + +// Generated as internal constructor for term reg_mem_imm_to_xmm. +pub fn constructor_reg_mem_imm_to_xmm( + ctx: &mut C, + arg0: &RegMemImm, +) -> Option { + let pattern0_0 = arg0; + match pattern0_0 { + &RegMemImm::Imm { simm32: pattern1_0 } => { + // Rule at src/isa/x64/inst.isle line 539. + return Some(pattern0_0.clone()); + } + &RegMemImm::Reg { reg: pattern1_0 } => { + // Rule at src/isa/x64/inst.isle line 540. + let expr0_0: Type = I8X16; + let expr1_0 = SseOpcode::Movd; + let expr2_0 = RegMem::Reg { reg: pattern1_0 }; + let expr3_0 = OperandSize::Size32; + let expr4_0 = constructor_gpr_to_xmm(ctx, expr0_0, &expr1_0, &expr2_0, &expr3_0)?; + let expr5_0 = RegMemImm::Reg { reg: expr4_0 }; + return Some(expr5_0); + } + &RegMemImm::Mem { + addr: ref pattern1_0, + } => { + // Rule at src/isa/x64/inst.isle line 538. + return Some(pattern0_0.clone()); + } + _ => {} + } + return None; +} + // Generated as internal constructor for term alu_rmi_r. pub fn constructor_alu_rmi_r( ctx: &mut C, @@ -408,7 +486,7 @@ pub fn constructor_alu_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 512. + // Rule at src/isa/x64/inst.isle line 562. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::AluRmiR { @@ -433,7 +511,7 @@ pub fn constructor_add( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 520. + // Rule at src/isa/x64/inst.isle line 570. let expr0_0 = AluRmiROpcode::Add; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -449,7 +527,7 @@ pub fn constructor_add_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 528. + // Rule at src/isa/x64/inst.isle line 578. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Add; @@ -478,7 +556,7 @@ pub fn constructor_adc( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 539. + // Rule at src/isa/x64/inst.isle line 589. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Adc; @@ -507,7 +585,7 @@ pub fn constructor_sub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 550. + // Rule at src/isa/x64/inst.isle line 600. let expr0_0 = AluRmiROpcode::Sub; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -523,7 +601,7 @@ pub fn constructor_sub_with_flags( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 558. + // Rule at src/isa/x64/inst.isle line 608. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sub; @@ -552,7 +630,7 @@ pub fn constructor_sbb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 569. + // Rule at src/isa/x64/inst.isle line 619. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sbb; @@ -581,7 +659,7 @@ pub fn constructor_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 580. + // Rule at src/isa/x64/inst.isle line 630. let expr0_0 = AluRmiROpcode::Mul; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -597,7 +675,7 @@ pub fn constructor_m_and( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 591. + // Rule at src/isa/x64/inst.isle line 641. let expr0_0 = AluRmiROpcode::And; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -613,7 +691,7 @@ pub fn constructor_or( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 599. + // Rule at src/isa/x64/inst.isle line 649. let expr0_0 = AluRmiROpcode::Or; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -629,7 +707,7 @@ pub fn constructor_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 607. + // Rule at src/isa/x64/inst.isle line 657. let expr0_0 = AluRmiROpcode::Xor; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -641,7 +719,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == I64 { let pattern2_0 = arg1; if let Some(pattern3_0) = C::nonzero_u64_fits_in_u32(ctx, pattern2_0) { - // Rule at src/isa/x64/inst.isle line 636. + // Rule at src/isa/x64/inst.isle line 686. let expr0_0: Type = I64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = OperandSize::Size32; @@ -658,7 +736,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 665. + // Rule at src/isa/x64/inst.isle line 715. let expr0_0: Type = F32; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -673,7 +751,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr2_0); } - // Rule at src/isa/x64/inst.isle line 624. + // Rule at src/isa/x64/inst.isle line 674. let expr0_0: Type = F32; let expr1_0 = SseOpcode::Movd; let expr2_0: Type = I32; @@ -686,7 +764,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 677. + // Rule at src/isa/x64/inst.isle line 727. let expr0_0: Type = F64; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -701,7 +779,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr2_0); } - // Rule at src/isa/x64/inst.isle line 628. + // Rule at src/isa/x64/inst.isle line 678. let expr0_0: Type = F64; let expr1_0 = SseOpcode::Movq; let expr2_0: Type = I64; @@ -714,7 +792,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 655. + // Rule at src/isa/x64/inst.isle line 705. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = constructor_sse_xor_op(ctx, pattern0_0)?; @@ -731,7 +809,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option } let pattern1_0 = arg1; if pattern1_0 == 0 { - // Rule at src/isa/x64/inst.isle line 642. + // Rule at src/isa/x64/inst.isle line 692. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); @@ -747,7 +825,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::emit(ctx, &expr5_0); return Some(expr1_0); } - // Rule at src/isa/x64/inst.isle line 617. + // Rule at src/isa/x64/inst.isle line 667. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Imm { @@ -772,7 +850,7 @@ pub fn constructor_shift_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 690. + // Rule at src/isa/x64/inst.isle line 740. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::raw_operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::ShiftR { @@ -797,7 +875,7 @@ pub fn constructor_m_rotl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 701. + // Rule at src/isa/x64/inst.isle line 751. let expr0_0 = ShiftKind::RotateLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -813,7 +891,7 @@ pub fn constructor_shl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 706. + // Rule at src/isa/x64/inst.isle line 756. let expr0_0 = ShiftKind::ShiftLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -829,7 +907,7 @@ pub fn constructor_shr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 711. + // Rule at src/isa/x64/inst.isle line 761. let expr0_0 = ShiftKind::ShiftRightLogical; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -845,7 +923,7 @@ pub fn constructor_sar( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 716. + // Rule at src/isa/x64/inst.isle line 766. let expr0_0 = ShiftKind::ShiftRightArithmetic; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -863,7 +941,7 @@ pub fn constructor_cmp_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 721. + // Rule at src/isa/x64/inst.isle line 771. let expr0_0 = MInst::CmpRmiR { size: pattern0_0.clone(), opcode: pattern1_0.clone(), @@ -888,7 +966,7 @@ pub fn constructor_cmp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 730. + // Rule at src/isa/x64/inst.isle line 780. let expr0_0 = CmpOpcode::Cmp; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -904,7 +982,7 @@ pub fn constructor_test( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 735. + // Rule at src/isa/x64/inst.isle line 785. let expr0_0 = CmpOpcode::Test; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -922,7 +1000,7 @@ pub fn constructor_cmove( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 740. + // Rule at src/isa/x64/inst.isle line 790. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Cmove { @@ -950,7 +1028,7 @@ pub fn constructor_movzx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 748. + // Rule at src/isa/x64/inst.isle line 798. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovzxRmR { ext_mode: pattern1_0.clone(), @@ -972,7 +1050,7 @@ pub fn constructor_movsx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 755. + // Rule at src/isa/x64/inst.isle line 805. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::MovsxRmR { ext_mode: pattern1_0.clone(), @@ -996,7 +1074,7 @@ pub fn constructor_xmm_rm_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 762. + // Rule at src/isa/x64/inst.isle line 812. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::XmmRmR { op: pattern1_0.clone(), @@ -1013,7 +1091,7 @@ pub fn constructor_xmm_rm_r( pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 769. + // Rule at src/isa/x64/inst.isle line 819. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1024,7 +1102,7 @@ pub fn constructor_paddb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 774. + // Rule at src/isa/x64/inst.isle line 824. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1035,7 +1113,7 @@ pub fn constructor_paddw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 779. + // Rule at src/isa/x64/inst.isle line 829. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Paddd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1046,7 +1124,7 @@ pub fn constructor_paddd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 784. + // Rule at src/isa/x64/inst.isle line 834. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Paddq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1057,7 +1135,7 @@ pub fn constructor_paddq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 789. + // Rule at src/isa/x64/inst.isle line 839. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1068,7 +1146,7 @@ pub fn constructor_paddsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 794. + // Rule at src/isa/x64/inst.isle line 844. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1079,7 +1157,7 @@ pub fn constructor_paddsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 799. + // Rule at src/isa/x64/inst.isle line 849. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1090,7 +1168,7 @@ pub fn constructor_paddusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 804. + // Rule at src/isa/x64/inst.isle line 854. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1101,7 +1179,7 @@ pub fn constructor_paddusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 809. + // Rule at src/isa/x64/inst.isle line 859. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1112,7 +1190,7 @@ pub fn constructor_psubb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 814. + // Rule at src/isa/x64/inst.isle line 864. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1123,7 +1201,7 @@ pub fn constructor_psubw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 819. + // Rule at src/isa/x64/inst.isle line 869. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Psubd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1134,7 +1212,7 @@ pub fn constructor_psubd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 824. + // Rule at src/isa/x64/inst.isle line 874. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Psubq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1145,7 +1223,7 @@ pub fn constructor_psubq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 829. + // Rule at src/isa/x64/inst.isle line 879. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1156,7 +1234,7 @@ pub fn constructor_psubsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 834. + // Rule at src/isa/x64/inst.isle line 884. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1167,7 +1245,7 @@ pub fn constructor_psubsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 839. + // Rule at src/isa/x64/inst.isle line 889. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1178,7 +1256,7 @@ pub fn constructor_psubusb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 844. + // Rule at src/isa/x64/inst.isle line 894. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1189,7 +1267,7 @@ pub fn constructor_psubusw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 849. + // Rule at src/isa/x64/inst.isle line 899. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pavgb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1200,7 +1278,7 @@ pub fn constructor_pavgb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 854. + // Rule at src/isa/x64/inst.isle line 904. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pavgw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1211,7 +1289,7 @@ pub fn constructor_pavgw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 859. + // Rule at src/isa/x64/inst.isle line 909. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Pand; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1222,7 +1300,7 @@ pub fn constructor_pand(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 864. + // Rule at src/isa/x64/inst.isle line 914. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1233,7 +1311,7 @@ pub fn constructor_andps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 869. + // Rule at src/isa/x64/inst.isle line 919. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1244,7 +1322,7 @@ pub fn constructor_andpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 874. + // Rule at src/isa/x64/inst.isle line 924. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Por; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1255,7 +1333,7 @@ pub fn constructor_por(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Opt pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 879. + // Rule at src/isa/x64/inst.isle line 929. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Orps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1266,7 +1344,7 @@ pub fn constructor_orps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 884. + // Rule at src/isa/x64/inst.isle line 934. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Orpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1277,7 +1355,7 @@ pub fn constructor_orpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 889. + // Rule at src/isa/x64/inst.isle line 939. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pxor; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1288,7 +1366,7 @@ pub fn constructor_pxor(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Op pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 894. + // Rule at src/isa/x64/inst.isle line 944. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Xorps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1299,7 +1377,7 @@ pub fn constructor_xorps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 899. + // Rule at src/isa/x64/inst.isle line 949. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Xorpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1310,7 +1388,7 @@ pub fn constructor_xorpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 904. + // Rule at src/isa/x64/inst.isle line 954. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmullw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1321,7 +1399,7 @@ pub fn constructor_pmullw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 909. + // Rule at src/isa/x64/inst.isle line 959. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulld; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1332,7 +1410,7 @@ pub fn constructor_pmulld(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 914. + // Rule at src/isa/x64/inst.isle line 964. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1343,7 +1421,7 @@ pub fn constructor_pmulhw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 919. + // Rule at src/isa/x64/inst.isle line 969. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1354,7 +1432,7 @@ pub fn constructor_pmulhuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 924. + // Rule at src/isa/x64/inst.isle line 974. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmuldq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1365,7 +1443,7 @@ pub fn constructor_pmuldq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 929. + // Rule at src/isa/x64/inst.isle line 979. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pmuludq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1376,7 +1454,7 @@ pub fn constructor_pmuludq(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 934. + // Rule at src/isa/x64/inst.isle line 984. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpckhwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1387,7 +1465,7 @@ pub fn constructor_punpckhwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 939. + // Rule at src/isa/x64/inst.isle line 989. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpcklwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1398,7 +1476,7 @@ pub fn constructor_punpcklwd(ctx: &mut C, arg0: Reg, arg1: &RegMem) pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 944. + // Rule at src/isa/x64/inst.isle line 994. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andnps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1409,7 +1487,7 @@ pub fn constructor_andnps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 949. + // Rule at src/isa/x64/inst.isle line 999. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andnpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1420,7 +1498,7 @@ pub fn constructor_andnpd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 954. + // Rule at src/isa/x64/inst.isle line 1004. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Pandn; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1431,17 +1509,17 @@ pub fn constructor_pandn(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 958. + // Rule at src/isa/x64/inst.isle line 1008. let expr0_0 = SseOpcode::Blendvps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 959. + // Rule at src/isa/x64/inst.isle line 1009. let expr0_0 = SseOpcode::Blendvpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 960. + // Rule at src/isa/x64/inst.isle line 1010. let expr0_0 = SseOpcode::Pblendvb; return Some(expr0_0); } @@ -1452,17 +1530,17 @@ pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 963. + // Rule at src/isa/x64/inst.isle line 1013. let expr0_0 = SseOpcode::Movaps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 964. + // Rule at src/isa/x64/inst.isle line 1014. let expr0_0 = SseOpcode::Movapd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 965. + // Rule at src/isa/x64/inst.isle line 1015. let expr0_0 = SseOpcode::Movdqa; return Some(expr0_0); } @@ -1481,7 +1559,7 @@ pub fn constructor_sse_blend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 969. + // Rule at src/isa/x64/inst.isle line 1019. let expr0_0 = C::xmm0(ctx); let expr1_0 = constructor_sse_mov_op(ctx, pattern0_0)?; let expr2_0 = MInst::XmmUnaryRmR { @@ -1505,7 +1583,7 @@ pub fn constructor_blendvpd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 981. + // Rule at src/isa/x64/inst.isle line 1031. let expr0_0 = C::xmm0(ctx); let expr1_0 = SseOpcode::Movapd; let expr2_0 = RegMem::Reg { reg: pattern2_0 }; @@ -1525,7 +1603,7 @@ pub fn constructor_blendvpd( pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 993. + // Rule at src/isa/x64/inst.isle line 1043. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1536,7 +1614,7 @@ pub fn constructor_movsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> O pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 998. + // Rule at src/isa/x64/inst.isle line 1048. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movlhps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1547,7 +1625,7 @@ pub fn constructor_movlhps(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1003. + // Rule at src/isa/x64/inst.isle line 1053. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1558,7 +1636,7 @@ pub fn constructor_pmaxsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1008. + // Rule at src/isa/x64/inst.isle line 1058. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1569,7 +1647,7 @@ pub fn constructor_pmaxsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1013. + // Rule at src/isa/x64/inst.isle line 1063. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1580,7 +1658,7 @@ pub fn constructor_pmaxsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1018. + // Rule at src/isa/x64/inst.isle line 1068. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1591,7 +1669,7 @@ pub fn constructor_pminsb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1023. + // Rule at src/isa/x64/inst.isle line 1073. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1602,7 +1680,7 @@ pub fn constructor_pminsw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1028. + // Rule at src/isa/x64/inst.isle line 1078. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1613,7 +1691,7 @@ pub fn constructor_pminsd(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1033. + // Rule at src/isa/x64/inst.isle line 1083. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1624,7 +1702,7 @@ pub fn constructor_pmaxub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1038. + // Rule at src/isa/x64/inst.isle line 1088. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1635,7 +1713,7 @@ pub fn constructor_pmaxuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1043. + // Rule at src/isa/x64/inst.isle line 1093. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1646,7 +1724,7 @@ pub fn constructor_pmaxud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1048. + // Rule at src/isa/x64/inst.isle line 1098. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1657,7 +1735,7 @@ pub fn constructor_pminub(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1053. + // Rule at src/isa/x64/inst.isle line 1103. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -1668,13 +1746,46 @@ pub fn constructor_pminuw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> pub fn constructor_pminud(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1058. + // Rule at src/isa/x64/inst.isle line 1108. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; return Some(expr2_0); } +// Generated as internal constructor for term punpcklbw. +pub fn constructor_punpcklbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1113. + let expr0_0: Type = I8X16; + let expr1_0 = SseOpcode::Punpcklbw; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term punpckhbw. +pub fn constructor_punpckhbw(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1118. + let expr0_0: Type = I8X16; + let expr1_0 = SseOpcode::Punpckhbw; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + +// Generated as internal constructor for term packsswb. +pub fn constructor_packsswb(ctx: &mut C, arg0: Reg, arg1: &RegMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1123. + let expr0_0: Type = I8X16; + let expr1_0 = SseOpcode::Packsswb; + let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; + return Some(expr2_0); +} + // Generated as internal constructor for term xmm_rm_r_imm. pub fn constructor_xmm_rm_r_imm( ctx: &mut C, @@ -1689,7 +1800,7 @@ pub fn constructor_xmm_rm_r_imm( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1063. + // Rule at src/isa/x64/inst.isle line 1128. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmRImm { @@ -1717,7 +1828,7 @@ pub fn constructor_palignr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1075. + // Rule at src/isa/x64/inst.isle line 1140. let expr0_0 = SseOpcode::Palignr; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -1735,7 +1846,7 @@ pub fn constructor_pshufd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1084. + // Rule at src/isa/x64/inst.isle line 1149. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = C::writable_reg_to_reg(ctx, expr1_0); @@ -1760,7 +1871,7 @@ pub fn constructor_xmm_unary_rm_r( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1097. + // Rule at src/isa/x64/inst.isle line 1162. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUnaryRmR { @@ -1776,7 +1887,7 @@ pub fn constructor_xmm_unary_rm_r( // Generated as internal constructor for term pmovsxbw. pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1104. + // Rule at src/isa/x64/inst.isle line 1169. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1785,7 +1896,7 @@ pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &RegMem) -> Option(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1109. + // Rule at src/isa/x64/inst.isle line 1174. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1794,7 +1905,7 @@ pub fn constructor_pmovzxbw(ctx: &mut C, arg0: &RegMem) -> Option(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1114. + // Rule at src/isa/x64/inst.isle line 1179. let expr0_0 = SseOpcode::Pabsb; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1803,7 +1914,7 @@ pub fn constructor_pabsb(ctx: &mut C, arg0: &RegMem) -> Option // Generated as internal constructor for term pabsw. pub fn constructor_pabsw(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1119. + // Rule at src/isa/x64/inst.isle line 1184. let expr0_0 = SseOpcode::Pabsw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1812,7 +1923,7 @@ pub fn constructor_pabsw(ctx: &mut C, arg0: &RegMem) -> Option // Generated as internal constructor for term pabsd. pub fn constructor_pabsd(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1124. + // Rule at src/isa/x64/inst.isle line 1189. let expr0_0 = SseOpcode::Pabsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1826,7 +1937,7 @@ pub fn constructor_xmm_unary_rm_r_evex( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1129. + // Rule at src/isa/x64/inst.isle line 1194. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUnaryRmREvex { @@ -1842,7 +1953,7 @@ pub fn constructor_xmm_unary_rm_r_evex( // Generated as internal constructor for term vpabsq. pub fn constructor_vpabsq(ctx: &mut C, arg0: &RegMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1136. + // Rule at src/isa/x64/inst.isle line 1201. let expr0_0 = Avx512Opcode::Vpabsq; let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -1858,7 +1969,7 @@ pub fn constructor_xmm_rm_r_evex( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1141. + // Rule at src/isa/x64/inst.isle line 1206. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmREvex { @@ -1876,7 +1987,7 @@ pub fn constructor_xmm_rm_r_evex( pub fn constructor_vpmullq(ctx: &mut C, arg0: &RegMem, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1153. + // Rule at src/isa/x64/inst.isle line 1218. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1892,7 +2003,7 @@ pub fn constructor_xmm_rmi_reg( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1160. + // Rule at src/isa/x64/inst.isle line 1225. let expr0_0: Type = I8X16; let expr1_0 = C::temp_writable_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmiReg { @@ -1910,7 +2021,7 @@ pub fn constructor_xmm_rmi_reg( pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1170. + // Rule at src/isa/x64/inst.isle line 1235. let expr0_0 = SseOpcode::Psllq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1920,7 +2031,7 @@ pub fn constructor_psllq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1175. + // Rule at src/isa/x64/inst.isle line 1240. let expr0_0 = SseOpcode::Psrld; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -1930,12 +2041,32 @@ pub fn constructor_psrld(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) - pub fn constructor_psrlq(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1180. + // Rule at src/isa/x64/inst.isle line 1245. let expr0_0 = SseOpcode::Psrlq; let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); } +// Generated as internal constructor for term psraw. +pub fn constructor_psraw(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1250. + let expr0_0 = SseOpcode::Psraw; + let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + +// Generated as internal constructor for term psrad. +pub fn constructor_psrad(ctx: &mut C, arg0: Reg, arg1: &RegMemImm) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1255. + let expr0_0 = SseOpcode::Psrad; + let expr1_0 = constructor_xmm_rmi_reg(ctx, &expr0_0, pattern0_0, pattern1_0)?; + return Some(expr1_0); +} + // Generated as internal constructor for term mul_hi. pub fn constructor_mul_hi( ctx: &mut C, @@ -1948,7 +2079,7 @@ pub fn constructor_mul_hi( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1187. + // Rule at src/isa/x64/inst.isle line 1262. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::temp_writable_reg(ctx, pattern0_0); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); @@ -1977,7 +2108,7 @@ pub fn constructor_mulhi_u( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1203. + // Rule at src/isa/x64/inst.isle line 1278. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1993,7 +2124,7 @@ pub fn constructor_cmpps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1208. + // Rule at src/isa/x64/inst.isle line 1283. let expr0_0 = SseOpcode::Cmpps; let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); let expr2_0 = OperandSize::Size32; @@ -2012,7 +2143,7 @@ pub fn constructor_cmppd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1221. + // Rule at src/isa/x64/inst.isle line 1296. let expr0_0 = SseOpcode::Cmppd; let expr1_0 = C::encode_fcmp_imm(ctx, pattern2_0); let expr2_0 = OperandSize::Size32; @@ -2033,7 +2164,7 @@ pub fn constructor_gpr_to_xmm( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1230. + // Rule at src/isa/x64/inst.isle line 1305. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = MInst::GprToXmm { op: pattern1_0.clone(), @@ -2056,7 +2187,7 @@ pub fn constructor_pinsrb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1237. + // Rule at src/isa/x64/inst.isle line 1312. let expr0_0 = SseOpcode::Pinsrb; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2074,7 +2205,7 @@ pub fn constructor_pinsrw( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1242. + // Rule at src/isa/x64/inst.isle line 1317. let expr0_0 = SseOpcode::Pinsrw; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2094,7 +2225,7 @@ pub fn constructor_pinsrd( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1247. + // Rule at src/isa/x64/inst.isle line 1322. let expr0_0 = SseOpcode::Pinsrd; let expr1_0 = constructor_xmm_rm_r_imm( ctx, &expr0_0, pattern0_0, pattern1_0, pattern2_0, pattern3_0, @@ -2112,7 +2243,7 @@ pub fn constructor_insertps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1252. + // Rule at src/isa/x64/inst.isle line 1327. let expr0_0 = SseOpcode::Insertps; let expr1_0 = OperandSize::Size32; let expr2_0 = @@ -2120,11 +2251,35 @@ pub fn constructor_insertps( return Some(expr2_0); } +// Generated as internal constructor for term pextrd. +pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Reg, arg2: u8) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1332. + let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); + let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); + let expr2_0 = SseOpcode::Pextrd; + let expr3_0 = RegMem::Reg { reg: pattern1_0 }; + let expr4_0 = C::lane_type(ctx, pattern0_0); + let expr5_0 = C::operand_size_of_type_32_64(ctx, expr4_0); + let expr6_0 = MInst::XmmRmRImm { + op: expr2_0, + src1: expr1_0, + src2: expr3_0, + dst: expr0_0, + imm: pattern2_0, + size: expr5_0, + }; + let expr7_0 = C::emit(ctx, &expr6_0); + return Some(expr1_0); +} + // Generated as internal constructor for term not. pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Reg) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1257. + // Rule at src/isa/x64/inst.isle line 1345. let expr0_0 = C::temp_writable_reg(ctx, pattern0_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Not { @@ -2177,7 +2332,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1031. + // Rule at src/isa/x64/lower.isle line 1148. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2366,7 +2521,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 719. + // Rule at src/isa/x64/lower.isle line 836. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -2457,7 +2612,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 652. + // Rule at src/isa/x64/lower.isle line 769. let expr0_0 = C::put_in_regs(ctx, pattern7_0); let expr1_0 = constructor_lo_reg(ctx, pattern7_1)?; let expr2_0 = constructor_shl_i128(ctx, expr0_0, expr1_0)?; @@ -2489,6 +2644,15 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 669. + let expr0_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr1_0 = C::put_in_regs(ctx, pattern7_0); + let expr2_0 = constructor_sar_i128(ctx, expr1_0, expr0_0)?; + return Some(expr2_0); + } _ => {} } } @@ -2497,7 +2661,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1028. + // Rule at src/isa/x64/lower.isle line 1145. let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; return Some(expr0_0); } @@ -2542,7 +2706,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1132. + // Rule at src/isa/x64/lower.isle line 1249. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsb(ctx, expr0_0, &expr1_0)?; @@ -2552,7 +2716,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1154. + // Rule at src/isa/x64/lower.isle line 1271. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminub(ctx, expr0_0, &expr1_0)?; @@ -2562,7 +2726,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1121. + // Rule at src/isa/x64/lower.isle line 1238. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsb(ctx, expr0_0, &expr1_0)?; @@ -2572,13 +2736,33 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1143. + // Rule at src/isa/x64/lower.isle line 1260. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxub(ctx, expr0_0, &expr1_0)?; let expr3_0 = C::value_reg(ctx, expr2_0); return Some(expr3_0); } + &Opcode::Sshr => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + let pattern8_0 = C::value_type(ctx, pattern7_1); + // Rule at src/isa/x64/lower.isle line 690. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); + let expr2_0 = + constructor_sshr_i8x16_bigger_shift(ctx, pattern8_0, &expr1_0)?; + let expr3_0 = RegMem::Reg { reg: expr0_0 }; + let expr4_0 = constructor_punpcklbw(ctx, expr0_0, &expr3_0)?; + let expr5_0 = constructor_psraw(ctx, expr4_0, &expr2_0)?; + let expr6_0 = RegMem::Reg { reg: expr0_0 }; + let expr7_0 = constructor_punpckhbw(ctx, expr0_0, &expr6_0)?; + let expr8_0 = constructor_psraw(ctx, expr7_0, &expr2_0)?; + let expr9_0 = RegMem::Reg { reg: expr8_0 }; + let expr10_0 = constructor_packsswb(ctx, expr5_0, &expr9_0)?; + let expr11_0 = C::value_reg(ctx, expr10_0); + return Some(expr11_0); + } _ => {} } } @@ -2587,7 +2771,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 974. + // Rule at src/isa/x64/lower.isle line 1091. let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); let expr1_0 = constructor_pabsb(ctx, &expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -2608,7 +2792,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1135. + // Rule at src/isa/x64/lower.isle line 1252. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsw(ctx, expr0_0, &expr1_0)?; @@ -2618,7 +2802,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1157. + // Rule at src/isa/x64/lower.isle line 1274. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminuw(ctx, expr0_0, &expr1_0)?; @@ -2628,7 +2812,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1124. + // Rule at src/isa/x64/lower.isle line 1241. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsw(ctx, expr0_0, &expr1_0)?; @@ -2638,13 +2822,24 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1146. + // Rule at src/isa/x64/lower.isle line 1263. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxuw(ctx, expr0_0, &expr1_0)?; let expr3_0 = C::value_reg(ctx, expr2_0); return Some(expr3_0); } + &Opcode::Sshr => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 712. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); + let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr3_0 = constructor_psraw(ctx, expr0_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } _ => {} } } @@ -2653,7 +2848,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 977. + // Rule at src/isa/x64/lower.isle line 1094. let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); let expr1_0 = constructor_pabsw(ctx, &expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -2674,7 +2869,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1138. + // Rule at src/isa/x64/lower.isle line 1255. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminsd(ctx, expr0_0, &expr1_0)?; @@ -2684,7 +2879,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1160. + // Rule at src/isa/x64/lower.isle line 1277. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pminud(ctx, expr0_0, &expr1_0)?; @@ -2694,7 +2889,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1127. + // Rule at src/isa/x64/lower.isle line 1244. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxsd(ctx, expr0_0, &expr1_0)?; @@ -2704,13 +2899,24 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1149. + // Rule at src/isa/x64/lower.isle line 1266. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = constructor_pmaxud(ctx, expr0_0, &expr1_0)?; let expr3_0 = C::value_reg(ctx, expr2_0); return Some(expr3_0); } + &Opcode::Sshr => { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 715. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0 = C::put_in_reg_mem_imm(ctx, pattern7_1); + let expr2_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr1_0)?; + let expr3_0 = constructor_psrad(ctx, expr0_0, &expr2_0)?; + let expr4_0 = C::value_reg(ctx, expr3_0); + return Some(expr4_0); + } _ => {} } } @@ -2719,7 +2925,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Iabs = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 980. + // Rule at src/isa/x64/lower.isle line 1097. let expr0_0 = C::put_in_reg_mem(ctx, pattern5_1); let expr1_0 = constructor_pabsd(ctx, &expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -2731,24 +2937,54 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + if let &Opcode::Sshr = &pattern5_0 { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, &pattern5_1); + // Rule at src/isa/x64/lower.isle line 727. + let expr0_0 = C::put_in_reg(ctx, pattern7_0); + let expr1_0: Type = I64; + let expr2_0: u8 = 0; + let expr3_0 = constructor_pextrd(ctx, expr1_0, expr0_0, expr2_0)?; + let expr4_0: Type = I64; + let expr5_0: u8 = 1; + let expr6_0 = constructor_pextrd(ctx, expr4_0, expr0_0, expr5_0)?; + let expr7_0 = C::put_in_imm8_reg(ctx, pattern7_1); + let expr8_0 = constructor_sshr_i64x2_shift_amount(ctx, &expr7_0)?; + let expr9_0: Type = I64; + let expr10_0 = constructor_sar(ctx, expr9_0, expr3_0, &expr8_0)?; + let expr11_0: Type = I64; + let expr12_0 = constructor_sar(ctx, expr11_0, expr6_0, &expr8_0)?; + let expr13_0 = RegMem::Reg { reg: expr10_0 }; + let expr14_0 = RegMem::Reg { reg: expr12_0 }; + let expr15_0 = + constructor_make_i64x2_from_lanes(ctx, &expr13_0, &expr14_0)?; + let expr16_0 = C::value_reg(ctx, expr15_0); + return Some(expr16_0); + } + } + &InstructionData::Unary { + opcode: ref pattern5_0, + arg: pattern5_1, + } => { + if let &Opcode::Iabs = &pattern5_0 { + // Rule at src/isa/x64/lower.isle line 1111. + let expr0_0 = C::put_in_reg(ctx, pattern5_1); + let expr1_0: Type = I64X2; + let expr2_0: u64 = 0; + let expr3_0 = constructor_imm(ctx, expr1_0, expr2_0)?; + let expr4_0 = RegMem::Reg { reg: expr0_0 }; + let expr5_0 = constructor_psubq(ctx, expr3_0, &expr4_0)?; + let expr6_0 = RegMem::Reg { reg: expr0_0 }; + let expr7_0 = constructor_blendvpd(ctx, expr5_0, &expr6_0, expr5_0)?; + let expr8_0 = C::value_reg(ctx, expr7_0); + return Some(expr8_0); + } } + _ => {} } } if pattern2_0 == F32X4 { @@ -2759,7 +2995,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::BandNot = &pattern4_0 { let (pattern6_0, pattern6_1) = C::unpack_value_array_2(ctx, &pattern4_1); - // Rule at src/isa/x64/lower.isle line 967. + // Rule at src/isa/x64/lower.isle line 1084. let expr0_0 = C::put_in_reg(ctx, pattern6_1); let expr1_0 = C::put_in_reg_mem(ctx, pattern6_0); let expr2_0 = constructor_sse_and_not(ctx, pattern2_0, expr0_0, &expr1_0)?; @@ -2838,7 +3074,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 662. + // Rule at src/isa/x64/lower.isle line 779. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); let expr2_0 = constructor_pavgb(ctx, expr0_0, &expr1_0)?; @@ -2966,7 +3202,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern9_0, pattern9_1) = C::unpack_value_array_2(ctx, &pattern7_1); - // Rule at src/isa/x64/lower.isle line 666. + // Rule at src/isa/x64/lower.isle line 783. let expr0_0 = C::put_in_reg(ctx, pattern9_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern9_1); let expr2_0 = constructor_pavgw(ctx, expr0_0, &expr1_0)?; @@ -3079,7 +3315,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1041. + // Rule at src/isa/x64/lower.isle line 1158. let expr0_0 = C::put_in_reg(ctx, pattern7_0); let expr1_0 = C::put_in_reg(ctx, pattern7_1); let expr2_0 = RegMem::Reg { reg: expr0_0 }; @@ -4026,7 +4262,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { let (pattern7_0, pattern7_1, pattern7_2) = C::unpack_value_array_3(ctx, &pattern5_1); - // Rule at src/isa/x64/lower.isle line 1055. + // Rule at src/isa/x64/lower.isle line 1172. let expr0_0 = C::put_in_reg_mem(ctx, pattern7_0); let expr1_0 = C::put_in_reg_mem(ctx, pattern7_1); let expr2_0 = C::put_in_reg(ctx, pattern7_2); @@ -4044,7 +4280,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1036. + // Rule at src/isa/x64/lower.isle line 1153. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0 = constructor_vector_all_ones(ctx, pattern2_0)?; let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -4171,7 +4407,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option { + let (pattern7_0, pattern7_1) = + C::unpack_value_array_2(ctx, &pattern5_1); + if let Some(pattern8_0) = C::imm8_from_value(ctx, pattern7_1) { + // Rule at src/isa/x64/lower.isle line 637. + let expr0_0 = ExtendKind::Sign; + let expr1_0 = constructor_extend_to_reg( + ctx, pattern7_0, pattern3_0, &expr0_0, + )?; + let expr2_0 = + constructor_sar(ctx, pattern3_0, expr1_0, &pattern8_0)?; + let expr3_0 = C::value_reg(ctx, expr2_0); + return Some(expr3_0); + } + // Rule at src/isa/x64/lower.isle line 630. + let expr0_0 = ExtendKind::Sign; + let expr1_0 = + constructor_extend_to_reg(ctx, pattern7_0, pattern3_0, &expr0_0)?; + let expr2_0 = constructor_lo_reg(ctx, pattern7_1)?; + let expr3_0 = Imm8Reg::Reg { reg: expr2_0 }; + let expr4_0 = constructor_sar(ctx, pattern3_0, expr1_0, &expr3_0)?; + let expr5_0 = C::value_reg(ctx, expr4_0); + return Some(expr5_0); + } _ => {} } } @@ -4450,7 +4710,7 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { if let &Opcode::Bnot = &pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1015. + // Rule at src/isa/x64/lower.isle line 1132. let expr0_0 = C::put_in_reg(ctx, pattern5_1); let expr1_0 = constructor_not(ctx, pattern3_0, expr0_0)?; let expr2_0 = C::value_reg(ctx, expr1_0); @@ -4486,13 +4746,13 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option(ctx: &mut C, arg0: Inst) -> Option( return Some(expr49_0); } +// Generated as internal constructor for term sar_i128. +pub fn constructor_sar_i128( + ctx: &mut C, + arg0: ValueRegs, + arg1: Reg, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/lower.isle line 644. + let expr0_0: usize = 0; + let expr1_0 = C::value_regs_get(ctx, pattern0_0, expr0_0); + let expr2_0: usize = 1; + let expr3_0 = C::value_regs_get(ctx, pattern0_0, expr2_0); + let expr4_0: Type = I64; + let expr5_0 = Imm8Reg::Reg { reg: pattern1_0 }; + let expr6_0 = constructor_shr(ctx, expr4_0, expr1_0, &expr5_0)?; + let expr7_0: Type = I64; + let expr8_0 = Imm8Reg::Reg { reg: pattern1_0 }; + let expr9_0 = constructor_sar(ctx, expr7_0, expr3_0, &expr8_0)?; + let expr10_0: Type = I64; + let expr11_0: Type = I64; + let expr12_0: Type = I64; + let expr13_0: u64 = 64; + let expr14_0 = constructor_imm(ctx, expr12_0, expr13_0)?; + let expr15_0 = RegMemImm::Reg { reg: pattern1_0 }; + let expr16_0 = constructor_sub(ctx, expr11_0, expr14_0, &expr15_0)?; + let expr17_0 = Imm8Reg::Reg { reg: expr16_0 }; + let expr18_0 = constructor_shl(ctx, expr10_0, expr3_0, &expr17_0)?; + let expr19_0 = OperandSize::Size64; + let expr20_0: u32 = 127; + let expr21_0 = RegMemImm::Imm { simm32: expr20_0 }; + let expr22_0 = constructor_test(ctx, &expr19_0, &expr21_0, pattern1_0)?; + let expr23_0: Type = I64; + let expr24_0 = CC::Z; + let expr25_0: Type = I64; + let expr26_0: u64 = 0; + let expr27_0 = constructor_imm(ctx, expr25_0, expr26_0)?; + let expr28_0 = RegMem::Reg { reg: expr27_0 }; + let expr29_0 = constructor_cmove(ctx, expr23_0, &expr24_0, &expr28_0, expr18_0)?; + let expr30_0 = constructor_with_flags_1(ctx, &expr22_0, &expr29_0)?; + let expr31_0: Type = I64; + let expr32_0 = RegMemImm::Reg { reg: expr30_0 }; + let expr33_0 = constructor_or(ctx, expr31_0, expr6_0, &expr32_0)?; + let expr34_0: Type = I64; + let expr35_0: u8 = 63; + let expr36_0 = Imm8Reg::Imm8 { imm: expr35_0 }; + let expr37_0 = constructor_sar(ctx, expr34_0, expr3_0, &expr36_0)?; + let expr38_0 = OperandSize::Size64; + let expr39_0: u32 = 64; + let expr40_0 = RegMemImm::Imm { simm32: expr39_0 }; + let expr41_0 = constructor_test(ctx, &expr38_0, &expr40_0, pattern1_0)?; + let expr42_0: Type = I64; + let expr43_0 = CC::Z; + let expr44_0 = RegMem::Reg { reg: expr33_0 }; + let expr45_0 = constructor_cmove(ctx, expr42_0, &expr43_0, &expr44_0, expr9_0)?; + let expr46_0: Type = I64; + let expr47_0 = CC::Z; + let expr48_0 = RegMem::Reg { reg: expr9_0 }; + let expr49_0 = constructor_cmove(ctx, expr46_0, &expr47_0, &expr48_0, expr37_0)?; + let expr50_0 = constructor_with_flags_2(ctx, &expr41_0, &expr45_0, &expr49_0)?; + return Some(expr50_0); +} + +// Generated as internal constructor for term sshr_i8x16_bigger_shift. +pub fn constructor_sshr_i8x16_bigger_shift( + ctx: &mut C, + arg0: Type, + arg1: &RegMemImm, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + match pattern1_0 { + &RegMemImm::Imm { simm32: pattern2_0 } => { + // Rule at src/isa/x64/lower.isle line 703. + let expr0_0: u32 = 8; + let expr1_0 = C::u32_add(ctx, pattern2_0, expr0_0); + let expr2_0 = RegMemImm::Imm { simm32: expr1_0 }; + return Some(expr2_0); + } + &RegMemImm::Reg { reg: pattern2_0 } => { + // Rule at src/isa/x64/lower.isle line 705. + let expr0_0: u32 = 8; + let expr1_0 = RegMemImm::Imm { simm32: expr0_0 }; + let expr2_0 = constructor_add(ctx, pattern0_0, pattern2_0, &expr1_0)?; + let expr3_0 = RegMemImm::Reg { reg: expr2_0 }; + let expr4_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr3_0)?; + return Some(expr4_0); + } + &RegMemImm::Mem { + addr: ref pattern2_0, + } => { + // Rule at src/isa/x64/lower.isle line 707. + let expr0_0: u64 = 8; + let expr1_0 = constructor_imm(ctx, pattern0_0, expr0_0)?; + let expr2_0 = constructor_add(ctx, pattern0_0, expr1_0, pattern1_0)?; + let expr3_0 = RegMemImm::Reg { reg: expr2_0 }; + let expr4_0 = constructor_reg_mem_imm_to_xmm(ctx, &expr3_0)?; + return Some(expr4_0); + } + _ => {} + } + return None; +} + +// Generated as internal constructor for term sshr_i64x2_shift_amount. +pub fn constructor_sshr_i64x2_shift_amount( + ctx: &mut C, + arg0: &Imm8Reg, +) -> Option { + let pattern0_0 = arg0; + match pattern0_0 { + &Imm8Reg::Imm8 { imm: pattern1_0 } => { + // Rule at src/isa/x64/lower.isle line 740. + let expr0_0: u8 = 63; + let expr1_0 = C::u8_and(ctx, pattern1_0, expr0_0); + let expr2_0 = Imm8Reg::Imm8 { imm: expr1_0 }; + return Some(expr2_0); + } + &Imm8Reg::Reg { reg: pattern1_0 } => { + // Rule at src/isa/x64/lower.isle line 739. + return Some(pattern0_0.clone()); + } + _ => {} + } + return None; +} + // Generated as internal constructor for term sse_and_not. pub fn constructor_sse_and_not( ctx: &mut C, @@ -4759,21 +5146,21 @@ pub fn constructor_sse_and_not( if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 956. + // Rule at src/isa/x64/lower.isle line 1073. let expr0_0 = constructor_andnps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 957. + // Rule at src/isa/x64/lower.isle line 1074. let expr0_0 = constructor_andnpd(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/lower.isle line 958. + // Rule at src/isa/x64/lower.isle line 1075. let expr0_0 = constructor_pandn(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } @@ -4783,7 +5170,7 @@ pub fn constructor_sse_and_not( // Generated as internal constructor for term i128_not. pub fn constructor_i128_not(ctx: &mut C, arg0: Value) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/lower.isle line 1021. + // Rule at src/isa/x64/lower.isle line 1138. let expr0_0 = C::put_in_regs(ctx, pattern0_0); let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -4810,7 +5197,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1075. + // Rule at src/isa/x64/lower.isle line 1192. let expr0_0 = constructor_pinsrb(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -4818,7 +5205,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1078. + // Rule at src/isa/x64/lower.isle line 1195. let expr0_0 = constructor_pinsrw(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -4826,7 +5213,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1081. + // Rule at src/isa/x64/lower.isle line 1198. let expr0_0 = OperandSize::Size32; let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; return Some(expr1_0); @@ -4835,7 +5222,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1084. + // Rule at src/isa/x64/lower.isle line 1201. let expr0_0 = OperandSize::Size64; let expr1_0 = constructor_pinsrd(ctx, pattern2_0, pattern3_0, pattern4_0, &expr0_0)?; return Some(expr1_0); @@ -4844,7 +5231,7 @@ pub fn constructor_vec_insert_lane( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/lower.isle line 1087. + // Rule at src/isa/x64/lower.isle line 1204. let expr0_0 = C::sse_insertps_lane_imm(ctx, pattern4_0); let expr1_0 = constructor_insertps(ctx, pattern2_0, pattern3_0, expr0_0)?; return Some(expr1_0); @@ -4855,7 +5242,7 @@ pub fn constructor_vec_insert_lane( if let &RegMem::Reg { reg: pattern4_0 } = pattern3_0 { let pattern5_0 = arg3; if pattern5_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1108. + // Rule at src/isa/x64/lower.isle line 1225. let expr0_0 = RegMem::Reg { reg: pattern4_0 }; let expr1_0 = constructor_movsd(ctx, pattern2_0, &expr0_0)?; return Some(expr1_0); @@ -4863,7 +5250,7 @@ pub fn constructor_vec_insert_lane( } let pattern4_0 = arg3; if pattern4_0 == 0 { - // Rule at src/isa/x64/lower.isle line 1109. + // Rule at src/isa/x64/lower.isle line 1226. let expr0_0 = SseOpcode::Movsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern3_0)?; let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -4871,7 +5258,7 @@ pub fn constructor_vec_insert_lane( return Some(expr3_0); } if pattern4_0 == 1 { - // Rule at src/isa/x64/lower.isle line 1117. + // Rule at src/isa/x64/lower.isle line 1234. let expr0_0 = constructor_movlhps(ctx, pattern2_0, pattern3_0)?; return Some(expr0_0); } diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index 1fe9b067fb33..a066eed76731 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -260,6 +260,18 @@ macro_rules! isle_prelude_methods { n => Some(n as u64), } } + + fn u32_add(&mut self, a: u32, b: u32) -> u32 { + a.wrapping_add(b) + } + + fn u8_and(&mut self, a: u8, b: u8) -> u8 { + a & b + } + + fn lane_type(&mut self, ty: Type) -> Type { + ty.lane_type() + } }; } diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index 9c1d5b72be10..4b7943649934 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -38,6 +38,12 @@ (type ValueList (primitive ValueList)) (type ValueRegs (primitive ValueRegs)) +(decl u32_add (u32 u32) u32) +(extern constructor u32_add u32_add) + +(decl u8_and (u8 u8) u8) +(extern constructor u8_and u8_and) + ;;;; Registers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (type Reg (primitive Reg)) @@ -146,6 +152,10 @@ (decl ty_bits_u16 (Type) u16) (extern constructor ty_bits_u16 ty_bits_u16) +;; Get the type of each lane in the given type. +(decl lane_type (Type) Type) +(extern constructor lane_type lane_type) + ;;;; Helper Clif Extractors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; An extractor that only matches types that can fit in 16 bits. diff --git a/cranelift/filetests/filetests/isa/x64/i128.clif b/cranelift/filetests/filetests/isa/x64/i128.clif index 57791f92fdc6..cf80b73dd963 100644 --- a/cranelift/filetests/filetests/isa/x64/i128.clif +++ b/cranelift/filetests/filetests/isa/x64/i128.clif @@ -965,29 +965,26 @@ block0(v0: i128, v1: i128): ; check: pushq %rbp ; nextln: movq %rsp, %rbp -; nextln: movq %rdi, %r8 -; nextln: movq %rsi, %rdi -; nextln: movq %rdi, %rsi +; nextln: movq %rdi, %rax ; nextln: movq %rdx, %rcx -; nextln: sarq %cl, %rsi +; nextln: shrq %cl, %rax +; nextln: movq %rsi, %rdi ; nextln: movq %rdx, %rcx -; nextln: shrq %cl, %r8 +; nextln: sarq %cl, %rdi ; nextln: movl $$64, %ecx ; nextln: subq %rdx, %rcx -; nextln: movq %rdi, %rax -; nextln: shlq %cl, %rax +; nextln: movq %rsi, %r8 +; nextln: shlq %cl, %r8 ; nextln: xorq %rcx, %rcx ; nextln: testq $$127, %rdx -; nextln: cmovzq %rcx, %rax +; nextln: cmovzq %rcx, %r8 ; nextln: orq %r8, %rax -; nextln: sarq $$63, %rdi -; nextln: xorq %rcx, %rcx -; nextln: andq $$64, %rdx -; nextln: cmovzq %rsi, %rdi -; nextln: cmovzq %rax, %rcx -; nextln: cmovnzq %rsi, %rcx -; nextln: movq %rcx, %rax -; nextln: movq %rdi, %rdx +; nextln: sarq $$63, %rsi +; nextln: testq $$64, %rdx +; nextln: cmovzq %rdi, %rsi +; nextln: cmovzq %rax, %rdi +; nextln: movq %rdi, %rax +; nextln: movq %rsi, %rdx ; nextln: movq %rbp, %rsp ; nextln: popq %rbp ; nextln: ret diff --git a/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif b/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif index 3626d55cd482..808a70f78551 100644 --- a/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif +++ b/cranelift/filetests/filetests/isa/x64/simd-bitwise-compile.clif @@ -160,25 +160,26 @@ block0(v0: i32): return v2 } ; check: addl $$8, %edi -; nextln: movd %edi, %xmm2 -; nextln: movdqa %xmm0, %xmm1 -; nextln: punpcklbw %xmm1, %xmm1 -; nextln: psraw %xmm2, %xmm1 -; nextln: punpckhbw %xmm0, %xmm0 -; nextln: psraw %xmm2, %xmm0 +; nextln: movd %edi, %xmm1 +; nextln: movdqa %xmm3, %xmm2 +; nextln: punpcklbw %xmm3, %xmm2 +; nextln: psraw %xmm1, %xmm2 +; nextln: movdqa %xmm3, %xmm0 +; nextln: punpckhbw %xmm3, %xmm0 +; nextln: psraw %xmm1, %xmm0 +; nextln: packsswb %xmm0, %xmm2 function %sshr_i8x16_imm(i8x16, i32) -> i8x16 { block0(v0: i8x16, v1: i32): v2 = sshr_imm v0, 3 return v2 } -; check: movdqa %xmm0, %xmm1 -; nextln: movdqa %xmm1, %xmm0 -; nextln: punpcklbw %xmm0, %xmm0 -; nextln: psraw $$11, %xmm0 -; nextln: punpckhbw %xmm1, %xmm1 +; check: punpcklbw %xmm2, %xmm1 ; nextln: psraw $$11, %xmm1 -; nextln: packsswb %xmm1, %xmm0 +; nextln: movdqa %xmm2, %xmm0 +; nextln: punpckhbw %xmm2, %xmm0 +; nextln: psraw $$11, %xmm0 +; nextln: packsswb %xmm0, %xmm1 @@ -195,6 +196,6 @@ block0(v0: i64x2, v1: i32): ; nextln: sarq %cl, %rsi ; nextln: movq %rdi, %rcx ; nextln: sarq %cl, %rax -; nextln: pinsrd.w $$0, %rsi, %xmm1 -; nextln: pinsrd.w $$1, %rax, %xmm1 -; nextln: movdqa %xmm1, %xmm0 +; nextln: uninit %xmm0 +; nextln: pinsrd.w $$0, %rsi, %xmm0 +; nextln: pinsrd.w $$1, %rax, %xmm0 diff --git a/cranelift/filetests/src/runner.rs b/cranelift/filetests/src/runner.rs index d3fa7595373d..f844f4c38819 100644 --- a/cranelift/filetests/src/runner.rs +++ b/cranelift/filetests/src/runner.rs @@ -13,7 +13,7 @@ use std::path::{Path, PathBuf}; use std::time; /// Timeout in seconds when we're not making progress. -const TIMEOUT_PANIC: usize = 10; +const TIMEOUT_PANIC: usize = 60; /// Timeout for reporting slow tests without panicking. const TIMEOUT_SLOW: usize = 3;