diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index 30cc41e9c471..df03815e0427 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle 9830498351ddf6a3 +src/prelude.isle 3a10bacb778efff0 src/isa/aarch64/inst.isle 3678d0a37bdb4cff src/isa/aarch64/lower.isle 90accbfcadaea46d diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index 33163212d390..02c97656d4f1 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -39,8 +39,12 @@ pub trait Context { fn u8_as_u64(&mut self, arg0: u8) -> u64; fn u16_as_u64(&mut self, arg0: u16) -> u64; fn u32_as_u64(&mut self, arg0: u32) -> u64; + fn i64_as_u64(&mut self, arg0: i64) -> u64; + fn u64_add(&mut self, arg0: u64, arg1: u64) -> u64; + fn u64_sub(&mut self, arg0: u64, arg1: u64) -> u64; fn ty_bits(&mut self, arg0: Type) -> u8; fn ty_bits_u16(&mut self, arg0: Type) -> u16; + fn ty_bits_u64(&mut self, arg0: Type) -> u64; fn ty_bytes(&mut self, arg0: Type) -> u16; fn lane_type(&mut self, arg0: Type) -> Type; fn fits_in_16(&mut self, arg0: Type) -> Option; @@ -110,13 +114,13 @@ pub trait Context { fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 363. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 378. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 385. +/// Internal type ProducesFlags: defined at src/prelude.isle line 400. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlagsSideEffect { inst: MInst }, @@ -124,7 +128,7 @@ pub enum ProducesFlags { ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 396. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 411. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlagsReturnsResultWithProducer { @@ -1050,7 +1054,7 @@ pub fn constructor_side_effect( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 368. + // Rule at src/prelude.isle line 383. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::output_none(ctx); return Some(expr1_0); @@ -1068,7 +1072,7 @@ pub fn constructor_safepoint( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 374. + // Rule at src/prelude.isle line 389. let expr0_0 = C::emit_safepoint(ctx, pattern1_0); let expr1_0 = C::output_none(ctx); return Some(expr1_0); @@ -1094,7 +1098,7 @@ pub fn constructor_consumes_flags_concat( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 408. + // Rule at src/prelude.isle line 423. let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1); let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs { inst1: pattern1_0.clone(), @@ -1124,7 +1128,7 @@ pub fn constructor_with_flags( inst: ref pattern3_0, result: pattern3_1, } => { - // Rule at src/prelude.isle line 433. + // Rule at src/prelude.isle line 448. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_reg(ctx, pattern3_1); @@ -1135,7 +1139,7 @@ pub fn constructor_with_flags( inst2: ref pattern3_1, result: pattern3_2, } => { - // Rule at src/prelude.isle line 439. + // Rule at src/prelude.isle line 454. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_1); let expr2_0 = C::emit(ctx, pattern3_0); @@ -1154,7 +1158,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 427. + // Rule at src/prelude.isle line 442. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1174,7 +1178,7 @@ pub fn constructor_with_flags_reg( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/prelude.isle line 452. + // Rule at src/prelude.isle line 467. let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?; let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest index 4ae1e1456980..25a03e1d1ba1 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle 9830498351ddf6a3 +src/prelude.isle 3a10bacb778efff0 src/isa/s390x/inst.isle d91a16074ab186a8 src/isa/s390x/lower.isle 1cc5a12adc8c75f9 diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs index e9e94db68195..1dbdbe1d2c6e 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs @@ -39,8 +39,12 @@ pub trait Context { fn u8_as_u64(&mut self, arg0: u8) -> u64; fn u16_as_u64(&mut self, arg0: u16) -> u64; fn u32_as_u64(&mut self, arg0: u32) -> u64; + fn i64_as_u64(&mut self, arg0: i64) -> u64; + fn u64_add(&mut self, arg0: u64, arg1: u64) -> u64; + fn u64_sub(&mut self, arg0: u64, arg1: u64) -> u64; fn ty_bits(&mut self, arg0: Type) -> u8; fn ty_bits_u16(&mut self, arg0: Type) -> u16; + fn ty_bits_u64(&mut self, arg0: Type) -> u64; fn ty_bytes(&mut self, arg0: Type) -> u16; fn lane_type(&mut self, arg0: Type) -> Type; fn fits_in_16(&mut self, arg0: Type) -> Option; @@ -144,13 +148,13 @@ pub trait Context { fn same_reg(&mut self, arg0: Reg, arg1: WritableReg) -> Option<()>; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 363. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 378. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 385. +/// Internal type ProducesFlags: defined at src/prelude.isle line 400. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlagsSideEffect { inst: MInst }, @@ -158,7 +162,7 @@ pub enum ProducesFlags { ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 396. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 411. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlagsReturnsResultWithProducer { @@ -941,7 +945,7 @@ pub fn constructor_side_effect( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 368. + // Rule at src/prelude.isle line 383. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::output_none(ctx); return Some(expr1_0); @@ -959,7 +963,7 @@ pub fn constructor_safepoint( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 374. + // Rule at src/prelude.isle line 389. let expr0_0 = C::emit_safepoint(ctx, pattern1_0); let expr1_0 = C::output_none(ctx); return Some(expr1_0); @@ -985,7 +989,7 @@ pub fn constructor_consumes_flags_concat( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 408. + // Rule at src/prelude.isle line 423. let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1); let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs { inst1: pattern1_0.clone(), @@ -1015,7 +1019,7 @@ pub fn constructor_with_flags( inst: ref pattern3_0, result: pattern3_1, } => { - // Rule at src/prelude.isle line 433. + // Rule at src/prelude.isle line 448. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_reg(ctx, pattern3_1); @@ -1026,7 +1030,7 @@ pub fn constructor_with_flags( inst2: ref pattern3_1, result: pattern3_2, } => { - // Rule at src/prelude.isle line 439. + // Rule at src/prelude.isle line 454. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_1); let expr2_0 = C::emit(ctx, pattern3_0); @@ -1045,7 +1049,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 427. + // Rule at src/prelude.isle line 442. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1065,7 +1069,7 @@ pub fn constructor_with_flags_reg( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/prelude.isle line 452. + // Rule at src/prelude.isle line 467. let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?; let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle index ac8566614798..9656bda0178a 100644 --- a/cranelift/codegen/src/isa/x64/inst.isle +++ b/cranelift/codegen/src/isa/x64/inst.isle @@ -1074,6 +1074,18 @@ (decl avx512f_enabled () Type) (extern extractor avx512f_enabled avx512f_enabled) +(decl avx512bitalg_enabled () Type) +(extern extractor avx512bitalg_enabled avx512bitalg_enabled) + +(decl use_lzcnt () Type) +(extern extractor use_lzcnt use_lzcnt) + +(decl use_bmi1 () Type) +(extern extractor use_bmi1 use_bmi1) + +(decl use_popcnt () Type) +(extern extractor use_popcnt use_popcnt) + ;;;; Helpers for Merging and Sinking Immediates/Loads ;;;;;;;;;;;;;;;;;;;;;;;;; ;; Extract a constant `Imm8Reg.Imm8` from a value operand. @@ -1266,6 +1278,13 @@ (xmm_unary_rm_r (SseOpcode.Movdqu) addr)) +;; Load a constant into an XMM register. +(decl xmm_load_const (Type VCodeConstant) Xmm) +(rule (xmm_load_const ty const) + (let ((dst WritableXmm (temp_writable_xmm)) + (_ Unit (emit (MInst.XmmLoadConst const dst ty)))) + dst)) + ;;;; Instruction Constructors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; ;; These constructors create SSA-style `MInst`s. It is their responsibility to @@ -1398,6 +1417,13 @@ (imm $I64 bits) (OperandSize.Size64))) +;; Helper fo remitting immediates with an `i64` value. Note that +;; integer constants in ISLE are always parsed as `i64`s; this enables +;; negative numbers to be used as immediates. +(decl imm_i64 (Type i64) Reg) +(rule (imm_i64 ty value) + (imm ty (i64_as_u64 value))) + (decl nonzero_u64_fits_in_u32 (u64) u64) (extern extractor nonzero_u64_fits_in_u32 nonzero_u64_fits_in_u32) @@ -1504,6 +1530,11 @@ (rule (cmp size src1 src2) (cmp_rmi_r size (CmpOpcode.Cmp) src1 src2)) +;; Helper for creating `cmp` instructions with an immediate. +(decl cmp_imm (OperandSize u32 Gpr) ProducesFlags) +(rule (cmp_imm size src1 src2) + (cmp_rmi_r size (CmpOpcode.Cmp) (RegMemImm.Imm src1) src2)) + ;; Helper for creating `MInst.XmmCmpRmR` instructions. (decl xmm_cmp_rm_r (SseOpcode XmmMem Xmm) ProducesFlags) (rule (xmm_cmp_rm_r opcode src1 src2) @@ -2027,6 +2058,16 @@ size)))) dst)) +;; Helper for creating `pshufb` instructions. +(decl pshufb (Xmm XmmMem) Xmm) +(rule (pshufb src1 src2) + (let ((dst WritableXmm (temp_writable_xmm)) + (_ Unit (emit (MInst.XmmRmR (SseOpcode.Pshufb) + src1 + src2 + dst)))) + dst)) + ;; Helper for creating `MInst.XmmUnaryRmR` instructions. (decl xmm_unary_rm_r (SseOpcode XmmMem) Xmm) (rule (xmm_unary_rm_r op src) @@ -2071,6 +2112,11 @@ (rule (vpabsq src) (xmm_unary_rm_r_evex (Avx512Opcode.Vpabsq) src)) +;; Helper for creating `vpopcntb` instructions. +(decl vpopcntb (XmmMem) Xmm) +(rule (vpopcntb src) + (xmm_unary_rm_r_evex (Avx512Opcode.Vpopcntb) src)) + ;; Helper for creating `MInst.XmmRmREvex` instructions. (decl xmm_rm_r_evex (Avx512Opcode XmmMem Xmm) Xmm) (rule (xmm_rm_r_evex op src1 src2) @@ -2221,6 +2267,68 @@ (rule (ud2 code) (SideEffectNoResult.Inst (MInst.Ud2 code))) +;; Helper for creating `lzcnt` instructions. +(decl lzcnt (Type Gpr) Gpr) +(rule (lzcnt ty src) + (let ((dst WritableGpr (temp_writable_gpr)) + (size OperandSize (operand_size_of_type_32_64 ty)) + (_ Unit (emit (MInst.UnaryRmR size (UnaryRmROpcode.Lzcnt) src dst)))) + dst)) + +;; Helper for creating `tzcnt` instructions. +(decl tzcnt (Type Gpr) Gpr) +(rule (tzcnt ty src) + (let ((dst WritableGpr (temp_writable_gpr)) + (size OperandSize (operand_size_of_type_32_64 ty)) + (_ Unit (emit (MInst.UnaryRmR size (UnaryRmROpcode.Tzcnt) src dst)))) + dst)) + +;; Helper for creating `bsr` instructions. +(decl bsr (Type Gpr) Gpr) +(rule (bsr ty src) + (let ((dst WritableGpr (temp_writable_gpr)) + (size OperandSize (operand_size_of_type_32_64 ty)) + (_ Unit (emit (MInst.UnaryRmR size (UnaryRmROpcode.Bsr) src dst)))) + dst)) + +;; Helper for creating `bsr + cmov` instruction pairs that produce the +;; result of the `bsr`, or `alt` if the input was zero. +(decl bsr_or_else (Type Gpr Gpr) Gpr) +(rule (bsr_or_else ty src alt) + (let ((dst WritableGpr (temp_writable_gpr)) + (tmp WritableGpr (temp_writable_gpr)) + (size OperandSize (operand_size_of_type_32_64 ty)) + (_1 Unit (emit (MInst.UnaryRmR size (UnaryRmROpcode.Bsr) src tmp))) + (_2 Unit (emit (MInst.Cmove size (CC.Z) alt tmp dst)))) + dst)) + +;; Helper for creating `bsf` instructions. +(decl bsf (Type Gpr) Gpr) +(rule (bsf ty src) + (let ((dst WritableGpr (temp_writable_gpr)) + (size OperandSize (operand_size_of_type_32_64 ty)) + (_ Unit (emit (MInst.UnaryRmR size (UnaryRmROpcode.Bsf) src dst)))) + dst)) + +;; Helper for creating `bsf + cmov` instruction pairs that produce the +;; result of the `bsf`, or `alt` if the input was zero. +(decl bsf_or_else (Type Gpr Gpr) Gpr) +(rule (bsf_or_else ty src alt) + (let ((dst WritableGpr (temp_writable_gpr)) + (tmp WritableGpr (temp_writable_gpr)) + (size OperandSize (operand_size_of_type_32_64 ty)) + (_1 Unit (emit (MInst.UnaryRmR size (UnaryRmROpcode.Bsf) src tmp))) + (_2 Unit (emit (MInst.Cmove size (CC.Z) alt tmp dst)))) + dst)) + +;; Helper for creating `popcnt` instructions. +(decl x64_popcnt (Type Gpr) Gpr) +(rule (x64_popcnt ty src) + (let ((dst WritableGpr (temp_writable_gpr)) + (size OperandSize (operand_size_of_type_32_64 ty)) + (_ Unit (emit (MInst.UnaryRmR size (UnaryRmROpcode.Popcnt) src dst)))) + dst)) + ;;;; Automatic conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (convert Gpr InstOutput output_gpr) @@ -2241,6 +2349,7 @@ (convert Reg GprMemImm reg_to_gpr_mem_imm) (convert WritableGpr WritableReg writable_gpr_to_reg) (convert WritableGpr Reg writable_gpr_to_r_reg) +(convert WritableGpr GprMem writable_gpr_to_gpr_mem) (convert Xmm InstOutput output_xmm) (convert Value Xmm put_in_xmm) @@ -2261,6 +2370,7 @@ (convert WritableXmm XmmMem writable_xmm_to_xmm_mem) (convert Gpr Imm8Gpr gpr_to_imm8_gpr) +(convert Imm8Reg Imm8Gpr imm8_reg_to_imm8_gpr) (convert Amode SyntheticAmode amode_to_synthetic_amode) (convert SyntheticAmode GprMem synthetic_amode_to_gpr_mem) @@ -2276,6 +2386,9 @@ (decl writable_gpr_to_r_reg (WritableGpr) Reg) (rule (writable_gpr_to_r_reg w_gpr) (writable_reg_to_reg (writable_gpr_to_reg w_gpr))) +(decl writable_gpr_to_gpr_mem (WritableGpr) GprMem) +(rule (writable_gpr_to_gpr_mem w_gpr) + (gpr_to_gpr_mem w_gpr)) (decl writable_xmm_to_r_reg (WritableXmm) Reg) (rule (writable_xmm_to_r_reg w_xmm) (writable_reg_to_reg (writable_xmm_to_reg w_xmm))) diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 6532f23273f5..10ade2cae347 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -141,6 +141,7 @@ impl Inst { } } + #[allow(dead_code)] pub(crate) fn unary_rm_r( size: OperandSize, op: UnaryRmROpcode, diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index 6efe4394843e..b7ec39672829 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -1467,22 +1467,22 @@ ;; - `CC.BE -> C = 1 OR Z = 1` (below or equal) ;; - `CC.NBE -> C = 0 AND Z = 0` (not below or equal) -(rule (lower (has_type ty (select (def_inst (fcmp (FloatCC.Ordered) a b)) x y))) +(rule (lower (has_type ty (select (fcmp (FloatCC.Ordered) a b) x y))) (with_flags (fpcmp b a) (cmove_from_values ty (CC.NP) x y))) -(rule (lower (has_type ty (select (def_inst (fcmp (FloatCC.Unordered) a b)) x y))) +(rule (lower (has_type ty (select (fcmp (FloatCC.Unordered) a b) x y))) (with_flags (fpcmp b a) (cmove_from_values ty (CC.P) x y))) -(rule (lower (has_type ty (select (def_inst (fcmp (FloatCC.GreaterThan) a b)) x y))) +(rule (lower (has_type ty (select (fcmp (FloatCC.GreaterThan) a b) x y))) (with_flags (fpcmp b a) (cmove_from_values ty (CC.NBE) x y))) -(rule (lower (has_type ty (select (def_inst (fcmp (FloatCC.GreaterThanOrEqual) a b)) x y))) +(rule (lower (has_type ty (select (fcmp (FloatCC.GreaterThanOrEqual) a b) x y))) (with_flags (fpcmp b a) (cmove_from_values ty (CC.NB) x y))) -(rule (lower (has_type ty (select (def_inst (fcmp (FloatCC.UnorderedOrLessThan) a b)) x y))) +(rule (lower (has_type ty (select (fcmp (FloatCC.UnorderedOrLessThan) a b) x y))) (with_flags (fpcmp b a) (cmove_from_values ty (CC.B) x y))) -(rule (lower (has_type ty (select (def_inst (fcmp (FloatCC.UnorderedOrLessThanOrEqual) a b)) x y))) +(rule (lower (has_type ty (select (fcmp (FloatCC.UnorderedOrLessThanOrEqual) a b) x y))) (with_flags (fpcmp b a) (cmove_from_values ty (CC.BE) x y))) ;; Certain FloatCC variants are implemented by flipping the operands of the @@ -1496,16 +1496,16 @@ ;; not `LT | UNO`. By flipping the operands AND inverting the comparison (e.g., ;; to `CC.NBE`), we also avoid these unordered cases. -(rule (lower (has_type ty (select (def_inst (fcmp (FloatCC.LessThan) a b)) x y))) +(rule (lower (has_type ty (select (fcmp (FloatCC.LessThan) a b) x y))) (with_flags (fpcmp a b) (cmove_from_values ty (CC.NBE) x y))) -(rule (lower (has_type ty (select (def_inst (fcmp (FloatCC.LessThanOrEqual) a b)) x y))) +(rule (lower (has_type ty (select (fcmp (FloatCC.LessThanOrEqual) a b) x y))) (with_flags (fpcmp a b) (cmove_from_values ty (CC.NB) x y))) -(rule (lower (has_type ty (select (def_inst (fcmp (FloatCC.UnorderedOrGreaterThan) a b)) x y))) +(rule (lower (has_type ty (select (fcmp (FloatCC.UnorderedOrGreaterThan) a b) x y))) (with_flags (fpcmp a b) (cmove_from_values ty (CC.B) x y))) -(rule (lower (has_type ty (select (def_inst (fcmp (FloatCC.UnorderedOrGreaterThanOrEqual) a b)) x y))) +(rule (lower (has_type ty (select (fcmp (FloatCC.UnorderedOrGreaterThanOrEqual) a b) x y))) (with_flags (fpcmp a b) (cmove_from_values ty (CC.BE) x y))) ;; `FloatCC.Equal` and `FloatCC.NotEqual` can only be implemented with multiple @@ -1521,8 +1521,222 @@ ;; More details about the CLIF semantics for `fcmp` are available at ;; https://docs.rs/cranelift-codegen/latest/cranelift_codegen/ir/trait.InstBuilder.html#method.fcmp. -(rule (lower (has_type ty (select (def_inst (fcmp (FloatCC.Equal) a b)) x y))) +(rule (lower (has_type ty (select (fcmp (FloatCC.Equal) a b) x y))) (with_flags (fpcmp a b) (cmove_or_from_values ty (CC.NZ) (CC.P) y x))) -(rule (lower (has_type ty (select (def_inst (fcmp (FloatCC.NotEqual) a b)) x y))) +(rule (lower (has_type ty (select (fcmp (FloatCC.NotEqual) a b) x y))) (with_flags (fpcmp a b) (cmove_or_from_values ty (CC.NZ) (CC.P) x y))) + +;; Rules for `clz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; If available, we can use a plain lzcnt instruction here. Note no +;; special handling is required for zero inputs, because the machine +;; instruction does what the CLIF expects for zero, i.e. it returns +;; zero. +(rule 1 (lower + (has_type (and + (ty_32_or_64 ty) + (use_lzcnt)) + (clz src))) + (lzcnt ty src)) + +(rule (lower + (has_type (ty_32_or_64 ty) + (clz src))) + (do_clz ty ty src)) + +(rule (lower + (has_type (ty_8_or_16 ty) + (clz src))) + (do_clz $I32 ty (extend_to_gpr src $I32 (ExtendKind.Zero)))) + +(rule (lower + (has_type $I128 + (clz src))) + (let ((upper Gpr (do_clz $I64 $I64 (value_regs_get_gpr src 1))) + (lower Gpr (add $I64 + (do_clz $I64 $I64 (value_regs_get_gpr src 0)) + (RegMemImm.Imm 64))) + (result_lo Gpr + (with_flags_reg + (cmp_imm (OperandSize.Size64) 64 upper) + (cmove $I64 (CC.NZ) upper lower)))) + (value_regs result_lo (imm $I64 0)))) + +;; Implementation helper for clz; operates on 32 or 64-bit units. +(decl do_clz (Type Type Gpr) Gpr) +(rule (do_clz ty orig_ty src) + (let ((highest_bit_index Reg (bsr_or_else ty src (imm_i64 $I64 -1))) + (bits_minus_1 Reg (imm ty (u64_sub (ty_bits_u64 orig_ty) 1)))) + (sub ty bits_minus_1 highest_bit_index))) + +;; Rules for `ctz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; Analogous to `clz` cases above, but using mirror instructions +;; (tzcnt vs lzcnt, bsf vs bsr). + +(rule 1 (lower + (has_type (and + (ty_32_or_64 ty) + (use_bmi1)) + (ctz src))) + (tzcnt ty src)) + +(rule (lower + (has_type (ty_32_or_64 ty) + (ctz src))) + (do_ctz ty ty src)) + +(rule (lower + (has_type (ty_8_or_16 ty) + (ctz src))) + (do_ctz $I32 ty (extend_to_gpr src $I32 (ExtendKind.Zero)))) + +(rule (lower + (has_type $I128 + (ctz src))) + (let ((lower Gpr (do_ctz $I64 $I64 (value_regs_get_gpr src 0))) + (upper Gpr (add $I64 + (do_ctz $I64 $I64 (value_regs_get_gpr src 1)) + (RegMemImm.Imm 64))) + (result_lo Gpr + (with_flags_reg + (cmp_imm (OperandSize.Size64) 64 lower) + (cmove $I64 (CC.Z) upper lower)))) + (value_regs result_lo (imm $I64 0)))) + +(decl do_ctz (Type Type Gpr) Gpr) +(rule (do_ctz ty orig_ty src) + (bsf_or_else ty src (imm $I64 (ty_bits_u64 orig_ty)))) + +;; Rules for `popcnt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(rule 1 (lower + (has_type (and + (ty_32_or_64 ty) + (use_popcnt)) + (popcnt src))) + (x64_popcnt ty src)) + +(rule 1 (lower + (has_type (and + (ty_8_or_16 ty) + (use_popcnt)) + (popcnt src))) + (x64_popcnt $I32 (extend_to_gpr src $I32 (ExtendKind.Zero)))) + +(rule 1 (lower + (has_type (and + $I128 + (use_popcnt)) + (popcnt src))) + (let ((lo_count Gpr (x64_popcnt $I64 (value_regs_get_gpr src 0))) + (hi_count Gpr (x64_popcnt $I64 (value_regs_get_gpr src 1)))) + (value_regs (add $I64 lo_count hi_count) (imm $I64 0)))) + +(rule (lower + (has_type (ty_32_or_64 ty) + (popcnt src))) + (do_popcnt ty src)) + +(rule (lower + (has_type (ty_8_or_16 ty) + (popcnt src))) + (do_popcnt $I32 (extend_to_gpr src $I32 (ExtendKind.Zero)))) + +(rule (lower + (has_type $I128 + (popcnt src))) + (let ((lo_count Gpr (do_popcnt $I64 (value_regs_get_gpr src 0))) + (hi_count Gpr (do_popcnt $I64 (value_regs_get_gpr src 1)))) + (value_regs (add $I64 lo_count hi_count) (imm $I64 0)))) + +(decl do_popcnt (Type Gpr) Gpr) +(rule (do_popcnt $I64 src) + (let ((shifted1 Gpr (shr $I64 src (Imm8Reg.Imm8 1))) + (sevens Gpr (imm $I64 0x7777777777777777)) + (masked1 Gpr (x64_and $I64 shifted1 sevens)) + (diff1 Gpr (sub $I64 src masked1)) + (shifted2 Gpr (shr $I64 masked1 (Imm8Reg.Imm8 1))) + (masked2 Gpr (x64_and $I64 shifted2 sevens)) + (diff2 Gpr (sub $I64 diff1 masked2)) + (shifted3 Gpr (shr $I64 masked2 (Imm8Reg.Imm8 1))) + (masked3 Gpr (x64_and $I64 shifted3 sevens)) + (diff3 Gpr (sub $I64 diff2 masked3)) + (sum1 Gpr (add $I64 + (shr $I64 diff3 (Imm8Reg.Imm8 4)) + diff3)) + (ofof Gpr (imm $I64 0x0f0f0f0f0f0f0f0f)) + (masked4 Gpr (x64_and $I64 sum1 ofof)) + (ones Gpr (imm $I64 0x0101010101010101)) + (mul Gpr (mul $I64 masked4 ones)) + (final Gpr (shr $I64 mul (Imm8Reg.Imm8 56)))) + final)) +(rule (do_popcnt $I32 src) + (let ((shifted1 Gpr (shr $I32 src (Imm8Reg.Imm8 1))) + (sevens Gpr (imm $I32 0x77777777)) + (masked1 Gpr (x64_and $I32 shifted1 sevens)) + (diff1 Gpr (sub $I32 src masked1)) + (shifted2 Gpr (shr $I32 masked1 (Imm8Reg.Imm8 1))) + (masked2 Gpr (x64_and $I32 shifted2 sevens)) + (diff2 Gpr (sub $I32 diff1 masked2)) + (shifted3 Gpr (shr $I32 masked2 (Imm8Reg.Imm8 1))) + (masked3 Gpr (x64_and $I32 shifted3 sevens)) + (diff3 Gpr (sub $I32 diff2 masked3)) + (sum1 Gpr (add $I32 + (shr $I32 diff3 (Imm8Reg.Imm8 4)) + diff3)) + (masked4 Gpr (x64_and $I32 sum1 (RegMemImm.Imm 0x0f0f0f0f))) + (mul Gpr (mul $I32 masked4 (RegMemImm.Imm 0x01010101))) + (final Gpr (shr $I32 mul (Imm8Reg.Imm8 24)))) + final)) + + +(rule 1 (lower (has_type (and + $I8X16 + (avx512vl_enabled) + (avx512bitalg_enabled)) + (popcnt src))) + (vpopcntb src)) + + + +;; For SIMD 4.4 we use Mula's algorithm (https://arxiv.org/pdf/1611.07612.pdf) +;; +;; __m128i count_bytes ( __m128i v) { +;; __m128i lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4); +;; __m128i low_mask = _mm_set1_epi8 (0x0f); +;; __m128i lo = _mm_and_si128 (v, low_mask); +;; __m128i hi = _mm_and_si128 (_mm_srli_epi16 (v, 4), low_mask); +;; __m128i cnt1 = _mm_shuffle_epi8 (lookup, lo); +;; __m128i cnt2 = _mm_shuffle_epi8 (lookup, hi); +;; return _mm_add_epi8 (cnt1, cnt2); +;; } +;; +;; Details of the above algorithm can be found in the reference noted above, but the basics +;; are to create a lookup table that pre populates the popcnt values for each number [0,15]. +;; The algorithm uses shifts to isolate 4 bit sections of the vector, pshufb as part of the +;; lookup process, and adds together the results. +;; +;; __m128i lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4); + +(decl popcount_4bit_table () VCodeConstant) ;; bits-per-nibble table `lookup` above +(extern constructor popcount_4bit_table popcount_4bit_table) + +(decl popcount_low_mask () VCodeConstant) ;; mask for low nibbles: 0x0f * 16 +(extern constructor popcount_low_mask popcount_low_mask) + +(rule (lower (has_type $I8X16 + (popcnt src))) + (let ((nibble_table_const VCodeConstant (popcount_4bit_table)) + (low_mask Xmm (xmm_load_const $I8X16 (popcount_low_mask))) + (low_nibbles Xmm (sse_and $I8X16 src low_mask)) + ;; Note that this is a 16x8 shift, but that's OK; we mask + ;; off anything that traverses from one byte to the next + ;; with the low_mask below. + (shifted_src Xmm (psrlw src (RegMemImm.Imm 4))) + (high_nibbles Xmm (sse_and $I8X16 shifted_src low_mask)) + (lookup Xmm (xmm_load_const $I8X16 (popcount_4bit_table))) + (bit_counts_low Xmm (pshufb lookup low_nibbles)) + (bit_counts_high Xmm (pshufb lookup high_nibbles))) + (paddb bit_counts_low bit_counts_high))) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index d5de0801321e..692dc6e6f358 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -21,7 +21,7 @@ use alloc::boxed::Box; use alloc::vec::Vec; use log::trace; use regalloc::{Reg, RegClass, Writable}; -use smallvec::{smallvec, SmallVec}; +use smallvec::SmallVec; use std::convert::TryFrom; use target_lexicon::Triple; @@ -1102,75 +1102,6 @@ fn emit_cmoves>( } } -fn emit_clz>( - ctx: &mut C, - orig_ty: Type, - ty: Type, - src: Reg, - dst: Writable, -) { - let src = RegMem::reg(src); - let tmp = ctx.alloc_tmp(ty).only_reg().unwrap(); - ctx.emit(Inst::imm(OperandSize::from_ty(ty), u64::max_value(), dst)); - - ctx.emit(Inst::unary_rm_r( - OperandSize::from_ty(ty), - UnaryRmROpcode::Bsr, - src, - tmp, - )); - - ctx.emit(Inst::cmove( - OperandSize::from_ty(ty), - CC::Z, - RegMem::reg(dst.to_reg()), - tmp, - )); - - ctx.emit(Inst::imm( - OperandSize::from_ty(ty), - orig_ty.bits() as u64 - 1, - dst, - )); - - ctx.emit(Inst::alu_rmi_r( - if ty == types::I64 { - OperandSize::Size64 - } else { - OperandSize::Size32 - }, - AluRmiROpcode::Sub, - RegMemImm::reg(tmp.to_reg()), - dst, - )); -} - -fn emit_ctz>( - ctx: &mut C, - orig_ty: Type, - ty: Type, - src: Reg, - dst: Writable, -) { - let src = RegMem::reg(src); - let tmp = ctx.alloc_tmp(ty).only_reg().unwrap(); - ctx.emit(Inst::imm(OperandSize::Size32, orig_ty.bits() as u64, tmp)); - - ctx.emit(Inst::unary_rm_r( - OperandSize::from_ty(ty), - UnaryRmROpcode::Bsf, - src, - dst, - )); - - ctx.emit(Inst::cmove( - OperandSize::from_ty(ty), - CC::Z, - RegMem::reg(tmp.to_reg()), - dst, - )); -} - //============================================================================= // Top-level instruction lowering entry point, for one instruction. @@ -1243,637 +1174,10 @@ fn lower_insn_to_regs>( | Opcode::Rotr | Opcode::Ineg | Opcode::Trap - | Opcode::ResumableTrap => implemented_in_isle(ctx), - - Opcode::Clz => { - let orig_ty = ty.unwrap(); - - if isa_flags.use_lzcnt() && (orig_ty == types::I32 || orig_ty == types::I64) { - // We can use a plain lzcnt instruction here. Note no special handling is required - // for zero inputs, because the machine instruction does what the CLIF expects for - // zero, i.e. it returns zero. - let src = input_to_reg_mem(ctx, inputs[0]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - ctx.emit(Inst::unary_rm_r( - OperandSize::from_ty(orig_ty), - UnaryRmROpcode::Lzcnt, - src, - dst, - )); - return Ok(()); - } - - // General formula using bit-scan reverse (BSR): - // mov -1, %dst - // bsr %src, %tmp - // cmovz %dst, %tmp - // mov $(size_bits - 1), %dst - // sub %tmp, %dst - - if orig_ty == types::I128 { - // clz upper, tmp1 - // clz lower, dst - // add dst, 64 - // cmp tmp1, 64 - // cmovnz tmp1, dst - let dsts = get_output_reg(ctx, outputs[0]); - let dst = dsts.regs()[0]; - let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let srcs = put_input_in_regs(ctx, inputs[0]); - let src_lo = srcs.regs()[0]; - let src_hi = srcs.regs()[1]; - emit_clz(ctx, types::I64, types::I64, src_hi, tmp1); - emit_clz(ctx, types::I64, types::I64, src_lo, dst); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Add, - RegMemImm::imm(64), - dst, - )); - ctx.emit(Inst::cmp_rmi_r( - OperandSize::Size64, - RegMemImm::imm(64), - tmp1.to_reg(), - )); - ctx.emit(Inst::cmove( - OperandSize::Size64, - CC::NZ, - RegMem::reg(tmp1.to_reg()), - dst, - )); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Xor, - RegMemImm::reg(dsts.regs()[1].to_reg()), - dsts.regs()[1], - )); - } else { - let (ext_spec, ty) = match orig_ty { - types::I8 | types::I16 => (Some(ExtSpec::ZeroExtendTo32), types::I32), - a if a == types::I32 || a == types::I64 => (None, a), - _ => unreachable!(), - }; - let src = if let Some(ext_spec) = ext_spec { - extend_input_to_reg(ctx, inputs[0], ext_spec) - } else { - put_input_in_reg(ctx, inputs[0]) - }; - - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - emit_clz(ctx, orig_ty, ty, src, dst); - } - } - - Opcode::Ctz => { - let orig_ty = ctx.input_ty(insn, 0); - - if isa_flags.use_bmi1() && (orig_ty == types::I32 || orig_ty == types::I64) { - // We can use a plain tzcnt instruction here. Note no special handling is required - // for zero inputs, because the machine instruction does what the CLIF expects for - // zero, i.e. it returns zero. - let src = input_to_reg_mem(ctx, inputs[0]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - ctx.emit(Inst::unary_rm_r( - OperandSize::from_ty(orig_ty), - UnaryRmROpcode::Tzcnt, - src, - dst, - )); - return Ok(()); - } - - // General formula using bit-scan forward (BSF): - // bsf %src, %dst - // mov $(size_bits), %tmp - // cmovz %tmp, %dst - if orig_ty == types::I128 { - // ctz src_lo, dst - // ctz src_hi, tmp1 - // add tmp1, 64 - // cmp dst, 64 - // cmovz tmp1, dst - let dsts = get_output_reg(ctx, outputs[0]); - let dst = dsts.regs()[0]; - let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let srcs = put_input_in_regs(ctx, inputs[0]); - let src_lo = srcs.regs()[0]; - let src_hi = srcs.regs()[1]; - emit_ctz(ctx, types::I64, types::I64, src_lo, dst); - emit_ctz(ctx, types::I64, types::I64, src_hi, tmp1); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Add, - RegMemImm::imm(64), - tmp1, - )); - ctx.emit(Inst::cmp_rmi_r( - OperandSize::Size64, - RegMemImm::imm(64), - dst.to_reg(), - )); - ctx.emit(Inst::cmove( - OperandSize::Size64, - CC::Z, - RegMem::reg(tmp1.to_reg()), - dst, - )); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Xor, - RegMemImm::reg(dsts.regs()[1].to_reg()), - dsts.regs()[1], - )); - } else { - let ty = if orig_ty.bits() < 32 { - types::I32 - } else { - orig_ty - }; - debug_assert!(ty == types::I32 || ty == types::I64); - - let src = put_input_in_reg(ctx, inputs[0]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - emit_ctz(ctx, orig_ty, ty, src, dst); - } - } - - Opcode::Popcnt => { - let ty_tmp = ty.unwrap(); - if !ty_tmp.is_vector() { - let ty = ctx.input_ty(insn, 0); - - if isa_flags.use_popcnt() { - match ty { - types::I8 | types::I16 => { - let src = RegMem::reg(extend_input_to_reg( - ctx, - inputs[0], - ExtSpec::ZeroExtendTo32, - )); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - ctx.emit(Inst::unary_rm_r( - OperandSize::from_ty(types::I32), - UnaryRmROpcode::Popcnt, - src, - dst, - )); - return Ok(()); - } - types::I32 | types::I64 => { - let src = input_to_reg_mem(ctx, inputs[0]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - ctx.emit(Inst::unary_rm_r( - OperandSize::from_ty(ty), - UnaryRmROpcode::Popcnt, - src, - dst, - )); - return Ok(()); - } - - types::I128 => { - // The number of ones in a 128-bits value is the plain sum of the number of - // ones in its low and high parts. No risk of overflow here. - let dsts = get_output_reg(ctx, outputs[0]); - let dst = dsts.regs()[0]; - let tmp = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let srcs = put_input_in_regs(ctx, inputs[0]); - let src_lo = srcs.regs()[0]; - let src_hi = srcs.regs()[1]; - - ctx.emit(Inst::unary_rm_r( - OperandSize::Size64, - UnaryRmROpcode::Popcnt, - RegMem::reg(src_lo), - dst, - )); - ctx.emit(Inst::unary_rm_r( - OperandSize::Size64, - UnaryRmROpcode::Popcnt, - RegMem::reg(src_hi), - tmp, - )); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Add, - RegMemImm::reg(tmp.to_reg()), - dst, - )); - - // Zero the result's high component. - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Xor, - RegMemImm::reg(dsts.regs()[1].to_reg()), - dsts.regs()[1], - )); - - return Ok(()); - } - _ => {} - } - } - - let (ext_spec, ty) = match ty { - types::I8 | types::I16 => (Some(ExtSpec::ZeroExtendTo32), types::I32), - a if a == types::I32 || a == types::I64 || a == types::I128 => (None, a), - _ => unreachable!(), - }; - - let (srcs, ty): (SmallVec<[RegMem; 2]>, Type) = if let Some(ext_spec) = ext_spec { - ( - smallvec![RegMem::reg(extend_input_to_reg(ctx, inputs[0], ext_spec))], - ty, - ) - } else if ty == types::I128 { - let regs = put_input_in_regs(ctx, inputs[0]); - ( - smallvec![RegMem::reg(regs.regs()[0]), RegMem::reg(regs.regs()[1])], - types::I64, - ) - } else { - // N.B.: explicitly put input in a reg here because the width of the instruction - // into which this RM op goes may not match the width of the input type (in fact, - // it won't for i32.popcnt), and we don't want a larger than necessary load. - (smallvec![RegMem::reg(put_input_in_reg(ctx, inputs[0]))], ty) - }; - - let mut dsts: SmallVec<[Reg; 2]> = smallvec![]; - for src in srcs { - let dst = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - dsts.push(dst.to_reg()); - if ty == types::I64 { - let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let cst = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - - // mov src, tmp1 - ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1)); - - // shr $1, tmp1 - ctx.emit(Inst::shift_r( - OperandSize::Size64, - ShiftKind::ShiftRightLogical, - Some(1), - tmp1, - )); - - // mov 0x7777_7777_7777_7777, cst - ctx.emit(Inst::imm(OperandSize::Size64, 0x7777777777777777, cst)); - - // andq cst, tmp1 - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::And, - RegMemImm::reg(cst.to_reg()), - tmp1, - )); - - // mov src, tmp2 - ctx.emit(Inst::mov64_rm_r(src, tmp2)); - - // sub tmp1, tmp2 - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Sub, - RegMemImm::reg(tmp1.to_reg()), - tmp2, - )); - - // shr $1, tmp1 - ctx.emit(Inst::shift_r( - OperandSize::Size64, - ShiftKind::ShiftRightLogical, - Some(1), - tmp1, - )); - - // and cst, tmp1 - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::And, - RegMemImm::reg(cst.to_reg()), - tmp1, - )); - - // sub tmp1, tmp2 - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Sub, - RegMemImm::reg(tmp1.to_reg()), - tmp2, - )); - - // shr $1, tmp1 - ctx.emit(Inst::shift_r( - OperandSize::Size64, - ShiftKind::ShiftRightLogical, - Some(1), - tmp1, - )); - - // and cst, tmp1 - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::And, - RegMemImm::reg(cst.to_reg()), - tmp1, - )); - - // sub tmp1, tmp2 - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Sub, - RegMemImm::reg(tmp1.to_reg()), - tmp2, - )); - - // mov tmp2, dst - ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst)); - - // shr $4, dst - ctx.emit(Inst::shift_r( - OperandSize::Size64, - ShiftKind::ShiftRightLogical, - Some(4), - dst, - )); - - // add tmp2, dst - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Add, - RegMemImm::reg(tmp2.to_reg()), - dst, - )); - - // mov $0x0F0F_0F0F_0F0F_0F0F, cst - ctx.emit(Inst::imm(OperandSize::Size64, 0x0F0F0F0F0F0F0F0F, cst)); - - // and cst, dst - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::And, - RegMemImm::reg(cst.to_reg()), - dst, - )); - - // mov $0x0101_0101_0101_0101, cst - ctx.emit(Inst::imm(OperandSize::Size64, 0x0101010101010101, cst)); - - // mul cst, dst - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Mul, - RegMemImm::reg(cst.to_reg()), - dst, - )); - - // shr $56, dst - ctx.emit(Inst::shift_r( - OperandSize::Size64, - ShiftKind::ShiftRightLogical, - Some(56), - dst, - )); - } else { - assert_eq!(ty, types::I32); - - let tmp1 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - let tmp2 = ctx.alloc_tmp(types::I64).only_reg().unwrap(); - - // mov src, tmp1 - ctx.emit(Inst::mov64_rm_r(src.clone(), tmp1)); - - // shr $1, tmp1 - ctx.emit(Inst::shift_r( - OperandSize::Size32, - ShiftKind::ShiftRightLogical, - Some(1), - tmp1, - )); - - // andq $0x7777_7777, tmp1 - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size32, - AluRmiROpcode::And, - RegMemImm::imm(0x77777777), - tmp1, - )); - - // mov src, tmp2 - ctx.emit(Inst::mov64_rm_r(src, tmp2)); - - // sub tmp1, tmp2 - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size32, - AluRmiROpcode::Sub, - RegMemImm::reg(tmp1.to_reg()), - tmp2, - )); - - // shr $1, tmp1 - ctx.emit(Inst::shift_r( - OperandSize::Size32, - ShiftKind::ShiftRightLogical, - Some(1), - tmp1, - )); - - // and 0x7777_7777, tmp1 - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size32, - AluRmiROpcode::And, - RegMemImm::imm(0x77777777), - tmp1, - )); - - // sub tmp1, tmp2 - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size32, - AluRmiROpcode::Sub, - RegMemImm::reg(tmp1.to_reg()), - tmp2, - )); - - // shr $1, tmp1 - ctx.emit(Inst::shift_r( - OperandSize::Size32, - ShiftKind::ShiftRightLogical, - Some(1), - tmp1, - )); - - // and $0x7777_7777, tmp1 - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size32, - AluRmiROpcode::And, - RegMemImm::imm(0x77777777), - tmp1, - )); - - // sub tmp1, tmp2 - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size32, - AluRmiROpcode::Sub, - RegMemImm::reg(tmp1.to_reg()), - tmp2, - )); - - // mov tmp2, dst - ctx.emit(Inst::mov64_rm_r(RegMem::reg(tmp2.to_reg()), dst)); - - // shr $4, dst - ctx.emit(Inst::shift_r( - OperandSize::Size32, - ShiftKind::ShiftRightLogical, - Some(4), - dst, - )); - - // add tmp2, dst - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size32, - AluRmiROpcode::Add, - RegMemImm::reg(tmp2.to_reg()), - dst, - )); - - // and $0x0F0F_0F0F, dst - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size32, - AluRmiROpcode::And, - RegMemImm::imm(0x0F0F0F0F), - dst, - )); - - // mul $0x0101_0101, dst - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size32, - AluRmiROpcode::Mul, - RegMemImm::imm(0x01010101), - dst, - )); - - // shr $24, dst - ctx.emit(Inst::shift_r( - OperandSize::Size32, - ShiftKind::ShiftRightLogical, - Some(24), - dst, - )); - } - } - - if dsts.len() == 1 { - let final_dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - ctx.emit(Inst::gen_move(final_dst, dsts[0], types::I64)); - } else { - assert!(dsts.len() == 2); - let final_dst = get_output_reg(ctx, outputs[0]); - ctx.emit(Inst::gen_move(final_dst.regs()[0], dsts[0], types::I64)); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Add, - RegMemImm::reg(dsts[1]), - final_dst.regs()[0], - )); - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Xor, - RegMemImm::reg(final_dst.regs()[1].to_reg()), - final_dst.regs()[1], - )); - } - } else { - // Lower `popcount` for vectors. - let ty = ty.unwrap(); - let src = put_input_in_reg(ctx, inputs[0]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - - if isa_flags.use_avx512vl_simd() && isa_flags.use_avx512bitalg_simd() { - // When AVX512VL and AVX512BITALG are available, - // `popcnt.i8x16` can be lowered to a single instruction. - assert_eq!(ty, types::I8X16); - ctx.emit(Inst::xmm_unary_rm_r_evex( - Avx512Opcode::Vpopcntb, - RegMem::reg(src), - dst, - )); - } else { - // For SIMD 4.4 we use Mula's algorithm (https://arxiv.org/pdf/1611.07612.pdf) - // - //__m128i count_bytes ( __m128i v) { - // __m128i lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4); - // __m128i low_mask = _mm_set1_epi8 (0x0f); - // __m128i lo = _mm_and_si128 (v, low_mask); - // __m128i hi = _mm_and_si128 (_mm_srli_epi16 (v, 4), low_mask); - // __m128i cnt1 = _mm_shuffle_epi8 (lookup, lo); - // __m128i cnt2 = _mm_shuffle_epi8 (lookup, hi); - // return _mm_add_epi8 (cnt1, cnt2); - //} - // - // Details of the above algorithm can be found in the reference noted above, but the basics - // are to create a lookup table that pre populates the popcnt values for each number [0,15]. - // The algorithm uses shifts to isolate 4 bit sections of the vector, pshufb as part of the - // lookup process, and adds together the results. - - // __m128i lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4); - static POPCOUNT_4BIT: [u8; 16] = [ - 0x00, 0x01, 0x01, 0x02, 0x01, 0x02, 0x02, 0x03, 0x01, 0x02, 0x02, 0x03, - 0x02, 0x03, 0x03, 0x04, - ]; - let lookup = ctx.use_constant(VCodeConstantData::WellKnown(&POPCOUNT_4BIT)); - - // Create a mask for lower 4bits of each subword. - static LOW_MASK: [u8; 16] = [0x0F; 16]; - let low_mask_const = ctx.use_constant(VCodeConstantData::WellKnown(&LOW_MASK)); - let low_mask = ctx.alloc_tmp(types::I8X16).only_reg().unwrap(); - ctx.emit(Inst::xmm_load_const(low_mask_const, low_mask, ty)); - - // __m128i lo = _mm_and_si128 (v, low_mask); - let lo = ctx.alloc_tmp(types::I8X16).only_reg().unwrap(); - ctx.emit(Inst::gen_move(lo, low_mask.to_reg(), types::I8X16)); - ctx.emit(Inst::xmm_rm_r(SseOpcode::Pand, RegMem::reg(src), lo)); - - // __m128i hi = _mm_and_si128 (_mm_srli_epi16 (v, 4), low_mask); - ctx.emit(Inst::gen_move(dst, src, ty)); - ctx.emit(Inst::xmm_rmi_reg(SseOpcode::Psrlw, RegMemImm::imm(4), dst)); - let tmp = ctx.alloc_tmp(types::I8X16).only_reg().unwrap(); - ctx.emit(Inst::gen_move(tmp, low_mask.to_reg(), types::I8X16)); - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Pand, - RegMem::reg(dst.to_reg()), - tmp, - )); - - // __m128i cnt1 = _mm_shuffle_epi8 (lookup, lo); - let tmp2 = ctx.alloc_tmp(types::I8X16).only_reg().unwrap(); - ctx.emit(Inst::xmm_load_const(lookup, tmp2, ty)); - ctx.emit(Inst::gen_move(dst, tmp2.to_reg(), types::I8X16)); - - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Pshufb, - RegMem::reg(lo.to_reg()), - dst, - )); - - // __m128i cnt2 = _mm_shuffle_epi8 (lookup , hi) ; - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Pshufb, - RegMem::reg(tmp.to_reg()), - tmp2, - )); - - // return _mm_add_epi8 (cnt1 , cnt2 ) ; - ctx.emit(Inst::xmm_rm_r( - SseOpcode::Paddb, - RegMem::reg(tmp2.to_reg()), - dst, - )); - } - } - } + | Opcode::ResumableTrap + | Opcode::Clz + | Opcode::Ctz + | Opcode::Popcnt => implemented_in_isle(ctx), Opcode::Bitrev => { let ty = ctx.input_ty(insn, 0); diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs index a0ae8be65756..c20bf3b1e188 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle.rs @@ -171,6 +171,42 @@ where } } + #[inline] + fn avx512bitalg_enabled(&mut self, _: Type) -> Option<()> { + if self.isa_flags.use_avx512bitalg_simd() { + Some(()) + } else { + None + } + } + + #[inline] + fn use_lzcnt(&mut self, _: Type) -> Option<()> { + if self.isa_flags.use_lzcnt() { + Some(()) + } else { + None + } + } + + #[inline] + fn use_bmi1(&mut self, _: Type) -> Option<()> { + if self.isa_flags.use_bmi1() { + Some(()) + } else { + None + } + } + + #[inline] + fn use_popcnt(&mut self, _: Type) -> Option<()> { + if self.isa_flags.use_popcnt() { + Some(()) + } else { + None + } + } + #[inline] fn imm8_from_value(&mut self, val: Value) -> Option { let inst = self.lower_ctx.dfg().value_def(val).inst()?; @@ -326,6 +362,16 @@ where SyntheticAmode::ConstantOffset(mask_table) } + fn popcount_4bit_table(&mut self) -> VCodeConstant { + self.lower_ctx + .use_constant(VCodeConstantData::WellKnown(&POPCOUNT_4BIT_TABLE)) + } + + fn popcount_low_mask(&mut self) -> VCodeConstant { + self.lower_ctx + .use_constant(VCodeConstantData::WellKnown(&POPCOUNT_LOW_MASK)) + } + #[inline] fn writable_reg_to_xmm(&mut self, r: WritableReg) -> WritableXmm { Writable::from_reg(Xmm::new(r.to_reg()).unwrap()) @@ -499,6 +545,18 @@ const I8X16_USHR_MASKS: [u8; 128] = [ 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, ]; +/// Number of bits set in a given nibble (4-bit value). Used in the +/// vector implementation of popcount. +#[rustfmt::skip] // Preserve 4x4 layout. +const POPCOUNT_4BIT_TABLE: [u8; 16] = [ + 0x00, 0x01, 0x01, 0x02, + 0x01, 0x02, 0x02, 0x03, + 0x01, 0x02, 0x02, 0x03, + 0x02, 0x03, 0x03, 0x04, +]; + +const POPCOUNT_LOW_MASK: [u8; 16] = [0x0f; 16]; + #[inline] fn to_simm32(constant: i64) -> Option { if constant == ((constant << 32) >> 32) { diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 3cef290a612b..9d0ac7920da8 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle 9830498351ddf6a3 -src/isa/x64/inst.isle 5ee89205e6e9a46b -src/isa/x64/lower.isle 348a808ea5de4cdb +src/prelude.isle 3a10bacb778efff0 +src/isa/x64/inst.isle ee587a01279f3496 +src/isa/x64/lower.isle 1de32c5a76dc2245 diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index 225f985bbbeb..d254f612efd2 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -39,8 +39,12 @@ pub trait Context { fn u8_as_u64(&mut self, arg0: u8) -> u64; fn u16_as_u64(&mut self, arg0: u16) -> u64; fn u32_as_u64(&mut self, arg0: u32) -> u64; + fn i64_as_u64(&mut self, arg0: i64) -> u64; + fn u64_add(&mut self, arg0: u64, arg1: u64) -> u64; + fn u64_sub(&mut self, arg0: u64, arg1: u64) -> u64; fn ty_bits(&mut self, arg0: Type) -> u8; fn ty_bits_u16(&mut self, arg0: Type) -> u16; + fn ty_bits_u64(&mut self, arg0: Type) -> u64; fn ty_bytes(&mut self, arg0: Type) -> u16; fn lane_type(&mut self, arg0: Type) -> Type; fn fits_in_16(&mut self, arg0: Type) -> Option; @@ -120,6 +124,10 @@ pub trait Context { fn avx512vl_enabled(&mut self, arg0: Type) -> Option<()>; fn avx512dq_enabled(&mut self, arg0: Type) -> Option<()>; fn avx512f_enabled(&mut self, arg0: Type) -> Option<()>; + fn avx512bitalg_enabled(&mut self, arg0: Type) -> Option<()>; + fn use_lzcnt(&mut self, arg0: Type) -> Option<()>; + fn use_bmi1(&mut self, arg0: Type) -> Option<()>; + fn use_popcnt(&mut self, arg0: Type) -> Option<()>; fn imm8_from_value(&mut self, arg0: Value) -> Option; fn const_to_type_masked_imm8(&mut self, arg0: u64, arg1: Type) -> Imm8Gpr; fn simm32_from_value(&mut self, arg0: Value) -> Option; @@ -133,15 +141,17 @@ pub trait Context { fn ushr_i8x16_mask_for_const(&mut self, arg0: u32) -> SyntheticAmode; fn ushr_i8x16_mask_table(&mut self) -> SyntheticAmode; fn sse_insertps_lane_imm(&mut self, arg0: u8) -> u8; + fn popcount_4bit_table(&mut self) -> VCodeConstant; + fn popcount_low_mask(&mut self) -> VCodeConstant; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 363. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 378. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 385. +/// Internal type ProducesFlags: defined at src/prelude.isle line 400. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlagsSideEffect { inst: MInst }, @@ -149,7 +159,7 @@ pub enum ProducesFlags { ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 396. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 411. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlagsReturnsResultWithProducer { @@ -495,7 +505,7 @@ pub enum MInst { }, } -/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 1125. +/// Internal type ExtendKind: defined at src/isa/x64/inst.isle line 1137. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum ExtendKind { Sign, @@ -549,7 +559,7 @@ pub fn constructor_side_effect( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 368. + // Rule at src/prelude.isle line 383. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::output_none(ctx); return Some(expr1_0); @@ -567,7 +577,7 @@ pub fn constructor_safepoint( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 374. + // Rule at src/prelude.isle line 389. let expr0_0 = C::emit_safepoint(ctx, pattern1_0); let expr1_0 = C::output_none(ctx); return Some(expr1_0); @@ -593,7 +603,7 @@ pub fn constructor_consumes_flags_concat( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 408. + // Rule at src/prelude.isle line 423. let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1); let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs { inst1: pattern1_0.clone(), @@ -623,7 +633,7 @@ pub fn constructor_with_flags( inst: ref pattern3_0, result: pattern3_1, } => { - // Rule at src/prelude.isle line 433. + // Rule at src/prelude.isle line 448. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_reg(ctx, pattern3_1); @@ -634,7 +644,7 @@ pub fn constructor_with_flags( inst2: ref pattern3_1, result: pattern3_2, } => { - // Rule at src/prelude.isle line 439. + // Rule at src/prelude.isle line 454. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_1); let expr2_0 = C::emit(ctx, pattern3_0); @@ -653,7 +663,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 427. + // Rule at src/prelude.isle line 442. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -673,7 +683,7 @@ pub fn constructor_with_flags_reg( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/prelude.isle line 452. + // Rule at src/prelude.isle line 467. let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?; let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -830,7 +840,7 @@ pub fn constructor_sink_load_to_gpr_mem_imm( arg0: &SinkableLoad, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1115. + // Rule at src/isa/x64/inst.isle line 1127. let expr0_0 = C::sink_load(ctx, pattern0_0); let expr1_0 = C::gpr_mem_imm_new(ctx, &expr0_0); return Some(expr1_0); @@ -848,12 +858,12 @@ pub fn constructor_extend_to_gpr( let pattern2_0 = arg1; if pattern2_0 == pattern1_0 { let pattern4_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1137. + // Rule at src/isa/x64/inst.isle line 1149. let expr0_0 = constructor_put_in_gpr(ctx, pattern0_0)?; return Some(expr0_0); } let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1140. + // Rule at src/isa/x64/inst.isle line 1152. let expr0_0 = C::ty_bits_u16(ctx, pattern1_0); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern2_0); let expr2_0 = constructor_operand_size_bits(ctx, &expr1_0)?; @@ -877,7 +887,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1160. + // Rule at src/isa/x64/inst.isle line 1172. let expr0_0 = constructor_movsx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -885,7 +895,7 @@ pub fn constructor_extend( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1156. + // Rule at src/isa/x64/inst.isle line 1168. let expr0_0 = constructor_movzx(ctx, pattern2_0, pattern3_0, pattern4_0)?; return Some(expr0_0); } @@ -898,17 +908,17 @@ pub fn constructor_extend( pub fn constructor_sse_xor_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1167. + // Rule at src/isa/x64/inst.isle line 1179. let expr0_0 = SseOpcode::Xorps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1168. + // Rule at src/isa/x64/inst.isle line 1180. let expr0_0 = SseOpcode::Xorpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1169. + // Rule at src/isa/x64/inst.isle line 1181. let expr0_0 = SseOpcode::Pxor; return Some(expr0_0); } @@ -925,7 +935,7 @@ pub fn constructor_sse_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1173. + // Rule at src/isa/x64/inst.isle line 1185. let expr0_0 = constructor_sse_xor_op(ctx, pattern0_0)?; let expr1_0 = constructor_xmm_rm_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -935,40 +945,40 @@ pub fn constructor_sse_xor( pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1182. + // Rule at src/isa/x64/inst.isle line 1194. let expr0_0 = SseOpcode::Cmpps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1183. + // Rule at src/isa/x64/inst.isle line 1195. let expr0_0 = SseOpcode::Cmppd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { if pattern1_0 == 8 { if pattern1_1 == 16 { - // Rule at src/isa/x64/inst.isle line 1178. + // Rule at src/isa/x64/inst.isle line 1190. let expr0_0 = SseOpcode::Pcmpeqb; return Some(expr0_0); } } if pattern1_0 == 16 { if pattern1_1 == 8 { - // Rule at src/isa/x64/inst.isle line 1179. + // Rule at src/isa/x64/inst.isle line 1191. let expr0_0 = SseOpcode::Pcmpeqw; return Some(expr0_0); } } if pattern1_0 == 32 { if pattern1_1 == 4 { - // Rule at src/isa/x64/inst.isle line 1180. + // Rule at src/isa/x64/inst.isle line 1192. let expr0_0 = SseOpcode::Pcmpeqd; return Some(expr0_0); } } if pattern1_0 == 64 { if pattern1_1 == 2 { - // Rule at src/isa/x64/inst.isle line 1181. + // Rule at src/isa/x64/inst.isle line 1193. let expr0_0 = SseOpcode::Pcmpeqq; return Some(expr0_0); } @@ -980,7 +990,7 @@ pub fn constructor_sse_cmp_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 1197. + // Rule at src/isa/x64/inst.isle line 1209. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0: Type = I32X4; let expr2_0 = constructor_sse_cmp_op(ctx, expr1_0)?; @@ -1005,7 +1015,7 @@ pub fn constructor_make_i64x2_from_lanes( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1207. + // Rule at src/isa/x64/inst.isle line 1219. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::writable_xmm_to_reg(ctx, expr0_0); let expr2_0 = MInst::XmmUninitializedValue { dst: expr0_0 }; @@ -1047,12 +1057,12 @@ pub fn constructor_mov_rmi_to_xmm(ctx: &mut C, arg0: &RegMemImm) -> let pattern0_0 = arg0; match pattern0_0 { &RegMemImm::Imm { simm32: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 1228. + // Rule at src/isa/x64/inst.isle line 1240. let expr0_0 = C::xmm_mem_imm_new(ctx, pattern0_0); return Some(expr0_0); } &RegMemImm::Reg { reg: pattern1_0 } => { - // Rule at src/isa/x64/inst.isle line 1229. + // Rule at src/isa/x64/inst.isle line 1241. let expr0_0 = SseOpcode::Movd; let expr1_0 = C::reg_to_gpr_mem(ctx, pattern1_0); let expr2_0 = OperandSize::Size32; @@ -1063,7 +1073,7 @@ pub fn constructor_mov_rmi_to_xmm(ctx: &mut C, arg0: &RegMemImm) -> &RegMemImm::Mem { addr: ref pattern1_0, } => { - // Rule at src/isa/x64/inst.isle line 1227. + // Rule at src/isa/x64/inst.isle line 1239. let expr0_0 = C::xmm_mem_imm_new(ctx, pattern0_0); return Some(expr0_0); } @@ -1083,7 +1093,7 @@ pub fn constructor_x64_load( if pattern0_0 == I64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1244. + // Rule at src/isa/x64/inst.isle line 1256. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = MInst::Mov64MR { src: pattern2_0.clone(), @@ -1096,7 +1106,7 @@ pub fn constructor_x64_load( if pattern0_0 == F32 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1249. + // Rule at src/isa/x64/inst.isle line 1261. let expr0_0 = SseOpcode::Movss; let expr1_0 = constructor_synthetic_amode_to_xmm_mem(ctx, pattern2_0)?; let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -1106,7 +1116,7 @@ pub fn constructor_x64_load( if pattern0_0 == F64 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1253. + // Rule at src/isa/x64/inst.isle line 1265. let expr0_0 = SseOpcode::Movsd; let expr1_0 = constructor_synthetic_amode_to_xmm_mem(ctx, pattern2_0)?; let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -1116,7 +1126,7 @@ pub fn constructor_x64_load( if pattern0_0 == F32X4 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1257. + // Rule at src/isa/x64/inst.isle line 1269. let expr0_0 = SseOpcode::Movups; let expr1_0 = constructor_synthetic_amode_to_xmm_mem(ctx, pattern2_0)?; let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -1126,7 +1136,7 @@ pub fn constructor_x64_load( if pattern0_0 == F64X2 { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1261. + // Rule at src/isa/x64/inst.isle line 1273. let expr0_0 = SseOpcode::Movupd; let expr1_0 = constructor_synthetic_amode_to_xmm_mem(ctx, pattern2_0)?; let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -1136,7 +1146,7 @@ pub fn constructor_x64_load( if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; let pattern3_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1265. + // Rule at src/isa/x64/inst.isle line 1277. let expr0_0 = SseOpcode::Movdqu; let expr1_0 = constructor_synthetic_amode_to_xmm_mem(ctx, pattern2_0)?; let expr2_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, &expr1_0)?; @@ -1147,7 +1157,7 @@ pub fn constructor_x64_load( let pattern2_0 = arg1; let pattern3_0 = arg2; if let &ExtKind::SignExtend = pattern3_0 { - // Rule at src/isa/x64/inst.isle line 1239. + // Rule at src/isa/x64/inst.isle line 1251. let expr0_0 = C::ty_bytes(ctx, pattern1_0); let expr1_0: u16 = 8; let expr2_0 = C::ext_mode(ctx, expr0_0, expr1_0); @@ -1160,6 +1170,27 @@ pub fn constructor_x64_load( return None; } +// Generated as internal constructor for term xmm_load_const. +pub fn constructor_xmm_load_const( + ctx: &mut C, + arg0: Type, + arg1: VCodeConstant, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1283. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = C::writable_xmm_to_reg(ctx, expr0_0); + let expr2_0 = MInst::XmmLoadConst { + src: pattern1_0, + dst: expr1_0, + ty: pattern0_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr4_0); +} + // Generated as internal constructor for term alu_rmi_r. pub fn constructor_alu_rmi_r( ctx: &mut C, @@ -1172,7 +1203,7 @@ pub fn constructor_alu_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1277. + // Rule at src/isa/x64/inst.isle line 1296. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::AluRmiR { @@ -1197,7 +1228,7 @@ pub fn constructor_add( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1285. + // Rule at src/isa/x64/inst.isle line 1304. let expr0_0 = AluRmiROpcode::Add; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1213,7 +1244,7 @@ pub fn constructor_add_with_flags_paired( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1293. + // Rule at src/isa/x64/inst.isle line 1312. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Add; @@ -1242,7 +1273,7 @@ pub fn constructor_adc_paired( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1305. + // Rule at src/isa/x64/inst.isle line 1324. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Adc; @@ -1271,7 +1302,7 @@ pub fn constructor_sub( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1317. + // Rule at src/isa/x64/inst.isle line 1336. let expr0_0 = AluRmiROpcode::Sub; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1287,7 +1318,7 @@ pub fn constructor_sub_with_flags_paired( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1325. + // Rule at src/isa/x64/inst.isle line 1344. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sub; @@ -1316,7 +1347,7 @@ pub fn constructor_sbb_paired( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1337. + // Rule at src/isa/x64/inst.isle line 1356. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = AluRmiROpcode::Sbb; @@ -1345,7 +1376,7 @@ pub fn constructor_mul( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1349. + // Rule at src/isa/x64/inst.isle line 1368. let expr0_0 = AluRmiROpcode::Mul; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1361,7 +1392,7 @@ pub fn constructor_x64_and( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1357. + // Rule at src/isa/x64/inst.isle line 1376. let expr0_0 = AluRmiROpcode::And; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1377,7 +1408,7 @@ pub fn constructor_or( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1365. + // Rule at src/isa/x64/inst.isle line 1384. let expr0_0 = AluRmiROpcode::Or; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1393,7 +1424,7 @@ pub fn constructor_xor( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1373. + // Rule at src/isa/x64/inst.isle line 1392. let expr0_0 = AluRmiROpcode::Xor; let expr1_0 = constructor_alu_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1405,7 +1436,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == I64 { let pattern2_0 = arg1; if let Some(pattern3_0) = C::nonzero_u64_fits_in_u32(ctx, pattern2_0) { - // Rule at src/isa/x64/inst.isle line 1406. + // Rule at src/isa/x64/inst.isle line 1432. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = OperandSize::Size32; let expr2_0 = MInst::Imm { @@ -1421,7 +1452,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F32 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 1435. + // Rule at src/isa/x64/inst.isle line 1461. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); let expr2_0 = SseOpcode::Xorps; @@ -1436,7 +1467,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::xmm_to_reg(ctx, expr1_0); return Some(expr6_0); } - // Rule at src/isa/x64/inst.isle line 1390. + // Rule at src/isa/x64/inst.isle line 1409. let expr0_0 = SseOpcode::Movd; let expr1_0: Type = I32; let expr2_0 = constructor_imm(ctx, expr1_0, pattern2_0)?; @@ -1449,7 +1480,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if pattern0_0 == F64 { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 1447. + // Rule at src/isa/x64/inst.isle line 1473. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); let expr2_0 = SseOpcode::Xorpd; @@ -1464,7 +1495,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr6_0 = C::xmm_to_reg(ctx, expr1_0); return Some(expr6_0); } - // Rule at src/isa/x64/inst.isle line 1396. + // Rule at src/isa/x64/inst.isle line 1415. let expr0_0 = SseOpcode::Movq; let expr1_0: Type = I64; let expr2_0 = constructor_imm(ctx, expr1_0, pattern2_0)?; @@ -1477,7 +1508,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 1425. + // Rule at src/isa/x64/inst.isle line 1451. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::writable_xmm_to_xmm(ctx, expr0_0); let expr2_0 = constructor_sse_xor_op(ctx, pattern0_0)?; @@ -1496,7 +1527,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option if let Some(pattern1_0) = C::fits_in_64(ctx, pattern0_0) { let pattern2_0 = arg1; if pattern2_0 == 0 { - // Rule at src/isa/x64/inst.isle line 1412. + // Rule at src/isa/x64/inst.isle line 1438. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::writable_gpr_to_gpr(ctx, expr0_0); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); @@ -1513,7 +1544,7 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option let expr7_0 = C::gpr_to_reg(ctx, expr1_0); return Some(expr7_0); } - // Rule at src/isa/x64/inst.isle line 1383. + // Rule at src/isa/x64/inst.isle line 1402. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern1_0); let expr2_0 = MInst::Imm { @@ -1528,6 +1559,16 @@ pub fn constructor_imm(ctx: &mut C, arg0: Type, arg1: u64) -> Option return None; } +// Generated as internal constructor for term imm_i64. +pub fn constructor_imm_i64(ctx: &mut C, arg0: Type, arg1: i64) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 1424. + let expr0_0 = C::i64_as_u64(ctx, pattern1_0); + let expr1_0 = constructor_imm(ctx, pattern0_0, expr0_0)?; + return Some(expr1_0); +} + // Generated as internal constructor for term shift_r. pub fn constructor_shift_r( ctx: &mut C, @@ -1540,7 +1581,7 @@ pub fn constructor_shift_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1460. + // Rule at src/isa/x64/inst.isle line 1486. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::raw_operand_size_of_type(ctx, pattern0_0); let expr2_0 = MInst::ShiftR { @@ -1565,7 +1606,7 @@ pub fn constructor_x64_rotl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1470. + // Rule at src/isa/x64/inst.isle line 1496. let expr0_0 = ShiftKind::RotateLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1581,7 +1622,7 @@ pub fn constructor_x64_rotr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1475. + // Rule at src/isa/x64/inst.isle line 1501. let expr0_0 = ShiftKind::RotateRight; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1597,7 +1638,7 @@ pub fn constructor_shl( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1480. + // Rule at src/isa/x64/inst.isle line 1506. let expr0_0 = ShiftKind::ShiftLeft; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1613,7 +1654,7 @@ pub fn constructor_shr( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1485. + // Rule at src/isa/x64/inst.isle line 1511. let expr0_0 = ShiftKind::ShiftRightLogical; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1629,7 +1670,7 @@ pub fn constructor_sar( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1490. + // Rule at src/isa/x64/inst.isle line 1516. let expr0_0 = ShiftKind::ShiftRightArithmetic; let expr1_0 = constructor_shift_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1647,7 +1688,7 @@ pub fn constructor_cmp_rmi_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1495. + // Rule at src/isa/x64/inst.isle line 1521. let expr0_0 = MInst::CmpRmiR { size: pattern0_0.clone(), opcode: pattern1_0.clone(), @@ -1668,12 +1709,30 @@ pub fn constructor_cmp( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1504. + // Rule at src/isa/x64/inst.isle line 1530. let expr0_0 = CmpOpcode::Cmp; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); } +// Generated as internal constructor for term cmp_imm. +pub fn constructor_cmp_imm( + ctx: &mut C, + arg0: &OperandSize, + arg1: u32, + arg2: Gpr, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 1535. + let expr0_0 = CmpOpcode::Cmp; + let expr1_0 = RegMemImm::Imm { simm32: pattern1_0 }; + let expr2_0 = C::gpr_mem_imm_new(ctx, &expr1_0); + let expr3_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, &expr2_0, pattern2_0)?; + return Some(expr3_0); +} + // Generated as internal constructor for term xmm_cmp_rm_r. pub fn constructor_xmm_cmp_rm_r( ctx: &mut C, @@ -1684,7 +1743,7 @@ pub fn constructor_xmm_cmp_rm_r( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1509. + // Rule at src/isa/x64/inst.isle line 1540. let expr0_0 = MInst::XmmCmpRmR { op: pattern0_0.clone(), src: pattern1_0.clone(), @@ -1704,7 +1763,7 @@ pub fn constructor_fpcmp( let pattern1_0 = C::value_type(ctx, pattern0_0); if pattern1_0 == F32 { let pattern3_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1516. + // Rule at src/isa/x64/inst.isle line 1547. let expr0_0 = SseOpcode::Ucomiss; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern0_0)?; let expr2_0 = constructor_put_in_xmm(ctx, pattern3_0)?; @@ -1713,7 +1772,7 @@ pub fn constructor_fpcmp( } if pattern1_0 == F64 { let pattern3_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1518. + // Rule at src/isa/x64/inst.isle line 1549. let expr0_0 = SseOpcode::Ucomisd; let expr1_0 = constructor_put_in_xmm_mem(ctx, pattern0_0)?; let expr2_0 = constructor_put_in_xmm(ctx, pattern3_0)?; @@ -1733,7 +1792,7 @@ pub fn constructor_test( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1523. + // Rule at src/isa/x64/inst.isle line 1554. let expr0_0 = CmpOpcode::Test; let expr1_0 = constructor_cmp_rmi_r(ctx, pattern0_0, &expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -1751,7 +1810,7 @@ pub fn constructor_cmove( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1530. + // Rule at src/isa/x64/inst.isle line 1561. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Cmove { @@ -1781,7 +1840,7 @@ pub fn constructor_cmove_xmm( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1538. + // Rule at src/isa/x64/inst.isle line 1569. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::XmmCmove { @@ -1812,7 +1871,7 @@ pub fn constructor_cmove_from_values( let pattern2_0 = arg1; let pattern3_0 = arg2; let pattern4_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1549. + // Rule at src/isa/x64/inst.isle line 1580. let expr0_0 = C::put_in_regs(ctx, pattern3_0); let expr1_0 = C::put_in_regs(ctx, pattern4_0); let expr2_0 = C::temp_writable_gpr(ctx); @@ -1857,7 +1916,7 @@ pub fn constructor_cmove_from_values( let pattern3_0 = arg1; let pattern4_0 = arg2; let pattern5_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1573. + // Rule at src/isa/x64/inst.isle line 1604. let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern5_0)?; let expr2_0 = constructor_cmove_xmm(ctx, pattern2_0, pattern3_0, &expr0_0, expr1_0)?; @@ -1869,7 +1928,7 @@ pub fn constructor_cmove_from_values( let pattern3_0 = arg1; let pattern4_0 = arg2; let pattern5_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1570. + // Rule at src/isa/x64/inst.isle line 1601. let expr0_0 = constructor_put_in_gpr_mem(ctx, pattern4_0)?; let expr1_0 = constructor_put_in_gpr(ctx, pattern5_0)?; let expr2_0 = constructor_cmove(ctx, pattern2_0, pattern3_0, &expr0_0, expr1_0)?; @@ -1893,7 +1952,7 @@ pub fn constructor_cmove_or( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1580. + // Rule at src/isa/x64/inst.isle line 1611. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::CmoveOr { @@ -1926,7 +1985,7 @@ pub fn constructor_cmove_or_xmm( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1588. + // Rule at src/isa/x64/inst.isle line 1619. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::XmmCmoveOr { @@ -1960,7 +2019,7 @@ pub fn constructor_cmove_or_from_values( let pattern3_0 = arg2; let pattern4_0 = arg3; let pattern5_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1599. + // Rule at src/isa/x64/inst.isle line 1630. let expr0_0 = C::put_in_regs(ctx, pattern4_0); let expr1_0 = C::put_in_regs(ctx, pattern5_0); let expr2_0 = C::temp_writable_gpr(ctx); @@ -2008,7 +2067,7 @@ pub fn constructor_cmove_or_from_values( let pattern4_0 = arg2; let pattern5_0 = arg3; let pattern6_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1615. + // Rule at src/isa/x64/inst.isle line 1646. let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern5_0)?; let expr1_0 = constructor_put_in_xmm(ctx, pattern6_0)?; let expr2_0 = constructor_cmove_or_xmm( @@ -2023,7 +2082,7 @@ pub fn constructor_cmove_or_from_values( let pattern4_0 = arg2; let pattern5_0 = arg3; let pattern6_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1612. + // Rule at src/isa/x64/inst.isle line 1643. let expr0_0 = constructor_put_in_gpr_mem(ctx, pattern5_0)?; let expr1_0 = constructor_put_in_gpr(ctx, pattern6_0)?; let expr2_0 = @@ -2044,7 +2103,7 @@ pub fn constructor_movzx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1620. + // Rule at src/isa/x64/inst.isle line 1651. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = MInst::MovzxRmR { ext_mode: pattern1_0.clone(), @@ -2066,7 +2125,7 @@ pub fn constructor_movsx( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1627. + // Rule at src/isa/x64/inst.isle line 1658. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = MInst::MovsxRmR { ext_mode: pattern1_0.clone(), @@ -2090,7 +2149,7 @@ pub fn constructor_xmm_rm_r( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1634. + // Rule at src/isa/x64/inst.isle line 1665. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmRmR { op: pattern1_0.clone(), @@ -2107,7 +2166,7 @@ pub fn constructor_xmm_rm_r( pub fn constructor_paddb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1641. + // Rule at src/isa/x64/inst.isle line 1672. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2118,7 +2177,7 @@ pub fn constructor_paddb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_paddw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1646. + // Rule at src/isa/x64/inst.isle line 1677. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2129,7 +2188,7 @@ pub fn constructor_paddw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_paddd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1651. + // Rule at src/isa/x64/inst.isle line 1682. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Paddd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2140,7 +2199,7 @@ pub fn constructor_paddd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_paddq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1656. + // Rule at src/isa/x64/inst.isle line 1687. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Paddq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2151,7 +2210,7 @@ pub fn constructor_paddq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_paddsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1661. + // Rule at src/isa/x64/inst.isle line 1692. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2162,7 +2221,7 @@ pub fn constructor_paddsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_paddsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1666. + // Rule at src/isa/x64/inst.isle line 1697. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2173,7 +2232,7 @@ pub fn constructor_paddsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_paddusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1671. + // Rule at src/isa/x64/inst.isle line 1702. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Paddusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2184,7 +2243,7 @@ pub fn constructor_paddusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_paddusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1676. + // Rule at src/isa/x64/inst.isle line 1707. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Paddusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2195,7 +2254,7 @@ pub fn constructor_paddusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_psubb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1681. + // Rule at src/isa/x64/inst.isle line 1712. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2206,7 +2265,7 @@ pub fn constructor_psubb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_psubw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1686. + // Rule at src/isa/x64/inst.isle line 1717. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2217,7 +2276,7 @@ pub fn constructor_psubw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_psubd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1691. + // Rule at src/isa/x64/inst.isle line 1722. let expr0_0: Type = I32X4; let expr1_0 = SseOpcode::Psubd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2228,7 +2287,7 @@ pub fn constructor_psubd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_psubq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1696. + // Rule at src/isa/x64/inst.isle line 1727. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Psubq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2239,7 +2298,7 @@ pub fn constructor_psubq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_psubsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1701. + // Rule at src/isa/x64/inst.isle line 1732. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2250,7 +2309,7 @@ pub fn constructor_psubsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_psubsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1706. + // Rule at src/isa/x64/inst.isle line 1737. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2261,7 +2320,7 @@ pub fn constructor_psubsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_psubusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1711. + // Rule at src/isa/x64/inst.isle line 1742. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Psubusb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2272,7 +2331,7 @@ pub fn constructor_psubusb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_psubusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1716. + // Rule at src/isa/x64/inst.isle line 1747. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Psubusw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2283,7 +2342,7 @@ pub fn constructor_psubusw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pavgb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1721. + // Rule at src/isa/x64/inst.isle line 1752. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pavgb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2294,7 +2353,7 @@ pub fn constructor_pavgb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_pavgw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1726. + // Rule at src/isa/x64/inst.isle line 1757. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pavgw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2305,7 +2364,7 @@ pub fn constructor_pavgw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_pand(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1731. + // Rule at src/isa/x64/inst.isle line 1762. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Pand; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2316,7 +2375,7 @@ pub fn constructor_pand(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Op pub fn constructor_andps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1736. + // Rule at src/isa/x64/inst.isle line 1767. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2327,7 +2386,7 @@ pub fn constructor_andps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_andpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1741. + // Rule at src/isa/x64/inst.isle line 1772. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2338,7 +2397,7 @@ pub fn constructor_andpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_por(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1746. + // Rule at src/isa/x64/inst.isle line 1777. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Por; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2349,7 +2408,7 @@ pub fn constructor_por(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Opt pub fn constructor_orps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1751. + // Rule at src/isa/x64/inst.isle line 1782. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Orps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2360,7 +2419,7 @@ pub fn constructor_orps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Op pub fn constructor_orpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1756. + // Rule at src/isa/x64/inst.isle line 1787. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Orpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2371,7 +2430,7 @@ pub fn constructor_orpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Op pub fn constructor_pxor(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1761. + // Rule at src/isa/x64/inst.isle line 1792. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pxor; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2382,7 +2441,7 @@ pub fn constructor_pxor(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Op pub fn constructor_xorps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1766. + // Rule at src/isa/x64/inst.isle line 1797. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Xorps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2393,7 +2452,7 @@ pub fn constructor_xorps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_xorpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1771. + // Rule at src/isa/x64/inst.isle line 1802. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Xorpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2404,7 +2463,7 @@ pub fn constructor_xorpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_pmullw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1776. + // Rule at src/isa/x64/inst.isle line 1807. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmullw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2415,7 +2474,7 @@ pub fn constructor_pmullw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmulld(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1781. + // Rule at src/isa/x64/inst.isle line 1812. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulld; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2426,7 +2485,7 @@ pub fn constructor_pmulld(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmulhw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1786. + // Rule at src/isa/x64/inst.isle line 1817. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2437,7 +2496,7 @@ pub fn constructor_pmulhw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmulhuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1791. + // Rule at src/isa/x64/inst.isle line 1822. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmulhuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2448,7 +2507,7 @@ pub fn constructor_pmulhuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmuldq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1796. + // Rule at src/isa/x64/inst.isle line 1827. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Pmuldq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2459,7 +2518,7 @@ pub fn constructor_pmuldq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmuludq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1801. + // Rule at src/isa/x64/inst.isle line 1832. let expr0_0: Type = I64X2; let expr1_0 = SseOpcode::Pmuludq; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2470,7 +2529,7 @@ pub fn constructor_pmuludq(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_punpckhwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1806. + // Rule at src/isa/x64/inst.isle line 1837. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpckhwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2481,7 +2540,7 @@ pub fn constructor_punpckhwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_punpcklwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1811. + // Rule at src/isa/x64/inst.isle line 1842. let expr0_0: Type = I16X8; let expr1_0 = SseOpcode::Punpcklwd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2492,7 +2551,7 @@ pub fn constructor_punpcklwd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_andnps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1816. + // Rule at src/isa/x64/inst.isle line 1847. let expr0_0: Type = F32X4; let expr1_0 = SseOpcode::Andnps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2503,7 +2562,7 @@ pub fn constructor_andnps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_andnpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1821. + // Rule at src/isa/x64/inst.isle line 1852. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Andnpd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2514,7 +2573,7 @@ pub fn constructor_andnpd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pandn(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1826. + // Rule at src/isa/x64/inst.isle line 1857. let expr0_0: Type = F64X2; let expr1_0 = SseOpcode::Pandn; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2525,17 +2584,17 @@ pub fn constructor_pandn(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1830. + // Rule at src/isa/x64/inst.isle line 1861. let expr0_0 = SseOpcode::Blendvps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1831. + // Rule at src/isa/x64/inst.isle line 1862. let expr0_0 = SseOpcode::Blendvpd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1832. + // Rule at src/isa/x64/inst.isle line 1863. let expr0_0 = SseOpcode::Pblendvb; return Some(expr0_0); } @@ -2546,17 +2605,17 @@ pub fn constructor_sse_blend_op(ctx: &mut C, arg0: Type) -> Option(ctx: &mut C, arg0: Type) -> Option { let pattern0_0 = arg0; if pattern0_0 == F32X4 { - // Rule at src/isa/x64/inst.isle line 1835. + // Rule at src/isa/x64/inst.isle line 1866. let expr0_0 = SseOpcode::Movaps; return Some(expr0_0); } if pattern0_0 == F64X2 { - // Rule at src/isa/x64/inst.isle line 1836. + // Rule at src/isa/x64/inst.isle line 1867. let expr0_0 = SseOpcode::Movapd; return Some(expr0_0); } if let Some((pattern1_0, pattern1_1)) = C::multi_lane(ctx, pattern0_0) { - // Rule at src/isa/x64/inst.isle line 1837. + // Rule at src/isa/x64/inst.isle line 1868. let expr0_0 = SseOpcode::Movdqa; return Some(expr0_0); } @@ -2575,7 +2634,7 @@ pub fn constructor_sse_blend( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1841. + // Rule at src/isa/x64/inst.isle line 1872. let expr0_0 = C::xmm0(ctx); let expr1_0 = constructor_sse_mov_op(ctx, pattern0_0)?; let expr2_0 = MInst::XmmUnaryRmR { @@ -2599,7 +2658,7 @@ pub fn constructor_blendvpd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1855. + // Rule at src/isa/x64/inst.isle line 1886. let expr0_0 = C::xmm0(ctx); let expr1_0 = SseOpcode::Movapd; let expr2_0 = C::xmm_to_xmm_mem(ctx, pattern2_0); @@ -2619,7 +2678,7 @@ pub fn constructor_blendvpd( pub fn constructor_movsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1869. + // Rule at src/isa/x64/inst.isle line 1900. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2630,7 +2689,7 @@ pub fn constructor_movsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> O pub fn constructor_movlhps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1874. + // Rule at src/isa/x64/inst.isle line 1905. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Movlhps; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2641,7 +2700,7 @@ pub fn constructor_movlhps(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmaxsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1879. + // Rule at src/isa/x64/inst.isle line 1910. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2652,7 +2711,7 @@ pub fn constructor_pmaxsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmaxsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1884. + // Rule at src/isa/x64/inst.isle line 1915. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2663,7 +2722,7 @@ pub fn constructor_pmaxsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmaxsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1889. + // Rule at src/isa/x64/inst.isle line 1920. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2674,7 +2733,7 @@ pub fn constructor_pmaxsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pminsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1894. + // Rule at src/isa/x64/inst.isle line 1925. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2685,7 +2744,7 @@ pub fn constructor_pminsb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pminsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1899. + // Rule at src/isa/x64/inst.isle line 1930. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2696,7 +2755,7 @@ pub fn constructor_pminsw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pminsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1904. + // Rule at src/isa/x64/inst.isle line 1935. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminsd; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2707,7 +2766,7 @@ pub fn constructor_pminsd(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmaxub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1909. + // Rule at src/isa/x64/inst.isle line 1940. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2718,7 +2777,7 @@ pub fn constructor_pmaxub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmaxuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1914. + // Rule at src/isa/x64/inst.isle line 1945. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2729,7 +2788,7 @@ pub fn constructor_pmaxuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pmaxud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1919. + // Rule at src/isa/x64/inst.isle line 1950. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pmaxud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2740,7 +2799,7 @@ pub fn constructor_pmaxud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pminub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1924. + // Rule at src/isa/x64/inst.isle line 1955. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminub; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2751,7 +2810,7 @@ pub fn constructor_pminub(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pminuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1929. + // Rule at src/isa/x64/inst.isle line 1960. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminuw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2762,7 +2821,7 @@ pub fn constructor_pminuw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_pminud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1934. + // Rule at src/isa/x64/inst.isle line 1965. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Pminud; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2773,7 +2832,7 @@ pub fn constructor_pminud(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> pub fn constructor_punpcklbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1939. + // Rule at src/isa/x64/inst.isle line 1970. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpcklbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2784,7 +2843,7 @@ pub fn constructor_punpcklbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_punpckhbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1944. + // Rule at src/isa/x64/inst.isle line 1975. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Punpckhbw; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2795,7 +2854,7 @@ pub fn constructor_punpckhbw(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) pub fn constructor_packsswb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 1949. + // Rule at src/isa/x64/inst.isle line 1980. let expr0_0: Type = I8X16; let expr1_0 = SseOpcode::Packsswb; let expr2_0 = constructor_xmm_rm_r(ctx, expr0_0, &expr1_0, pattern0_0, pattern1_0)?; @@ -2816,7 +2875,7 @@ pub fn constructor_xmm_rm_r_imm( let pattern2_0 = arg2; let pattern3_0 = arg3; let pattern4_0 = arg4; - // Rule at src/isa/x64/inst.isle line 1954. + // Rule at src/isa/x64/inst.isle line 1985. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = C::writable_xmm_to_reg(ctx, expr0_0); let expr2_0 = MInst::XmmRmRImm { @@ -2844,7 +2903,7 @@ pub fn constructor_palignr( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 1966. + // Rule at src/isa/x64/inst.isle line 1997. let expr0_0 = SseOpcode::Palignr; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); @@ -2863,7 +2922,7 @@ pub fn constructor_cmpps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1975. + // Rule at src/isa/x64/inst.isle line 2006. let expr0_0 = SseOpcode::Cmpps; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); @@ -2883,7 +2942,7 @@ pub fn constructor_pinsrb( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1984. + // Rule at src/isa/x64/inst.isle line 2015. let expr0_0 = SseOpcode::Pinsrb; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); @@ -2902,7 +2961,7 @@ pub fn constructor_pinsrw( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 1993. + // Rule at src/isa/x64/inst.isle line 2024. let expr0_0 = SseOpcode::Pinsrw; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); @@ -2923,7 +2982,7 @@ pub fn constructor_pinsrd( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 2002. + // Rule at src/isa/x64/inst.isle line 2033. let expr0_0 = SseOpcode::Pinsrd; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::gpr_mem_to_reg_mem(ctx, pattern1_0); @@ -2942,7 +3001,7 @@ pub fn constructor_insertps( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2011. + // Rule at src/isa/x64/inst.isle line 2042. let expr0_0 = SseOpcode::Insertps; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); @@ -2961,7 +3020,7 @@ pub fn constructor_pshufd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2020. + // Rule at src/isa/x64/inst.isle line 2051. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = SseOpcode::Pshufd; let expr2_0 = constructor_writable_xmm_to_r_reg(ctx, expr0_0)?; @@ -2980,6 +3039,24 @@ pub fn constructor_pshufd( return Some(expr7_0); } +// Generated as internal constructor for term pshufb. +pub fn constructor_pshufb(ctx: &mut C, arg0: Xmm, arg1: &XmmMem) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 2063. + let expr0_0 = C::temp_writable_xmm(ctx); + let expr1_0 = SseOpcode::Pshufb; + let expr2_0 = MInst::XmmRmR { + op: expr1_0, + src1: pattern0_0, + src2: pattern1_0.clone(), + dst: expr0_0, + }; + let expr3_0 = C::emit(ctx, &expr2_0); + let expr4_0 = C::writable_xmm_to_xmm(ctx, expr0_0); + return Some(expr4_0); +} + // Generated as internal constructor for term xmm_unary_rm_r. pub fn constructor_xmm_unary_rm_r( ctx: &mut C, @@ -2988,7 +3065,7 @@ pub fn constructor_xmm_unary_rm_r( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2032. + // Rule at src/isa/x64/inst.isle line 2073. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmUnaryRmR { op: pattern0_0.clone(), @@ -3003,7 +3080,7 @@ pub fn constructor_xmm_unary_rm_r( // Generated as internal constructor for term pmovsxbw. pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2039. + // Rule at src/isa/x64/inst.isle line 2080. let expr0_0 = SseOpcode::Pmovsxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3012,7 +3089,7 @@ pub fn constructor_pmovsxbw(ctx: &mut C, arg0: &XmmMem) -> Option(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2044. + // Rule at src/isa/x64/inst.isle line 2085. let expr0_0 = SseOpcode::Pmovzxbw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3021,7 +3098,7 @@ pub fn constructor_pmovzxbw(ctx: &mut C, arg0: &XmmMem) -> Option(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2049. + // Rule at src/isa/x64/inst.isle line 2090. let expr0_0 = SseOpcode::Pabsb; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3030,7 +3107,7 @@ pub fn constructor_pabsb(ctx: &mut C, arg0: &XmmMem) -> Option // Generated as internal constructor for term pabsw. pub fn constructor_pabsw(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2054. + // Rule at src/isa/x64/inst.isle line 2095. let expr0_0 = SseOpcode::Pabsw; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3039,7 +3116,7 @@ pub fn constructor_pabsw(ctx: &mut C, arg0: &XmmMem) -> Option // Generated as internal constructor for term pabsd. pub fn constructor_pabsd(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2059. + // Rule at src/isa/x64/inst.isle line 2100. let expr0_0 = SseOpcode::Pabsd; let expr1_0 = constructor_xmm_unary_rm_r(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); @@ -3053,7 +3130,7 @@ pub fn constructor_xmm_unary_rm_r_evex( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2064. + // Rule at src/isa/x64/inst.isle line 2105. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmUnaryRmREvex { op: pattern0_0.clone(), @@ -3068,12 +3145,21 @@ pub fn constructor_xmm_unary_rm_r_evex( // Generated as internal constructor for term vpabsq. pub fn constructor_vpabsq(ctx: &mut C, arg0: &XmmMem) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2071. + // Rule at src/isa/x64/inst.isle line 2112. let expr0_0 = Avx512Opcode::Vpabsq; let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; return Some(expr1_0); } +// Generated as internal constructor for term vpopcntb. +pub fn constructor_vpopcntb(ctx: &mut C, arg0: &XmmMem) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 2117. + let expr0_0 = Avx512Opcode::Vpopcntb; + let expr1_0 = constructor_xmm_unary_rm_r_evex(ctx, &expr0_0, pattern0_0)?; + return Some(expr1_0); +} + // Generated as internal constructor for term xmm_rm_r_evex. pub fn constructor_xmm_rm_r_evex( ctx: &mut C, @@ -3084,7 +3170,7 @@ pub fn constructor_xmm_rm_r_evex( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2076. + // Rule at src/isa/x64/inst.isle line 2122. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmRmREvex { op: pattern0_0.clone(), @@ -3101,7 +3187,7 @@ pub fn constructor_xmm_rm_r_evex( pub fn constructor_vpmullq(ctx: &mut C, arg0: &XmmMem, arg1: Xmm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2088. + // Rule at src/isa/x64/inst.isle line 2134. let expr0_0 = Avx512Opcode::Vpmullq; let expr1_0 = constructor_xmm_rm_r_evex(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3119,7 +3205,7 @@ pub fn constructor_mul_hi( let pattern1_0 = arg1; let pattern2_0 = arg2; let pattern3_0 = arg3; - // Rule at src/isa/x64/inst.isle line 2097. + // Rule at src/isa/x64/inst.isle line 2143. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::temp_writable_gpr(ctx); let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); @@ -3148,7 +3234,7 @@ pub fn constructor_mulhi_u( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2112. + // Rule at src/isa/x64/inst.isle line 2158. let expr0_0: bool = false; let expr1_0 = constructor_mul_hi(ctx, pattern0_0, expr0_0, pattern1_0, pattern2_0)?; return Some(expr1_0); @@ -3164,7 +3250,7 @@ pub fn constructor_xmm_rmi_xmm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2117. + // Rule at src/isa/x64/inst.isle line 2163. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::XmmRmiReg { opcode: pattern0_0.clone(), @@ -3181,7 +3267,7 @@ pub fn constructor_xmm_rmi_xmm( pub fn constructor_psllw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2127. + // Rule at src/isa/x64/inst.isle line 2173. let expr0_0 = SseOpcode::Psllw; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3191,7 +3277,7 @@ pub fn constructor_psllw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_pslld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2132. + // Rule at src/isa/x64/inst.isle line 2178. let expr0_0 = SseOpcode::Pslld; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3201,7 +3287,7 @@ pub fn constructor_pslld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_psllq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2137. + // Rule at src/isa/x64/inst.isle line 2183. let expr0_0 = SseOpcode::Psllq; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3211,7 +3297,7 @@ pub fn constructor_psllq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_psrlw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2142. + // Rule at src/isa/x64/inst.isle line 2188. let expr0_0 = SseOpcode::Psrlw; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3221,7 +3307,7 @@ pub fn constructor_psrlw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_psrld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2147. + // Rule at src/isa/x64/inst.isle line 2193. let expr0_0 = SseOpcode::Psrld; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3231,7 +3317,7 @@ pub fn constructor_psrld(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_psrlq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2152. + // Rule at src/isa/x64/inst.isle line 2198. let expr0_0 = SseOpcode::Psrlq; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3241,7 +3327,7 @@ pub fn constructor_psrlq(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_psraw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2157. + // Rule at src/isa/x64/inst.isle line 2203. let expr0_0 = SseOpcode::Psraw; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3251,7 +3337,7 @@ pub fn constructor_psraw(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) - pub fn constructor_psrad(ctx: &mut C, arg0: Xmm, arg1: &XmmMemImm) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2162. + // Rule at src/isa/x64/inst.isle line 2208. let expr0_0 = SseOpcode::Psrad; let expr1_0 = constructor_xmm_rmi_xmm(ctx, &expr0_0, pattern0_0, pattern1_0)?; return Some(expr1_0); @@ -3262,7 +3348,7 @@ pub fn constructor_pextrd(ctx: &mut C, arg0: Type, arg1: Xmm, arg2: let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2167. + // Rule at src/isa/x64/inst.isle line 2213. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = SseOpcode::Pextrd; let expr2_0 = constructor_writable_gpr_to_r_reg(ctx, expr0_0)?; @@ -3295,7 +3381,7 @@ pub fn constructor_cmppd( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2183. + // Rule at src/isa/x64/inst.isle line 2229. let expr0_0 = SseOpcode::Cmppd; let expr1_0 = C::xmm_to_reg(ctx, pattern0_0); let expr2_0 = C::xmm_mem_to_reg_mem(ctx, pattern1_0); @@ -3315,7 +3401,7 @@ pub fn constructor_gpr_to_xmm( let pattern0_0 = arg0; let pattern1_0 = arg1; let pattern2_0 = arg2; - // Rule at src/isa/x64/inst.isle line 2192. + // Rule at src/isa/x64/inst.isle line 2238. let expr0_0 = C::temp_writable_xmm(ctx); let expr1_0 = MInst::GprToXmm { op: pattern0_0.clone(), @@ -3332,7 +3418,7 @@ pub fn constructor_gpr_to_xmm( pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2199. + // Rule at src/isa/x64/inst.isle line 2245. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Not { @@ -3349,7 +3435,7 @@ pub fn constructor_not(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/isa/x64/inst.isle line 2207. + // Rule at src/isa/x64/inst.isle line 2253. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); let expr2_0 = MInst::Neg { @@ -3365,7 +3451,7 @@ pub fn constructor_neg(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option // Generated as internal constructor for term lea. pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2214. + // Rule at src/isa/x64/inst.isle line 2260. let expr0_0 = C::temp_writable_gpr(ctx); let expr1_0 = MInst::LoadEffectiveAddress { addr: pattern0_0.clone(), @@ -3379,7 +3465,7 @@ pub fn constructor_lea(ctx: &mut C, arg0: &SyntheticAmode) -> Option // Generated as internal constructor for term ud2. pub fn constructor_ud2(ctx: &mut C, arg0: &TrapCode) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2221. + // Rule at src/isa/x64/inst.isle line 2267. let expr0_0 = MInst::Ud2 { trap_code: pattern0_0.clone(), }; @@ -3387,10 +3473,186 @@ pub fn constructor_ud2(ctx: &mut C, arg0: &TrapCode) -> Option(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 2272. + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); + let expr2_0 = UnaryRmROpcode::Lzcnt; + let expr3_0 = C::gpr_to_gpr_mem(ctx, pattern1_0); + let expr4_0 = MInst::UnaryRmR { + size: expr1_0, + op: expr2_0, + src: expr3_0, + dst: expr0_0, + }; + let expr5_0 = C::emit(ctx, &expr4_0); + let expr6_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + return Some(expr6_0); +} + +// Generated as internal constructor for term tzcnt. +pub fn constructor_tzcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 2280. + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); + let expr2_0 = UnaryRmROpcode::Tzcnt; + let expr3_0 = C::gpr_to_gpr_mem(ctx, pattern1_0); + let expr4_0 = MInst::UnaryRmR { + size: expr1_0, + op: expr2_0, + src: expr3_0, + dst: expr0_0, + }; + let expr5_0 = C::emit(ctx, &expr4_0); + let expr6_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + return Some(expr6_0); +} + +// Generated as internal constructor for term bsr. +pub fn constructor_bsr(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 2288. + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); + let expr2_0 = UnaryRmROpcode::Bsr; + let expr3_0 = C::gpr_to_gpr_mem(ctx, pattern1_0); + let expr4_0 = MInst::UnaryRmR { + size: expr1_0, + op: expr2_0, + src: expr3_0, + dst: expr0_0, + }; + let expr5_0 = C::emit(ctx, &expr4_0); + let expr6_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + return Some(expr6_0); +} + +// Generated as internal constructor for term bsr_or_else. +pub fn constructor_bsr_or_else( + ctx: &mut C, + arg0: Type, + arg1: Gpr, + arg2: Gpr, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 2297. + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = C::temp_writable_gpr(ctx); + let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); + let expr3_0 = UnaryRmROpcode::Bsr; + let expr4_0 = C::gpr_to_gpr_mem(ctx, pattern1_0); + let expr5_0 = MInst::UnaryRmR { + size: expr2_0, + op: expr3_0, + src: expr4_0, + dst: expr1_0, + }; + let expr6_0 = C::emit(ctx, &expr5_0); + let expr7_0 = CC::Z; + let expr8_0 = C::gpr_to_gpr_mem(ctx, pattern2_0); + let expr9_0 = C::writable_gpr_to_gpr(ctx, expr1_0); + let expr10_0 = MInst::Cmove { + size: expr2_0, + cc: expr7_0, + consequent: expr8_0, + alternative: expr9_0, + dst: expr0_0, + }; + let expr11_0 = C::emit(ctx, &expr10_0); + let expr12_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + return Some(expr12_0); +} + +// Generated as internal constructor for term bsf. +pub fn constructor_bsf(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 2307. + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); + let expr2_0 = UnaryRmROpcode::Bsf; + let expr3_0 = C::gpr_to_gpr_mem(ctx, pattern1_0); + let expr4_0 = MInst::UnaryRmR { + size: expr1_0, + op: expr2_0, + src: expr3_0, + dst: expr0_0, + }; + let expr5_0 = C::emit(ctx, &expr4_0); + let expr6_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + return Some(expr6_0); +} + +// Generated as internal constructor for term bsf_or_else. +pub fn constructor_bsf_or_else( + ctx: &mut C, + arg0: Type, + arg1: Gpr, + arg2: Gpr, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/inst.isle line 2316. + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = C::temp_writable_gpr(ctx); + let expr2_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); + let expr3_0 = UnaryRmROpcode::Bsf; + let expr4_0 = C::gpr_to_gpr_mem(ctx, pattern1_0); + let expr5_0 = MInst::UnaryRmR { + size: expr2_0, + op: expr3_0, + src: expr4_0, + dst: expr1_0, + }; + let expr6_0 = C::emit(ctx, &expr5_0); + let expr7_0 = CC::Z; + let expr8_0 = C::gpr_to_gpr_mem(ctx, pattern2_0); + let expr9_0 = C::writable_gpr_to_gpr(ctx, expr1_0); + let expr10_0 = MInst::Cmove { + size: expr2_0, + cc: expr7_0, + consequent: expr8_0, + alternative: expr9_0, + dst: expr0_0, + }; + let expr11_0 = C::emit(ctx, &expr10_0); + let expr12_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + return Some(expr12_0); +} + +// Generated as internal constructor for term x64_popcnt. +pub fn constructor_x64_popcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + // Rule at src/isa/x64/inst.isle line 2326. + let expr0_0 = C::temp_writable_gpr(ctx); + let expr1_0 = C::operand_size_of_type_32_64(ctx, pattern0_0); + let expr2_0 = UnaryRmROpcode::Popcnt; + let expr3_0 = C::gpr_to_gpr_mem(ctx, pattern1_0); + let expr4_0 = MInst::UnaryRmR { + size: expr1_0, + op: expr2_0, + src: expr3_0, + dst: expr0_0, + }; + let expr5_0 = C::emit(ctx, &expr4_0); + let expr6_0 = C::writable_gpr_to_gpr(ctx, expr0_0); + return Some(expr6_0); +} + // Generated as internal constructor for term reg_to_xmm_mem. pub fn constructor_reg_to_xmm_mem(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2270. + // Rule at src/isa/x64/inst.isle line 2380. let expr0_0 = C::xmm_new(ctx, pattern0_0); let expr1_0 = C::xmm_to_xmm_mem(ctx, expr0_0); return Some(expr1_0); @@ -3399,7 +3661,7 @@ pub fn constructor_reg_to_xmm_mem(ctx: &mut C, arg0: Reg) -> Option< // Generated as internal constructor for term xmm_to_reg_mem. pub fn constructor_xmm_to_reg_mem(ctx: &mut C, arg0: Reg) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2273. + // Rule at src/isa/x64/inst.isle line 2383. let expr0_0 = C::xmm_new(ctx, pattern0_0); let expr1_0 = C::xmm_to_reg(ctx, expr0_0); let expr2_0 = RegMem::Reg { reg: expr1_0 }; @@ -3413,19 +3675,31 @@ pub fn constructor_writable_gpr_to_r_reg( arg0: WritableGpr, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2277. + // Rule at src/isa/x64/inst.isle line 2387. let expr0_0 = C::writable_gpr_to_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); } +// Generated as internal constructor for term writable_gpr_to_gpr_mem. +pub fn constructor_writable_gpr_to_gpr_mem( + ctx: &mut C, + arg0: WritableGpr, +) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/inst.isle line 2390. + let expr0_0 = C::writable_gpr_to_gpr(ctx, pattern0_0); + let expr1_0 = C::gpr_to_gpr_mem(ctx, expr0_0); + return Some(expr1_0); +} + // Generated as internal constructor for term writable_xmm_to_r_reg. pub fn constructor_writable_xmm_to_r_reg( ctx: &mut C, arg0: WritableXmm, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2280. + // Rule at src/isa/x64/inst.isle line 2393. let expr0_0 = C::writable_xmm_to_reg(ctx, pattern0_0); let expr1_0 = C::writable_reg_to_reg(ctx, expr0_0); return Some(expr1_0); @@ -3437,7 +3711,7 @@ pub fn constructor_writable_xmm_to_xmm_mem( arg0: WritableXmm, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2283. + // Rule at src/isa/x64/inst.isle line 2396. let expr0_0 = C::writable_xmm_to_xmm(ctx, pattern0_0); let expr1_0 = C::xmm_to_xmm_mem(ctx, expr0_0); return Some(expr1_0); @@ -3449,7 +3723,7 @@ pub fn constructor_synthetic_amode_to_gpr_mem( arg0: &SyntheticAmode, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2287. + // Rule at src/isa/x64/inst.isle line 2400. let expr0_0 = C::synthetic_amode_to_reg_mem(ctx, pattern0_0); let expr1_0 = C::reg_mem_to_gpr_mem(ctx, &expr0_0); return Some(expr1_0); @@ -3461,7 +3735,7 @@ pub fn constructor_synthetic_amode_to_xmm_mem( arg0: &SyntheticAmode, ) -> Option { let pattern0_0 = arg0; - // Rule at src/isa/x64/inst.isle line 2290. + // Rule at src/isa/x64/inst.isle line 2403. let expr0_0 = C::synthetic_amode_to_reg_mem(ctx, pattern0_0); let expr1_0 = C::reg_mem_to_xmm_mem(ctx, &expr0_0); return Some(expr1_0); @@ -3469,6 +3743,136 @@ pub fn constructor_synthetic_amode_to_xmm_mem( // Generated as internal constructor for term lower. pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let pattern0_0 = arg0; + if let Some(pattern1_0) = C::first_result(ctx, pattern0_0) { + let pattern2_0 = C::value_type(ctx, pattern1_0); + if pattern2_0 == I128 { + if let Some(()) = C::use_popcnt(ctx, pattern2_0) { + let pattern5_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Unary { + opcode: ref pattern6_0, + arg: pattern6_1, + } = &pattern5_0 + { + if let &Opcode::Popcnt = pattern6_0 { + // Rule at src/isa/x64/lower.isle line 1628. + let expr0_0: Type = I64; + let expr1_0 = C::put_in_regs(ctx, pattern6_1); + let expr2_0: usize = 0; + let expr3_0 = constructor_value_regs_get_gpr(ctx, expr1_0, expr2_0)?; + let expr4_0 = constructor_x64_popcnt(ctx, expr0_0, expr3_0)?; + let expr5_0: Type = I64; + let expr6_0 = C::put_in_regs(ctx, pattern6_1); + let expr7_0: usize = 1; + let expr8_0 = constructor_value_regs_get_gpr(ctx, expr6_0, expr7_0)?; + let expr9_0 = constructor_x64_popcnt(ctx, expr5_0, expr8_0)?; + let expr10_0: Type = I64; + let expr11_0 = C::gpr_to_gpr_mem_imm(ctx, expr9_0); + let expr12_0 = constructor_add(ctx, expr10_0, expr4_0, &expr11_0)?; + let expr13_0 = C::gpr_to_reg(ctx, expr12_0); + let expr14_0: Type = I64; + let expr15_0: u64 = 0; + let expr16_0 = constructor_imm(ctx, expr14_0, expr15_0)?; + let expr17_0 = C::value_regs(ctx, expr13_0, expr16_0); + let expr18_0 = C::output(ctx, expr17_0); + return Some(expr18_0); + } + } + } + } + if pattern2_0 == I8X16 { + if let Some(()) = C::avx512vl_enabled(ctx, pattern2_0) { + if let Some(()) = C::avx512bitalg_enabled(ctx, pattern2_0) { + let pattern6_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Unary { + opcode: ref pattern7_0, + arg: pattern7_1, + } = &pattern6_0 + { + if let &Opcode::Popcnt = pattern7_0 { + // Rule at src/isa/x64/lower.isle line 1695. + let expr0_0 = constructor_put_in_xmm_mem(ctx, pattern7_1)?; + let expr1_0 = constructor_vpopcntb(ctx, &expr0_0)?; + let expr2_0 = constructor_output_xmm(ctx, expr1_0)?; + return Some(expr2_0); + } + } + } + } + } + if let Some(pattern3_0) = C::ty_32_or_64(ctx, pattern2_0) { + if let Some(()) = C::use_lzcnt(ctx, pattern2_0) { + let pattern5_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Unary { + opcode: ref pattern6_0, + arg: pattern6_1, + } = &pattern5_0 + { + if let &Opcode::Clz = pattern6_0 { + // Rule at src/isa/x64/lower.isle line 1536. + let expr0_0 = constructor_put_in_gpr(ctx, pattern6_1)?; + let expr1_0 = constructor_lzcnt(ctx, pattern3_0, expr0_0)?; + let expr2_0 = constructor_output_gpr(ctx, expr1_0)?; + return Some(expr2_0); + } + } + } + if let Some(()) = C::use_bmi1(ctx, pattern2_0) { + let pattern5_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Unary { + opcode: ref pattern6_0, + arg: pattern6_1, + } = &pattern5_0 + { + if let &Opcode::Ctz = pattern6_0 { + // Rule at src/isa/x64/lower.isle line 1578. + let expr0_0 = constructor_put_in_gpr(ctx, pattern6_1)?; + let expr1_0 = constructor_tzcnt(ctx, pattern3_0, expr0_0)?; + let expr2_0 = constructor_output_gpr(ctx, expr1_0)?; + return Some(expr2_0); + } + } + } + if let Some(()) = C::use_popcnt(ctx, pattern2_0) { + let pattern5_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Unary { + opcode: ref pattern6_0, + arg: pattern6_1, + } = &pattern5_0 + { + if let &Opcode::Popcnt = pattern6_0 { + // Rule at src/isa/x64/lower.isle line 1614. + let expr0_0 = constructor_put_in_gpr(ctx, pattern6_1)?; + let expr1_0 = constructor_x64_popcnt(ctx, pattern3_0, expr0_0)?; + let expr2_0 = constructor_output_gpr(ctx, expr1_0)?; + return Some(expr2_0); + } + } + } + } + if let Some(pattern3_0) = C::ty_8_or_16(ctx, pattern2_0) { + if let Some(()) = C::use_popcnt(ctx, pattern2_0) { + let pattern5_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Unary { + opcode: ref pattern6_0, + arg: pattern6_1, + } = &pattern5_0 + { + if let &Opcode::Popcnt = pattern6_0 { + // Rule at src/isa/x64/lower.isle line 1621. + let expr0_0: Type = I32; + let expr1_0: Type = I32; + let expr2_0 = ExtendKind::Zero; + let expr3_0 = + constructor_extend_to_gpr(ctx, pattern6_1, expr1_0, &expr2_0)?; + let expr4_0 = constructor_x64_popcnt(ctx, expr0_0, expr3_0)?; + let expr5_0 = constructor_output_gpr(ctx, expr4_0)?; + return Some(expr5_0); + } + } + } + } + } let pattern0_0 = arg0; let pattern1_0 = C::inst_data(ctx, pattern0_0); match &pattern1_0 { @@ -3882,11 +4286,111 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - if let &Opcode::Bnot = pattern5_0 { - // Rule at src/isa/x64/lower.isle line 1266. - let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; - let expr1_0 = C::output(ctx, expr0_0); - return Some(expr1_0); + match pattern5_0 { + &Opcode::Bnot => { + // Rule at src/isa/x64/lower.isle line 1266. + let expr0_0 = constructor_i128_not(ctx, pattern5_1)?; + let expr1_0 = C::output(ctx, expr0_0); + return Some(expr1_0); + } + &Opcode::Clz => { + // Rule at src/isa/x64/lower.isle line 1553. + let expr0_0: Type = I64; + let expr1_0: Type = I64; + let expr2_0 = C::put_in_regs(ctx, pattern5_1); + let expr3_0: usize = 1; + let expr4_0 = constructor_value_regs_get_gpr(ctx, expr2_0, expr3_0)?; + let expr5_0 = constructor_do_clz(ctx, expr0_0, expr1_0, expr4_0)?; + let expr6_0: Type = I64; + let expr7_0: Type = I64; + let expr8_0: Type = I64; + let expr9_0 = C::put_in_regs(ctx, pattern5_1); + let expr10_0: usize = 0; + let expr11_0 = constructor_value_regs_get_gpr(ctx, expr9_0, expr10_0)?; + let expr12_0 = constructor_do_clz(ctx, expr7_0, expr8_0, expr11_0)?; + let expr13_0: u32 = 64; + let expr14_0 = RegMemImm::Imm { simm32: expr13_0 }; + let expr15_0 = C::gpr_mem_imm_new(ctx, &expr14_0); + let expr16_0 = constructor_add(ctx, expr6_0, expr12_0, &expr15_0)?; + let expr17_0 = OperandSize::Size64; + let expr18_0: u32 = 64; + let expr19_0 = constructor_cmp_imm(ctx, &expr17_0, expr18_0, expr5_0)?; + let expr20_0: Type = I64; + let expr21_0 = CC::NZ; + let expr22_0 = C::gpr_to_gpr_mem(ctx, expr5_0); + let expr23_0 = + constructor_cmove(ctx, expr20_0, &expr21_0, &expr22_0, expr16_0)?; + let expr24_0 = constructor_with_flags_reg(ctx, &expr19_0, &expr23_0)?; + let expr25_0 = C::gpr_new(ctx, expr24_0); + let expr26_0 = C::gpr_to_reg(ctx, expr25_0); + let expr27_0: Type = I64; + let expr28_0: u64 = 0; + let expr29_0 = constructor_imm(ctx, expr27_0, expr28_0)?; + let expr30_0 = C::value_regs(ctx, expr26_0, expr29_0); + let expr31_0 = C::output(ctx, expr30_0); + return Some(expr31_0); + } + &Opcode::Ctz => { + // Rule at src/isa/x64/lower.isle line 1595. + let expr0_0: Type = I64; + let expr1_0: Type = I64; + let expr2_0 = C::put_in_regs(ctx, pattern5_1); + let expr3_0: usize = 0; + let expr4_0 = constructor_value_regs_get_gpr(ctx, expr2_0, expr3_0)?; + let expr5_0 = constructor_do_ctz(ctx, expr0_0, expr1_0, expr4_0)?; + let expr6_0: Type = I64; + let expr7_0: Type = I64; + let expr8_0: Type = I64; + let expr9_0 = C::put_in_regs(ctx, pattern5_1); + let expr10_0: usize = 1; + let expr11_0 = constructor_value_regs_get_gpr(ctx, expr9_0, expr10_0)?; + let expr12_0 = constructor_do_ctz(ctx, expr7_0, expr8_0, expr11_0)?; + let expr13_0: u32 = 64; + let expr14_0 = RegMemImm::Imm { simm32: expr13_0 }; + let expr15_0 = C::gpr_mem_imm_new(ctx, &expr14_0); + let expr16_0 = constructor_add(ctx, expr6_0, expr12_0, &expr15_0)?; + let expr17_0 = OperandSize::Size64; + let expr18_0: u32 = 64; + let expr19_0 = constructor_cmp_imm(ctx, &expr17_0, expr18_0, expr5_0)?; + let expr20_0: Type = I64; + let expr21_0 = CC::Z; + let expr22_0 = C::gpr_to_gpr_mem(ctx, expr16_0); + let expr23_0 = + constructor_cmove(ctx, expr20_0, &expr21_0, &expr22_0, expr5_0)?; + let expr24_0 = constructor_with_flags_reg(ctx, &expr19_0, &expr23_0)?; + let expr25_0 = C::gpr_new(ctx, expr24_0); + let expr26_0 = C::gpr_to_reg(ctx, expr25_0); + let expr27_0: Type = I64; + let expr28_0: u64 = 0; + let expr29_0 = constructor_imm(ctx, expr27_0, expr28_0)?; + let expr30_0 = C::value_regs(ctx, expr26_0, expr29_0); + let expr31_0 = C::output(ctx, expr30_0); + return Some(expr31_0); + } + &Opcode::Popcnt => { + // Rule at src/isa/x64/lower.isle line 1647. + let expr0_0: Type = I64; + let expr1_0 = C::put_in_regs(ctx, pattern5_1); + let expr2_0: usize = 0; + let expr3_0 = constructor_value_regs_get_gpr(ctx, expr1_0, expr2_0)?; + let expr4_0 = constructor_do_popcnt(ctx, expr0_0, expr3_0)?; + let expr5_0: Type = I64; + let expr6_0 = C::put_in_regs(ctx, pattern5_1); + let expr7_0: usize = 1; + let expr8_0 = constructor_value_regs_get_gpr(ctx, expr6_0, expr7_0)?; + let expr9_0 = constructor_do_popcnt(ctx, expr5_0, expr8_0)?; + let expr10_0: Type = I64; + let expr11_0 = C::gpr_to_gpr_mem_imm(ctx, expr9_0); + let expr12_0 = constructor_add(ctx, expr10_0, expr4_0, &expr11_0)?; + let expr13_0 = C::gpr_to_reg(ctx, expr12_0); + let expr14_0: Type = I64; + let expr15_0: u64 = 0; + let expr16_0 = constructor_imm(ctx, expr14_0, expr15_0)?; + let expr17_0 = C::value_regs(ctx, expr13_0, expr16_0); + let expr18_0 = C::output(ctx, expr17_0); + return Some(expr18_0); + } + _ => {} } } &InstructionData::BinaryImm64 { @@ -4049,6 +4553,36 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + // Rule at src/isa/x64/lower.isle line 1729. + let expr0_0 = C::popcount_4bit_table(ctx); + let expr1_0: Type = I8X16; + let expr2_0 = C::popcount_low_mask(ctx); + let expr3_0 = constructor_xmm_load_const(ctx, expr1_0, expr2_0)?; + let expr4_0: Type = I8X16; + let expr5_0 = constructor_put_in_xmm(ctx, pattern5_1)?; + let expr6_0 = C::xmm_to_xmm_mem(ctx, expr3_0); + let expr7_0 = constructor_sse_and(ctx, expr4_0, expr5_0, &expr6_0)?; + let expr8_0 = constructor_put_in_xmm(ctx, pattern5_1)?; + let expr9_0: u32 = 4; + let expr10_0 = RegMemImm::Imm { simm32: expr9_0 }; + let expr11_0 = constructor_mov_rmi_to_xmm(ctx, &expr10_0)?; + let expr12_0 = constructor_psrlw(ctx, expr8_0, &expr11_0)?; + let expr13_0: Type = I8X16; + let expr14_0 = C::xmm_to_xmm_mem(ctx, expr3_0); + let expr15_0 = constructor_sse_and(ctx, expr13_0, expr12_0, &expr14_0)?; + let expr16_0: Type = I8X16; + let expr17_0 = C::popcount_4bit_table(ctx); + let expr18_0 = constructor_xmm_load_const(ctx, expr16_0, expr17_0)?; + let expr19_0 = C::xmm_to_xmm_mem(ctx, expr7_0); + let expr20_0 = constructor_pshufb(ctx, expr18_0, &expr19_0)?; + let expr21_0 = C::xmm_to_xmm_mem(ctx, expr15_0); + let expr22_0 = constructor_pshufb(ctx, expr18_0, &expr21_0)?; + let expr23_0 = C::xmm_to_xmm_mem(ctx, expr22_0); + let expr24_0 = constructor_paddb(ctx, expr20_0, &expr23_0)?; + let expr25_0 = constructor_output_xmm(ctx, expr24_0)?; + return Some(expr25_0); + } _ => {} } } @@ -6243,152 +6777,242 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { - let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { - let pattern9_0 = C::inst_data(ctx, pattern8_0); - if let &InstructionData::UnaryImm { - opcode: ref pattern10_0, - imm: pattern10_1, - } = &pattern9_0 - { - if let &Opcode::Iconst = pattern10_0 { - let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); - // Rule at src/isa/x64/lower.isle line 823. - let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; - let expr1_0 = - C::const_to_type_masked_imm8(ctx, pattern12_0, pattern3_0); - let expr2_0 = - constructor_x64_rotl(ctx, pattern3_0, expr0_0, &expr1_0)?; - let expr3_0 = constructor_output_gpr(ctx, expr2_0)?; - return Some(expr3_0); + match &pattern4_0 { + &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } => { + match pattern5_0 { + &Opcode::Rotl => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Iconst = pattern10_0 { + let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); + // Rule at src/isa/x64/lower.isle line 823. + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr1_0 = C::const_to_type_masked_imm8( + ctx, + pattern12_0, + pattern3_0, + ); + let expr2_0 = constructor_x64_rotl( + ctx, pattern3_0, expr0_0, &expr1_0, + )?; + let expr3_0 = constructor_output_gpr(ctx, expr2_0)?; + return Some(expr3_0); + } } } + // Rule at src/isa/x64/lower.isle line 817. + let expr0_0 = constructor_lo_gpr(ctx, pattern7_1)?; + let expr1_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr2_0 = C::gpr_to_imm8_gpr(ctx, expr0_0); + let expr3_0 = constructor_x64_rotl(ctx, pattern3_0, expr1_0, &expr2_0)?; + let expr4_0 = constructor_output_gpr(ctx, expr3_0)?; + return Some(expr4_0); } - // Rule at src/isa/x64/lower.isle line 817. - let expr0_0 = constructor_lo_gpr(ctx, pattern7_1)?; - let expr1_0 = constructor_put_in_gpr(ctx, pattern7_0)?; - let expr2_0 = C::gpr_to_imm8_gpr(ctx, expr0_0); - let expr3_0 = constructor_x64_rotl(ctx, pattern3_0, expr1_0, &expr2_0)?; - let expr4_0 = constructor_output_gpr(ctx, expr3_0)?; - return Some(expr4_0); - } - &Opcode::Rotr => { - let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { - let pattern9_0 = C::inst_data(ctx, pattern8_0); - if let &InstructionData::UnaryImm { - opcode: ref pattern10_0, - imm: pattern10_1, - } = &pattern9_0 - { - if let &Opcode::Iconst = pattern10_0 { - let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); - // Rule at src/isa/x64/lower.isle line 863. - let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; - let expr1_0 = - C::const_to_type_masked_imm8(ctx, pattern12_0, pattern3_0); - let expr2_0 = - constructor_x64_rotr(ctx, pattern3_0, expr0_0, &expr1_0)?; - let expr3_0 = constructor_output_gpr(ctx, expr2_0)?; - return Some(expr3_0); + &Opcode::Rotr => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Iconst = pattern10_0 { + let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); + // Rule at src/isa/x64/lower.isle line 863. + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr1_0 = C::const_to_type_masked_imm8( + ctx, + pattern12_0, + pattern3_0, + ); + let expr2_0 = constructor_x64_rotr( + ctx, pattern3_0, expr0_0, &expr1_0, + )?; + let expr3_0 = constructor_output_gpr(ctx, expr2_0)?; + return Some(expr3_0); + } } } + // Rule at src/isa/x64/lower.isle line 857. + let expr0_0 = constructor_lo_gpr(ctx, pattern7_1)?; + let expr1_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr2_0 = C::gpr_to_imm8_gpr(ctx, expr0_0); + let expr3_0 = constructor_x64_rotr(ctx, pattern3_0, expr1_0, &expr2_0)?; + let expr4_0 = constructor_output_gpr(ctx, expr3_0)?; + return Some(expr4_0); } - // Rule at src/isa/x64/lower.isle line 857. - let expr0_0 = constructor_lo_gpr(ctx, pattern7_1)?; - let expr1_0 = constructor_put_in_gpr(ctx, pattern7_0)?; - let expr2_0 = C::gpr_to_imm8_gpr(ctx, expr0_0); - let expr3_0 = constructor_x64_rotr(ctx, pattern3_0, expr1_0, &expr2_0)?; - let expr4_0 = constructor_output_gpr(ctx, expr3_0)?; - return Some(expr4_0); + _ => {} } - _ => {} } + &InstructionData::Unary { + opcode: ref pattern5_0, + arg: pattern5_1, + } => { + match pattern5_0 { + &Opcode::Clz => { + // Rule at src/isa/x64/lower.isle line 1543. + let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; + let expr1_0 = constructor_do_clz(ctx, pattern3_0, pattern3_0, expr0_0)?; + let expr2_0 = constructor_output_gpr(ctx, expr1_0)?; + return Some(expr2_0); + } + &Opcode::Ctz => { + // Rule at src/isa/x64/lower.isle line 1585. + let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; + let expr1_0 = constructor_do_ctz(ctx, pattern3_0, pattern3_0, expr0_0)?; + let expr2_0 = constructor_output_gpr(ctx, expr1_0)?; + return Some(expr2_0); + } + &Opcode::Popcnt => { + // Rule at src/isa/x64/lower.isle line 1637. + let expr0_0 = constructor_put_in_gpr(ctx, pattern5_1)?; + let expr1_0 = constructor_do_popcnt(ctx, pattern3_0, expr0_0)?; + let expr2_0 = constructor_output_gpr(ctx, expr1_0)?; + return Some(expr2_0); + } + _ => {} + } + } + _ => {} } } if let Some(pattern3_0) = C::ty_8_or_16(ctx, pattern2_0) { let pattern4_0 = C::inst_data(ctx, pattern0_0); - if let &InstructionData::Binary { - opcode: ref pattern5_0, - args: ref pattern5_1, - } = &pattern4_0 - { - match pattern5_0 { - &Opcode::Rotl => { - let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { - let pattern9_0 = C::inst_data(ctx, pattern8_0); - if let &InstructionData::UnaryImm { - opcode: ref pattern10_0, - imm: pattern10_1, - } = &pattern9_0 - { - if let &Opcode::Iconst = pattern10_0 { - let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); - // Rule at src/isa/x64/lower.isle line 809. - let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; - let expr1_0 = - C::const_to_type_masked_imm8(ctx, pattern12_0, pattern3_0); - let expr2_0 = - constructor_x64_rotl(ctx, pattern3_0, expr0_0, &expr1_0)?; - let expr3_0 = constructor_output_gpr(ctx, expr2_0)?; - return Some(expr3_0); + match &pattern4_0 { + &InstructionData::Binary { + opcode: ref pattern5_0, + args: ref pattern5_1, + } => { + match pattern5_0 { + &Opcode::Rotl => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Iconst = pattern10_0 { + let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); + // Rule at src/isa/x64/lower.isle line 809. + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr1_0 = C::const_to_type_masked_imm8( + ctx, + pattern12_0, + pattern3_0, + ); + let expr2_0 = constructor_x64_rotl( + ctx, pattern3_0, expr0_0, &expr1_0, + )?; + let expr3_0 = constructor_output_gpr(ctx, expr2_0)?; + return Some(expr3_0); + } } } + // Rule at src/isa/x64/lower.isle line 805. + let expr0_0: Type = I32; + let expr1_0 = ExtendKind::Zero; + let expr2_0 = + constructor_extend_to_gpr(ctx, pattern7_1, expr0_0, &expr1_0)?; + let expr3_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr4_0 = C::gpr_to_imm8_gpr(ctx, expr2_0); + let expr5_0 = constructor_x64_rotl(ctx, pattern3_0, expr3_0, &expr4_0)?; + let expr6_0 = constructor_output_gpr(ctx, expr5_0)?; + return Some(expr6_0); } - // Rule at src/isa/x64/lower.isle line 805. - let expr0_0: Type = I32; - let expr1_0 = ExtendKind::Zero; - let expr2_0 = - constructor_extend_to_gpr(ctx, pattern7_1, expr0_0, &expr1_0)?; - let expr3_0 = constructor_put_in_gpr(ctx, pattern7_0)?; - let expr4_0 = C::gpr_to_imm8_gpr(ctx, expr2_0); - let expr5_0 = constructor_x64_rotl(ctx, pattern3_0, expr3_0, &expr4_0)?; - let expr6_0 = constructor_output_gpr(ctx, expr5_0)?; - return Some(expr6_0); - } - &Opcode::Rotr => { - let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); - if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { - let pattern9_0 = C::inst_data(ctx, pattern8_0); - if let &InstructionData::UnaryImm { - opcode: ref pattern10_0, - imm: pattern10_1, - } = &pattern9_0 - { - if let &Opcode::Iconst = pattern10_0 { - let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); - // Rule at src/isa/x64/lower.isle line 849. - let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; - let expr1_0 = - C::const_to_type_masked_imm8(ctx, pattern12_0, pattern3_0); - let expr2_0 = - constructor_x64_rotr(ctx, pattern3_0, expr0_0, &expr1_0)?; - let expr3_0 = constructor_output_gpr(ctx, expr2_0)?; - return Some(expr3_0); + &Opcode::Rotr => { + let (pattern7_0, pattern7_1) = C::unpack_value_array_2(ctx, pattern5_1); + if let Some(pattern8_0) = C::def_inst(ctx, pattern7_1) { + let pattern9_0 = C::inst_data(ctx, pattern8_0); + if let &InstructionData::UnaryImm { + opcode: ref pattern10_0, + imm: pattern10_1, + } = &pattern9_0 + { + if let &Opcode::Iconst = pattern10_0 { + let pattern12_0 = C::u64_from_imm64(ctx, pattern10_1); + // Rule at src/isa/x64/lower.isle line 849. + let expr0_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr1_0 = C::const_to_type_masked_imm8( + ctx, + pattern12_0, + pattern3_0, + ); + let expr2_0 = constructor_x64_rotr( + ctx, pattern3_0, expr0_0, &expr1_0, + )?; + let expr3_0 = constructor_output_gpr(ctx, expr2_0)?; + return Some(expr3_0); + } } } + // Rule at src/isa/x64/lower.isle line 845. + let expr0_0: Type = I32; + let expr1_0 = ExtendKind::Zero; + let expr2_0 = + constructor_extend_to_gpr(ctx, pattern7_1, expr0_0, &expr1_0)?; + let expr3_0 = constructor_put_in_gpr(ctx, pattern7_0)?; + let expr4_0 = C::gpr_to_imm8_gpr(ctx, expr2_0); + let expr5_0 = constructor_x64_rotr(ctx, pattern3_0, expr3_0, &expr4_0)?; + let expr6_0 = constructor_output_gpr(ctx, expr5_0)?; + return Some(expr6_0); } - // Rule at src/isa/x64/lower.isle line 845. - let expr0_0: Type = I32; - let expr1_0 = ExtendKind::Zero; - let expr2_0 = - constructor_extend_to_gpr(ctx, pattern7_1, expr0_0, &expr1_0)?; - let expr3_0 = constructor_put_in_gpr(ctx, pattern7_0)?; - let expr4_0 = C::gpr_to_imm8_gpr(ctx, expr2_0); - let expr5_0 = constructor_x64_rotr(ctx, pattern3_0, expr3_0, &expr4_0)?; - let expr6_0 = constructor_output_gpr(ctx, expr5_0)?; - return Some(expr6_0); + _ => {} + } + } + &InstructionData::Unary { + opcode: ref pattern5_0, + arg: pattern5_1, + } => { + match pattern5_0 { + &Opcode::Clz => { + // Rule at src/isa/x64/lower.isle line 1548. + let expr0_0: Type = I32; + let expr1_0: Type = I32; + let expr2_0 = ExtendKind::Zero; + let expr3_0 = + constructor_extend_to_gpr(ctx, pattern5_1, expr1_0, &expr2_0)?; + let expr4_0 = constructor_do_clz(ctx, expr0_0, pattern3_0, expr3_0)?; + let expr5_0 = constructor_output_gpr(ctx, expr4_0)?; + return Some(expr5_0); + } + &Opcode::Ctz => { + // Rule at src/isa/x64/lower.isle line 1590. + let expr0_0: Type = I32; + let expr1_0: Type = I32; + let expr2_0 = ExtendKind::Zero; + let expr3_0 = + constructor_extend_to_gpr(ctx, pattern5_1, expr1_0, &expr2_0)?; + let expr4_0 = constructor_do_ctz(ctx, expr0_0, pattern3_0, expr3_0)?; + let expr5_0 = constructor_output_gpr(ctx, expr4_0)?; + return Some(expr5_0); + } + &Opcode::Popcnt => { + // Rule at src/isa/x64/lower.isle line 1642. + let expr0_0: Type = I32; + let expr1_0: Type = I32; + let expr2_0 = ExtendKind::Zero; + let expr3_0 = + constructor_extend_to_gpr(ctx, pattern5_1, expr1_0, &expr2_0)?; + let expr4_0 = constructor_do_popcnt(ctx, expr0_0, expr3_0)?; + let expr5_0 = constructor_output_gpr(ctx, expr4_0)?; + return Some(expr5_0); + } + _ => {} } - _ => {} } + _ => {} } } } @@ -6996,3 +7620,189 @@ pub fn constructor_cmp_and_choose( } return None; } + +// Generated as internal constructor for term do_clz. +pub fn constructor_do_clz( + ctx: &mut C, + arg0: Type, + arg1: Type, + arg2: Gpr, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/lower.isle line 1568. + let expr0_0: Type = I64; + let expr1_0: i64 = -1; + let expr2_0 = constructor_imm_i64(ctx, expr0_0, expr1_0)?; + let expr3_0 = C::gpr_new(ctx, expr2_0); + let expr4_0 = constructor_bsr_or_else(ctx, pattern0_0, pattern2_0, expr3_0)?; + let expr5_0 = C::gpr_to_reg(ctx, expr4_0); + let expr6_0 = C::ty_bits_u64(ctx, pattern1_0); + let expr7_0: u64 = 1; + let expr8_0 = C::u64_sub(ctx, expr6_0, expr7_0); + let expr9_0 = constructor_imm(ctx, pattern0_0, expr8_0)?; + let expr10_0 = C::gpr_new(ctx, expr9_0); + let expr11_0 = constructor_reg_to_gpr_mem_imm(ctx, expr5_0)?; + let expr12_0 = constructor_sub(ctx, pattern0_0, expr10_0, &expr11_0)?; + return Some(expr12_0); +} + +// Generated as internal constructor for term do_ctz. +pub fn constructor_do_ctz( + ctx: &mut C, + arg0: Type, + arg1: Type, + arg2: Gpr, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + let pattern2_0 = arg2; + // Rule at src/isa/x64/lower.isle line 1609. + let expr0_0: Type = I64; + let expr1_0 = C::ty_bits_u64(ctx, pattern1_0); + let expr2_0 = constructor_imm(ctx, expr0_0, expr1_0)?; + let expr3_0 = C::gpr_new(ctx, expr2_0); + let expr4_0 = constructor_bsf_or_else(ctx, pattern0_0, pattern2_0, expr3_0)?; + return Some(expr4_0); +} + +// Generated as internal constructor for term do_popcnt. +pub fn constructor_do_popcnt(ctx: &mut C, arg0: Type, arg1: Gpr) -> Option { + let pattern0_0 = arg0; + if pattern0_0 == I32 { + let pattern2_0 = arg1; + // Rule at src/isa/x64/lower.isle line 1675. + let expr0_0: Type = I32; + let expr1_0: u8 = 1; + let expr2_0 = Imm8Reg::Imm8 { imm: expr1_0 }; + let expr3_0 = C::imm8_reg_to_imm8_gpr(ctx, &expr2_0); + let expr4_0 = constructor_shr(ctx, expr0_0, pattern2_0, &expr3_0)?; + let expr5_0: Type = I32; + let expr6_0: u64 = 2004318071; + let expr7_0 = constructor_imm(ctx, expr5_0, expr6_0)?; + let expr8_0 = C::gpr_new(ctx, expr7_0); + let expr9_0: Type = I32; + let expr10_0 = C::gpr_to_gpr_mem_imm(ctx, expr8_0); + let expr11_0 = constructor_x64_and(ctx, expr9_0, expr4_0, &expr10_0)?; + let expr12_0: Type = I32; + let expr13_0 = C::gpr_to_gpr_mem_imm(ctx, expr11_0); + let expr14_0 = constructor_sub(ctx, expr12_0, pattern2_0, &expr13_0)?; + let expr15_0: Type = I32; + let expr16_0: u8 = 1; + let expr17_0 = Imm8Reg::Imm8 { imm: expr16_0 }; + let expr18_0 = C::imm8_reg_to_imm8_gpr(ctx, &expr17_0); + let expr19_0 = constructor_shr(ctx, expr15_0, expr11_0, &expr18_0)?; + let expr20_0: Type = I32; + let expr21_0 = C::gpr_to_gpr_mem_imm(ctx, expr8_0); + let expr22_0 = constructor_x64_and(ctx, expr20_0, expr19_0, &expr21_0)?; + let expr23_0: Type = I32; + let expr24_0 = C::gpr_to_gpr_mem_imm(ctx, expr22_0); + let expr25_0 = constructor_sub(ctx, expr23_0, expr14_0, &expr24_0)?; + let expr26_0: Type = I32; + let expr27_0: u8 = 1; + let expr28_0 = Imm8Reg::Imm8 { imm: expr27_0 }; + let expr29_0 = C::imm8_reg_to_imm8_gpr(ctx, &expr28_0); + let expr30_0 = constructor_shr(ctx, expr26_0, expr22_0, &expr29_0)?; + let expr31_0: Type = I32; + let expr32_0 = C::gpr_to_gpr_mem_imm(ctx, expr8_0); + let expr33_0 = constructor_x64_and(ctx, expr31_0, expr30_0, &expr32_0)?; + let expr34_0: Type = I32; + let expr35_0 = C::gpr_to_gpr_mem_imm(ctx, expr33_0); + let expr36_0 = constructor_sub(ctx, expr34_0, expr25_0, &expr35_0)?; + let expr37_0: Type = I32; + let expr38_0: Type = I32; + let expr39_0: u8 = 4; + let expr40_0 = Imm8Reg::Imm8 { imm: expr39_0 }; + let expr41_0 = C::imm8_reg_to_imm8_gpr(ctx, &expr40_0); + let expr42_0 = constructor_shr(ctx, expr38_0, expr36_0, &expr41_0)?; + let expr43_0 = C::gpr_to_gpr_mem_imm(ctx, expr36_0); + let expr44_0 = constructor_add(ctx, expr37_0, expr42_0, &expr43_0)?; + let expr45_0: Type = I32; + let expr46_0: u32 = 252645135; + let expr47_0 = RegMemImm::Imm { simm32: expr46_0 }; + let expr48_0 = C::gpr_mem_imm_new(ctx, &expr47_0); + let expr49_0 = constructor_x64_and(ctx, expr45_0, expr44_0, &expr48_0)?; + let expr50_0: Type = I32; + let expr51_0: u32 = 16843009; + let expr52_0 = RegMemImm::Imm { simm32: expr51_0 }; + let expr53_0 = C::gpr_mem_imm_new(ctx, &expr52_0); + let expr54_0 = constructor_mul(ctx, expr50_0, expr49_0, &expr53_0)?; + let expr55_0: Type = I32; + let expr56_0: u8 = 24; + let expr57_0 = Imm8Reg::Imm8 { imm: expr56_0 }; + let expr58_0 = C::imm8_reg_to_imm8_gpr(ctx, &expr57_0); + let expr59_0 = constructor_shr(ctx, expr55_0, expr54_0, &expr58_0)?; + return Some(expr59_0); + } + if pattern0_0 == I64 { + let pattern2_0 = arg1; + // Rule at src/isa/x64/lower.isle line 1655. + let expr0_0: Type = I64; + let expr1_0: u8 = 1; + let expr2_0 = Imm8Reg::Imm8 { imm: expr1_0 }; + let expr3_0 = C::imm8_reg_to_imm8_gpr(ctx, &expr2_0); + let expr4_0 = constructor_shr(ctx, expr0_0, pattern2_0, &expr3_0)?; + let expr5_0: Type = I64; + let expr6_0: u64 = 8608480567731124087; + let expr7_0 = constructor_imm(ctx, expr5_0, expr6_0)?; + let expr8_0 = C::gpr_new(ctx, expr7_0); + let expr9_0: Type = I64; + let expr10_0 = C::gpr_to_gpr_mem_imm(ctx, expr8_0); + let expr11_0 = constructor_x64_and(ctx, expr9_0, expr4_0, &expr10_0)?; + let expr12_0: Type = I64; + let expr13_0 = C::gpr_to_gpr_mem_imm(ctx, expr11_0); + let expr14_0 = constructor_sub(ctx, expr12_0, pattern2_0, &expr13_0)?; + let expr15_0: Type = I64; + let expr16_0: u8 = 1; + let expr17_0 = Imm8Reg::Imm8 { imm: expr16_0 }; + let expr18_0 = C::imm8_reg_to_imm8_gpr(ctx, &expr17_0); + let expr19_0 = constructor_shr(ctx, expr15_0, expr11_0, &expr18_0)?; + let expr20_0: Type = I64; + let expr21_0 = C::gpr_to_gpr_mem_imm(ctx, expr8_0); + let expr22_0 = constructor_x64_and(ctx, expr20_0, expr19_0, &expr21_0)?; + let expr23_0: Type = I64; + let expr24_0 = C::gpr_to_gpr_mem_imm(ctx, expr22_0); + let expr25_0 = constructor_sub(ctx, expr23_0, expr14_0, &expr24_0)?; + let expr26_0: Type = I64; + let expr27_0: u8 = 1; + let expr28_0 = Imm8Reg::Imm8 { imm: expr27_0 }; + let expr29_0 = C::imm8_reg_to_imm8_gpr(ctx, &expr28_0); + let expr30_0 = constructor_shr(ctx, expr26_0, expr22_0, &expr29_0)?; + let expr31_0: Type = I64; + let expr32_0 = C::gpr_to_gpr_mem_imm(ctx, expr8_0); + let expr33_0 = constructor_x64_and(ctx, expr31_0, expr30_0, &expr32_0)?; + let expr34_0: Type = I64; + let expr35_0 = C::gpr_to_gpr_mem_imm(ctx, expr33_0); + let expr36_0 = constructor_sub(ctx, expr34_0, expr25_0, &expr35_0)?; + let expr37_0: Type = I64; + let expr38_0: Type = I64; + let expr39_0: u8 = 4; + let expr40_0 = Imm8Reg::Imm8 { imm: expr39_0 }; + let expr41_0 = C::imm8_reg_to_imm8_gpr(ctx, &expr40_0); + let expr42_0 = constructor_shr(ctx, expr38_0, expr36_0, &expr41_0)?; + let expr43_0 = C::gpr_to_gpr_mem_imm(ctx, expr36_0); + let expr44_0 = constructor_add(ctx, expr37_0, expr42_0, &expr43_0)?; + let expr45_0: Type = I64; + let expr46_0: u64 = 1085102592571150095; + let expr47_0 = constructor_imm(ctx, expr45_0, expr46_0)?; + let expr48_0 = C::gpr_new(ctx, expr47_0); + let expr49_0: Type = I64; + let expr50_0 = C::gpr_to_gpr_mem_imm(ctx, expr48_0); + let expr51_0 = constructor_x64_and(ctx, expr49_0, expr44_0, &expr50_0)?; + let expr52_0: Type = I64; + let expr53_0: u64 = 72340172838076673; + let expr54_0 = constructor_imm(ctx, expr52_0, expr53_0)?; + let expr55_0 = C::gpr_new(ctx, expr54_0); + let expr56_0: Type = I64; + let expr57_0 = C::gpr_to_gpr_mem_imm(ctx, expr55_0); + let expr58_0 = constructor_mul(ctx, expr56_0, expr51_0, &expr57_0)?; + let expr59_0: Type = I64; + let expr60_0: u8 = 56; + let expr61_0 = Imm8Reg::Imm8 { imm: expr60_0 }; + let expr62_0 = C::imm8_reg_to_imm8_gpr(ctx, &expr61_0); + let expr63_0 = constructor_shr(ctx, expr59_0, expr58_0, &expr62_0)?; + return Some(expr63_0); + } + return None; +} diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index a7259c6d3ad6..9436ea975ae9 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -140,6 +140,21 @@ macro_rules! isle_prelude_methods { x.into() } + #[inline] + fn i64_as_u64(&mut self, x: i64) -> u64 { + x as u64 + } + + #[inline] + fn u64_add(&mut self, x: u64, y: u64) -> u64 { + x.wrapping_add(y) + } + + #[inline] + fn u64_sub(&mut self, x: u64, y: u64) -> u64 { + x.wrapping_sub(y) + } + #[inline] fn ty_bits(&mut self, ty: Type) -> u8 { use std::convert::TryInto; @@ -151,6 +166,11 @@ macro_rules! isle_prelude_methods { ty.bits() } + #[inline] + fn ty_bits_u64(&mut self, ty: Type) -> u64 { + ty.bits() as u64 + } + #[inline] fn ty_bytes(&mut self, ty: Type) -> u16 { u16::try_from(ty.bytes()).unwrap() diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index 0be9c856f574..410a7f7170a4 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -167,6 +167,17 @@ (decl u32_as_u64 (u32) u64) (extern constructor u32_as_u64 u32_as_u64) +(decl i64_as_u64 (i64) u64) +(extern constructor i64_as_u64 i64_as_u64) + +;;;; Primitive Arithmetic ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl u64_add (u64 u64) u64) +(extern constructor u64_add u64_add) + +(decl u64_sub (u64 u64) u64) +(extern constructor u64_sub u64_sub) + ;;;; `cranelift_codegen::ir::Type` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (extern const $B1 Type) @@ -209,6 +220,10 @@ (decl ty_bits_u16 (Type) u16) (extern constructor ty_bits_u16 ty_bits_u16) +;; Get the bit width of a given type. +(decl ty_bits_u64 (Type) u64) +(extern constructor ty_bits_u64 ty_bits_u64) + ;; Get the byte width of a given type. (decl ty_bytes (Type) u16) (extern constructor ty_bytes ty_bytes) diff --git a/cranelift/filetests/filetests/isa/x64/i128.clif b/cranelift/filetests/filetests/isa/x64/i128.clif index 20064dac82e8..16ba97d4f9c6 100644 --- a/cranelift/filetests/filetests/isa/x64/i128.clif +++ b/cranelift/filetests/filetests/isa/x64/i128.clif @@ -600,57 +600,55 @@ block0(v0: i128): ; Entry block: 0 ; Block 0: ; (original IR block: block0) -; (instruction range: 0 .. 50) +; (instruction range: 0 .. 48) ; Inst 0: pushq %rbp ; Inst 1: movq %rsp, %rbp -; Inst 2: movq %rsi, %rdx -; Inst 3: movq %rdi, %rsi -; Inst 4: shrq $1, %rsi -; Inst 5: movabsq $8608480567731124087, %rcx -; Inst 6: andq %rcx, %rsi -; Inst 7: movq %rdi, %rax -; Inst 8: subq %rsi, %rax -; Inst 9: shrq $1, %rsi -; Inst 10: andq %rcx, %rsi -; Inst 11: subq %rsi, %rax -; Inst 12: shrq $1, %rsi -; Inst 13: andq %rcx, %rsi -; Inst 14: subq %rsi, %rax -; Inst 15: movq %rax, %rsi -; Inst 16: shrq $4, %rsi -; Inst 17: addq %rax, %rsi -; Inst 18: movabsq $1085102592571150095, %rdi -; Inst 19: andq %rdi, %rsi -; Inst 20: movabsq $72340172838076673, %rdi -; Inst 21: imulq %rdi, %rsi -; Inst 22: shrq $56, %rsi -; Inst 23: movq %rdx, %rax -; Inst 24: shrq $1, %rax -; Inst 25: movabsq $8608480567731124087, %rcx -; Inst 26: andq %rcx, %rax -; Inst 27: movq %rdx, %rdi -; Inst 28: subq %rax, %rdi -; Inst 29: shrq $1, %rax -; Inst 30: andq %rcx, %rax -; Inst 31: subq %rax, %rdi -; Inst 32: shrq $1, %rax -; Inst 33: andq %rcx, %rax -; Inst 34: subq %rax, %rdi -; Inst 35: movq %rdi, %rax -; Inst 36: shrq $4, %rax -; Inst 37: addq %rdi, %rax -; Inst 38: movabsq $1085102592571150095, %rdi -; Inst 39: andq %rdi, %rax -; Inst 40: movabsq $72340172838076673, %rdi -; Inst 41: imulq %rdi, %rax -; Inst 42: shrq $56, %rax -; Inst 43: addq %rax, %rsi -; Inst 44: xorq %rdi, %rdi -; Inst 45: movq %rsi, %rax -; Inst 46: movq %rdi, %rdx -; Inst 47: movq %rbp, %rsp -; Inst 48: popq %rbp -; Inst 49: ret +; Inst 2: movq %rdi, %rax +; Inst 3: movq %rax, %rcx +; Inst 4: shrq $1, %rcx +; Inst 5: movabsq $8608480567731124087, %rdi +; Inst 6: andq %rdi, %rcx +; Inst 7: subq %rcx, %rax +; Inst 8: shrq $1, %rcx +; Inst 9: andq %rdi, %rcx +; Inst 10: subq %rcx, %rax +; Inst 11: shrq $1, %rcx +; Inst 12: andq %rdi, %rcx +; Inst 13: subq %rcx, %rax +; Inst 14: movq %rax, %rdi +; Inst 15: shrq $4, %rdi +; Inst 16: addq %rax, %rdi +; Inst 17: movabsq $1085102592571150095, %rax +; Inst 18: andq %rax, %rdi +; Inst 19: movabsq $72340172838076673, %rax +; Inst 20: imulq %rax, %rdi +; Inst 21: shrq $56, %rdi +; Inst 22: movq %rsi, %rcx +; Inst 23: shrq $1, %rcx +; Inst 24: movabsq $8608480567731124087, %rax +; Inst 25: andq %rax, %rcx +; Inst 26: subq %rcx, %rsi +; Inst 27: shrq $1, %rcx +; Inst 28: andq %rax, %rcx +; Inst 29: subq %rcx, %rsi +; Inst 30: shrq $1, %rcx +; Inst 31: andq %rax, %rcx +; Inst 32: subq %rcx, %rsi +; Inst 33: movq %rsi, %rax +; Inst 34: shrq $4, %rax +; Inst 35: addq %rsi, %rax +; Inst 36: movabsq $1085102592571150095, %rsi +; Inst 37: andq %rsi, %rax +; Inst 38: movabsq $72340172838076673, %rsi +; Inst 39: imulq %rsi, %rax +; Inst 40: shrq $56, %rax +; Inst 41: addq %rax, %rdi +; Inst 42: xorq %rsi, %rsi +; Inst 43: movq %rdi, %rax +; Inst 44: movq %rsi, %rdx +; Inst 45: movq %rbp, %rsp +; Inst 46: popq %rbp +; Inst 47: ret ; }} function %f20(i128) -> i128 { @@ -1020,11 +1018,11 @@ block0(v0: i128): ; Inst 4: cmovzq %rcx, %rax ; Inst 5: movl $63, %esi ; Inst 6: subq %rax, %rsi -; Inst 7: movabsq $-1, %rcx -; Inst 8: bsrq %rdi, %rax -; Inst 9: cmovzq %rcx, %rax +; Inst 7: movabsq $-1, %rax +; Inst 8: bsrq %rdi, %rcx +; Inst 9: cmovzq %rax, %rcx ; Inst 10: movl $63, %edi -; Inst 11: subq %rax, %rdi +; Inst 11: subq %rcx, %rdi ; Inst 12: addq $64, %rdi ; Inst 13: cmpq $64, %rsi ; Inst 14: cmovnzq %rsi, %rdi diff --git a/cranelift/filetests/filetests/isa/x64/popcnt.clif b/cranelift/filetests/filetests/isa/x64/popcnt.clif index 8f03f279513f..549fa9c4bf9a 100644 --- a/cranelift/filetests/filetests/isa/x64/popcnt.clif +++ b/cranelift/filetests/filetests/isa/x64/popcnt.clif @@ -14,17 +14,17 @@ block0(v0: i64): ; (instruction range: 0 .. 25) ; Inst 0: pushq %rbp ; Inst 1: movq %rsp, %rbp -; Inst 2: movq %rdi, %rsi -; Inst 3: shrq $1, %rsi -; Inst 4: movabsq $8608480567731124087, %rax -; Inst 5: andq %rax, %rsi -; Inst 6: subq %rsi, %rdi -; Inst 7: shrq $1, %rsi -; Inst 8: andq %rax, %rsi -; Inst 9: subq %rsi, %rdi -; Inst 10: shrq $1, %rsi -; Inst 11: andq %rax, %rsi -; Inst 12: subq %rsi, %rdi +; Inst 2: movq %rdi, %rax +; Inst 3: shrq $1, %rax +; Inst 4: movabsq $8608480567731124087, %rsi +; Inst 5: andq %rsi, %rax +; Inst 6: subq %rax, %rdi +; Inst 7: shrq $1, %rax +; Inst 8: andq %rsi, %rax +; Inst 9: subq %rax, %rdi +; Inst 10: shrq $1, %rax +; Inst 11: andq %rsi, %rax +; Inst 12: subq %rax, %rdi ; Inst 13: movq %rdi, %rsi ; Inst 14: shrq $4, %rsi ; Inst 15: addq %rdi, %rsi @@ -54,17 +54,17 @@ block0(v0: i64): ; Inst 0: pushq %rbp ; Inst 1: movq %rsp, %rbp ; Inst 2: movq 0(%rdi), %rdi -; Inst 3: movq %rdi, %rsi -; Inst 4: shrq $1, %rsi -; Inst 5: movabsq $8608480567731124087, %rax -; Inst 6: andq %rax, %rsi -; Inst 7: subq %rsi, %rdi -; Inst 8: shrq $1, %rsi -; Inst 9: andq %rax, %rsi -; Inst 10: subq %rsi, %rdi -; Inst 11: shrq $1, %rsi -; Inst 12: andq %rax, %rsi -; Inst 13: subq %rsi, %rdi +; Inst 3: movq %rdi, %rax +; Inst 4: shrq $1, %rax +; Inst 5: movabsq $8608480567731124087, %rsi +; Inst 6: andq %rsi, %rax +; Inst 7: subq %rax, %rdi +; Inst 8: shrq $1, %rax +; Inst 9: andq %rsi, %rax +; Inst 10: subq %rax, %rdi +; Inst 11: shrq $1, %rax +; Inst 12: andq %rsi, %rax +; Inst 13: subq %rax, %rdi ; Inst 14: movq %rdi, %rsi ; Inst 15: shrq $4, %rsi ; Inst 16: addq %rdi, %rsi @@ -89,29 +89,30 @@ block0(v0: i32): ; Entry block: 0 ; Block 0: ; (original IR block: block0) -; (instruction range: 0 .. 22) +; (instruction range: 0 .. 23) ; Inst 0: pushq %rbp ; Inst 1: movq %rsp, %rbp -; Inst 2: movq %rdi, %rsi -; Inst 3: shrl $1, %esi -; Inst 4: andl $2004318071, %esi -; Inst 5: subl %esi, %edi -; Inst 6: shrl $1, %esi -; Inst 7: andl $2004318071, %esi -; Inst 8: subl %esi, %edi -; Inst 9: shrl $1, %esi -; Inst 10: andl $2004318071, %esi -; Inst 11: subl %esi, %edi -; Inst 12: movq %rdi, %rsi -; Inst 13: shrl $4, %esi -; Inst 14: addl %edi, %esi -; Inst 15: andl $252645135, %esi -; Inst 16: imull $16843009, %esi -; Inst 17: shrl $24, %esi -; Inst 18: movq %rsi, %rax -; Inst 19: movq %rbp, %rsp -; Inst 20: popq %rbp -; Inst 21: ret +; Inst 2: movq %rdi, %rax +; Inst 3: shrl $1, %eax +; Inst 4: movl $2004318071, %esi +; Inst 5: andl %esi, %eax +; Inst 6: subl %eax, %edi +; Inst 7: shrl $1, %eax +; Inst 8: andl %esi, %eax +; Inst 9: subl %eax, %edi +; Inst 10: shrl $1, %eax +; Inst 11: andl %esi, %eax +; Inst 12: subl %eax, %edi +; Inst 13: movq %rdi, %rsi +; Inst 14: shrl $4, %esi +; Inst 15: addl %edi, %esi +; Inst 16: andl $252645135, %esi +; Inst 17: imull $16843009, %esi +; Inst 18: shrl $24, %esi +; Inst 19: movq %rsi, %rax +; Inst 20: movq %rbp, %rsp +; Inst 21: popq %rbp +; Inst 22: ret ; }} function %popcnt32load(i64) -> i32 { @@ -125,29 +126,30 @@ block0(v0: i64): ; Entry block: 0 ; Block 0: ; (original IR block: block0) -; (instruction range: 0 .. 23) +; (instruction range: 0 .. 24) ; Inst 0: pushq %rbp ; Inst 1: movq %rsp, %rbp ; Inst 2: movl 0(%rdi), %edi -; Inst 3: movq %rdi, %rsi -; Inst 4: shrl $1, %esi -; Inst 5: andl $2004318071, %esi -; Inst 6: subl %esi, %edi -; Inst 7: shrl $1, %esi -; Inst 8: andl $2004318071, %esi -; Inst 9: subl %esi, %edi -; Inst 10: shrl $1, %esi -; Inst 11: andl $2004318071, %esi -; Inst 12: subl %esi, %edi -; Inst 13: movq %rdi, %rsi -; Inst 14: shrl $4, %esi -; Inst 15: addl %edi, %esi -; Inst 16: andl $252645135, %esi -; Inst 17: imull $16843009, %esi -; Inst 18: shrl $24, %esi -; Inst 19: movq %rsi, %rax -; Inst 20: movq %rbp, %rsp -; Inst 21: popq %rbp -; Inst 22: ret +; Inst 3: movq %rdi, %rax +; Inst 4: shrl $1, %eax +; Inst 5: movl $2004318071, %esi +; Inst 6: andl %esi, %eax +; Inst 7: subl %eax, %edi +; Inst 8: shrl $1, %eax +; Inst 9: andl %esi, %eax +; Inst 10: subl %eax, %edi +; Inst 11: shrl $1, %eax +; Inst 12: andl %esi, %eax +; Inst 13: subl %eax, %edi +; Inst 14: movq %rdi, %rsi +; Inst 15: shrl $4, %esi +; Inst 16: addl %edi, %esi +; Inst 17: andl $252645135, %esi +; Inst 18: imull $16843009, %esi +; Inst 19: shrl $24, %esi +; Inst 20: movq %rsi, %rax +; Inst 21: movq %rbp, %rsp +; Inst 22: popq %rbp +; Inst 23: ret ; }}