Migrate clz, ctz, and popcnt on x64 to ISLE.

cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest

@@ -1,4 +1,4 @@
 src/clif.isle 9ea75a6f790b5c03
-src/prelude.isle 9830498351ddf6a3
+src/prelude.isle 3a10bacb778efff0
 src/isa/aarch64/inst.isle 3678d0a37bdb4cff
 src/isa/aarch64/lower.isle 90accbfcadaea46d

cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest

@@ -1,4 +1,4 @@
 src/clif.isle 9ea75a6f790b5c03
-src/prelude.isle 9830498351ddf6a3
+src/prelude.isle 3a10bacb778efff0
 src/isa/s390x/inst.isle d91a16074ab186a8
 src/isa/s390x/lower.isle 1cc5a12adc8c75f9

cranelift/codegen/src/isa/x64/inst.isle

@@ -1074,6 +1074,18 @@
 (decl avx512f_enabled () Type)
 (extern extractor avx512f_enabled avx512f_enabled)
 
+(decl avx512bitalg_enabled () Type)
+(extern extractor avx512bitalg_enabled avx512bitalg_enabled)
+
+(decl use_lzcnt () Type)
+(extern extractor use_lzcnt use_lzcnt)
+
+(decl use_bmi1 () Type)
+(extern extractor use_bmi1 use_bmi1)
+
+(decl use_popcnt () Type)
+(extern extractor use_popcnt use_popcnt)
+
 ;;;; Helpers for Merging and Sinking Immediates/Loads  ;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Extract a constant `Imm8Reg.Imm8` from a value operand.
@@ -1266,6 +1278,13 @@
       (xmm_unary_rm_r (SseOpcode.Movdqu)
                       addr))
 
+;; Load a constant into an XMM register.
+(decl xmm_load_const (Type VCodeConstant) Xmm)
+(rule (xmm_load_const ty const)
+      (let ((dst WritableXmm (temp_writable_xmm))
+            (_ Unit (emit (MInst.XmmLoadConst const dst ty))))
+        dst))
+
 ;;;; Instruction Constructors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;
 ;; These constructors create SSA-style `MInst`s. It is their responsibility to
@@ -1398,6 +1417,13 @@
                   (imm $I64 bits)
                   (OperandSize.Size64)))
 
+;; Helper fo remitting immediates with an `i64` value. Note that
+;; integer constants in ISLE are always parsed as `i64`s; this enables
+;; negative numbers to be used as immediates.
+(decl imm_i64 (Type i64) Reg)
+(rule (imm_i64 ty value)
+      (imm ty (i64_as_u64 value)))
+
 (decl nonzero_u64_fits_in_u32 (u64) u64)
 (extern extractor nonzero_u64_fits_in_u32 nonzero_u64_fits_in_u32)
 
@@ -1504,6 +1530,11 @@
 (rule (cmp size src1 src2)
       (cmp_rmi_r size (CmpOpcode.Cmp) src1 src2))
 
+;; Helper for creating `cmp` instructions with an immediate.
+(decl cmp_imm (OperandSize u32 Gpr) ProducesFlags)
+(rule (cmp_imm size src1 src2)
+      (cmp_rmi_r size (CmpOpcode.Cmp) (RegMemImm.Imm src1) src2))
+
 ;; Helper for creating `MInst.XmmCmpRmR` instructions.
 (decl xmm_cmp_rm_r (SseOpcode XmmMem Xmm) ProducesFlags)
 (rule (xmm_cmp_rm_r opcode src1 src2)
@@ -2027,6 +2058,16 @@
                                            size))))
         dst))
 
+;; Helper for creating `pshufb` instructions.
+(decl pshufb (Xmm XmmMem) Xmm)
+(rule (pshufb src1 src2)
+      (let ((dst WritableXmm (temp_writable_xmm))
+            (_ Unit (emit (MInst.XmmRmR (SseOpcode.Pshufb)
+                                        src1
+                                        src2
+                                        dst))))
+        dst))
+
 ;; Helper for creating `MInst.XmmUnaryRmR` instructions.
 (decl xmm_unary_rm_r (SseOpcode XmmMem) Xmm)
 (rule (xmm_unary_rm_r op src)
@@ -2071,6 +2112,11 @@
 (rule (vpabsq src)
       (xmm_unary_rm_r_evex (Avx512Opcode.Vpabsq) src))
 
+;; Helper for creating `vpopcntb` instructions.
+(decl vpopcntb (XmmMem) Xmm)
+(rule (vpopcntb src)
+      (xmm_unary_rm_r_evex (Avx512Opcode.Vpopcntb) src))
+
 ;; Helper for creating `MInst.XmmRmREvex` instructions.
 (decl xmm_rm_r_evex (Avx512Opcode XmmMem Xmm) Xmm)
 (rule (xmm_rm_r_evex op src1 src2)
@@ -2221,6 +2267,68 @@
 (rule (ud2 code)
       (SideEffectNoResult.Inst (MInst.Ud2 code)))
 
+;; Helper for creating `lzcnt` instructions.
+(decl lzcnt (Type Gpr) Gpr)
+(rule (lzcnt ty src)
+      (let ((dst WritableGpr (temp_writable_gpr))
+            (size OperandSize (operand_size_of_type_32_64 ty))
+            (_ Unit (emit (MInst.UnaryRmR size (UnaryRmROpcode.Lzcnt) src dst))))
+        dst))
+
+;; Helper for creating `tzcnt` instructions.
+(decl tzcnt (Type Gpr) Gpr)
+(rule (tzcnt ty src)
+      (let ((dst WritableGpr (temp_writable_gpr))
+            (size OperandSize (operand_size_of_type_32_64 ty))
+            (_ Unit (emit (MInst.UnaryRmR size (UnaryRmROpcode.Tzcnt) src dst))))
+        dst))
+
+;; Helper for creating `bsr` instructions.
+(decl bsr (Type Gpr) Gpr)
+(rule (bsr ty src)
+      (let ((dst WritableGpr (temp_writable_gpr))
+            (size OperandSize (operand_size_of_type_32_64 ty))
+            (_ Unit (emit (MInst.UnaryRmR size (UnaryRmROpcode.Bsr) src dst))))
+        dst))
+
+;; Helper for creating `bsr + cmov` instruction pairs that produce the
+;; result of the `bsr`, or `alt` if the input was zero.
+(decl bsr_or_else (Type Gpr Gpr) Gpr)
+(rule (bsr_or_else ty src alt)
+      (let ((dst WritableGpr (temp_writable_gpr))
+            (tmp WritableGpr (temp_writable_gpr))
+            (size OperandSize (operand_size_of_type_32_64 ty))
+            (_1 Unit (emit (MInst.UnaryRmR size (UnaryRmROpcode.Bsr) src tmp)))
+            (_2 Unit (emit (MInst.Cmove size (CC.Z) alt tmp dst))))
+        dst))
+
+;; Helper for creating `bsf` instructions.
+(decl bsf (Type Gpr) Gpr)
+(rule (bsf ty src)
+      (let ((dst WritableGpr (temp_writable_gpr))
+            (size OperandSize (operand_size_of_type_32_64 ty))
+            (_ Unit (emit (MInst.UnaryRmR size (UnaryRmROpcode.Bsf) src dst))))
+        dst))
+
+;; Helper for creating `bsf + cmov` instruction pairs that produce the
+;; result of the `bsf`, or `alt` if the input was zero.
+(decl bsf_or_else (Type Gpr Gpr) Gpr)
+(rule (bsf_or_else ty src alt)
+      (let ((dst WritableGpr (temp_writable_gpr))
+            (tmp WritableGpr (temp_writable_gpr))
+            (size OperandSize (operand_size_of_type_32_64 ty))
+            (_1 Unit (emit (MInst.UnaryRmR size (UnaryRmROpcode.Bsf) src tmp)))
+            (_2 Unit (emit (MInst.Cmove size (CC.Z) alt tmp dst))))
+        dst))
+
+;; Helper for creating `popcnt` instructions.
+(decl x64_popcnt (Type Gpr) Gpr)
+(rule (x64_popcnt ty src)
+      (let ((dst WritableGpr (temp_writable_gpr))
+            (size OperandSize (operand_size_of_type_32_64 ty))
+            (_ Unit (emit (MInst.UnaryRmR size (UnaryRmROpcode.Popcnt) src dst))))
+        dst))
+
 ;;;; Automatic conversions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (convert Gpr InstOutput output_gpr)
@@ -2241,6 +2349,7 @@
 (convert Reg GprMemImm reg_to_gpr_mem_imm)
 (convert WritableGpr WritableReg writable_gpr_to_reg)
 (convert WritableGpr Reg writable_gpr_to_r_reg)
+(convert WritableGpr GprMem writable_gpr_to_gpr_mem)
 
 (convert Xmm InstOutput output_xmm)
 (convert Value Xmm put_in_xmm)
@@ -2261,6 +2370,7 @@
 (convert WritableXmm XmmMem writable_xmm_to_xmm_mem)
 
 (convert Gpr Imm8Gpr gpr_to_imm8_gpr)
+(convert Imm8Reg Imm8Gpr imm8_reg_to_imm8_gpr)
 
 (convert Amode SyntheticAmode amode_to_synthetic_amode)
 (convert SyntheticAmode GprMem synthetic_amode_to_gpr_mem)
@@ -2276,6 +2386,9 @@
 (decl writable_gpr_to_r_reg (WritableGpr) Reg)
 (rule (writable_gpr_to_r_reg w_gpr)
       (writable_reg_to_reg (writable_gpr_to_reg w_gpr)))
+(decl writable_gpr_to_gpr_mem (WritableGpr) GprMem)
+(rule (writable_gpr_to_gpr_mem w_gpr)
+      (gpr_to_gpr_mem w_gpr))
 (decl writable_xmm_to_r_reg (WritableXmm) Reg)
 (rule (writable_xmm_to_r_reg w_xmm)
       (writable_reg_to_reg (writable_xmm_to_reg w_xmm)))