diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest index 78ce678fcc4e..a9fae6d4d2ec 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle 6b0160bfcac86902 +src/prelude.isle b2bc986bcbbbb77 src/isa/aarch64/inst.isle 3678d0a37bdb4cff src/isa/aarch64/lower.isle 90accbfcadaea46d diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs index 85d9957a9a00..817d95a04511 100644 --- a/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/aarch64/lower/isle/generated_code.rs @@ -54,6 +54,8 @@ pub trait Context { fn fits_in_64(&mut self, arg0: Type) -> Option; fn ty_32_or_64(&mut self, arg0: Type) -> Option; fn ty_8_or_16(&mut self, arg0: Type) -> Option; + fn ty_int_bool_64(&mut self, arg0: Type) -> Option; + fn ty_int_bool_128(&mut self, arg0: Type) -> Option; fn vec128(&mut self, arg0: Type) -> Option; fn not_i64x2(&mut self, arg0: Type) -> Option<()>; fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; @@ -116,13 +118,13 @@ pub trait Context { fn rotr_opposite_amount(&mut self, arg0: Type, arg1: ImmShift) -> ImmShift; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 385. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 393. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 407. +/// Internal type ProducesFlags: defined at src/prelude.isle line 415. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlagsSideEffect { inst: MInst }, @@ -130,7 +132,7 @@ pub enum ProducesFlags { ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 418. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 426. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlagsReturnsResultWithProducer { @@ -1063,7 +1065,7 @@ pub fn constructor_side_effect( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 390. + // Rule at src/prelude.isle line 398. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::output_none(ctx); return Some(expr1_0); @@ -1081,7 +1083,7 @@ pub fn constructor_safepoint( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 396. + // Rule at src/prelude.isle line 404. let expr0_0 = C::emit_safepoint(ctx, pattern1_0); let expr1_0 = C::output_none(ctx); return Some(expr1_0); @@ -1100,7 +1102,7 @@ pub fn constructor_produces_flags_get_reg( result: pattern1_1, } = pattern0_0 { - // Rule at src/prelude.isle line 434. + // Rule at src/prelude.isle line 442. return Some(pattern1_1); } return None; @@ -1117,7 +1119,7 @@ pub fn constructor_produces_flags_ignore( inst: ref pattern1_0, result: pattern1_1, } => { - // Rule at src/prelude.isle line 439. + // Rule at src/prelude.isle line 447. let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { inst: pattern1_0.clone(), }; @@ -1127,7 +1129,7 @@ pub fn constructor_produces_flags_ignore( inst: ref pattern1_0, result: pattern1_1, } => { - // Rule at src/prelude.isle line 441. + // Rule at src/prelude.isle line 449. let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { inst: pattern1_0.clone(), }; @@ -1156,7 +1158,7 @@ pub fn constructor_consumes_flags_concat( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 448. + // Rule at src/prelude.isle line 456. let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1); let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs { inst1: pattern1_0.clone(), @@ -1186,7 +1188,7 @@ pub fn constructor_with_flags( inst: ref pattern3_0, result: pattern3_1, } => { - // Rule at src/prelude.isle line 473. + // Rule at src/prelude.isle line 481. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_reg(ctx, pattern3_1); @@ -1197,7 +1199,7 @@ pub fn constructor_with_flags( inst2: ref pattern3_1, result: pattern3_2, } => { - // Rule at src/prelude.isle line 479. + // Rule at src/prelude.isle line 487. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::emit(ctx, pattern3_1); @@ -1210,7 +1212,7 @@ pub fn constructor_with_flags( inst4: ref pattern3_3, result: pattern3_4, } => { - // Rule at src/prelude.isle line 491. + // Rule at src/prelude.isle line 499. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::emit(ctx, pattern3_1); @@ -1231,7 +1233,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 467. + // Rule at src/prelude.isle line 475. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1251,7 +1253,7 @@ pub fn constructor_with_flags_reg( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/prelude.isle line 508. + // Rule at src/prelude.isle line 516. let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?; let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest index b7a8bb8367ca..f72035cd1f0c 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle 6b0160bfcac86902 +src/prelude.isle b2bc986bcbbbb77 src/isa/s390x/inst.isle d91a16074ab186a8 src/isa/s390x/lower.isle 1cc5a12adc8c75f9 diff --git a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs index 844ff61c94dc..09fbd7b4cb38 100644 --- a/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/s390x/lower/isle/generated_code.rs @@ -54,6 +54,8 @@ pub trait Context { fn fits_in_64(&mut self, arg0: Type) -> Option; fn ty_32_or_64(&mut self, arg0: Type) -> Option; fn ty_8_or_16(&mut self, arg0: Type) -> Option; + fn ty_int_bool_64(&mut self, arg0: Type) -> Option; + fn ty_int_bool_128(&mut self, arg0: Type) -> Option; fn vec128(&mut self, arg0: Type) -> Option; fn not_i64x2(&mut self, arg0: Type) -> Option<()>; fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; @@ -150,13 +152,13 @@ pub trait Context { fn same_reg(&mut self, arg0: Reg, arg1: WritableReg) -> Option<()>; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 385. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 393. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 407. +/// Internal type ProducesFlags: defined at src/prelude.isle line 415. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlagsSideEffect { inst: MInst }, @@ -164,7 +166,7 @@ pub enum ProducesFlags { ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 418. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 426. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlagsReturnsResultWithProducer { @@ -954,7 +956,7 @@ pub fn constructor_side_effect( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 390. + // Rule at src/prelude.isle line 398. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::output_none(ctx); return Some(expr1_0); @@ -972,7 +974,7 @@ pub fn constructor_safepoint( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 396. + // Rule at src/prelude.isle line 404. let expr0_0 = C::emit_safepoint(ctx, pattern1_0); let expr1_0 = C::output_none(ctx); return Some(expr1_0); @@ -991,7 +993,7 @@ pub fn constructor_produces_flags_get_reg( result: pattern1_1, } = pattern0_0 { - // Rule at src/prelude.isle line 434. + // Rule at src/prelude.isle line 442. return Some(pattern1_1); } return None; @@ -1008,7 +1010,7 @@ pub fn constructor_produces_flags_ignore( inst: ref pattern1_0, result: pattern1_1, } => { - // Rule at src/prelude.isle line 439. + // Rule at src/prelude.isle line 447. let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { inst: pattern1_0.clone(), }; @@ -1018,7 +1020,7 @@ pub fn constructor_produces_flags_ignore( inst: ref pattern1_0, result: pattern1_1, } => { - // Rule at src/prelude.isle line 441. + // Rule at src/prelude.isle line 449. let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { inst: pattern1_0.clone(), }; @@ -1047,7 +1049,7 @@ pub fn constructor_consumes_flags_concat( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 448. + // Rule at src/prelude.isle line 456. let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1); let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs { inst1: pattern1_0.clone(), @@ -1077,7 +1079,7 @@ pub fn constructor_with_flags( inst: ref pattern3_0, result: pattern3_1, } => { - // Rule at src/prelude.isle line 473. + // Rule at src/prelude.isle line 481. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_reg(ctx, pattern3_1); @@ -1088,7 +1090,7 @@ pub fn constructor_with_flags( inst2: ref pattern3_1, result: pattern3_2, } => { - // Rule at src/prelude.isle line 479. + // Rule at src/prelude.isle line 487. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::emit(ctx, pattern3_1); @@ -1101,7 +1103,7 @@ pub fn constructor_with_flags( inst4: ref pattern3_3, result: pattern3_4, } => { - // Rule at src/prelude.isle line 491. + // Rule at src/prelude.isle line 499. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::emit(ctx, pattern3_1); @@ -1122,7 +1124,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 467. + // Rule at src/prelude.isle line 475. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -1142,7 +1144,7 @@ pub fn constructor_with_flags_reg( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/prelude.isle line 508. + // Rule at src/prelude.isle line 516. let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?; let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle index bc8b25db0ce2..83679d585418 100644 --- a/cranelift/codegen/src/isa/x64/lower.isle +++ b/cranelift/codegen/src/isa/x64/lower.isle @@ -1859,3 +1859,130 @@ (cmp_imm (OperandSize.Size64) 0xffffffff src) ;; simm32 0xffff_ffff is sign-extended to -1. (setcc (CC.Z)))) + +;; Rules for `uextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; T -> T is a no-op. +(rule (lower (has_type ty (uextend src @ (value_type =ty)))) + src) + +;; I64 -> I128. +(rule (lower (has_type $I128 (uextend src @ (value_type $I64)))) + (value_regs src (imm $I64 0))) + +;; I{8,16,32} -> I128. +(rule (lower (has_type $I128 (uextend src @ (value_type (fits_in_32 src_ty))))) + (value_regs (extend_to_gpr src $I64 (ExtendKind.Zero)) (imm $I64 0))) + +;; I{8,16,32} -> I64. +(rule (lower (has_type $I64 (uextend src @ (value_type (fits_in_32 src_ty))))) + (extend_to_gpr src $I64 (ExtendKind.Zero))) + +;; I8 -> I{16,32}, I16 -> I32. +(rule (lower (has_type (fits_in_32 dst_ty) (uextend src @ (value_type (fits_in_32 src_ty))))) + (extend_to_gpr src $I32 (ExtendKind.Zero))) + +;; I32 -> I64 with op that produces a zero-extended value in a register. +;; +;; As a particular x64 extra-pattern matching opportunity, all the ALU +;; opcodes on 32-bits will zero-extend the upper 32-bits, so we can +;; even not generate a zero-extended move in this case. +;; +;; (Note that we unfortunately can't factor out the +;; insts-that-zero-upper-32 pattern into a separate extractor until we +;; can write internal extractors with multiple rules; and we'd rather +;; keep these here than write an external extractor containing bits of +;; the instruction pattern.s) +(rule (lower (has_type $I64 + (uextend src @ (iadd _ _)))) + src) +(rule (lower (has_type $I64 + (uextend src @ (iadd_ifcout _ _)))) + src) +(rule (lower (has_type $I64 + (uextend src @ (isub _ _)))) + src) +(rule (lower (has_type $I64 + (uextend src @ (imul _ _)))) + src) +(rule (lower (has_type $I64 + (uextend src @ (band _ _)))) + src) +(rule (lower (has_type $I64 + (uextend src @ (bor _ _)))) + src) +(rule (lower (has_type $I64 + (uextend src @ (bxor _ _)))) + src) +(rule (lower (has_type $I64 + (uextend src @ (ishl _ _)))) + src) +(rule (lower (has_type $I64 + (uextend src @ (ushr _ _)))) + src) +(rule (lower (has_type $I64 + (uextend src @ (uload32 _ _ _)))) + src) + +;; Rules for `sextend` / `bextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(decl generic_sextend (Value Type Type) InstOutput) + +;; T -> T is a no-op. +(rule (generic_sextend src ty =ty) + src) + +;; Produce upper 64 bits sign-extended from lower 64: shift right by +;; 63 bits to spread the sign bit across the result. +(decl spread_sign_bit (Gpr) Gpr) +(rule (spread_sign_bit src) + (sar $I64 src (Imm8Reg.Imm8 63))) + +;; I64 -> I128. +(rule (generic_sextend src (ty_int_bool_64 _) (ty_int_bool_128 _)) + (value_regs src (spread_sign_bit src))) + +;; I{8,16,32} -> I128. +(rule (generic_sextend src (fits_in_32 src_ty) (ty_int_bool_128 _)) + (let ((lo Gpr (extend_to_gpr src $I64 (ExtendKind.Sign))) + (hi Gpr (spread_sign_bit lo))) + (value_regs lo hi))) + +;; I{8,16,32} -> I64. +(rule (generic_sextend src (fits_in_32 src_ty) (ty_int_bool_64 _)) + (extend_to_gpr src $I64 (ExtendKind.Sign))) + +;; I8 -> I{16,32}, I16 -> I32. +(rule (generic_sextend src (fits_in_32 src_ty) (fits_in_32 dst_ty)) + (extend_to_gpr src $I32 (ExtendKind.Sign))) + +(rule (lower + (has_type dst_ty + (sextend src @ (value_type src_ty)))) + (generic_sextend src src_ty dst_ty)) + +;; Bools are stored as 0/-1 so extends must sign-extend as well. +(rule (lower + (has_type dst_ty + (bextend src @ (value_type src_ty)))) + (generic_sextend src src_ty dst_ty)) + +;; Rules for `ireduce` / `breduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; T -> T is always a no-op, even I128 -> I128. +(rule (lower (has_type ty (ireduce src @ (value_type =ty)))) + src) + +;; T -> I{64,32,16,8}: We can simply pass through the value: values +;; are always stored with high bits undefined, so we can just leave +;; them be. +(rule (lower (has_type (fits_in_64 ty) (ireduce src))) + (value_regs_get_gpr src 0)) + +;; Likewise for breduce. + +(rule (lower (has_type ty (breduce src @ (value_type =ty)))) + src) + +(rule (lower (has_type (fits_in_64 ty) (breduce src))) + (value_regs_get_gpr src 0)) diff --git a/cranelift/codegen/src/isa/x64/lower.rs b/cranelift/codegen/src/isa/x64/lower.rs index 024556ee1f1f..71aa48fff049 100644 --- a/cranelift/codegen/src/isa/x64/lower.rs +++ b/cranelift/codegen/src/isa/x64/lower.rs @@ -71,25 +71,6 @@ fn matches_input>( }) } -/// Returns whether the given specified `input` is a result produced by an instruction with any of -/// the opcodes specified in `ops`. -fn matches_input_any>( - ctx: &mut C, - input: InsnInput, - ops: &[Opcode], -) -> Option { - let inputs = ctx.get_input_as_source_or_const(input.insn, input.input); - inputs.inst.and_then(|(src_inst, _)| { - let data = ctx.data(src_inst); - for &op in ops { - if data.opcode() == op { - return Some(src_inst); - } - } - None - }) -} - /// Emits instruction(s) to generate the given 64-bit constant value into a newly-allocated /// temporary register, returning that register. fn generate_constant>(ctx: &mut C, ty: Type, c: u64) -> ValueRegs { @@ -917,122 +898,12 @@ fn lower_insn_to_regs>( | Opcode::Popcnt | Opcode::Bitrev | Opcode::IsNull - | Opcode::IsInvalid => implemented_in_isle(ctx), - - Opcode::Uextend | Opcode::Sextend | Opcode::Breduce | Opcode::Bextend | Opcode::Ireduce => { - let src_ty = ctx.input_ty(insn, 0); - let dst_ty = ctx.output_ty(insn, 0); - - if src_ty == types::I128 { - assert!(dst_ty.bits() <= 64); - assert!(op == Opcode::Ireduce); - let src = put_input_in_regs(ctx, inputs[0]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - ctx.emit(Inst::gen_move(dst, src.regs()[0], types::I64)); - } else if dst_ty == types::I128 { - assert!(src_ty.bits() <= 64); - let src = put_input_in_reg(ctx, inputs[0]); - let dst = get_output_reg(ctx, outputs[0]); - assert!(op == Opcode::Uextend || op == Opcode::Sextend); - // Extend to 64 bits first. - - let ext_mode = ExtMode::new(src_ty.bits(), /* dst bits = */ 64); - if let Some(ext_mode) = ext_mode { - if op == Opcode::Sextend { - ctx.emit(Inst::movsx_rm_r(ext_mode, RegMem::reg(src), dst.regs()[0])); - } else { - ctx.emit(Inst::movzx_rm_r(ext_mode, RegMem::reg(src), dst.regs()[0])); - } - } else { - ctx.emit(Inst::mov64_rm_r(RegMem::reg(src), dst.regs()[0])); - } - - // Now generate the top 64 bits. - if op == Opcode::Sextend { - // Sign-extend: move dst[0] into dst[1] and arithmetic-shift right by 63 bits - // to spread the sign bit across all bits. - ctx.emit(Inst::gen_move( - dst.regs()[1], - dst.regs()[0].to_reg(), - types::I64, - )); - ctx.emit(Inst::shift_r( - OperandSize::Size64, - ShiftKind::ShiftRightArithmetic, - Some(63), - dst.regs()[1], - )); - } else { - // Zero-extend: just zero the top word. - ctx.emit(Inst::alu_rmi_r( - OperandSize::Size64, - AluRmiROpcode::Xor, - RegMemImm::reg(dst.regs()[1].to_reg()), - dst.regs()[1], - )); - } - } else { - // Sextend requires a sign-extended move, but all the other opcodes are simply a move - // from a zero-extended source. Here is why this works, in each case: - // - // - Breduce, Bextend: changing width of a boolean. We - // represent a bool as a 0 or -1, so Breduce can mask, while - // Bextend must sign-extend. - // - // - Ireduce: changing width of an integer. Smaller ints are stored with undefined - // high-order bits, so we can simply do a copy. - let is_sextend = match op { - Opcode::Sextend | Opcode::Bextend => true, - _ => false, - }; - if src_ty == types::I32 && dst_ty == types::I64 && !is_sextend { - // As a particular x64 extra-pattern matching opportunity, all the ALU opcodes on - // 32-bits will zero-extend the upper 32-bits, so we can even not generate a - // zero-extended move in this case. - // TODO add loads and shifts here. - if let Some(_) = matches_input_any( - ctx, - inputs[0], - &[ - Opcode::Iadd, - Opcode::IaddIfcout, - Opcode::Isub, - Opcode::Imul, - Opcode::Band, - Opcode::Bor, - Opcode::Bxor, - ], - ) { - let src = put_input_in_reg(ctx, inputs[0]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - ctx.emit(Inst::gen_move(dst, src, types::I64)); - return Ok(()); - } - } - - let src = input_to_reg_mem(ctx, inputs[0]); - let dst = get_output_reg(ctx, outputs[0]).only_reg().unwrap(); - - let ext_mode = ExtMode::new(src_ty.bits(), dst_ty.bits()); - assert_eq!( - src_ty.bits() < dst_ty.bits(), - ext_mode.is_some(), - "unexpected extension: {} -> {}", - src_ty, - dst_ty - ); - - if let Some(ext_mode) = ext_mode { - if is_sextend { - ctx.emit(Inst::movsx_rm_r(ext_mode, src, dst)); - } else { - ctx.emit(Inst::movzx_rm_r(ext_mode, src, dst)); - } - } else { - ctx.emit(Inst::mov64_rm_r(src, dst)); - } - } - } + | Opcode::IsInvalid + | Opcode::Uextend + | Opcode::Sextend + | Opcode::Breduce + | Opcode::Bextend + | Opcode::Ireduce => implemented_in_isle(ctx), Opcode::Bint => { // Booleans are stored as all-zeroes (0) or all-ones (-1). We AND diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest index 99952d3c158f..6fc00080cf2f 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.manifest @@ -1,4 +1,4 @@ src/clif.isle 9ea75a6f790b5c03 -src/prelude.isle 6b0160bfcac86902 +src/prelude.isle b2bc986bcbbbb77 src/isa/x64/inst.isle 67eb719e568c2a81 -src/isa/x64/lower.isle 142626fe062fd7d7 +src/isa/x64/lower.isle 2d06b233fb3a1e1c diff --git a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs index 8ffdff98a5bb..d2bfa465b2a4 100644 --- a/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs +++ b/cranelift/codegen/src/isa/x64/lower/isle/generated_code.rs @@ -54,6 +54,8 @@ pub trait Context { fn fits_in_64(&mut self, arg0: Type) -> Option; fn ty_32_or_64(&mut self, arg0: Type) -> Option; fn ty_8_or_16(&mut self, arg0: Type) -> Option; + fn ty_int_bool_64(&mut self, arg0: Type) -> Option; + fn ty_int_bool_128(&mut self, arg0: Type) -> Option; fn vec128(&mut self, arg0: Type) -> Option; fn not_i64x2(&mut self, arg0: Type) -> Option<()>; fn value_list_slice(&mut self, arg0: ValueList) -> ValueSlice; @@ -147,13 +149,13 @@ pub trait Context { fn popcount_low_mask(&mut self) -> VCodeConstant; } -/// Internal type SideEffectNoResult: defined at src/prelude.isle line 385. +/// Internal type SideEffectNoResult: defined at src/prelude.isle line 393. #[derive(Clone, Debug)] pub enum SideEffectNoResult { Inst { inst: MInst }, } -/// Internal type ProducesFlags: defined at src/prelude.isle line 407. +/// Internal type ProducesFlags: defined at src/prelude.isle line 415. #[derive(Clone, Debug)] pub enum ProducesFlags { ProducesFlagsSideEffect { inst: MInst }, @@ -161,7 +163,7 @@ pub enum ProducesFlags { ProducesFlagsReturnsResultWithConsumer { inst: MInst, result: Reg }, } -/// Internal type ConsumesFlags: defined at src/prelude.isle line 418. +/// Internal type ConsumesFlags: defined at src/prelude.isle line 426. #[derive(Clone, Debug)] pub enum ConsumesFlags { ConsumesFlagsReturnsResultWithProducer { @@ -552,7 +554,7 @@ pub fn constructor_side_effect( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 390. + // Rule at src/prelude.isle line 398. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::output_none(ctx); return Some(expr1_0); @@ -570,7 +572,7 @@ pub fn constructor_safepoint( inst: ref pattern1_0, } = pattern0_0 { - // Rule at src/prelude.isle line 396. + // Rule at src/prelude.isle line 404. let expr0_0 = C::emit_safepoint(ctx, pattern1_0); let expr1_0 = C::output_none(ctx); return Some(expr1_0); @@ -589,7 +591,7 @@ pub fn constructor_produces_flags_get_reg( result: pattern1_1, } = pattern0_0 { - // Rule at src/prelude.isle line 434. + // Rule at src/prelude.isle line 442. return Some(pattern1_1); } return None; @@ -606,7 +608,7 @@ pub fn constructor_produces_flags_ignore( inst: ref pattern1_0, result: pattern1_1, } => { - // Rule at src/prelude.isle line 439. + // Rule at src/prelude.isle line 447. let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { inst: pattern1_0.clone(), }; @@ -616,7 +618,7 @@ pub fn constructor_produces_flags_ignore( inst: ref pattern1_0, result: pattern1_1, } => { - // Rule at src/prelude.isle line 441. + // Rule at src/prelude.isle line 449. let expr0_0 = ProducesFlags::ProducesFlagsSideEffect { inst: pattern1_0.clone(), }; @@ -645,7 +647,7 @@ pub fn constructor_consumes_flags_concat( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 448. + // Rule at src/prelude.isle line 456. let expr0_0 = C::value_regs(ctx, pattern1_1, pattern3_1); let expr1_0 = ConsumesFlags::ConsumesFlagsTwiceReturnsValueRegs { inst1: pattern1_0.clone(), @@ -675,7 +677,7 @@ pub fn constructor_with_flags( inst: ref pattern3_0, result: pattern3_1, } => { - // Rule at src/prelude.isle line 473. + // Rule at src/prelude.isle line 481. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_reg(ctx, pattern3_1); @@ -686,7 +688,7 @@ pub fn constructor_with_flags( inst2: ref pattern3_1, result: pattern3_2, } => { - // Rule at src/prelude.isle line 479. + // Rule at src/prelude.isle line 487. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::emit(ctx, pattern3_1); @@ -699,7 +701,7 @@ pub fn constructor_with_flags( inst4: ref pattern3_3, result: pattern3_4, } => { - // Rule at src/prelude.isle line 491. + // Rule at src/prelude.isle line 499. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::emit(ctx, pattern3_1); @@ -720,7 +722,7 @@ pub fn constructor_with_flags( result: pattern3_1, } = pattern2_0 { - // Rule at src/prelude.isle line 467. + // Rule at src/prelude.isle line 475. let expr0_0 = C::emit(ctx, pattern1_0); let expr1_0 = C::emit(ctx, pattern3_0); let expr2_0 = C::value_regs(ctx, pattern1_1, pattern3_1); @@ -740,7 +742,7 @@ pub fn constructor_with_flags_reg( ) -> Option { let pattern0_0 = arg0; let pattern1_0 = arg1; - // Rule at src/prelude.isle line 508. + // Rule at src/prelude.isle line 516. let expr0_0 = constructor_with_flags(ctx, pattern0_0, pattern1_0)?; let expr1_0: usize = 0; let expr2_0 = C::value_regs_get(ctx, expr0_0, expr1_0); @@ -4260,13 +4262,126 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + // Rule at src/isa/x64/lower.isle line 1780. + let expr0_0: Type = I64; + let expr1_0 = constructor_put_in_gpr(ctx, pattern5_1)?; + let expr2_0 = constructor_do_bitrev64(ctx, expr0_0, expr1_0)?; + let expr3_0 = constructor_output_gpr(ctx, expr2_0)?; + return Some(expr3_0); + } + &Opcode::Uextend => { + if let Some(pattern7_0) = C::def_inst(ctx, pattern5_1) { + let pattern8_0 = C::inst_data(ctx, pattern7_0); + match &pattern8_0 { + &InstructionData::Binary { + opcode: ref pattern9_0, + args: ref pattern9_1, + } => { + match pattern9_0 { + &Opcode::Iadd => { + let (pattern11_0, pattern11_1) = + C::unpack_value_array_2(ctx, pattern9_1); + // Rule at src/isa/x64/lower.isle line 1896. + let expr0_0 = + constructor_output_value(ctx, pattern5_1)?; + return Some(expr0_0); + } + &Opcode::Isub => { + let (pattern11_0, pattern11_1) = + C::unpack_value_array_2(ctx, pattern9_1); + // Rule at src/isa/x64/lower.isle line 1902. + let expr0_0 = + constructor_output_value(ctx, pattern5_1)?; + return Some(expr0_0); + } + &Opcode::Imul => { + let (pattern11_0, pattern11_1) = + C::unpack_value_array_2(ctx, pattern9_1); + // Rule at src/isa/x64/lower.isle line 1905. + let expr0_0 = + constructor_output_value(ctx, pattern5_1)?; + return Some(expr0_0); + } + &Opcode::IaddIfcout => { + let (pattern11_0, pattern11_1) = + C::unpack_value_array_2(ctx, pattern9_1); + // Rule at src/isa/x64/lower.isle line 1899. + let expr0_0 = + constructor_output_value(ctx, pattern5_1)?; + return Some(expr0_0); + } + &Opcode::Band => { + let (pattern11_0, pattern11_1) = + C::unpack_value_array_2(ctx, pattern9_1); + // Rule at src/isa/x64/lower.isle line 1908. + let expr0_0 = + constructor_output_value(ctx, pattern5_1)?; + return Some(expr0_0); + } + &Opcode::Bor => { + let (pattern11_0, pattern11_1) = + C::unpack_value_array_2(ctx, pattern9_1); + // Rule at src/isa/x64/lower.isle line 1911. + let expr0_0 = + constructor_output_value(ctx, pattern5_1)?; + return Some(expr0_0); + } + &Opcode::Bxor => { + let (pattern11_0, pattern11_1) = + C::unpack_value_array_2(ctx, pattern9_1); + // Rule at src/isa/x64/lower.isle line 1914. + let expr0_0 = + constructor_output_value(ctx, pattern5_1)?; + return Some(expr0_0); + } + &Opcode::Ishl => { + let (pattern11_0, pattern11_1) = + C::unpack_value_array_2(ctx, pattern9_1); + // Rule at src/isa/x64/lower.isle line 1917. + let expr0_0 = + constructor_output_value(ctx, pattern5_1)?; + return Some(expr0_0); + } + &Opcode::Ushr => { + let (pattern11_0, pattern11_1) = + C::unpack_value_array_2(ctx, pattern9_1); + // Rule at src/isa/x64/lower.isle line 1920. + let expr0_0 = + constructor_output_value(ctx, pattern5_1)?; + return Some(expr0_0); + } + _ => {} + } + } + &InstructionData::Load { + opcode: ref pattern9_0, + arg: pattern9_1, + flags: pattern9_2, + offset: pattern9_3, + } => { + if let &Opcode::Uload32 = pattern9_0 { + // Rule at src/isa/x64/lower.isle line 1923. + let expr0_0 = constructor_output_value(ctx, pattern5_1)?; + return Some(expr0_0); + } + } + _ => {} + } + } + let pattern7_0 = C::value_type(ctx, pattern5_1); + if let Some(pattern8_0) = C::fits_in_32(ctx, pattern7_0) { + // Rule at src/isa/x64/lower.isle line 1878. + let expr0_0: Type = I64; + let expr1_0 = ExtendKind::Zero; + let expr2_0 = + constructor_extend_to_gpr(ctx, pattern5_1, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_gpr(ctx, expr2_0)?; + return Some(expr3_0); + } + } + _ => {} } } } @@ -4630,6 +4745,33 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + let pattern7_0 = C::value_type(ctx, pattern5_1); + if pattern7_0 == I64 { + // Rule at src/isa/x64/lower.isle line 1870. + let expr0_0 = C::put_in_reg(ctx, pattern5_1); + let expr1_0: Type = I64; + let expr2_0: u64 = 0; + let expr3_0 = constructor_imm(ctx, expr1_0, expr2_0)?; + let expr4_0 = C::value_regs(ctx, expr0_0, expr3_0); + let expr5_0 = C::output(ctx, expr4_0); + return Some(expr5_0); + } + if let Some(pattern8_0) = C::fits_in_32(ctx, pattern7_0) { + // Rule at src/isa/x64/lower.isle line 1874. + let expr0_0: Type = I64; + let expr1_0 = ExtendKind::Zero; + let expr2_0 = + constructor_extend_to_gpr(ctx, pattern5_1, expr0_0, &expr1_0)?; + let expr3_0 = C::gpr_to_reg(ctx, expr2_0); + let expr4_0: Type = I64; + let expr5_0: u64 = 0; + let expr6_0 = constructor_imm(ctx, expr4_0, expr5_0)?; + let expr7_0 = C::value_regs(ctx, expr3_0, expr6_0); + let expr8_0 = C::output(ctx, expr7_0); + return Some(expr8_0); + } + } _ => {} } } @@ -5390,6 +5532,52 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + match pattern4_0 { + &Opcode::Breduce => { + let pattern6_0 = C::value_type(ctx, pattern4_1); + if pattern6_0 == pattern2_0 { + // Rule at src/isa/x64/lower.isle line 1984. + let expr0_0 = constructor_output_value(ctx, pattern4_1)?; + return Some(expr0_0); + } + } + &Opcode::Bextend => { + let pattern6_0 = C::value_type(ctx, pattern4_1); + // Rule at src/isa/x64/lower.isle line 1965. + let expr0_0 = + constructor_generic_sextend(ctx, pattern4_1, pattern6_0, pattern2_0)?; + return Some(expr0_0); + } + &Opcode::Ireduce => { + let pattern6_0 = C::value_type(ctx, pattern4_1); + if pattern6_0 == pattern2_0 { + // Rule at src/isa/x64/lower.isle line 1973. + let expr0_0 = constructor_output_value(ctx, pattern4_1)?; + return Some(expr0_0); + } + } + &Opcode::Uextend => { + let pattern6_0 = C::value_type(ctx, pattern4_1); + if pattern6_0 == pattern2_0 { + // Rule at src/isa/x64/lower.isle line 1866. + let expr0_0 = constructor_output_value(ctx, pattern4_1)?; + return Some(expr0_0); + } + } + &Opcode::Sextend => { + let pattern6_0 = C::value_type(ctx, pattern4_1); + // Rule at src/isa/x64/lower.isle line 1959. + let expr0_0 = + constructor_generic_sextend(ctx, pattern4_1, pattern6_0, pattern2_0)?; + return Some(expr0_0); + } + _ => {} + } + } _ => {} } if let Some(()) = C::avx512vl_enabled(ctx, pattern2_0) { @@ -6568,6 +6756,27 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option {} } } + if let Some(pattern3_0) = C::fits_in_32(ctx, pattern2_0) { + let pattern4_0 = C::inst_data(ctx, pattern0_0); + if let &InstructionData::Unary { + opcode: ref pattern5_0, + arg: pattern5_1, + } = &pattern4_0 + { + if let &Opcode::Uextend = pattern5_0 { + let pattern7_0 = C::value_type(ctx, pattern5_1); + if let Some(pattern8_0) = C::fits_in_32(ctx, pattern7_0) { + // Rule at src/isa/x64/lower.isle line 1882. + let expr0_0: Type = I32; + let expr1_0 = ExtendKind::Zero; + let expr2_0 = + constructor_extend_to_gpr(ctx, pattern5_1, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_gpr(ctx, expr2_0)?; + return Some(expr3_0); + } + } + } + } if let Some(pattern3_0) = C::fits_in_64(ctx, pattern2_0) { let pattern4_0 = C::inst_data(ctx, pattern0_0); match &pattern4_0 { @@ -6993,6 +7202,22 @@ pub fn constructor_lower(ctx: &mut C, arg0: Inst) -> Option { + // Rule at src/isa/x64/lower.isle line 1987. + let expr0_0 = C::put_in_regs(ctx, pattern5_1); + let expr1_0: usize = 0; + let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?; + let expr3_0 = constructor_output_gpr(ctx, expr2_0)?; + return Some(expr3_0); + } + &Opcode::Ireduce => { + // Rule at src/isa/x64/lower.isle line 1979. + let expr0_0 = C::put_in_regs(ctx, pattern5_1); + let expr1_0: usize = 0; + let expr2_0 = constructor_value_regs_get_gpr(ctx, expr0_0, expr1_0)?; + let expr3_0 = constructor_output_gpr(ctx, expr2_0)?; + return Some(expr3_0); + } _ => {} } } @@ -8192,3 +8417,77 @@ pub fn constructor_do_bitrev64(ctx: &mut C, arg0: Type, arg1: Gpr) - } return None; } + +// Generated as internal constructor for term generic_sextend. +pub fn constructor_generic_sextend( + ctx: &mut C, + arg0: Value, + arg1: Type, + arg2: Type, +) -> Option { + let pattern0_0 = arg0; + let pattern1_0 = arg1; + if let Some(pattern2_0) = C::fits_in_32(ctx, pattern1_0) { + let pattern3_0 = arg2; + if let Some(pattern4_0) = C::fits_in_32(ctx, pattern3_0) { + // Rule at src/isa/x64/lower.isle line 1956. + let expr0_0: Type = I32; + let expr1_0 = ExtendKind::Sign; + let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_gpr(ctx, expr2_0)?; + return Some(expr3_0); + } + if let Some(pattern4_0) = C::ty_int_bool_64(ctx, pattern3_0) { + // Rule at src/isa/x64/lower.isle line 1952. + let expr0_0: Type = I64; + let expr1_0 = ExtendKind::Sign; + let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?; + let expr3_0 = constructor_output_gpr(ctx, expr2_0)?; + return Some(expr3_0); + } + if let Some(pattern4_0) = C::ty_int_bool_128(ctx, pattern3_0) { + // Rule at src/isa/x64/lower.isle line 1946. + let expr0_0: Type = I64; + let expr1_0 = ExtendKind::Sign; + let expr2_0 = constructor_extend_to_gpr(ctx, pattern0_0, expr0_0, &expr1_0)?; + let expr3_0 = constructor_spread_sign_bit(ctx, expr2_0)?; + let expr4_0 = C::gpr_to_reg(ctx, expr2_0); + let expr5_0 = C::gpr_to_reg(ctx, expr3_0); + let expr6_0 = C::value_regs(ctx, expr4_0, expr5_0); + let expr7_0 = C::output(ctx, expr6_0); + return Some(expr7_0); + } + } + if let Some(pattern2_0) = C::ty_int_bool_64(ctx, pattern1_0) { + let pattern3_0 = arg2; + if let Some(pattern4_0) = C::ty_int_bool_128(ctx, pattern3_0) { + // Rule at src/isa/x64/lower.isle line 1942. + let expr0_0 = C::put_in_reg(ctx, pattern0_0); + let expr1_0 = constructor_put_in_gpr(ctx, pattern0_0)?; + let expr2_0 = constructor_spread_sign_bit(ctx, expr1_0)?; + let expr3_0 = C::gpr_to_reg(ctx, expr2_0); + let expr4_0 = C::value_regs(ctx, expr0_0, expr3_0); + let expr5_0 = C::output(ctx, expr4_0); + return Some(expr5_0); + } + } + let pattern2_0 = arg2; + if pattern2_0 == pattern1_0 { + // Rule at src/isa/x64/lower.isle line 1932. + let expr0_0 = constructor_output_value(ctx, pattern0_0)?; + return Some(expr0_0); + } + return None; +} + +// Generated as internal constructor for term spread_sign_bit. +pub fn constructor_spread_sign_bit(ctx: &mut C, arg0: Gpr) -> Option { + let pattern0_0 = arg0; + // Rule at src/isa/x64/lower.isle line 1938. + let expr0_0: Type = I64; + let expr1_0: u8 = 63; + let expr2_0 = Imm8Reg::Imm8 { imm: expr1_0 }; + let expr3_0 = C::imm8_reg_to_imm8_gpr(ctx, &expr2_0); + let expr4_0 = constructor_sar(ctx, expr0_0, pattern0_0, &expr3_0)?; + return Some(expr4_0); +} diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rs index c07fc393cb69..3afb02f8a56d 100644 --- a/cranelift/codegen/src/machinst/isle.rs +++ b/cranelift/codegen/src/machinst/isle.rs @@ -237,6 +237,22 @@ macro_rules! isle_prelude_methods { } } + #[inline] + fn ty_int_bool_64(&mut self, ty: Type) -> Option { + match ty { + I64 | B64 => Some(ty), + _ => None, + } + } + + #[inline] + fn ty_int_bool_128(&mut self, ty: Type) -> Option { + match ty { + I128 | B128 => Some(ty), + _ => None, + } + } + fn vec128(&mut self, ty: Type) -> Option { if ty.is_vector() && ty.bits() == 128 { Some(ty) diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle index c631bdc0e606..397c5b0b88e2 100644 --- a/cranelift/codegen/src/prelude.isle +++ b/cranelift/codegen/src/prelude.isle @@ -253,14 +253,22 @@ (decl fits_in_64 (Type) Type) (extern extractor fits_in_64 fits_in_64) -;; An extractor that maches 32- and 64-bit types only. +;; An extractor that matches 32- and 64-bit types only. (decl ty_32_or_64 (Type) Type) (extern extractor ty_32_or_64 ty_32_or_64) -;; An extractor that maches 8- and 16-bit types only. +;; An extractor that matches 8- and 16-bit types only. (decl ty_8_or_16 (Type) Type) (extern extractor ty_8_or_16 ty_8_or_16) +;; An extractor that matches I64 or B64. +(decl ty_int_bool_64 (Type) Type) +(extern extractor ty_int_bool_64 ty_int_bool_64) + +;; An extractor that matches I128 or B128. +(decl ty_int_bool_128 (Type) Type) +(extern extractor ty_int_bool_128 ty_int_bool_128) + ;; An extractor that only matches 128-bit vector types. (decl vec128 (Type) Type) (extern extractor vec128 vec128)