Skip to content

Commit

Permalink
Auto merge of #120500 - oli-obk:intrinsics2.0, r=WaffleLapkin
Browse files Browse the repository at this point in the history 
Implement intrinsics with fallback bodies

fixes #93145 (though we can port many more intrinsics)
cc #63585

The way this works is that the backend logic for generating custom code for intrinsics has been made fallible. The only failure path is "this intrinsic is unknown". The `Instance` (that was `InstanceDef::Intrinsic`) then gets converted to `InstanceDef::Item`, which represents the fallback body. A regular function call to that body is then codegenned. This is currently implemented for

* codegen_ssa (so llvm and gcc)
* codegen_cranelift

other backends will need to adjust, but they can just keep doing what they were doing if they prefer (though adding new intrinsics to the compiler will then require them to implement them, instead of getting the fallback body).

cc `@scottmcm` `@WaffleLapkin`

### todo

* [ ] miri support
* [x] default intrinsic name to name of function instead of requiring it to be specified in attribute
* [x] make sure that the bodies are always available (must be collected for metadata)
  • Loading branch information
@bors
bors committed Feb 16, 2024
2 parents 1be4688 + 164b9c3 commit dfa88b3
Show file tree
Hide file tree
Showing 49 changed files with 620 additions and 451 deletions.
13 changes: 3 additions & 10 deletions compiler/rustc_borrowck/src/type_check/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1666,16 +1666,9 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {

let func_ty = func.ty(body, self.infcx.tcx);
if let ty::FnDef(def_id, _) = *func_ty.kind() {
if self.tcx().is_intrinsic(def_id) {
match self.tcx().item_name(def_id) {
sym::simd_shuffle => {
if !matches!(args[2], Spanned { node: Operand::Constant(_), .. }) {
self.tcx()
.dcx()
.emit_err(SimdShuffleLastConst { span: term.source_info.span });
}
}
_ => {}
if let Some(sym::simd_shuffle) = self.tcx().intrinsic(def_id) {
if !matches!(args[2], Spanned { node: Operand::Constant(_), .. }) {
self.tcx().dcx().emit_err(SimdShuffleLastConst { span: term.source_info.span });
}
}
}
Expand Down
8 changes: 5 additions & 3 deletions compiler/rustc_codegen_cranelift/src/abi/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -387,15 +387,17 @@ pub(crate) fn codegen_terminator_call<'tcx>(

match instance.def {
InstanceDef::Intrinsic(_) => {
crate::intrinsics::codegen_intrinsic_call(
match crate::intrinsics::codegen_intrinsic_call(
fx,
instance,
args,
ret_place,
target,
source_info,
);
return;
) {
Ok(()) => return,
Err(instance) => Some(instance),
}
}
InstanceDef::DropGlue(_, None) => {
// empty drop glue - a nop.
Expand Down
72 changes: 27 additions & 45 deletions compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
destination: CPlace<'tcx>,
target: Option<BasicBlock>,
source_info: mir::SourceInfo,
) {
) -> Result<(), Instance<'tcx>> {
let intrinsic = fx.tcx.item_name(instance.def_id());
let instance_args = instance.args;

Expand All @@ -295,8 +295,9 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
destination,
target,
source_info,
);
)?;
}
Ok(())
}

fn codegen_float_intrinsic_call<'tcx>(
Expand Down Expand Up @@ -430,25 +431,20 @@ fn codegen_regular_intrinsic_call<'tcx>(
ret: CPlace<'tcx>,
destination: Option<BasicBlock>,
source_info: mir::SourceInfo,
) {
) -> Result<(), Instance<'tcx>> {
assert_eq!(generic_args, instance.args);
let usize_layout = fx.layout_of(fx.tcx.types.usize);

match intrinsic {
sym::abort => {
fx.bcx.ins().trap(TrapCode::User(0));
return;
return Ok(());
}
sym::likely | sym::unlikely => {
intrinsic_args!(fx, args => (a); intrinsic);

ret.write_cvalue(fx, a);
}
sym::is_val_statically_known => {
intrinsic_args!(fx, args => (_a); intrinsic);

let res = fx.bcx.ins().iconst(types::I8, 0);
ret.write_cvalue(fx, CValue::by_val(res, ret.layout()));
}
sym::breakpoint => {
intrinsic_args!(fx, args => (); intrinsic);

Expand Down Expand Up @@ -697,7 +693,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
})
});
crate::base::codegen_panic_nounwind(fx, &msg_str, Some(source_info.span));
return;
return Ok(());
}
}
}
Expand Down Expand Up @@ -792,7 +788,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
// special case for compiler-builtins to avoid having to patch it
crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
return;
return Ok(());
} else {
fx.tcx
.dcx()
Expand All @@ -802,7 +798,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
_ => {
report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty);
return;
return Ok(());
}
}
let clif_ty = fx.clif_type(ty).unwrap();
Expand All @@ -823,7 +819,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
// special case for compiler-builtins to avoid having to patch it
crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
return;
return Ok(());
} else {
fx.tcx
.dcx()
Expand All @@ -833,7 +829,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
_ => {
report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty);
return;
return Ok(());
}
}

Expand All @@ -850,7 +846,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
_ => {
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
return;
return Ok(());
}
}
let ty = fx.clif_type(layout.ty).unwrap();
Expand All @@ -872,7 +868,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
_ => {
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
return;
return Ok(());
}
}

Expand All @@ -895,7 +891,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
_ => {
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
return;
return Ok(());
}
}
let ty = fx.clif_type(layout.ty).unwrap();
Expand All @@ -917,7 +913,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
_ => {
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
return;
return Ok(());
}
}
let ty = fx.clif_type(layout.ty).unwrap();
Expand All @@ -939,7 +935,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
_ => {
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
return;
return Ok(());
}
}
let ty = fx.clif_type(layout.ty).unwrap();
Expand All @@ -960,7 +956,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
_ => {
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
return;
return Ok(());
}
}
let ty = fx.clif_type(layout.ty).unwrap();
Expand All @@ -981,7 +977,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
_ => {
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
return;
return Ok(());
}
}
let ty = fx.clif_type(layout.ty).unwrap();
Expand All @@ -1002,7 +998,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
_ => {
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
return;
return Ok(());
}
}
let ty = fx.clif_type(layout.ty).unwrap();
Expand All @@ -1023,7 +1019,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
_ => {
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
return;
return Ok(());
}
}
let ty = fx.clif_type(layout.ty).unwrap();
Expand All @@ -1044,7 +1040,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
_ => {
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
return;
return Ok(());
}
}
let ty = fx.clif_type(layout.ty).unwrap();
Expand All @@ -1065,7 +1061,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
_ => {
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
return;
return Ok(());
}
}
let ty = fx.clif_type(layout.ty).unwrap();
Expand All @@ -1086,7 +1082,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
_ => {
report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
return;
return Ok(());
}
}
let ty = fx.clif_type(layout.ty).unwrap();
Expand Down Expand Up @@ -1233,19 +1229,6 @@ fn codegen_regular_intrinsic_call<'tcx>(
ret.write_cvalue(fx, CValue::by_val(cmp, ret.layout()));
}

sym::const_allocate => {
intrinsic_args!(fx, args => (_size, _align); intrinsic);

// returns a null pointer at runtime.
let null = fx.bcx.ins().iconst(fx.pointer_type, 0);
ret.write_cvalue(fx, CValue::by_val(null, ret.layout()));
}

sym::const_deallocate => {
intrinsic_args!(fx, args => (_ptr, _size, _align); intrinsic);
// nop at runtime.
}

sym::black_box => {
intrinsic_args!(fx, args => (a); intrinsic);

Expand All @@ -1261,13 +1244,12 @@ fn codegen_regular_intrinsic_call<'tcx>(
);
}

_ => {
fx.tcx
.dcx()
.span_fatal(source_info.span, format!("unsupported intrinsic {}", intrinsic));
}
// Unimplemented intrinsics must have a fallback body. The fallback body is obtained
// by converting the `InstanceDef::Intrinsic` to an `InstanceDef::Item`.
_ => return Err(Instance::new(instance.def_id(), instance.args)),
}

let ret_block = fx.get_block(destination.unwrap());
fx.bcx.ins().jump(ret_block, &[]);
Ok(())
}
18 changes: 10 additions & 8 deletions compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ fn get_simple_intrinsic<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, name: Symbol) ->
}

impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
fn codegen_intrinsic_call(&mut self, instance: Instance<'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, args: &[OperandRef<'tcx, RValue<'gcc>>], llresult: RValue<'gcc>, span: Span) {
fn codegen_intrinsic_call(&mut self, instance: Instance<'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, args: &[OperandRef<'tcx, RValue<'gcc>>], llresult: RValue<'gcc>, span: Span) -> Result<(), Instance<'tcx>> {
let tcx = self.tcx;
let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all());

Expand Down Expand Up @@ -137,7 +137,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
args[2].immediate(),
llresult,
);
return;
return Ok(());
}
sym::breakpoint => {
unimplemented!();
Expand Down Expand Up @@ -166,12 +166,12 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
sym::volatile_store => {
let dst = args[0].deref(self.cx());
args[1].val.volatile_store(self, dst);
return;
return Ok(());
}
sym::unaligned_volatile_store => {
let dst = args[0].deref(self.cx());
args[1].val.unaligned_volatile_store(self, dst);
return;
return Ok(());
}
sym::prefetch_read_data
| sym::prefetch_write_data
Expand Down Expand Up @@ -269,7 +269,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
},
None => {
tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType { span, name, ty });
return;
return Ok(());
}
}
}
Expand Down Expand Up @@ -339,7 +339,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
extended_asm.set_volatile_flag(true);

// We have copied the value to `result` already.
return;
return Ok(());
}

sym::ptr_mask => {
Expand All @@ -357,11 +357,12 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
_ if name_str.starts_with("simd_") => {
match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) {
Ok(llval) => llval,
Err(()) => return,
Err(()) => return Ok(()),
}
}

_ => bug!("unknown intrinsic '{}'", name),
// Fall back to default body
_ => return Err(Instance::new(instance.def_id(), instance.args)),
};

if !fn_abi.ret.is_ignore() {
Expand All @@ -376,6 +377,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
.store(self, result);
}
}
Ok(())
}

fn abort(&mut self) {
Expand Down
Loading

0 comments on commit dfa88b3

Please sign in to comment.