From 09400e4f5c3e31ba528c2dbcf8e1ccef2a66ce3d Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 21 Mar 2024 13:10:08 +0100 Subject: [PATCH] Switch LLVM codegen of Ptr{T} to an actual pointer type. (#53687) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR switches our code generation for `Ptr{T}` from `i64` to an actual LLVM pointer type (`ptr` when using opaque pointers, an untyped `i8*` otherwise). The main motivation is to simplify `llvmcall` usage (doing away with the `inttoptr`/`ptrtoint` conversions), and also make it possible to simply use `ccall` to call intrinsics with `Ptr`-valued arguments (where we currently always need `llvmcall` for converting to an actual pointer). Changing codegen like this is a breaking change for `llvmcall` users, but I've added backwards compatibility and a deprecation warning. Before: ```llvm julia> @code_llvm pointer([]) define i64 @julia_pointer_1542(ptr noundef nonnull align 8 dereferenceable(24) %"x::Array") #0 { top: ; ┌ @ pointer.jl:65 within `cconvert` %0 = load ptr, ptr %"x::Array", align 8 ; └ ; ┌ @ pointer.jl:90 within `unsafe_convert` ; │┌ @ pointer.jl:30 within `convert` %bitcast_coercion = ptrtoint ptr %0 to i64 ret i64 %bitcast_coercion ; └└ } ``` After: ```llvm julia> @code_llvm pointer([]) define ptr @julia_pointer_3880(ptr noundef nonnull align 8 dereferenceable(24) %"x::Array") #0 { top: ; ┌ @ pointer.jl:65 within `cconvert` %0 = load ptr, ptr %"x::Array", align 8 ; └ ; ┌ @ pointer.jl:90 within `unsafe_convert` ; │┌ @ pointer.jl:30 within `convert` ret ptr %0 ; └└ } ``` This also simplifies "real code", e.g., when `ccall` converts an Array to a pointer, resulting in some more optimization opportunities. --- NEWS.md | 5 + base/atomics.jl | 12 +- base/compiler/tfuncs.jl | 7 +- base/pointer.jl | 4 +- base/task.jl | 3 +- src/ccall.cpp | 214 ++++++++++++++++++++++++--------- src/cgutils.cpp | 2 + src/codegen.cpp | 4 +- src/intrinsics.cpp | 67 +++++++---- src/intrinsics.h | 5 +- src/llvm-remove-addrspaces.cpp | 9 +- src/runtime_intrinsics.c | 18 ++- 12 files changed, 253 insertions(+), 97 deletions(-) diff --git a/NEWS.md b/NEWS.md index 695427d7eba8e..72b4629fe4174 100644 --- a/NEWS.md +++ b/NEWS.md @@ -26,6 +26,11 @@ Language changes Compiler/Runtime improvements ----------------------------- +- Generated LLVM IR now uses actual pointer types instead of passing pointers as integers. + This affects `llvmcall`: Inline LLVM IR should be updated to use `i8*` or `ptr` instead of + `i32` or `i64`, and remove unneeded `ptrtoint`/`inttoptr` conversions. For compatibility, + IR with integer pointers is still supported, but generates a deprecation warning. ([#53687]) + Command-line option changes --------------------------- diff --git a/base/atomics.jl b/base/atomics.jl index dde981b8c0e6a..e6f3a5654cbf7 100644 --- a/base/atomics.jl +++ b/base/atomics.jl @@ -364,13 +364,13 @@ for typ in atomictypes irt = "$ilt, $ilt*" @eval getindex(x::Atomic{$typ}) = GC.@preserve x llvmcall($""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %ptr = bitcast i8* %0 to $lt* %rv = load atomic $rt %ptr acquire, align $(gc_alignment(typ)) ret $lt %rv """, $typ, Tuple{Ptr{$typ}}, unsafe_convert(Ptr{$typ}, x)) @eval setindex!(x::Atomic{$typ}, v::$typ) = GC.@preserve x llvmcall($""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %ptr = bitcast i8* %0 to $lt* store atomic $lt %1, $lt* %ptr release, align $(gc_alignment(typ)) ret void """, Cvoid, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v) @@ -379,7 +379,7 @@ for typ in atomictypes if typ <: Integer @eval atomic_cas!(x::Atomic{$typ}, cmp::$typ, new::$typ) = GC.@preserve x llvmcall($""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %ptr = bitcast i8* %0 to $lt* %rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 acq_rel acquire %rv = extractvalue { $lt, i1 } %rs, 0 ret $lt %rv @@ -388,7 +388,7 @@ for typ in atomictypes else @eval atomic_cas!(x::Atomic{$typ}, cmp::$typ, new::$typ) = GC.@preserve x llvmcall($""" - %iptr = inttoptr i$WORD_SIZE %0 to $ilt* + %iptr = bitcast i8* %0 to $ilt* %icmp = bitcast $lt %1 to $ilt %inew = bitcast $lt %2 to $ilt %irs = cmpxchg $ilt* %iptr, $ilt %icmp, $ilt %inew acq_rel acquire @@ -411,7 +411,7 @@ for typ in atomictypes if typ <: Integer @eval $fn(x::Atomic{$typ}, v::$typ) = GC.@preserve x llvmcall($""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %ptr = bitcast i8* %0 to $lt* %rv = atomicrmw $rmw $lt* %ptr, $lt %1 acq_rel ret $lt %rv """, $typ, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v) @@ -419,7 +419,7 @@ for typ in atomictypes rmwop === :xchg || continue @eval $fn(x::Atomic{$typ}, v::$typ) = GC.@preserve x llvmcall($""" - %iptr = inttoptr i$WORD_SIZE %0 to $ilt* + %iptr = bitcast i8* %0 to $ilt* %ival = bitcast $lt %1 to $ilt %irv = atomicrmw $rmw $ilt* %iptr, $ilt %ival acq_rel %rv = bitcast $ilt %irv to $lt diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index d697ec56bf55f..090be7e91fad9 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -184,8 +184,6 @@ add_tfunc(sdiv_int, 2, 2, math_tfunc, 20) add_tfunc(udiv_int, 2, 2, math_tfunc, 20) add_tfunc(srem_int, 2, 2, math_tfunc, 20) add_tfunc(urem_int, 2, 2, math_tfunc, 20) -add_tfunc(add_ptr, 2, 2, math_tfunc, 1) -add_tfunc(sub_ptr, 2, 2, math_tfunc, 1) add_tfunc(neg_float, 1, 1, math_tfunc, 1) add_tfunc(add_float, 2, 2, math_tfunc, 2) add_tfunc(sub_float, 2, 2, math_tfunc, 2) @@ -662,6 +660,9 @@ function pointer_eltype(@nospecialize(ptr)) return Any end +@nospecs function pointerarith_tfunc(𝕃::AbstractLattice, ptr, offset) + return ptr +end @nospecs function pointerref_tfunc(𝕃::AbstractLattice, a, i, align) return pointer_eltype(a) end @@ -705,6 +706,8 @@ end end return ccall(:jl_apply_cmpswap_type, Any, (Any,), T) where T end +add_tfunc(add_ptr, 2, 2, pointerarith_tfunc, 1) +add_tfunc(sub_ptr, 2, 2, pointerarith_tfunc, 1) add_tfunc(pointerref, 3, 3, pointerref_tfunc, 4) add_tfunc(pointerset, 4, 4, pointerset_tfunc, 5) add_tfunc(atomic_fence, 1, 1, atomic_fence_tfunc, 4) diff --git a/base/pointer.jl b/base/pointer.jl index 2f67c8a6757cb..494e87996355c 100644 --- a/base/pointer.jl +++ b/base/pointer.jl @@ -313,8 +313,8 @@ isless(x::Ptr{T}, y::Ptr{T}) where {T} = x < y <(x::Ptr, y::Ptr) = UInt(x) < UInt(y) -(x::Ptr, y::Ptr) = UInt(x) - UInt(y) -+(x::Ptr, y::Integer) = oftype(x, add_ptr(UInt(x), (y % UInt) % UInt)) --(x::Ptr, y::Integer) = oftype(x, sub_ptr(UInt(x), (y % UInt) % UInt)) ++(x::Ptr, y::Integer) = add_ptr(x, (y % UInt) % UInt) +-(x::Ptr, y::Integer) = sub_ptr(x, (y % UInt) % UInt) +(x::Integer, y::Ptr) = y + x unsigned(x::Ptr) = UInt(x) diff --git a/base/task.jl b/base/task.jl index 2b758d342b169..90f74441c7fbd 100644 --- a/base/task.jl +++ b/base/task.jl @@ -154,8 +154,7 @@ const _state_index = findfirst(==(:_state), fieldnames(Task)) @eval function load_state_acquire(t) # TODO: Replace this by proper atomic operations when available @GC.preserve t llvmcall($(""" - %ptr = inttoptr i$(Sys.WORD_SIZE) %0 to i8* - %rv = load atomic i8, i8* %ptr acquire, align 8 + %rv = load atomic i8, i8* %0 acquire, align 8 ret i8 %rv """), UInt8, Tuple{Ptr{UInt8}}, Ptr{UInt8}(pointer_from_objref(t) + fieldoffset(Task, _state_index))) diff --git a/src/ccall.cpp b/src/ccall.cpp index 53714cebcbe2a..2d4343a00a2a5 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -616,7 +616,7 @@ static void interpret_symbol_arg(jl_codectx_t &ctx, native_sym_arg_t &out, jl_va emit_cpointercheck(ctx, arg1, errmsg); } arg1 = update_julia_type(ctx, arg1, (jl_value_t*)jl_voidpointer_type); - jl_ptr = emit_unbox(ctx, ctx.types().T_size, arg1, (jl_value_t*)jl_voidpointer_type); + jl_ptr = emit_unbox(ctx, ctx.types().T_ptr, arg1, (jl_value_t*)jl_voidpointer_type); } else { out.gcroot = ptr; @@ -696,7 +696,7 @@ static jl_cgval_t emit_cglobal(jl_codectx_t &ctx, jl_value_t **args, size_t narg else { rt = (jl_value_t*)jl_voidpointer_type; } - Type *lrt = ctx.types().T_size; + Type *lrt = ctx.types().T_ptr; assert(lrt == julia_type_to_llvm(ctx, rt)); interpret_symbol_arg(ctx, sym, args[1], /*ccall=*/false, false); @@ -723,7 +723,6 @@ static jl_cgval_t emit_cglobal(jl_codectx_t &ctx, jl_value_t **args, size_t narg } else /*if (ctx.emission_context.imaging) */{ res = runtime_sym_lookup(ctx, cast(getInt8PtrTy(ctx.builder.getContext())), sym.f_lib, NULL, sym.f_name, ctx.f); - res = ctx.builder.CreatePtrToInt(res, lrt); } } @@ -814,18 +813,13 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar JL_TYPECHK(llvmcall, type, rt); JL_TYPECHK(llvmcall, type, at); - // Generate arguments - std::string arguments; - raw_string_ostream argstream(arguments); - jl_svec_t *tt = ((jl_datatype_t*)at)->parameters; - jl_value_t *rtt = rt; + // Determine argument types + // + // Semantics for arguments are as follows: + // If the argument type is immutable (including bitstype), we pass the loaded llvm value + // type. Otherwise we pass a pointer to a jl_value_t. + jl_svec_t *tt = ((jl_datatype_t *)at)->parameters; size_t nargt = jl_svec_len(tt); - - /* - * Semantics for arguments are as follows: - * If the argument type is immutable (including bitstype), we pass the loaded llvm value - * type. Otherwise we pass a pointer to a jl_value_t. - */ SmallVector argtypes; SmallVector argvals(nargt); for (size_t i = 0; i < nargt; ++i) { @@ -846,45 +840,87 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar argvals[i] = llvm_type_rewrite(ctx, v, t, issigned); } + // Determine return type + jl_value_t *rtt = rt; bool retboxed; Type *rettype = julia_type_to_llvm(ctx, rtt, &retboxed); // Make sure to find a unique name std::string ir_name; while (true) { - raw_string_ostream(ir_name) << (ctx.f->getName().str()) << "u" << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); + raw_string_ostream(ir_name) + << (ctx.f->getName().str()) << "u" + << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); if (jl_Module->getFunction(ir_name) == NULL) break; } // generate a temporary module that contains our IR std::unique_ptr Mod; + Function *f; if (entry == NULL) { // we only have function IR, which we should put in a function - bool first = true; + // stringify arguments + std::string arguments; + raw_string_ostream argstream(arguments); for (SmallVector::iterator it = argtypes.begin(); it != argtypes.end(); ++it) { - if (!first) + if (it != argtypes.begin()) argstream << ","; - else - first = false; (*it)->print(argstream); argstream << " "; } + // stringify return type std::string rstring; raw_string_ostream rtypename(rstring); rettype->print(rtypename); - std::map localDecls; + // generate IR function definition std::string ir_string; raw_string_ostream ir_stream(ir_string); - ir_stream << "; Number of arguments: " << nargt << "\n" - << "define "< 0) + compat_argstream << ","; + jl_value_t *tti = jl_svecref(tt, i); + Type *t; + if (jl_is_cpointer_type(tti)) + t = ctx.types().T_size; + else + t = argtypes[i]; + t->print(compat_argstream); + compat_argstream << " "; + } + + std::string compat_rstring; + raw_string_ostream compat_rtypename(compat_rstring); + if (jl_is_cpointer_type(rtt)) + ctx.types().T_size->print(compat_rtypename); + else + rettype->print(compat_rtypename); + + std::string compat_ir_string; + raw_string_ostream compat_ir_stream(compat_ir_string); + compat_ir_stream << "define " << compat_rtypename.str() << " @\"" << ir_name + << "\"(" << compat_argstream.str() << ") {\n" + << jl_string_data(ir) << "\n}"; + + SMDiagnostic Err = SMDiagnostic(); + Mod = + parseAssemblyString(compat_ir_stream.str(), Err, ctx.builder.getContext()); + } + if (!Mod) { std::string message = "Failed to parse LLVM assembly: \n"; raw_string_ostream stream(message); @@ -894,7 +930,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar return jl_cgval_t(); } - Function *f = Mod->getFunction(ir_name); + f = Mod->getFunction(ir_name); f->addFnAttr(Attribute::AlwaysInline); } else { @@ -932,21 +968,96 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar Mod = std::move(ModuleOrErr.get()); } - Function *f = Mod->getFunction(jl_string_data(entry)); + f = Mod->getFunction(jl_string_data(entry)); if (!f) { emit_error(ctx, "Module IR does not contain specified entry function"); JL_GC_POP(); return jl_cgval_t(); } + assert(!f->isDeclaration()); f->setName(ir_name); + } - // verify the function type - assert(!f->isDeclaration()); - assert(f->getReturnType() == rettype); - int i = 0; - for (SmallVector::iterator it = argtypes.begin(); - it != argtypes.end(); ++it, ++i) - assert(*it == f->getFunctionType()->getParamType(i)); + // backwards compatibility: support for IR with integer pointers + bool mismatched_pointers = false; + for (size_t i = 0; i < nargt; ++i) { + jl_value_t *tti = jl_svecref(tt, i); + if (jl_is_cpointer_type(tti) && + !f->getFunctionType()->getParamType(i)->isPointerTy()) { + mismatched_pointers = true; + break; + } + } + if (mismatched_pointers) { + if (jl_options.depwarn) { + if (jl_options.depwarn == JL_OPTIONS_DEPWARN_ERROR) + jl_error("llvmcall with integer pointers is deprecated, " + "use an actual pointer type instead."); + + // ensure we only depwarn once per method + // TODO: lift this into a reusable codegen-level depwarn utility + static std::set llvmcall_depwarns; + jl_method_t *m = ctx.linfo->def.method; + if (llvmcall_depwarns.find(m) == llvmcall_depwarns.end()) { + llvmcall_depwarns.insert(m); + jl_printf(JL_STDERR, + "WARNING: llvmcall with integer pointers is deprecated.\n" + "Use actual pointers instead, replacing i32 or i64 with i8* or ptr\n" + "in "); + jl_static_show(JL_STDERR, (jl_value_t*) ctx.linfo->def.method); + jl_printf(JL_STDERR, " at %s\n", ctx.file.str().c_str()); + } + } + + // wrap the function, performing the necessary pointer conversion + + Function *inner = f; + inner->setName(ir_name + ".inner"); + + FunctionType *wrapper_ft = FunctionType::get(rettype, argtypes, false); + Function *wrapper = + Function::Create(wrapper_ft, inner->getLinkage(), ir_name, *Mod); + + wrapper->copyAttributesFrom(inner); + inner->addFnAttr(Attribute::AlwaysInline); + + BasicBlock *entry = BasicBlock::Create(ctx.builder.getContext(), "", wrapper); + IRBuilder<> irbuilder(entry); + SmallVector wrapper_args; + for (size_t i = 0; i < nargt; ++i) { + jl_value_t *tti = jl_svecref(tt, i); + Value *v = wrapper->getArg(i); + if (jl_is_cpointer_type(tti)) + v = irbuilder.CreatePtrToInt(v, ctx.types().T_size); + wrapper_args.push_back(v); + } + Value *call = irbuilder.CreateCall(inner, wrapper_args); + // check if void + if (rettype->isVoidTy()) + irbuilder.CreateRetVoid(); + else { + if (jl_is_cpointer_type(rtt)) + call = irbuilder.CreateIntToPtr(call, ctx.types().T_ptr); + irbuilder.CreateRet(call); + } + + f = wrapper; + } + + // verify the function type + assert(f->getReturnType() == rettype); + int i = 0; + for (SmallVector::iterator it = argtypes.begin(); it != argtypes.end(); + ++it, ++i) { + if (*it != f->getFunctionType()->getParamType(i)) { + std::string message; + raw_string_ostream stream(message); + stream << "Malformed llvmcall: argument " << i + 1 << " type " + << *f->getFunctionType()->getParamType(i) + << " does not match expected argument type " << **it; + emit_error(ctx, stream.str()); + return jl_cgval_t(); + } } // copy module properties that should always match @@ -984,7 +1095,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar if (inst->getType() != rettype) { std::string message; raw_string_ostream stream(message); - stream << "llvmcall return type " << *inst->getType() + stream << "Malformed llvmcall: return type " << *inst->getType() << " does not match declared return type" << *rettype; emit_error(ctx, stream.str()); return jl_cgval_t(); @@ -1462,14 +1573,14 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) (void)isVa; // prevent compiler warning if (is_libjulia_func(jl_value_ptr)) { ++CCALL_STAT(jl_value_ptr); - assert(retboxed ? lrt == ctx.types().T_prjlvalue : lrt == ctx.types().T_size); + assert(retboxed ? lrt == ctx.types().T_prjlvalue : lrt == ctx.types().T_ptr); assert(!isVa && !llvmcall && nccallargs == 1); jl_value_t *tti = jl_svecref(at, 0); Type *largty; bool isboxed; if (jl_is_abstract_ref_type(tti)) { tti = (jl_value_t*)jl_voidpointer_type; - largty = ctx.types().T_size; + largty = ctx.types().T_ptr; isboxed = false; } else { @@ -1482,7 +1593,6 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) } else { retval = emit_unbox(ctx, largty, argv[0], tti); - retval = emit_inttoptr(ctx, retval, ctx.types().T_pjlvalue); } // retval is now an untracked jl_value_t* if (retboxed) @@ -1561,9 +1671,7 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) assert(lrt == ctx.types().T_size); assert(!isVa && !llvmcall && nccallargs == 0); JL_GC_POP(); - return mark_or_box_ccall_result(ctx, - ctx.builder.CreatePtrToInt(get_current_ptls(ctx), lrt), - retboxed, rt, unionall, static_rt); + return mark_or_box_ccall_result(ctx, get_current_ptls(ctx), retboxed, rt, unionall, static_rt); } else if (is_libjulia_func(jl_threadid)) { ++CCALL_STAT(jl_threadid); @@ -1683,21 +1791,20 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) } else if (is_libjulia_func(jl_string_ptr)) { ++CCALL_STAT(jl_string_ptr); - assert(lrt == ctx.types().T_size); + assert(lrt == ctx.types().T_ptr); assert(!isVa && !llvmcall && nccallargs == 1); auto obj = emit_bitcast(ctx, emit_pointer_from_objref(ctx, boxed(ctx, argv[0])), ctx.types().T_pprjlvalue); // The inbounds gep makes it more clear to LLVM that the resulting value is not // a null pointer. auto strp = ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_prjlvalue, obj, 1); - strp = ctx.builder.CreatePtrToInt(strp, ctx.types().T_size); setName(ctx.emission_context, strp, "string_ptr"); JL_GC_POP(); return mark_or_box_ccall_result(ctx, strp, retboxed, rt, unionall, static_rt); } else if (is_libjulia_func(jl_symbol_name)) { ++CCALL_STAT(jl_symbol_name); - assert(lrt == ctx.types().T_size); + assert(lrt == ctx.types().T_ptr); assert(!isVa && !llvmcall && nccallargs == 1); auto obj = emit_bitcast(ctx, emit_pointer_from_objref(ctx, boxed(ctx, argv[0])), ctx.types().T_pprjlvalue); @@ -1705,7 +1812,6 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) // a null pointer. auto strp = ctx.builder.CreateConstInBoundsGEP1_32( ctx.types().T_prjlvalue, obj, (sizeof(jl_sym_t) + sizeof(void*) - 1) / sizeof(void*)); - strp = ctx.builder.CreatePtrToInt(strp, ctx.types().T_size); setName(ctx.emission_context, strp, "symbol_name"); JL_GC_POP(); return mark_or_box_ccall_result(ctx, strp, retboxed, rt, unionall, static_rt); @@ -1750,14 +1856,12 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) const jl_cgval_t &dst = argv[0]; const jl_cgval_t &src = argv[1]; const jl_cgval_t &n = argv[2]; - Value *destp = emit_unbox(ctx, ctx.types().T_size, dst, (jl_value_t*)jl_voidpointer_type); + Value *destp = emit_unbox(ctx, ctx.types().T_ptr, dst, (jl_value_t*)jl_voidpointer_type); ctx.builder.CreateMemCpy( - emit_inttoptr(ctx, destp, getInt8PtrTy(ctx.builder.getContext())), + destp, MaybeAlign(1), - emit_inttoptr(ctx, - emit_unbox(ctx, ctx.types().T_size, src, (jl_value_t*)jl_voidpointer_type), - getInt8PtrTy(ctx.builder.getContext())), + emit_unbox(ctx, ctx.types().T_ptr, src, (jl_value_t*)jl_voidpointer_type), MaybeAlign(1), emit_unbox(ctx, ctx.types().T_size, n, (jl_value_t*)jl_ulong_type), false); @@ -1770,11 +1874,11 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) const jl_cgval_t &dst = argv[0]; const jl_cgval_t &val = argv[1]; const jl_cgval_t &n = argv[2]; - Value *destp = emit_unbox(ctx, ctx.types().T_size, dst, (jl_value_t*)jl_voidpointer_type); + Value *destp = emit_unbox(ctx, ctx.types().T_ptr, dst, (jl_value_t*)jl_voidpointer_type); Value *val32 = emit_unbox(ctx, getInt32Ty(ctx.builder.getContext()), val, (jl_value_t*)jl_uint32_type); Value *val8 = ctx.builder.CreateTrunc(val32, getInt8Ty(ctx.builder.getContext()), "memset_val"); ctx.builder.CreateMemSet( - emit_inttoptr(ctx, destp, getInt8PtrTy(ctx.builder.getContext())), + destp, val8, emit_unbox(ctx, ctx.types().T_size, n, (jl_value_t*)jl_ulong_type), MaybeAlign(1) @@ -1788,14 +1892,12 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) const jl_cgval_t &dst = argv[0]; const jl_cgval_t &src = argv[1]; const jl_cgval_t &n = argv[2]; - Value *destp = emit_unbox(ctx, ctx.types().T_size, dst, (jl_value_t*)jl_voidpointer_type); + Value *destp = emit_unbox(ctx, ctx.types().T_ptr, dst, (jl_value_t*)jl_voidpointer_type); ctx.builder.CreateMemMove( - emit_inttoptr(ctx, destp, getInt8PtrTy(ctx.builder.getContext())), + destp, MaybeAlign(0), - emit_inttoptr(ctx, - emit_unbox(ctx, ctx.types().T_size, src, (jl_value_t*)jl_voidpointer_type), - getInt8PtrTy(ctx.builder.getContext())), + emit_unbox(ctx, ctx.types().T_ptr, src, (jl_value_t*)jl_voidpointer_type), MaybeAlign(0), emit_unbox(ctx, ctx.types().T_size, n, (jl_value_t*)jl_ulong_type), false); @@ -1999,7 +2101,7 @@ jl_cgval_t function_sig_t::emit_a_ccall( ++LiteralCCalls; null_pointer_check(ctx, symarg.jl_ptr, nullptr); Type *funcptype = PointerType::get(functype, 0); - llvmf = emit_inttoptr(ctx, symarg.jl_ptr, funcptype); + llvmf = ctx.builder.CreateBitCast(symarg.jl_ptr, funcptype); } else if (symarg.fptr != NULL) { ++LiteralCCalls; diff --git a/src/cgutils.cpp b/src/cgutils.cpp index b17b174dd8a8d..8385b5682a0bc 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -690,6 +690,8 @@ static Type *bitstype_to_llvm(jl_value_t *bt, LLVMContext &ctxt, bool llvmcall = return getDoubleTy(ctxt); if (bt == (jl_value_t*)jl_bfloat16_type) return getBFloatTy(ctxt); + if (jl_is_cpointer_type(bt)) + return PointerType::get(getInt8Ty(ctxt), 0); if (jl_is_llvmpointer_type(bt)) { jl_value_t *as_param = jl_tparam1(bt); int as; diff --git a/src/codegen.cpp b/src/codegen.cpp index cf5122df295eb..2e2afc02080a1 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -283,6 +283,7 @@ extern void _chkstk(void); // types struct jl_typecache_t { + Type *T_ptr; Type *T_size; Type *T_jlvalue; Type *T_pjlvalue; @@ -304,7 +305,7 @@ struct jl_typecache_t { bool initialized; jl_typecache_t() : - T_jlvalue(nullptr), T_pjlvalue(nullptr), T_prjlvalue(nullptr), + T_ptr(nullptr), T_jlvalue(nullptr), T_pjlvalue(nullptr), T_prjlvalue(nullptr), T_ppjlvalue(nullptr), T_pprjlvalue(nullptr), T_jlgenericmemory(nullptr), T_jlarray(nullptr), T_pjlarray(nullptr), T_jlfunc(nullptr), T_jlfuncparams(nullptr), T_sigatomic(nullptr), T_ppint8(nullptr), @@ -315,6 +316,7 @@ struct jl_typecache_t { return; } initialized = true; + T_ptr = getInt8PtrTy(context); T_ppint8 = PointerType::get(getInt8PtrTy(context), 0); T_sigatomic = Type::getIntNTy(context, sizeof(sig_atomic_t) * 8); T_size = DL.getIntPtrType(context); diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 3a98850ddca68..c2450d359be45 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -17,6 +17,7 @@ STATISTIC(EmittedRuntimeCalls, "Number of runtime intrinsic calls emitted"); STATISTIC(EmittedIntrinsics, "Number of intrinsic calls emitted"); STATISTIC(Emitted_pointerref, "Number of pointerref calls emitted"); STATISTIC(Emitted_pointerset, "Number of pointerset calls emitted"); +STATISTIC(Emitted_pointerarith, "Number of pointer arithmetic calls emitted"); STATISTIC(Emitted_atomic_fence, "Number of atomic_fence calls emitted"); STATISTIC(Emitted_atomic_pointerref, "Number of atomic_pointerref calls emitted"); STATISTIC(Emitted_atomic_pointerop, "Number of atomic_pointerop calls emitted"); @@ -625,10 +626,14 @@ static jl_cgval_t generic_bitcast(jl_codectx_t &ctx, ArrayRef argv) vx = ctx.builder.CreateZExt(vx, llvmt); } else if (vxt->isPointerTy() && !llvmt->isPointerTy()) { vx = ctx.builder.CreatePtrToInt(vx, llvmt); - setName(ctx.emission_context, vx, "bitcast_coercion"); + if (isa(vx) && !vx->hasName()) + // CreatePtrToInt may undo an IntToPtr + setName(ctx.emission_context, vx, "bitcast_coercion"); } else if (!vxt->isPointerTy() && llvmt->isPointerTy()) { vx = emit_inttoptr(ctx, vx, llvmt); - setName(ctx.emission_context, vx, "bitcast_coercion"); + if (isa(vx) && !vx->hasName()) + // emit_inttoptr may undo an PtrToInt + setName(ctx.emission_context, vx, "bitcast_coercion"); } else { vx = emit_bitcast(ctx, vx, llvmt); setName(ctx.emission_context, vx, "bitcast_coercion"); @@ -741,7 +746,8 @@ static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, ArrayRef argv) if (ety == (jl_value_t*)jl_any_type) { Value *thePtr = emit_unbox(ctx, ctx.types().T_pprjlvalue, e, e.typ); - setName(ctx.emission_context, thePtr, "unbox_any_ptr"); + if (isa(thePtr) && !thePtr->hasName()) + setName(ctx.emission_context, thePtr, "unbox_any_ptr"); LoadInst *load = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, ctx.builder.CreateInBoundsGEP(ctx.types().T_prjlvalue, thePtr, im1), Align(align_nb)); setName(ctx.emission_context, load, "any_unbox"); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_data); @@ -853,6 +859,34 @@ static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, ArrayRef argv) return e; } +// ptr + offset +// ptr - offset +static jl_cgval_t emit_pointerarith(jl_codectx_t &ctx, intrinsic f, + ArrayRef argv) +{ + jl_value_t *ptrtyp = argv[0].typ; + jl_value_t *offtyp = argv[1].typ; + if (!jl_is_cpointer_type(ptrtyp) || offtyp != (jl_value_t *)jl_ulong_type) + return emit_runtime_call(ctx, f, argv, argv.size()); + assert(f == add_ptr || f == sub_ptr); + + Value *ptr = emit_unbox(ctx, ctx.types().T_ptr, argv[0], ptrtyp); + Value *off = emit_unbox(ctx, ctx.types().T_size, argv[1], offtyp); + if (f == sub_ptr) + off = ctx.builder.CreateNeg(off); + Value *ans = ctx.builder.CreateGEP(getInt8Ty(ctx.builder.getContext()), ptr, off); + + if (jl_is_concrete_type(ptrtyp)) { + return mark_julia_type(ctx, ans, false, ptrtyp); + } + else { + Value *box = emit_allocobj(ctx, (jl_datatype_t *)ptrtyp, true); + setName(ctx.emission_context, box, "ptr_box"); + init_bits_value(ctx, box, ans, ctx.tbaa().tbaa_immut); + return mark_julia_type(ctx, box, true, (jl_datatype_t *)ptrtyp); + } +} + static jl_cgval_t emit_atomicfence(jl_codectx_t &ctx, ArrayRef argv) { const jl_cgval_t &ord = argv[0]; @@ -1270,6 +1304,13 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar ++Emitted_pointerset; assert(nargs == 4); return emit_pointerset(ctx, argv); + + case add_ptr: + case sub_ptr: + ++Emitted_pointerarith; + assert(nargs == 2); + return emit_pointerarith(ctx, f, argv); + case atomic_fence: ++Emitted_atomic_fence; assert(nargs == 1); @@ -1437,26 +1478,6 @@ static Value *emit_untyped_intrinsic(jl_codectx_t &ctx, intrinsic f, ArrayRefreplaceAllUsesWith(ASC->getOperand(0)); + if (ASC->getType() == ASC->getOperand(0)->getType()) { + ASC->replaceAllUsesWith(ASC->getOperand(0)); + } else { + // uncanonicalized addrspacecast; demote to bitcast + llvm::IRBuilder<> builder(ASC); + auto BC = builder.CreateBitCast(ASC->getOperand(0), ASC->getType()); + ASC->replaceAllUsesWith(BC); + } NoopCasts.push_back(ASC); } } diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c index ff479a4d30959..9abe9828fb07f 100644 --- a/src/runtime_intrinsics.c +++ b/src/runtime_intrinsics.c @@ -1384,10 +1384,8 @@ JL_DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b, jl_value_t *c) un_iintrinsic_fast(LLVMNeg, neg, neg_int, u) #define add(a,b) a + b bi_iintrinsic_fast(LLVMAdd, add, add_int, u) -bi_iintrinsic_fast(LLVMAdd, add, add_ptr, u) #define sub(a,b) a - b bi_iintrinsic_fast(LLVMSub, sub, sub_int, u) -bi_iintrinsic_fast(LLVMSub, sub, sub_ptr, u) #define mul(a,b) a * b bi_iintrinsic_fast(LLVMMul, mul, mul_int, u) #define div(a,b) a / b @@ -1696,3 +1694,19 @@ JL_DLLEXPORT jl_value_t *jl_have_fma(jl_value_t *typ) else return jl_false; } + +JL_DLLEXPORT jl_value_t *jl_add_ptr(jl_value_t *ptr, jl_value_t *offset) +{ + JL_TYPECHK(add_ptr, pointer, ptr); + JL_TYPECHK(add_ptr, ulong, offset); + char *ptrval = (char*)jl_unbox_long(ptr) + jl_unbox_ulong(offset); + return jl_new_bits(jl_typeof(ptr), &ptrval); +} + +JL_DLLEXPORT jl_value_t *jl_sub_ptr(jl_value_t *ptr, jl_value_t *offset) +{ + JL_TYPECHK(sub_ptr, pointer, ptr); + JL_TYPECHK(sub_ptr, ulong, offset); + char *ptrval = (char*)jl_unbox_long(ptr) - jl_unbox_ulong(offset); + return jl_new_bits(jl_typeof(ptr), &ptrval); +}