diff --git a/src/ccall.cpp b/src/ccall.cpp index 196ec82ee802f..dbe075c431772 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -130,7 +130,7 @@ static Value *runtime_sym_lookup(PointerType *funcptype, const char *f_lib, cons Value *llvm_type_rewrite(Value *v, Type *from_type, Type *target_type, bool tojulia, /* only matters if byref is set (declares the direction of the byref attribute) */ - bool byref, /* only applies to arguments, set false for return values -- effectively the same as jl_cgval_t.ispointer */ + bool byref, /* only applies to arguments, set false for return values -- effectively the same as jl_cgval_t.ispointer() */ bool issigned, /* determines whether an integer value should be zero or sign extended */ jl_codectx_t *ctx) { @@ -316,7 +316,7 @@ static Value *julia_to_native(Type *to, bool toboxed, jl_value_t *jlto, const jl if (addressOf) to = to->getContainedType(0); Value *slot = emit_static_alloca(to, ctx); - if (!jvinfo.ispointer) { + if (!jvinfo.ispointer()) { builder.CreateStore(emit_unbox(to, jvinfo, ety), slot); } else { @@ -856,7 +856,7 @@ static jl_cgval_t mark_or_box_ccall_result(Value *result, bool isboxed, jl_value Value *runtime_bt = boxed(emit_expr(rt_expr, ctx), ctx); int nb = sizeof(void*); return mark_julia_type( - init_bits_value(emit_allocobj(nb), runtime_bt, result), + init_bits_value(emit_allocobj(nb), runtime_bt, result, tbaa_user), true, (jl_value_t*)jl_pointer_type, ctx); } @@ -1359,7 +1359,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) if (sret) { jl_cgval_t sret_val = emit_new_struct(rt,1,NULL,ctx); // TODO: is it valid to be creating an incomplete type this way? assert(sret_val.typ != NULL && "Type was not concrete"); - if (!sret_val.ispointer) { + if (!sret_val.ispointer()) { Value *mem = emit_static_alloca(lrt, ctx); builder.CreateStore(sret_val.V, mem); result = mem; @@ -1544,7 +1544,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) assert(newst.typ != NULL && "Type was not concrete"); assert(newst.isboxed); // copy the data from the return value to the new struct - builder.CreateAlignedStore(result, builder.CreateBitCast(newst.V, prt->getPointerTo()), 16); // julia gc is aligned 16 + tbaa_decorate(newst.tbaa, builder.CreateAlignedStore(result, builder.CreateBitCast(newst.V, prt->getPointerTo()), 16)); // julia gc is aligned 16 return newst; } else if (jlrt != prt) { diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 021b811b7d195..7455b7e71d64d 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -179,7 +179,7 @@ static Value *julia_gv(const char *cname, void *addr) // first see if there already is a GlobalVariable for this address it = jl_value_to_llvm.find(addr); if (it != jl_value_to_llvm.end()) - return builder.CreateLoad(prepare_global((llvm::GlobalVariable*)it->second.gv)); + return tbaa_decorate(tbaa_const, builder.CreateLoad(prepare_global((llvm::GlobalVariable*)it->second.gv))); std::stringstream gvname; gvname << cname << globalUnique++; @@ -189,7 +189,7 @@ static Value *julia_gv(const char *cname, void *addr) NULL, gvname.str()); addComdat(gv); *(void**)jl_emit_and_add_to_shadow(gv, addr) = addr; - return builder.CreateLoad(gv); + return tbaa_decorate(tbaa_const, builder.CreateLoad(gv)); } static Value *julia_gv(const char *prefix, jl_sym_t *name, jl_module_t *mod, void *addr) @@ -491,7 +491,7 @@ static Value *emit_typeof(Value *tt) { // given p, a jl_value_t*, compute its type tag assert(tt->getType() == T_pjlvalue); - tt = builder.CreateLoad(emit_typeptr_addr(tt), false); + tt = tbaa_decorate(tbaa_tag, builder.CreateLoad(emit_typeptr_addr(tt), false)); tt = builder.CreateIntToPtr(builder.CreateAnd( builder.CreatePtrToInt(tt, T_size), ConstantInt::get(T_size,~(uintptr_t)15)), @@ -517,22 +517,22 @@ static Value *emit_typeof_boxed(const jl_cgval_t &p, jl_codectx_t *ctx) static Value *emit_datatype_types(Value *dt) { - return builder. + return tbaa_decorate(tbaa_datatype, builder. CreateLoad(builder. CreateBitCast(builder. CreateGEP(builder.CreateBitCast(dt, T_pint8), ConstantInt::get(T_size, offsetof(jl_datatype_t, types))), - T_ppjlvalue)); + T_ppjlvalue))); } static Value *emit_datatype_nfields(Value *dt) { - Value *nf = builder. + Value *nf = tbaa_decorate(tbaa_datatype, builder. CreateLoad(builder. CreateBitCast(builder. CreateGEP(builder.CreateBitCast(dt, T_pint8), ConstantInt::get(T_size, offsetof(jl_datatype_t, nfields))), - T_pint32)); + T_pint32))); #ifdef _P64 nf = builder.CreateSExt(nf, T_int64); #endif @@ -541,28 +541,28 @@ static Value *emit_datatype_nfields(Value *dt) static Value *emit_datatype_size(Value *dt) { - Value *size = builder. + Value *size = tbaa_decorate(tbaa_datatype, builder. CreateLoad(builder. CreateBitCast(builder. CreateGEP(builder.CreateBitCast(dt, T_pint8), ConstantInt::get(T_size, offsetof(jl_datatype_t, size))), - T_pint32)); + T_pint32))); return size; } static Value *emit_datatype_mutabl(Value *dt) { - Value *mutabl = builder. + Value *mutabl = tbaa_decorate(tbaa_datatype, builder. CreateLoad(builder.CreateGEP(builder.CreateBitCast(dt, T_pint8), - ConstantInt::get(T_size, offsetof(jl_datatype_t, mutabl)))); + ConstantInt::get(T_size, offsetof(jl_datatype_t, mutabl))))); return builder.CreateTrunc(mutabl, T_int1); } static Value *emit_datatype_abstract(Value *dt) { - Value *abstract = builder. + Value *abstract = tbaa_decorate(tbaa_datatype, builder. CreateLoad(builder.CreateGEP(builder.CreateBitCast(dt, T_pint8), - ConstantInt::get(T_size, offsetof(jl_datatype_t, abstract)))); + ConstantInt::get(T_size, offsetof(jl_datatype_t, abstract))))); return builder.CreateTrunc(abstract, T_int1); } @@ -634,7 +634,7 @@ static void raise_exception_if(Value *cond, Value *exc, jl_codectx_t *ctx) static void raise_exception_if(Value *cond, GlobalVariable *exc, jl_codectx_t *ctx) { - raise_exception_if(cond, (Value*)builder.CreateLoad(exc, false), ctx); + raise_exception_if(cond, (Value*)tbaa_decorate(tbaa_const, builder.CreateLoad(exc, false)), ctx); } static void null_pointer_check(Value *v, jl_codectx_t *ctx) @@ -737,7 +737,7 @@ static Value *emit_bounds_check(const jl_cgval_t &ainfo, jl_value_t *ty, Value * if (ainfo.isghost) { a = Constant::getNullValue(T_pint8); } - else if (!ainfo.ispointer) { + else if (!ainfo.ispointer()) { // CreateAlloca is OK here since we are on an error branch Value *tempSpace = builder.CreateAlloca(a->getType()); builder.CreateStore(a, tempSpace); @@ -963,10 +963,10 @@ static Value *data_pointer(const jl_cgval_t &x, jl_codectx_t *ctx, Type *astype static bool emit_getfield_unknownidx(jl_cgval_t *ret, const jl_cgval_t &strct, Value *idx, jl_datatype_t *stt, jl_codectx_t *ctx) { size_t nfields = jl_datatype_nfields(stt); - if (strct.ispointer) { // boxed or stack + if (strct.ispointer()) { // boxed or stack if (is_datatype_all_pointers(stt)) { idx = emit_bounds_check(strct, (jl_value_t*)stt, idx, ConstantInt::get(T_size, nfields), ctx); - Value *fld = tbaa_decorate(tbaa_user, builder.CreateLoad( + Value *fld = tbaa_decorate(strct.tbaa, builder.CreateLoad( builder.CreateGEP(data_pointer(strct, ctx), idx))); if ((unsigned)stt->ninitialized != nfields) null_pointer_check(fld, ctx); @@ -982,12 +982,12 @@ static bool emit_getfield_unknownidx(jl_cgval_t *ret, const jl_cgval_t &strct, V // just compute the pointer and let user load it when necessary Type *fty = julia_type_to_llvm(jt); Value *addr = builder.CreateGEP(builder.CreatePointerCast(ptr, PointerType::get(fty,0)), idx); - *ret = mark_julia_slot(addr, jt); + *ret = mark_julia_slot(addr, jt, strct.tbaa); ret->gcroot = strct.gcroot; ret->isimmutable = strct.isimmutable; return true; } - *ret = typed_load(ptr, idx, jt, ctx, stt->mutabl ? tbaa_user : tbaa_immut); + *ret = typed_load(ptr, idx, jt, ctx, strct.tbaa); return true; } else if (strct.isboxed) { @@ -1045,7 +1045,7 @@ static jl_cgval_t emit_getfield_knownidx(const jl_cgval_t &strct, unsigned idx, Value *addr = builder.CreateGEP(builder.CreateBitCast(boxed(strct, ctx), T_pint8), ConstantInt::get(T_size, jl_field_offset(jt,idx))); - MDNode *tbaa = jt->mutabl ? tbaa_user : tbaa_immut; + MDNode *tbaa = strct.tbaa; if (jl_field_isptr(jt, idx)) { Value *fldv = tbaa_decorate(tbaa, builder.CreateLoad(builder.CreateBitCast(addr, T_ppjlvalue))); if (idx >= (unsigned)jt->ninitialized) @@ -1060,7 +1060,7 @@ static jl_cgval_t emit_getfield_knownidx(const jl_cgval_t &strct, unsigned idx, return typed_load(addr, ConstantInt::get(T_size, 0), jfty, ctx, tbaa, align); } } - else if (strct.ispointer) { // something stack allocated + else if (strct.ispointer()) { // something stack allocated Value *addr; if (is_vecelement_type((jl_value_t*)jt)) // VecElement types are unwrapped in LLVM. @@ -1070,7 +1070,7 @@ static jl_cgval_t emit_getfield_knownidx(const jl_cgval_t &strct, unsigned idx, LLVM37_param(julia_type_to_llvm(strct.typ)) strct.V, 0, idx); assert(!jt->mutabl); - jl_cgval_t fieldval = mark_julia_slot(addr, jfty); + jl_cgval_t fieldval = mark_julia_slot(addr, jfty, strct.tbaa); fieldval.isimmutable = strct.isimmutable; fieldval.gcroot = strct.gcroot; return fieldval; @@ -1305,10 +1305,10 @@ static Value *emit_array_nd_index(const jl_cgval_t &ainfo, jl_value_t *ex, size_ // --- boxing --- static Value *emit_allocobj(size_t static_size); -static Value *init_bits_value(Value *newv, Value *jt, Value *v) +static Value *init_bits_value(Value *newv, Value *jt, Value *v, MDNode *tbaa) { - builder.CreateStore(jt, emit_typeptr_addr(newv)); - builder.CreateAlignedStore(v, builder.CreateBitCast(newv, PointerType::get(v->getType(),0)), sizeof(void*)); // min alignment in julia's gc is pointer-aligned + tbaa_decorate(tbaa_tag, builder.CreateStore(jt, emit_typeptr_addr(newv))); + tbaa_decorate(tbaa, builder.CreateAlignedStore(v, builder.CreateBitCast(newv, PointerType::get(v->getType(),0)), sizeof(void*))); // min alignment in julia's gc is pointer-aligned return newv; } @@ -1411,8 +1411,8 @@ static Value *boxed(const jl_cgval_t &vinfo, jl_codectx_t *ctx, bool gcrooted) Type *t = julia_type_to_llvm(vinfo.typ); assert(!type_is_ghost(t)); // should have been handled by isghost above! - if (vinfo.ispointer) - v = build_load( builder.CreatePointerCast(v, t->getPointerTo()), vinfo.typ ); + if (vinfo.ispointer()) + v = tbaa_decorate(vinfo.tbaa, build_load(builder.CreatePointerCast(v, t->getPointerTo()), vinfo.typ)); if (t == T_int1) return julia_bool(v); @@ -1468,7 +1468,7 @@ static Value *boxed(const jl_cgval_t &vinfo, jl_codectx_t *ctx, bool gcrooted) return literal_pointer_val(jb->instance); } else { - box = init_bits_value(emit_allocobj(jl_datatype_size(jt)), literal_pointer_val(jt), v); + box = init_bits_value(emit_allocobj(jl_datatype_size(jt)), literal_pointer_val(jt), v, jb->mutabl ? tbaa_user : tbaa_immut); } if (gcrooted) { @@ -1531,7 +1531,7 @@ static Value *emit_allocobj(size_t static_size) static void emit_write_barrier(jl_codectx_t *ctx, Value *parent, Value *ptr) { Value *parenttag = builder.CreateBitCast(emit_typeptr_addr(parent), T_psize); - Value *parent_type = builder.CreateLoad(parenttag); + Value *parent_type = tbaa_decorate(tbaa_tag, builder.CreateLoad(parenttag)); Value *parent_mark_bits = builder.CreateAnd(parent_type, 1); // the branch hint does not seem to make it to the generated code @@ -1544,7 +1544,7 @@ static void emit_write_barrier(jl_codectx_t *ctx, Value *parent, Value *ptr) builder.CreateCondBr(parent_marked, barrier_may_trigger, cont); builder.SetInsertPoint(barrier_may_trigger); - Value *ptr_mark_bit = builder.CreateAnd(builder.CreateLoad(builder.CreateBitCast(emit_typeptr_addr(ptr), T_psize)), 1); + Value *ptr_mark_bit = builder.CreateAnd(tbaa_decorate(tbaa_tag, builder.CreateLoad(builder.CreateBitCast(emit_typeptr_addr(ptr), T_psize))), 1); Value *ptr_not_marked = builder.CreateICmpEQ(ptr_mark_bit, ConstantInt::get(T_size, 0)); builder.CreateCondBr(ptr_not_marked, barrier_trigger, cont); builder.SetInsertPoint(barrier_trigger); @@ -1572,13 +1572,13 @@ static void emit_setfield(jl_datatype_t *sty, const jl_cgval_t &strct, size_t id const jl_cgval_t &rhs, jl_codectx_t *ctx, bool checked, bool wb) { if (sty->mutabl || !checked) { - assert(strct.ispointer); + assert(strct.ispointer()); Value *addr = builder.CreateGEP(data_pointer(strct, ctx, T_pint8), ConstantInt::get(T_size, jl_field_offset(sty, idx0))); jl_value_t *jfty = jl_svecref(sty->types, idx0); if (jl_field_isptr(sty, idx0)) { Value *r = boxed(rhs, ctx, false); // don't need a temporary gcroot since it'll be rooted by strct (but should ensure strct is rooted via mark_gc_use) - builder.CreateStore(r, builder.CreateBitCast(addr, T_ppjlvalue)); + tbaa_decorate(strct.tbaa, builder.CreateStore(r, builder.CreateBitCast(addr, T_ppjlvalue))); if (wb && strct.isboxed) emit_checked_write_barrier(ctx, boxed(strct, ctx), r); mark_gc_use(strct); } @@ -1586,7 +1586,7 @@ static void emit_setfield(jl_datatype_t *sty, const jl_cgval_t &strct, size_t id int align = jl_field_offset(sty, idx0); align |= 16; align &= -align; - typed_store(addr, ConstantInt::get(T_size, 0), rhs, jfty, ctx, sty->mutabl ? tbaa_user : tbaa_immut, data_pointer(strct, ctx, T_pjlvalue), align); + typed_store(addr, ConstantInt::get(T_size, 0), rhs, jfty, ctx, strct.tbaa, data_pointer(strct, ctx, T_pjlvalue), align); } } else { @@ -1652,8 +1652,8 @@ static jl_cgval_t emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **arg } Value *strct = emit_allocobj(sty->size); jl_cgval_t strctinfo = mark_julia_type(strct, true, ty, ctx); - builder.CreateStore(literal_pointer_val((jl_value_t*)ty), - emit_typeptr_addr(strct)); + tbaa_decorate(tbaa_tag, builder.CreateStore(literal_pointer_val((jl_value_t*)ty), + emit_typeptr_addr(strct))); if (f1) { jl_cgval_t f1info = mark_julia_type(f1, true, jl_any_type, ctx); if (!jl_subtype(expr_type(args[1],ctx), jl_field_type(sty,0), 0)) @@ -1662,12 +1662,12 @@ static jl_cgval_t emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **arg } for(size_t i=j; i < nf; i++) { if (jl_field_isptr(sty, i)) { - builder.CreateStore( + tbaa_decorate(strctinfo.tbaa, builder.CreateStore( V_null, builder.CreatePointerCast( builder.CreateGEP(builder.CreateBitCast(strct, T_pint8), ConstantInt::get(T_size, jl_field_offset(sty,i))), - T_ppjlvalue)); + T_ppjlvalue))); } } bool need_wb = false; diff --git a/src/codegen.cpp b/src/codegen.cpp index 68362e76dff3d..9bc4eaf6886e9 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -267,7 +267,10 @@ static Type *T_void; // type-based alias analysis nodes. Indentation of comments indicates hierarchy. static MDNode *tbaa_gcframe; // GC frame +static MDNode *tbaa_stack; // stack static MDNode *tbaa_user; // User data that is mutable +static MDNode *tbaa_tag; // Type tag +static MDNode *tbaa_binding; // jl_binding_t::value static MDNode *tbaa_immut; // User data inside a heap-allocated immutable static MDNode *tbaa_value; // Julia value static MDNode *tbaa_array; // Julia array @@ -441,8 +444,12 @@ struct jl_cgval_t { //Type *T; // cached result of julia_type_to_llvm(typ) bool isboxed; // whether this value is a jl_value_t* allocated on the heap with the right type tag bool isghost; // whether this value is "ghost" - bool ispointer; // whether this value is actually pointer to the value bool isimmutable; // V points to something that is definitely immutable (e.g. single-assignment, but including memory) + MDNode *tbaa; // The related tbaa node. Non-NULL iff this is not a pointer. + bool ispointer() const + { + return tbaa != nullptr; + } jl_cgval_t(Value *V, Value *gcroot, bool isboxed, jl_value_t *typ) : // general constructor (with pointer type auto-detect) V(V), // V is allowed to be NULL in a jl_varinfo_t context, but not during codegen contexts constant(NULL), @@ -451,8 +458,9 @@ struct jl_cgval_t { //T(julia_type_to_llvm(typ)), isboxed(isboxed), isghost(false), - ispointer(isboxed), - isimmutable(isboxed && jl_is_immutable_datatype(typ)) + isimmutable(isboxed && jl_is_immutable_datatype(typ)), + tbaa(isboxed ? (jl_is_leaf_type(typ) && !jl_is_mutable(typ) ? + tbaa_immut : tbaa_user) : nullptr) { } jl_cgval_t(jl_value_t *typ) : // ghost value constructor @@ -463,8 +471,8 @@ struct jl_cgval_t { //T(T_void), isboxed(false), isghost(true), - ispointer(false), - isimmutable(true) + isimmutable(true), + tbaa(nullptr) { assert(jl_is_datatype(typ)); assert(constant); @@ -477,8 +485,8 @@ struct jl_cgval_t { //T(V.T), isboxed(v.isboxed), isghost(v.isghost), - ispointer(v.ispointer), - isimmutable(v.isimmutable) + isimmutable(v.isimmutable), + tbaa(v.tbaa) { assert(isboxed || v.typ == typ); // expect a badly or equivalently typed version } @@ -489,8 +497,8 @@ struct jl_cgval_t { typ(jl_bottom_type), isboxed(false), isghost(true), - ispointer(false), - isimmutable(true) + isimmutable(true), + tbaa(nullptr) { } }; @@ -592,7 +600,7 @@ static void mark_gc_use(const jl_cgval_t &v); static Value *make_jlcall(ArrayRef args, jl_codectx_t *ctx); static Value *global_binding_pointer(jl_module_t *m, jl_sym_t *s, jl_binding_t **pbnd, bool assign, jl_codectx_t *ctx); -static jl_cgval_t emit_checked_var(Value *bp, jl_sym_t *name, jl_codectx_t *ctx, bool isvol=false); +static jl_cgval_t emit_checked_var(Value *bp, jl_sym_t *name, jl_codectx_t *ctx, bool isvol=false, MDNode *tbaa=nullptr); static Value *emit_condition(jl_value_t *cond, const std::string &msg, jl_codectx_t *ctx); static void allocate_gc_frame(BasicBlock *b0, jl_codectx_t *ctx); static void jl_finalize_module(std::unique_ptr m, bool shadow); @@ -626,12 +634,13 @@ static inline jl_cgval_t ghostValue(jl_datatype_t *typ) return ghostValue((jl_value_t*)typ); } -static inline jl_cgval_t mark_julia_slot(Value *v, jl_value_t *typ) +static inline jl_cgval_t mark_julia_slot(Value *v, jl_value_t *typ, MDNode *tbaa) { // eagerly put this back onto the stack assert(v->getType() != T_pjlvalue); + assert(tbaa); jl_cgval_t tagval(v, NULL, false, typ); - tagval.ispointer = true; + tagval.tbaa = tbaa; tagval.isimmutable = true; return tagval; } @@ -648,7 +657,7 @@ static inline jl_cgval_t mark_julia_type(Value *v, bool isboxed, jl_value_t *typ // llvm mem2reg pass will remove this if unneeded Value *loc = emit_static_alloca(T); builder.CreateStore(v, loc); - return mark_julia_slot(loc, typ); + return mark_julia_slot(loc, typ, tbaa_stack); } Value *froot = NULL; if (needsroot && isboxed) { @@ -754,7 +763,7 @@ static Value *alloc_local(int s, jl_codectx_t *ctx) } // CreateAlloca is OK here because alloc_local is only called during prologue setup Value *lv = builder.CreateAlloca(vtype, 0, jl_symbol_name(slot_symbol(s,ctx))); - vi.value = mark_julia_slot(lv, jt); + vi.value = mark_julia_slot(lv, jt, tbaa_stack); // slot is not immutable if there are multiple assignments vi.value.isimmutable &= (vi.isSA && s >= ctx->linfo->nargs); assert(vi.value.isboxed == false); @@ -1884,10 +1893,10 @@ static jl_cgval_t emit_getfield(jl_value_t *expr, jl_sym_t *name, jl_codectx_t * if (bnd->constp) { return mark_julia_const(bnd->value); } - return mark_julia_type(builder.CreateLoad(bp), true, (jl_value_t*)jl_any_type, ctx); + return mark_julia_type(tbaa_decorate(tbaa_binding, builder.CreateLoad(bp)), true, (jl_value_t*)jl_any_type, ctx); } // todo: use type info to avoid undef check - return emit_checked_var(bp, name, ctx); + return emit_checked_var(bp, name, ctx, false, tbaa_binding); } jl_datatype_t *sty = (jl_datatype_t*)expr_type(expr, ctx); @@ -1954,7 +1963,7 @@ static Value *emit_bits_compare(const jl_cgval_t &arg1, const jl_cgval_t &arg2, } if (at->isAggregateType()) { // Struct or Array - assert(arg1.ispointer && arg2.ispointer); + assert(arg1.ispointer() && arg2.ispointer()); size_t sz = jl_datatype_size(arg1.typ); if (sz > 512 && !((jl_datatype_t*)arg1.typ)->haspadding) { #ifdef LLVM37 @@ -1986,7 +1995,7 @@ static Value *emit_bits_compare(const jl_cgval_t &arg1, const jl_cgval_t &arg2, fld2 = builder.CreateConstGEP2_32(LLVM37_param(at) varg2, 0, i); if (type_is_ghost(fld1->getType()->getPointerElementType())) continue; - subAns = emit_bits_compare(mark_julia_slot(fld1, fldty), mark_julia_slot(fld2, fldty), ctx); + subAns = emit_bits_compare(mark_julia_slot(fld1, fldty, arg1.tbaa), mark_julia_slot(fld2, fldty, arg2.tbaa), ctx); answer = builder.CreateAnd(answer, subAns); } return answer; @@ -2375,12 +2384,12 @@ static bool emit_builtin_call(jl_cgval_t *ret, jl_value_t *f, jl_value_t **args, // load owner pointer Value *own_ptr; if (jl_is_long(ndp)) { - own_ptr = builder.CreateLoad( + own_ptr = tbaa_decorate(tbaa_const, builder.CreateLoad( builder.CreateBitCast( builder.CreateConstGEP1_32( builder.CreateBitCast(aryv, T_pint8), jl_array_data_owner_offset(nd)), - T_ppjlvalue)); + T_ppjlvalue))); } else { #ifdef LLVM37 @@ -2540,7 +2549,7 @@ static bool emit_builtin_call(jl_cgval_t *ret, jl_value_t *f, jl_value_t **args, Value *types_len = emit_datatype_nfields(tyv); Value *idx = emit_unbox(T_size, emit_expr(args[2], ctx), (jl_value_t*)jl_long_type); emit_bounds_check(ty, (jl_value_t*)jl_datatype_type, idx, types_len, ctx); - Value *fieldtyp = builder.CreateLoad(builder.CreateGEP(builder.CreateBitCast(types_svec, T_ppjlvalue), idx)); + Value *fieldtyp = tbaa_decorate(tbaa_const, builder.CreateLoad(builder.CreateGEP(builder.CreateBitCast(types_svec, T_ppjlvalue), idx))); *ret = mark_julia_type(fieldtyp, true, expr_type(expr, ctx), ctx); JL_GC_POP(); return true; @@ -2664,7 +2673,7 @@ static jl_cgval_t emit_call_function_object(jl_lambda_info_t *li, const jl_cgval // can lazy load on demand, no copy needed assert(at == PointerType::get(et, 0)); jl_cgval_t arg = i==0 ? theF : emit_expr(args[i], ctx); - assert(arg.ispointer); + assert(arg.ispointer()); argvals[idx] = data_pointer(arg, ctx, at); mark_gc_use(arg); // TODO: must be after the jlcall } @@ -2680,7 +2689,7 @@ static jl_cgval_t emit_call_function_object(jl_lambda_info_t *li, const jl_cgval assert(idx == nfargs); CallInst *call = builder.CreateCall(prepare_call(cf), ArrayRef(&argvals[0], nfargs)); call->setAttributes(cf->getAttributes()); - return sret ? mark_julia_slot(result, jlretty) : mark_julia_type(call, retboxed, jlretty, ctx); + return sret ? mark_julia_slot(result, jlretty, tbaa_stack) : mark_julia_type(call, retboxed, jlretty, ctx); } return mark_julia_type(emit_jlcall(theFptr, boxed(theF,ctx), &args[1], nargs, ctx), true, expr_type(callexpr, ctx), ctx); @@ -2855,10 +2864,12 @@ static Value *global_binding_pointer(jl_module_t *m, jl_sym_t *s, return julia_binding_gv(b); } -static jl_cgval_t emit_checked_var(Value *bp, jl_sym_t *name, jl_codectx_t *ctx, bool isvol) +static jl_cgval_t emit_checked_var(Value *bp, jl_sym_t *name, jl_codectx_t *ctx, bool isvol, MDNode *tbaa) { assert(bp->getType() == T_ppjlvalue); - Value *v = builder.CreateLoad(bp, isvol); + Instruction *v = builder.CreateLoad(bp, isvol); + if (tbaa) + tbaa_decorate(tbaa, v); undef_var_error_if_null(v, name, ctx); return mark_julia_type(v, true, jl_any_type, ctx); } @@ -2888,9 +2899,9 @@ static jl_cgval_t emit_global(jl_sym_t *sym, jl_codectx_t *ctx) // double-check that a global variable is actually defined. this // can be a problem in parallel when a definition is missing on // one machine. - return mark_julia_type(builder.CreateLoad(bp), true, jl_any_type, ctx); + return mark_julia_type(tbaa_decorate(tbaa_binding, builder.CreateLoad(bp)), true, jl_any_type, ctx); } - return emit_checked_var(bp, sym, ctx); + return emit_checked_var(bp, sym, ctx, tbaa_binding); } static jl_cgval_t emit_local(jl_value_t *slotload, jl_codectx_t *ctx) @@ -3026,7 +3037,7 @@ static void emit_assignment(jl_value_t *l, jl_value_t *r, jl_codectx_t *ctx) } else { // store unboxed - assert(vi.value.ispointer); + assert(vi.value.ispointer()); builder.CreateStore( emit_unbox(julia_type_to_llvm(vi.value.typ), rval_info, vi.value.typ), vi.value.V, vi.isVolatile); @@ -3143,7 +3154,7 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx) if (b->constp && b->value != NULL) { return mark_julia_const(b->value); } - return emit_checked_var(julia_binding_gv(b), var, ctx); + return emit_checked_var(julia_binding_gv(b), var, ctx, tbaa_binding); } if (!jl_is_expr(expr)) { int needroot = true; @@ -3667,11 +3678,12 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t if (isboxed) { // passed an unboxed T, but want something boxed Value *mem = emit_allocobj(jl_datatype_size(jargty)); - builder.CreateStore(literal_pointer_val((jl_value_t*)jargty), - emit_typeptr_addr(mem)); - builder.CreateAlignedStore(val, - builder.CreateBitCast(mem, val->getType()->getPointerTo()), - 16); // julia's gc gives 16-byte aligned addresses + tbaa_decorate(tbaa_tag, builder.CreateStore(literal_pointer_val((jl_value_t*)jargty), + emit_typeptr_addr(mem))); + tbaa_decorate(jl_is_mutable(jargty) ? tbaa_user : tbaa_immut, + builder.CreateAlignedStore(val, + builder.CreateBitCast(mem, val->getType()->getPointerTo()), + 16)); // julia's gc gives 16-byte aligned addresses inputarg = mark_julia_type(mem, true, jargty, &ctx); } else { @@ -3955,7 +3967,7 @@ static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, Function *f, bool sre bool retboxed; (void)julia_type_to_llvm(jlretty, &retboxed); if (sret) { assert(!retboxed); } - jl_cgval_t retval = sret ? mark_julia_slot(result, jlretty) : mark_julia_type(call, retboxed, jlretty, &ctx); + jl_cgval_t retval = sret ? mark_julia_slot(result, jlretty, tbaa_stack) : mark_julia_type(call, retboxed, jlretty, &ctx); builder.CreateRet(boxed(retval, &ctx, false)); // no gcroot needed since this on the return path return w; @@ -4519,7 +4531,7 @@ static std::unique_ptr emit_function(jl_lambda_info_t *lam, jl_llvm_func theArg = ghostValue(argType); } else if (llvmArgType->isAggregateType()) { - theArg = mark_julia_slot(&*AI++, argType); // this argument is by-pointer + theArg = mark_julia_slot(&*AI++, argType, tbaa_stack); // this argument is by-pointer theArg.isimmutable = true; } else { @@ -4555,10 +4567,11 @@ static std::unique_ptr emit_function(jl_lambda_info_t *lam, jl_llvm_func if (vi.memloc == NULL) { if (vi.value.V) { // copy theArg into its local variable slot (unboxed) - assert(vi.isAssigned && vi.value.ispointer); - builder.CreateStore(emit_unbox(vi.value.V->getType()->getContainedType(0), - theArg, vi.value.typ), - vi.value.V); + assert(vi.isAssigned && vi.value.ispointer()); + tbaa_decorate(vi.value.tbaa, + builder.CreateStore(emit_unbox(vi.value.V->getType()->getContainedType(0), + theArg, vi.value.typ), + vi.value.V)); } else { // keep track of original (possibly boxed) value to avoid re-boxing or moving @@ -4567,7 +4580,7 @@ static std::unique_ptr emit_function(jl_lambda_info_t *lam, jl_llvm_func #ifdef LLVM36 if (specsig && theArg.V && ctx.debug_enabled) { SmallVector addr; - if ((Metadata*)vi.dinfo->getType() != jl_pvalue_dillvmt && theArg.ispointer) + if ((Metadata*)vi.dinfo->getType() != jl_pvalue_dillvmt && theArg.ispointer()) addr.push_back(llvm::dwarf::DW_OP_deref); AllocaInst *parg = dyn_cast(theArg.V); if (!parg) { @@ -4941,8 +4954,11 @@ extern "C" void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specs static void init_julia_llvm_env(Module *m) { MDNode *tbaa_root = mbuilder->createTBAARoot("jtbaa"); + tbaa_stack = tbaa_make_child("jtbaa_stack",tbaa_root); tbaa_gcframe = tbaa_make_child("jtbaa_gcframe",tbaa_root); tbaa_user = tbaa_make_child("jtbaa_user",tbaa_root); + tbaa_tag = tbaa_make_child("jtbaa_tag",tbaa_root); + tbaa_binding = tbaa_make_child("jtbaa_binding",tbaa_root); tbaa_value = tbaa_make_child("jtbaa_value",tbaa_root); tbaa_immut = tbaa_make_child("jtbaa_immut",tbaa_root); tbaa_array = tbaa_make_child("jtbaa_array",tbaa_value); diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 8de0bbfad6873..8c77ab42908ab 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -271,7 +271,7 @@ static Value *emit_unbox(Type *to, const jl_cgval_t &x, jl_value_t *jt) } Constant *c = x.constant ? julia_const_to_llvm(x.constant) : NULL; - if (!x.ispointer || c) { // already unboxed, but sometimes need conversion + if (!x.ispointer() || c) { // already unboxed, but sometimes need conversion Value *unboxed = c ? c : x.V; Type *ty = unboxed->getType(); // bools are stored internally as int8 (for now) @@ -301,17 +301,22 @@ static Value *emit_unbox(Type *to, const jl_cgval_t &x, jl_value_t *jt) if (p->getType() != ptype) p = builder.CreateBitCast(p, ptype); if (to == T_int1) - return builder.CreateTrunc(builder.CreateLoad(p), T_int1); + return builder.CreateTrunc(tbaa_decorate(x.tbaa, builder.CreateLoad(p)), T_int1); if (jt == (jl_value_t*)jl_bool_type) - return builder.CreateZExt(builder.CreateTrunc(builder.CreateLoad(p), T_int1), to); + return builder.CreateZExt(builder.CreateTrunc(tbaa_decorate(x.tbaa, builder.CreateLoad(p)), T_int1), to); - if (x.isboxed) - return builder.CreateAlignedLoad(p, 16); // julia's gc gives 16-byte aligned addresses - else if (jt) - return build_load(p, jt); - else + Instruction *load; + if (x.isboxed) { + load = builder.CreateAlignedLoad(p, 16); // julia's gc gives 16-byte aligned addresses + } + else if (jt) { + load = build_load(p, jt); + } + else { // stack has default alignment - return builder.CreateLoad(p); + load = builder.CreateLoad(p); + } + return tbaa_decorate(x.tbaa, load); } // unbox, trying to determine correct bitstype automatically @@ -440,18 +445,18 @@ static jl_cgval_t generic_box(jl_value_t *targ, jl_value_t *x, jl_codectx_t *ctx assert(!v.isghost); Value *vx = NULL; - if (!v.ispointer) + if (!v.ispointer()) vx = v.V; else if (v.constant) vx = julia_const_to_llvm(v.constant); - if (v.ispointer && vx == NULL) { + if (v.ispointer() && vx == NULL) { // try to load as original Type, to preserve llvm optimizations // but if the v.typ is not well known, use llvmt if (isboxed) vxt = llvmt; - vx = builder.CreateLoad(data_pointer(v, ctx, - vxt == T_int1 ? T_pint8 : vxt->getPointerTo())); + vx = tbaa_decorate(v.tbaa, builder.CreateLoad(data_pointer(v, ctx, + vxt == T_int1 ? T_pint8 : vxt->getPointerTo()))); } vxt = vx->getType(); @@ -509,8 +514,8 @@ static jl_cgval_t generic_unbox(jl_value_t *targ, jl_value_t *x, jl_codectx_t *c // XXX: emit type validity check on runtime_bt (bitstype of size nb) Value *newobj = emit_allocobj(nb); - builder.CreateStore(runtime_bt, emit_typeptr_addr(newobj)); - if (!v.ispointer) { + tbaa_decorate(tbaa_tag, builder.CreateStore(runtime_bt, emit_typeptr_addr(newobj))); + if (!v.ispointer()) { builder.CreateAlignedStore(emit_unbox(llvmt, v, v.typ), builder.CreatePointerCast(newobj, llvmt->getPointerTo()), alignment); } else { @@ -531,8 +536,8 @@ static jl_cgval_t generic_unbox(jl_value_t *targ, jl_value_t *x, jl_codectx_t *c return v; Value *vx; - if (v.ispointer) { - vx = builder.CreateLoad(data_pointer(v, ctx, llvmt->getPointerTo())); + if (v.ispointer()) { + vx = tbaa_decorate(v.tbaa, builder.CreateLoad(data_pointer(v, ctx, llvmt->getPointerTo()))); } else { vx = v.V; @@ -733,8 +738,8 @@ static jl_cgval_t emit_pointerref(jl_value_t *e, jl_value_t *i, jl_codectx_t *ct assert(jl_is_datatype(ety)); uint64_t size = jl_datatype_size(ety); Value *strct = emit_allocobj(size); - builder.CreateStore(literal_pointer_val((jl_value_t*)ety), - emit_typeptr_addr(strct)); + tbaa_decorate(tbaa_tag, builder.CreateStore(literal_pointer_val((jl_value_t*)ety), + emit_typeptr_addr(strct))); im1 = builder.CreateMul(im1, ConstantInt::get(T_size, LLT_ALIGN(size, ((jl_datatype_t*)ety)->alignment))); thePtr = builder.CreateGEP(builder.CreateBitCast(thePtr, T_pint8), im1);