From fe3789c15e14f522d8e2123d9dddbf4a54169e43 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 25 Sep 2023 13:58:39 +0000 Subject: [PATCH] add Core.memoryrefoffset intrinsic --- base/array.jl | 8 +- base/compiler/tfuncs.jl | 15 +- base/deepcopy.jl | 4 +- base/genericmemory.jl | 4 +- base/pointer.jl | 16 --- src/builtin_proto.h | 1 + src/builtins.c | 20 +++ src/cgutils.cpp | 167 ++++++++++------------ src/codegen.cpp | 58 ++++---- src/staticdata.c | 5 +- stdlib/Serialization/src/Serialization.jl | 4 +- 11 files changed, 155 insertions(+), 147 deletions(-) diff --git a/base/array.jl b/base/array.jl index ccf901efa7fc64..480da74619c079 100644 --- a/base/array.jl +++ b/base/array.jl @@ -1055,7 +1055,7 @@ function _growbeg!(a::Vector, delta::Integer) ref = a.ref mem = ref.mem len = length(a) - offset = memoffset(ref) + offset = memoryrefoffset(ref) - 1 newlen = len + delta a.size = (newlen,) # if offset is far enough advanced to fit data in existing memory without copying @@ -1093,7 +1093,7 @@ function _growend!(a::Vector, delta::Integer) memlen = length(mem) len = length(a) newlen = len + delta - offset = memoffset(ref) + offset = memoryrefoffset(ref) - 1 a.size = (newlen,) newmemlen = offset + newlen if memlen < newmemlen @@ -1132,7 +1132,7 @@ function _growat!(a::Vector, i::Integer, delta::Integer) mem = ref.mem memlen = length(mem) newlen = len + delta - offset = memoffset(ref) + offset = memoryrefoffset(ref) - 1 a.size = (newlen,) newmemlen = offset + newlen @@ -1475,7 +1475,7 @@ function sizehint!(a::Vector, sz::Integer) ref = a.ref mem = ref.mem memlen = length(mem) - offset = memoffset(ref) + offset = memoryrefoffset(ref) - 1 sz = max(Int(sz), offset+len) if sz <= memlen # if we don't save at least 1/8th memlen then its not worth it to shrink diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index f100914758286f..9bbe3f7241083c 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -2012,6 +2012,13 @@ end end add_tfunc(memoryref, 1, 3, memoryref_tfunc, 1) +@nospecs function memoryrefoffset_tfunc(๐•ƒ::AbstractLattice, mem) + hasintersect(widenconst(mem), MemoryRef) || return Bottom + return Int +end +add_tfunc(memoryrefoffset, 1, 1, memoryrefoffset_tfunc, 5) + + @nospecs function memoryref_builtin_common_errorcheck(mem, order, boundscheck) hasintersect(widenconst(mem), MemoryRef) || return false @@ -2140,6 +2147,10 @@ end โŠ‘ = Core.Compiler.:โŠ‘(๐•ƒ) if f === memoryref return memoryref_builtin_common_nothrow(argtypes) + elseif f === memoryrefoffset + length(argtypes) == 1 || return false + memtype = widenconst(argtypes[1]) + return memtype โŠ‘ MemoryRef elseif f === memoryrefset return memoryrefop_builtin_common_nothrow(argtypes, f) elseif f === memoryrefget @@ -2259,6 +2270,7 @@ const _EFFECT_FREE_BUILTINS = [ UnionAll, getfield, memoryref, + memoryrefoffset, memoryrefget, memoryref_isassigned, isdefined, @@ -2293,6 +2305,7 @@ const _INACCESSIBLEMEM_BUILTINS = Any[ const _ARGMEM_BUILTINS = Any[ memoryref, + memoryrefoffset, memoryrefget, memoryref_isassigned, memoryrefset, @@ -2466,7 +2479,7 @@ function builtin_effects(๐•ƒ::AbstractLattice, @nospecialize(f::Builtin), argin else if contains_is(_CONSISTENT_BUILTINS, f) consistent = ALWAYS_TRUE - elseif f === memoryref + elseif f === memoryref || f === memoryrefoffset consistent = ALWAYS_TRUE elseif f === memoryrefget || f === memoryrefset || f === memoryref_isassigned consistent = CONSISTENT_IF_INACCESSIBLEMEMONLY diff --git a/base/deepcopy.jl b/base/deepcopy.jl index d6438fa37d3585..ff097ae259e00c 100644 --- a/base/deepcopy.jl +++ b/base/deepcopy.jl @@ -125,8 +125,8 @@ function deepcopy_internal(x::MemoryRef, stackdict::IdDict) end mem = getfield(x, :mem) dest = MemoryRef(deepcopy_internal(mem, stackdict)::typeof(mem)) - i = memoffset(x) - i == 0 || (dest = Core.memoryref(dest, i + 1, true)) + i = memoryrefoffset(x) + i == 1 || (dest = Core.memoryref(dest, i, true)) return dest end diff --git a/base/genericmemory.jl b/base/genericmemory.jl index c782ccaeb8ddee..c0302e33bde85a 100644 --- a/base/genericmemory.jl +++ b/base/genericmemory.jl @@ -17,7 +17,7 @@ Memory ## Basic functions ## -using Core: memoryref_isassigned # import more functions which were not essential +using Core: memoryrefoffset, memoryref_isassigned # import more functions which were not essential size(a::GenericMemory, d::Int) = d < 1 ? error("dimension out of range") : @@ -72,7 +72,7 @@ end function unsafe_copyto!(dest::MemoryRef, src::MemoryRef, n) n == 0 && return dest @boundscheck MemoryRef(dest, n), MemoryRef(src, n) - unsafe_copyto!(dest.mem, memoffset(dest)+1, src.mem, memoffset(src)+1, n) + unsafe_copyto!(dest.mem, memoryrefoffset(dest), src.mem, memoryrefoffset(src), n) return dest end diff --git a/base/pointer.jl b/base/pointer.jl index a2ae036712cf5b..a57474af869d9b 100644 --- a/base/pointer.jl +++ b/base/pointer.jl @@ -84,22 +84,6 @@ function unsafe_convert(::Type{Ptr{Cvoid}}, a::MemoryRef{<:Any,T}) where {T} end unsafe_convert(::Type{Ptr{T}}, a::MemoryRef) where {T} = convert(Ptr{T}, unsafe_convert(Ptr{Cvoid}, a)) -function memoffset(a::MemoryRef{<:Any,T}) where {T} # n.b. offset + 1 = index - mem = getfield(a, :mem) - offset = getfield(a, :ptr) - MemT = typeof(mem) - arrayelem = datatype_arrayelem(MemT) - elsz = datatype_layoutsize(MemT) - isboxed = 1; isunion = 2 - if arrayelem != isunion && elsz != 0 - offset -= unsafe_convert(Ptr{Cvoid}, mem) - offset รท= elsz - else - offset = UInt(offset) - end - return offset % Int -end - # unsafe pointer to array conversions """ unsafe_wrap(Array, pointer::Ptr{T}, dims; own = false) diff --git a/src/builtin_proto.h b/src/builtin_proto.h index ef9188c054529b..e6e91207b2fdb5 100644 --- a/src/builtin_proto.h +++ b/src/builtin_proto.h @@ -26,6 +26,7 @@ DECLARE_BUILTIN(_apply_iterate); DECLARE_BUILTIN(_apply_pure); DECLARE_BUILTIN(apply_type); DECLARE_BUILTIN(memoryref); +DECLARE_BUILTIN(memoryrefoffset); DECLARE_BUILTIN(memoryrefget); DECLARE_BUILTIN(memoryrefset); DECLARE_BUILTIN(memoryref_isassigned); diff --git a/src/builtins.c b/src/builtins.c index d79deca4c8eb3e..8aa2148772245b 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -1553,6 +1553,25 @@ JL_CALLABLE(jl_f_memoryref) } } +JL_CALLABLE(jl_f_memoryrefoffset) +{ + JL_NARGS(memoryrefoffset, 1, 1); + JL_TYPECHK(memoryref, genericmemoryref, args[0]); + jl_genericmemoryref_t m = *(jl_genericmemoryref_t*)args[0]; + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; + size_t offset; + if (layout->arrayelem_isboxed) { + offset = (((char*)m.data - (char*)m.mem->data) / sizeof(jl_value_t*)); + } + else if (layout->arrayelem_isunion || layout->size == 0) { + offset = (size_t)m.data; + } + else { + offset = ((char*)m.data - (char*)m.mem->data) / layout->size; + } + return (jl_value_t*)jl_box_long(offset + 1); +} + JL_CALLABLE(jl_f_memoryrefget) { JL_NARGS(memoryrefget, 3, 3); @@ -2099,6 +2118,7 @@ void jl_init_primitives(void) JL_GC_DISABLED // memory primitives jl_builtin_memoryref = add_builtin_func("memoryref", jl_f_memoryref); + jl_builtin_memoryrefoffset = add_builtin_func("memoryrefoffset", jl_f_memoryrefoffset); jl_builtin_memoryrefget = add_builtin_func("memoryrefget", jl_f_memoryrefget); jl_builtin_memoryrefset = add_builtin_func("memoryrefset", jl_f_memoryrefset); jl_builtin_memoryref_isassigned = add_builtin_func("memoryref_isassigned", jl_f_memoryref_isassigned); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index a8ae35c62a75f0..40a422f5c1a5ae 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -699,9 +699,12 @@ static llvm::StructType* get_jlmemoryunionref(llvm::LLVMContext &C, llvm::Type * JuliaType::get_prjlvalue_ty(C), }); } -static StructType *get_memoryref_type(LLVMContext &ctxt, Type *T_size, bool isboxed, bool isunion, bool isghost, unsigned AS) +static StructType *get_memoryref_type(LLVMContext &ctxt, Type *T_size, const jl_datatype_layout_t *layout, unsigned AS) { // TODO: try to remove this fairly weird special case + bool isboxed = layout->arrayelem_isboxed; + bool isunion = layout->arrayelem_isunion; + bool isghost = layout->size == 0; if (isboxed) return get_jlmemoryboxedref(ctxt, AS); if (isunion || isghost) @@ -724,11 +727,8 @@ static Type *_julia_struct_to_llvm(jl_codegen_params_t *ctx, LLVMContext &ctxt, if (jl_is_genericmemoryref_type(jst)) { jl_value_t *mty_dt = jl_field_type_concrete(jst, 1); const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; - bool isboxed = layout->arrayelem_isboxed; - bool isunion = layout->arrayelem_isunion; - bool isghost = layout->size == 0; Type *T_size = bitstype_to_llvm((jl_value_t*)jl_long_type, ctxt); - return get_memoryref_type(ctxt, T_size, isboxed, isunion, isghost, 0); + return get_memoryref_type(ctxt, T_size, layout, 0); } bool isTuple = jl_is_tuple_type(jt); jl_svec_t *ftypes = jl_get_fieldtypes(jst); @@ -2763,8 +2763,6 @@ static Value *emit_genericmemoryelsize(jl_codectx_t &ctx, Value *v, jl_value_t * ++EmittedArrayElsize; jl_datatype_t *sty = (jl_datatype_t*)jl_unwrap_unionall(typ); if (jl_is_datatype(sty) && !jl_has_free_typevars((jl_value_t*)sty) && sty->layout) { - if (jl_is_array_type(sty)) - sty = (jl_datatype_t*)jl_field_type_concrete(sty, 0); if (jl_is_genericmemoryref_type(sty)) sty = (jl_datatype_t*)jl_field_type_concrete(sty, 1); return ConstantInt::get(ctx.types().T_size, sty->layout->size); @@ -2811,7 +2809,7 @@ static Value *emit_genericmemorylen(jl_codectx_t &ctx, Value *addr, jl_value_t * return LI; } -static Value *emit_genericmemoryptr(jl_codectx_t &ctx, Value *mem, bool isboxed, bool isunion, bool isghost, unsigned AS) +static Value *emit_genericmemoryptr(jl_codectx_t &ctx, Value *mem, const jl_datatype_layout_t *layout, unsigned AS) { ++EmittedArrayptr; PointerType *PT = cast(mem->getType()); @@ -2825,22 +2823,16 @@ static Value *emit_genericmemoryptr(jl_codectx_t &ctx, Value *mem, bool isboxed, LI->setMetadata(LLVMContext::MD_nonnull, MDNode::get(ctx.builder.getContext(), None)); jl_aliasinfo_t aliasinfo = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); aliasinfo.decorateInst(LI); - Value *ptr = LI; if (AS) { assert(AS == AddressSpace::Loaded); ptr = ctx.builder.CreateCall(prepare_call(gc_loaded_func), { mem, ptr }); } - if (!isboxed) + if (!layout->arrayelem_isboxed) ptr = ctx.builder.CreateBitCast(ptr, PointerType::get(getInt8Ty(ctx.builder.getContext()), AS)); return ptr; } -static Value *emit_genericmemoryptr(jl_codectx_t &ctx, const jl_cgval_t &mem, bool isboxed, bool isunion, bool isghost, unsigned AS) -{ - return emit_genericmemoryptr(ctx, boxed(ctx, mem), isboxed, isunion, isghost, AS); -} - static Value *emit_genericmemoryowner(jl_codectx_t &ctx, Value *t) { Value *m = emit_bitcast(ctx, decay_derived(ctx, t), ctx.types().T_jlgenericmemory->getPointerTo(0)); @@ -3927,30 +3919,33 @@ static int compare_cgparams(const jl_cgparams_t *a, const jl_cgparams_t *b) } #endif -static jl_cgval_t _emit_memoryref(jl_codectx_t &ctx, Value *mem, Value *data, bool isboxed, bool isunion, bool isghost, jl_value_t *typ) +static jl_cgval_t _emit_memoryref(jl_codectx_t &ctx, Value *mem, Value *data, const jl_datatype_layout_t *layout, jl_value_t *typ) { //jl_cgval_t argv[] = { // mark_julia_type(ctx, mem, true, jl_any_type), // mark_julia_type(ctx, data, false, jl_voidpointer_type) //}; //return emit_new_struct(ctx, typ, 3, argv); - Value *ref = Constant::getNullValue(get_memoryref_type(ctx.builder.getContext(), ctx.types().T_size, isboxed, isunion, isghost, 0)); + Value *ref = Constant::getNullValue(get_memoryref_type(ctx.builder.getContext(), ctx.types().T_size, layout, 0)); ref = ctx.builder.CreateInsertValue(ref, data, 0); ref = ctx.builder.CreateInsertValue(ref, mem, 1); return mark_julia_type(ctx, ref, false, typ); } -static jl_cgval_t _emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &mem, bool isboxed, bool isunion, bool isghost, jl_value_t *typ) +static jl_cgval_t _emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &mem, const jl_datatype_layout_t *layout, jl_value_t *typ) { - Value *data = (!isboxed && isunion) || isghost ? ConstantInt::get(ctx.types().T_size, 0) : emit_genericmemoryptr(ctx, mem, isboxed, isunion, isghost, 0); - return _emit_memoryref(ctx, boxed(ctx, mem), data, isboxed, isunion, isghost, typ); + bool isboxed = layout->arrayelem_isboxed; + bool isunion = layout->arrayelem_isunion; + bool isghost = layout->size == 0; + Value *data = (!isboxed && isunion) || isghost ? ConstantInt::get(ctx.types().T_size, 0) : emit_genericmemoryptr(ctx, boxed(ctx, mem), layout, 0); + return _emit_memoryref(ctx, boxed(ctx, mem), data, layout, typ); } -static Value *emit_memoryref_FCA(jl_codectx_t &ctx, const jl_cgval_t &ref, bool isboxed, bool isunion, bool isghost) +static Value *emit_memoryref_FCA(jl_codectx_t &ctx, const jl_cgval_t &ref, const jl_datatype_layout_t *layout) { if (ref.ispointer()) { LLVMContext &C = ctx.builder.getContext(); - Type *type = get_memoryref_type(C, ctx.types().T_size, isboxed, isunion, isghost, 0); + Type *type = get_memoryref_type(C, ctx.types().T_size, layout, 0); LoadInst *load = ctx.builder.CreateLoad(type, emit_bitcast(ctx, data_pointer(ctx, ref), PointerType::get(type, 0))); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ref.tbaa); ai.decorateInst(load); @@ -3961,14 +3956,14 @@ static Value *emit_memoryref_FCA(jl_codectx_t &ctx, const jl_cgval_t &ref, bool } } -static jl_cgval_t emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &ref, jl_cgval_t idx, jl_value_t *inbounds, bool isboxed, bool isunion, bool isghost) +static jl_cgval_t emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &ref, jl_cgval_t idx, jl_value_t *inbounds, const jl_datatype_layout_t *layout) { ++EmittedArrayNdIndex; emit_typecheck(ctx, idx, (jl_value_t*)jl_long_type, "memoryref"); idx = update_julia_type(ctx, idx, (jl_value_t*)jl_long_type); if (idx.typ == jl_bottom_type) return jl_cgval_t(); - Value *V = emit_memoryref_FCA(ctx, ref, isboxed, isunion, isghost); + Value *V = emit_memoryref_FCA(ctx, ref, layout); Value *data = CreateSimplifiedExtractValue(ctx, V, 0); Value *mem = CreateSimplifiedExtractValue(ctx, V, 1); Value *i = emit_unbox(ctx, ctx.types().T_size, idx, (jl_value_t*)jl_long_type); @@ -3979,6 +3974,9 @@ static jl_cgval_t emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &ref, jl_cg Value *ovflw = nullptr; #endif Value *newdata; + bool isboxed = layout->arrayelem_isboxed; + bool isunion = layout->arrayelem_isunion; + bool isghost = layout->size == 0; if ((!isboxed && isunion) || isghost) { newdata = ctx.builder.CreateAdd(data, offset); if (bc) { @@ -4032,7 +4030,7 @@ static jl_cgval_t emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &ref, jl_cg failBB = BasicBlock::Create(ctx.builder.getContext(), "oob"); endBB = BasicBlock::Create(ctx.builder.getContext(), "idxend"); Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); - Value *mptr = emit_genericmemoryptr(ctx, mem, isboxed, isunion, isghost, 0); + Value *mptr = emit_genericmemoryptr(ctx, mem, layout, 0); mptr = emit_bitcast(ctx, mptr, newdata->getType()); #if 0 Value *mend = emit_bitcast(ctx, mptr, getInt8PtrTy(ctx.builder.getContext())); @@ -4070,88 +4068,69 @@ static jl_cgval_t emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &ref, jl_cg ctx.builder.SetInsertPoint(endBB); } } - return _emit_memoryref(ctx, mem, newdata, isboxed, isunion, isghost, ref.typ); + return _emit_memoryref(ctx, mem, newdata, layout, ref.typ); } -static Value *emit_memoryref_mem(jl_codectx_t &ctx, const jl_cgval_t &ref, bool isboxed, bool isunion, bool isghost) +static jl_cgval_t emit_memoryref_offset(jl_codectx_t &ctx, const jl_cgval_t &ref, const jl_datatype_layout_t *layout) { - Value *V = emit_memoryref_FCA(ctx, ref, isboxed, isunion, isghost); + Value *offset; + Value *V = emit_memoryref_FCA(ctx, ref, layout); + Value *data = CreateSimplifiedExtractValue(ctx, V, 0); + if (layout->arrayelem_isunion || layout->size == 0) { + offset = data; + } + else { + Value *mem = CreateSimplifiedExtractValue(ctx, V, 1); + Value *mptr = emit_genericmemoryptr(ctx, mem, layout, 0); + mptr = emit_bitcast(ctx, mptr, mem->getType()); + // (data - mptr) / elsz + offset = ctx.builder.CreateSub( + ctx.builder.CreatePtrToInt(data, ctx.types().T_size), + ctx.builder.CreatePtrToInt(mptr, ctx.types().T_size)); + Value *elsz = emit_genericmemoryelsize(ctx, mem, ref.typ); + offset = ctx.builder.CreateExactUDiv(offset, elsz); + } + offset = ctx.builder.CreateAdd(offset, ConstantInt::get(ctx.types().T_size, 1)); + return mark_julia_type(ctx, offset, false, jl_long_type); +} + +static Value *emit_memoryref_mem(jl_codectx_t &ctx, const jl_cgval_t &ref, const jl_datatype_layout_t *layout) +{ + Value *V = emit_memoryref_FCA(ctx, ref, layout); return CreateSimplifiedExtractValue(ctx, V, 1); } -static Value *emit_memoryref_ptr(jl_codectx_t &ctx, const jl_cgval_t &ref, bool isboxed, bool isunion, bool isghost) +static Value *emit_memoryref_ptr(jl_codectx_t &ctx, const jl_cgval_t &ref, const jl_datatype_layout_t *layout) { - assert((isboxed || !isunion) && !isghost); - Value *newref = emit_memoryref_FCA(ctx, ref, isboxed, isunion, isghost); + assert(!layout->arrayelem_isunion && layout->size != 0); + Value *newref = emit_memoryref_FCA(ctx, ref, layout); Value *data = CreateSimplifiedExtractValue(ctx, newref, 0); - if (isboxed || !(isunion && isghost)) { - unsigned AS = AddressSpace::Loaded; - Value *mem = CreateSimplifiedExtractValue(ctx, newref, 1); - // rebuild GEP on data, so that we manually hoist this gc_loaded_func call over it, back to the original load - // we should add this to llvm-julia-licm too, so we can attempt hoisting over PhiNodes too (which aren't defined yet here) - IRBuilder<>::InsertPointGuard resetIP(ctx.builder); - std::vector GEPlist; - data = data->stripPointerCastsSameRepresentation(); - while (GetElementPtrInst *GEP = dyn_cast(data)) { // ignoring bitcast will not be required with opaque pointers - GEPlist.push_back(GEP); - data = GEP->getPointerOperand()->stripPointerCastsSameRepresentation(); - } - // Under the common assumption that data and mem will come from the same basic block and both will dominate the first GEP, - // manually hoist these right away to improve the IR structure. But tolerate the case where we could not verify that. - bool moveip = false; - //Instruction *defmem = dyn_cast(mem); - //if (Instruction *def = dyn_cast(data)) { - // if (defmem == nullptr || def->getParent() == defmem->getParent()) { - // if (defmem != nullptr && !defmem->comesBefore(def)) - // def = defmem; - // ctx.builder.SetInsertPoint(def->getParent(), std::next(def->getIterator())); - // ctx.builder.SetCurrentDebugLocation(def->getDebugLoc()); - // moveip = true; - // } - // else if (LoadInst *defbase = dyn_cast(def)) { - // def = defbase; - // if (defmem != nullptr && !defmem->comesBefore(def)) - // def = defmem; - // ctx.builder.SetInsertPoint(def->getParent(), std::next(def->getIterator())); - // ctx.builder.SetCurrentDebugLocation(def->getDebugLoc()); - // moveip = true; - // } - //} - //// if we couldn't find the def Instruction, we might still be able to put it right before the GEP chain - //if (!moveip && !GEPlist.empty()) { - // Instruction *defgep = GEPlist[GEPlist.size() - 1]; - // if (defmem == nullptr || defmem->getParent() == defgep->getParent()) { - // if (defmem == nullptr || defmem->comesBefore(defgep)) { - // ctx.builder.SetInsertPoint(GEPlist[GEPlist.size() - 1]); - // moveip = true; - // } - // } - //} - data = ctx.builder.CreateBitCast(data, ctx.types().T_pprjlvalue); -#if 0 // XXX(jwn): do not do this - (void)mem; - data = ctx.builder.CreateAddrSpaceCast(data, prepare_call(gc_loaded_func)->getReturnType()); -#else - data = ctx.builder.CreateCall(prepare_call(gc_loaded_func), { mem, data }); -#endif - if (!GEPlist.empty()) { - for (auto &GEP : make_range(GEPlist.rbegin(), GEPlist.rend())) { - if (moveip) - ctx.builder.SetInsertPoint(GEP); - data = ctx.builder.CreateBitCast(data, PointerType::get(GEP->getSourceElementType(), AS)); - Instruction *GEP2 = GEP->clone(); - GEP2->mutateType(PointerType::get(GEP->getResultElementType(), AS)); - GEP2->setOperand(GetElementPtrInst::getPointerOperandIndex(), data); - ctx.builder.Insert(GEP2); - data = GEP2; - } + unsigned AS = AddressSpace::Loaded; + Value *mem = CreateSimplifiedExtractValue(ctx, newref, 1); + // rebuild GEP on data, so that we manually hoist this gc_loaded_func call over it, back to the original load + // we should add this to llvm-julia-licm too, so we can attempt hoisting over PhiNodes too (which aren't defined yet here) + IRBuilder<>::InsertPointGuard resetIP(ctx.builder); + std::vector GEPlist; + data = data->stripPointerCastsSameRepresentation(); + while (GetElementPtrInst *GEP = dyn_cast(data)) { // ignoring bitcast will not be required with opaque pointers + GEPlist.push_back(GEP); + data = GEP->getPointerOperand()->stripPointerCastsSameRepresentation(); + } + data = ctx.builder.CreateBitCast(data, ctx.types().T_pprjlvalue); + data = ctx.builder.CreateCall(prepare_call(gc_loaded_func), { mem, data }); + if (!GEPlist.empty()) { + for (auto &GEP : make_range(GEPlist.rbegin(), GEPlist.rend())) { + data = ctx.builder.CreateBitCast(data, PointerType::get(GEP->getSourceElementType(), AS)); + Instruction *GEP2 = GEP->clone(); + GEP2->mutateType(PointerType::get(GEP->getResultElementType(), AS)); + GEP2->setOperand(GetElementPtrInst::getPointerOperandIndex(), data); + ctx.builder.Insert(GEP2); + data = GEP2; } } return data; } - - // Reset us back to codegen debug type #undef DEBUG_TYPE #define DEBUG_TYPE "julia_irgen_codegen" diff --git a/src/codegen.cpp b/src/codegen.cpp index 7bad41dc053f79..e6e9896cc05894 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1340,7 +1340,8 @@ static const auto &builtin_func_map() { { jl_f_nfields_addr, new JuliaFunction<>{XSTR(jl_f_nfields), get_func_sig, get_func_attrs} }, { jl_f__expr_addr, new JuliaFunction<>{XSTR(jl_f__expr), get_func_sig, get_func_attrs} }, { jl_f__typevar_addr, new JuliaFunction<>{XSTR(jl_f__typevar), get_func_sig, get_func_attrs} }, - { jl_f_memoryrefget_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefget), get_func_sig, get_func_attrs} }, + { jl_f_memoryref_addr, new JuliaFunction<>{XSTR(jl_f_memoryref), get_func_sig, get_func_attrs} }, + { jl_f_memoryrefoffset_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefoffset), get_func_sig, get_func_attrs} }, { jl_f_memoryrefset_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefset), get_func_sig, get_func_attrs} }, { jl_f_memoryref_isassigned_addr,new JuliaFunction<>{XSTR(jl_f_memoryref_isassigned), get_func_sig, get_func_attrs} }, { jl_f_apply_type_addr, new JuliaFunction<>{XSTR(jl_f_apply_type), get_func_sig, get_func_attrs} }, @@ -3489,7 +3490,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, jl_value_t *ety = jl_tparam1(mty_dt); const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; jl_value_t *typ = jl_apply_type2((jl_value_t*)jl_genericmemoryref_type, isatomic, ety); - *ret = _emit_memoryref(ctx, mem, layout->arrayelem_isboxed, layout->arrayelem_isunion, layout->size == 0, typ); + *ret = _emit_memoryref(ctx, mem, layout, typ); return true; } } @@ -3503,7 +3504,18 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, jl_value_t *boundscheck = nargs == 3 ? argv[3].constant : nullptr; if (nargs == 3) emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryref"); - *ret = emit_memoryref(ctx, ref, argv[2], boundscheck, layout->arrayelem_isboxed, layout->arrayelem_isunion, layout->size == 0); + *ret = emit_memoryref(ctx, ref, argv[2], boundscheck, layout); + return true; + } + } + + else if (f == jl_builtin_memoryrefoffset && nargs == 1) { + const jl_cgval_t &ref = argv[1]; + jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); + if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { + mty_dt = jl_field_type_concrete((jl_datatype_t*)mty_dt, 1); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; + *ret = emit_memoryref_offset(ctx, ref, layout); return true; } } @@ -3521,11 +3533,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, jl_value_t *boundscheck = argv[3].constant; emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryref"); const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; - bool isboxed = layout->arrayelem_isboxed; - bool isunion = layout->arrayelem_isunion; - size_t elsz = layout->size; - size_t al = layout->alignment; - Value *mem = emit_memoryref_mem(ctx, ref, isboxed, isunion, elsz == 0); + Value *mem = emit_memoryref_mem(ctx, ref, layout); Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); if (bounds_check_enabled(ctx, boundscheck)) { BasicBlock *failBB, *endBB; @@ -3539,15 +3547,19 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, ctx.f->getBasicBlockList().push_back(endBB); ctx.builder.SetInsertPoint(endBB); } + bool isboxed = layout->arrayelem_isboxed; + bool isunion = layout->arrayelem_isunion; + size_t elsz = layout->size; + size_t al = layout->alignment; if (!isboxed && !isunion && elsz == 0) { assert(jl_is_datatype(ety) && jl_is_datatype_singleton((jl_datatype_t*)ety)); *ret = ghostValue(ctx, ety); } else if (isunion) { - Value *V = emit_memoryref_FCA(ctx, ref, isboxed, isunion, elsz == 0); + Value *V = emit_memoryref_FCA(ctx, ref, layout); Value *idx0 = CreateSimplifiedExtractValue(ctx, V, 0); Value *mem = CreateSimplifiedExtractValue(ctx, V, 1); - Value *data = emit_genericmemoryptr(ctx, mem, isboxed, isunion, elsz == 0, AddressSpace::Loaded); + Value *data = emit_genericmemoryptr(ctx, mem, layout, AddressSpace::Loaded); Value *ptindex; if (elsz == 0) { ptindex = data; @@ -3569,7 +3581,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, else { MDNode *aliasscope = nullptr; *ret = typed_load(ctx, - emit_memoryref_ptr(ctx, ref, isboxed, isunion, elsz == 0), + emit_memoryref_ptr(ctx, ref, layout), nullptr, ety, isboxed ? ctx.tbaa().tbaa_ptrarraybuf : ctx.tbaa().tbaa_arraybuf, aliasscope, @@ -3594,11 +3606,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, jl_value_t *boundscheck = argv[4].constant; emit_typecheck(ctx, argv[4], (jl_value_t*)jl_bool_type, "memoryset"); const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; - bool isboxed = layout->arrayelem_isboxed; - bool isunion = layout->arrayelem_isunion; - size_t elsz = layout->size; - size_t al = layout->alignment; - Value *mem = emit_memoryref_mem(ctx, ref, isboxed, isunion, elsz == 0); + Value *mem = emit_memoryref_mem(ctx, ref, layout); Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); if (bounds_check_enabled(ctx, boundscheck)) { BasicBlock *failBB, *endBB; @@ -3616,6 +3624,10 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, val = update_julia_type(ctx, val, ety); if (val.typ == jl_bottom_type) return true; + bool isboxed = layout->arrayelem_isboxed; + bool isunion = layout->arrayelem_isunion; + size_t elsz = layout->size; + size_t al = layout->alignment; if (isboxed) ety = (jl_value_t*)jl_any_type; if (!isboxed && !isunion && elsz == 0) { @@ -3623,7 +3635,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, // no-op } else { - Value *V = emit_memoryref_FCA(ctx, ref, isboxed, isunion, elsz == 0); + Value *V = emit_memoryref_FCA(ctx, ref, layout); Value *data_owner = NULL; // owner object against which the write barrier must check if (isboxed || layout->first_ptr >= 0) { // if elements are just bits, don't need a write barrier data_owner = emit_genericmemoryowner(ctx, CreateSimplifiedExtractValue(ctx, V, 1)); @@ -3631,7 +3643,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, if (isunion) { Value *idx0 = CreateSimplifiedExtractValue(ctx, V, 0); Value *mem = CreateSimplifiedExtractValue(ctx, V, 1); - Value *data = emit_genericmemoryptr(ctx, mem, isboxed, isunion, elsz == 0, AddressSpace::Loaded); + Value *data = emit_genericmemoryptr(ctx, mem, layout, AddressSpace::Loaded); Type *AT = ArrayType::get(IntegerType::get(ctx.builder.getContext(), 8 * al), (elsz + al - 1) / al); data = emit_bitcast(ctx, data, AT->getPointerTo()); // compute tindex from val @@ -3659,7 +3671,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } else { typed_store(ctx, - emit_memoryref_ptr(ctx, ref, isboxed, isunion, elsz == 0), + emit_memoryref_ptr(ctx, ref, layout), nullptr, val, jl_cgval_t(), ety, isboxed ? ctx.tbaa().tbaa_ptrarraybuf : ctx.tbaa().tbaa_arraybuf, ctx.noalias().aliasscope.current, @@ -3695,12 +3707,10 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, jl_value_t *boundscheck = argv[3].constant; emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memory_isassigned"); const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; - bool isboxed = layout->arrayelem_isboxed; - bool isunion = layout->arrayelem_isunion; - size_t elsz = layout->size; - Value *mem = emit_memoryref_mem(ctx, ref, isboxed, isunion, elsz == 0); + Value *mem = emit_memoryref_mem(ctx, ref, layout); Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); Value *oob = bounds_check_enabled(ctx, boundscheck) ? ctx.builder.CreateIsNull(mlen) : nullptr; + bool isboxed = layout->arrayelem_isboxed; if (isboxed || layout->first_ptr >= 0) { PHINode *result = nullptr; if (oob) { @@ -3716,7 +3726,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, setName(ctx.emission_context, result, "arraysize"); ctx.builder.SetInsertPoint(passBB); } - Value *elem = emit_memoryref_ptr(ctx, ref, isboxed, isunion, elsz == 0); + Value *elem = emit_memoryref_ptr(ctx, ref, layout); elem = emit_bitcast(ctx, elem, ctx.types().T_pprjlvalue); if (!isboxed) elem = ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_prjlvalue, elem, layout->first_ptr); diff --git a/src/staticdata.c b/src/staticdata.c index 21f24669fff933..5c86482a272e61 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -99,7 +99,7 @@ extern "C" { // TODO: put WeakRefs on the weak_refs list during deserialization // TODO: handle finalizers -#define NUM_TAGS 168 +#define NUM_TAGS 169 // An array of references that need to be restored from the sysimg // This is a manually constructed dual of the gvars array, which would be produced by codegen for Julia code, for C. @@ -274,6 +274,7 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_builtin_replacefield); INSERT_TAG(jl_builtin_fieldtype); INSERT_TAG(jl_builtin_memoryref); + INSERT_TAG(jl_builtin_memoryrefoffset); INSERT_TAG(jl_builtin_memoryrefget); INSERT_TAG(jl_builtin_memoryrefset); INSERT_TAG(jl_builtin_memoryref_isassigned); @@ -454,7 +455,7 @@ static const jl_fptr_args_t id_to_fptrs[] = { &jl_f_tuple, &jl_f_svec, &jl_f_intrinsic_call, &jl_f_getfield, &jl_f_setfield, &jl_f_swapfield, &jl_f_modifyfield, &jl_f_replacefield, &jl_f_fieldtype, &jl_f_nfields, &jl_f_apply_type, - &jl_f_memoryref, &jl_f_memoryrefget, &jl_f_memoryrefset, &jl_f_memoryref_isassigned, + &jl_f_memoryref, &jl_f_memoryrefoffset, &jl_f_memoryrefget, &jl_f_memoryrefset, &jl_f_memoryref_isassigned, &jl_f_applicable, &jl_f_invoke, &jl_f_sizeof, &jl_f__expr, &jl_f__typevar, &jl_f_ifelse, &jl_f__structtype, &jl_f__abstracttype, &jl_f__primitivetype, &jl_f__typebody, &jl_f__setsuper, &jl_f__equiv_typedef, &jl_f_get_binding_type, diff --git a/stdlib/Serialization/src/Serialization.jl b/stdlib/Serialization/src/Serialization.jl index d23c1667452d36..778ca94caae216 100644 --- a/stdlib/Serialization/src/Serialization.jl +++ b/stdlib/Serialization/src/Serialization.jl @@ -310,7 +310,7 @@ end function serialize(s::AbstractSerializer, x::MemoryRef) serialize_type(s, typeof(x)) serialize(s, getfield(x, :mem)) - serialize(s, Base.memoffset(x)) + serialize(s, Base.memoryrefoffset(x)) end function serialize(s::AbstractSerializer, ss::String) @@ -1345,7 +1345,7 @@ end function deserialize(s::AbstractSerializer, X::Type{MemoryRef{:not_atomic,T}} where T) x = Core.memoryref(deserialize(s))::X i = deserialize(s)::Int - i == 0 || (x = Core.memoryref(x, i + 1, true)) + i == 2 || (x = Core.memoryref(x, i, true)) return x::X end