[WIP] allocation complement to #55045

Author: vtjnash

base/reflection.jl

@@ -642,7 +642,7 @@ struct FieldDescStorage{T}
     offset::T
 end
 FieldDesc(fd::FieldDescStorage{T}) where {T} =
-    FieldDesc(false, fd.ptrsize & 1 != 0,
+    FieldDesc(false, (fd.ptrsize & 3) == 1,
               fd.ptrsize >> 1, fd.offset)
 
 struct DataTypeFieldDesc

src/builtins.c

@@ -85,23 +85,28 @@ static int NOINLINE compare_fields(const jl_value_t *a, const jl_value_t *b, jl_
         size_t offs = jl_field_offset(dt, f);
         char *ao = (char*)a + offs;
         char *bo = (char*)b + offs;
-        if (jl_field_isptr(dt, f)) {
+        enum jl_fieldkind_t kind = jl_field_kind(dt, i);
+        if (kind == JL_FIELDKIND_ISPTR) {
             // Save ptr recursion until the end -- only recurse if otherwise equal
             // Note that we also skip comparing the pointers for null here, because
             // null fields are rare so it can save CPU to delay this read too.
             continue;
         }
         else {
             jl_datatype_t *ft = (jl_datatype_t*)jl_field_type_concrete(dt, f);
-            if (jl_is_uniontype(ft)) {
+            if (kind == JL_FIELDKIND_UNION) {
+                assert(jl_is_uniontype(ft));
                 size_t idx = jl_field_size(dt, f) - 1;
                 uint8_t asel = ((uint8_t*)ao)[idx];
                 uint8_t bsel = ((uint8_t*)bo)[idx];
                 if (asel != bsel)
                     return 0;
                 ft = (jl_datatype_t*)jl_nth_union_component((jl_value_t*)ft, asel);
             }
-            else if (ft->layout->first_ptr >= 0) {
+            else if (kind != JL_FIELDKIND_ISBITS) {
+                if (jl_is_uniontype(ft))
+                    ft = (jl_datatype_t*)((jl_uniontype_t*)ft)->b;
+                assert(ft->layout->first_ptr >= 0);
                 // If the field is a inline immutable that can be undef
                 // we need to check for undef first since undef struct
                 // may have fields that are different but should still be treated as equal.
@@ -408,19 +413,25 @@ static uintptr_t immut_id_(jl_datatype_t *dt, jl_value_t *v, uintptr_t h) JL_NOT
         size_t offs = jl_field_offset(dt, f);
         char *vo = (char*)v + offs;
         uintptr_t u;
-        if (jl_field_isptr(dt, f)) {
+        enum jl_fieldkind_t kind = jl_field_kind(dt, f);
+        if (kind == JL_FIELDKIND_ISPTR) {
             jl_value_t *f = *(jl_value_t**)vo;
             u = (f == NULL) ? 0 : jl_object_id(f);
         }
         else {
             jl_datatype_t *fieldtype = (jl_datatype_t*)jl_field_type_concrete(dt, f);
-            if (jl_is_uniontype(fieldtype)) {
+            if (kind == JL_FIELDKIND_ISUNION) {
+                assert(jl_is_uniontype(fieldtype));
                 uint8_t sel = ((uint8_t*)vo)[jl_field_size(dt, f) - 1];
                 fieldtype = (jl_datatype_t*)jl_nth_union_component((jl_value_t*)fieldtype, sel);
             }
+            else if (kind != JL_FIELDKIND_ISBITS) {
+                if (jl_is_uniontype(fieldtype))
+                    fieldtype = (jl_datatype_t*)((jl_uniontype_t*)fieldtype)->b;
+                assert(fieldtype->layout->first_ptr >= 0);
+            }
             assert(jl_is_datatype(fieldtype) && !fieldtype->name->abstract && !fieldtype->name->mutabl);
-            int32_t first_ptr = fieldtype->layout->first_ptr;
-            if (first_ptr >= 0 && ((jl_value_t**)vo)[first_ptr] == NULL) {
+            if (kind == JL_FIELDKIND_OTHER && ((jl_value_t**)vo)[fieldtype->layout->first_ptr] == NULL) {
                 // If the field is a inline immutable that can be can be undef
                 // we need to check to check for undef first since undef struct
                 // may have fields that are different but should still be treated as equal.

src/cgutils.cpp

@@ -788,11 +788,12 @@ static Type *_julia_struct_to_llvm(jl_codegen_params_t *ctx, LLVMContext &ctxt,
                 abort();
             }
             Type *lty;
-            if (jl_field_isptr(jst, i)) {
+            enum jl_fieldkind_t kind = jl_field_kind(jst, i);
+            if (kind == JL_FIELDKIND_ISPTR) {
                 lty = JuliaType::get_prjlvalue_ty(ctxt);
                 isvector = false;
             }
-            else if (jl_is_uniontype(ty)) {
+            else if (kind == JL_FIELDKIND_ISUNION) {
                 // pick an Integer type size such that alignment will generally be correct,
                 // and always end with an Int8 (selector byte).
                 // We may need to insert padding first to get to the right offset
@@ -822,6 +823,8 @@ static Type *_julia_struct_to_llvm(jl_codegen_params_t *ctx, LLVMContext &ctxt,
                 continue;
             }
             else {
+                if (kind == JL_FIELDKIND_ISOTHER && jl_is_uniontype(ty))
+                    ty = ((jl_uniontype_t*)ty)->b;
                 bool isptr;
                 lty = _julia_struct_to_llvm(ctx, ctxt, ty, &isptr, llvmcall);
                 assert(lty && !isptr);
@@ -1943,11 +1946,19 @@ static void emit_lockstate_value(jl_codectx_t &ctx, Value *strct, bool newstate)
 
 // If `nullcheck` is not NULL and a pointer NULL check is necessary
 // store the pointer to be checked in `*nullcheck` instead of checking it
-static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, jl_value_t *jltype,
+static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, jl_value_t *loadtype,
                              MDNode *tbaa, MDNode *aliasscope, bool isboxed, AtomicOrdering Order,
                              bool maybe_null_if_boxed = true, unsigned alignment = 0,
                              Value **nullcheck = nullptr)
 {
+    jl_value_t *jltype = loadtype;
+    Value *otherval = nullptr;
+    bool isother = !isboxed && jl_is_uniontype(jltype);
+    if (isother) {
+        nullcheck = &otherval;
+        maybe_null = true;
+        jltype = ((jl_uniontype_t*)jltype)->b;
+    }
     // TODO: we should use unordered loads for anything with CountTrackedPointers(elty).count > 0 (if not otherwise locked)
     Type *elty = isboxed ? ctx.types().T_prjlvalue : julia_type_to_llvm(ctx, jltype);
     if (type_is_ghost(elty)) {
@@ -2042,10 +2053,16 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j
             instr = ctx.builder.CreateLoad(intcast->getAllocatedType(), intcast);
         instr = ctx.builder.CreateTrunc(instr, getInt1Ty(ctx.builder.getContext()));
     }
+    Value *tindex = nullptr;
+    if (isother) {
+        tindex = ctx.builder.CreateZExt(null_pointer_cmp(ctx, otherval), getInt8Ty(ctx.builder.getContext()));
+        assert(ret.Vboxed == nullptr);
+        ret = jl_cgval_t(ret, jl_field_type(jt, idx), tindex);
+    }
     if (instr)
-        return mark_julia_type(ctx, instr, isboxed, jltype);
+        return mark_julia_type(ctx, instr, isboxed, loadtype);
     else
-        return mark_julia_slot(intcast, jltype, NULL, ctx.tbaa().tbaa_stack);
+        return mark_julia_slot(intcast, loadtype, tindex, ctx.tbaa().tbaa_stack);
 }
 
 static jl_cgval_t typed_store(jl_codectx_t &ctx,
@@ -2495,7 +2512,8 @@ static bool field_may_be_null(const jl_cgval_t &strct, jl_datatype_t *stt, size_
     size_t nfields = jl_datatype_nfields(stt);
     if (idx < nfields - (unsigned)stt->name->n_uninitialized)
         return false;
-    if (!jl_field_isptr(stt, idx) && !jl_type_hasptr(jl_field_type(stt, idx)))
+    enum jl_fieldkind_t kind = jl_field_kind(stt, idx);
+    if (kind == JL_FIELDKIND_ISUNION || kind == JL_FIELDKIND_ISBITS)
         return false;
     if (strct.constant) {
         if ((jl_is_immutable(stt) || jl_field_isconst(stt, idx)) && jl_field_isdefined(strct.constant, idx))
@@ -2839,7 +2857,8 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st
     if (type_is_ghost(julia_type_to_llvm(ctx, jfty)))
         return ghostValue(ctx, jfty);
     Value *needlock = nullptr;
-    if (isatomic && !jl_field_isptr(jt, idx) && jl_datatype_size(jfty) > MAX_ATOMIC_SIZE) {
+    enum jl_fieldkind_t kind = jl_field_kind(jt, idx);
+    if (isatomic && kind != JL_FIELDKIND_ISPTR && jl_datatype_size(jfty) > MAX_ATOMIC_SIZE) {
         assert(strct.isboxed);
         needlock = boxed(ctx, strct);
     }
@@ -2876,7 +2895,7 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st
                 setNameWithField(ctx.emission_context, addr, get_objname, jt, idx, Twine("_ptr"));
             }
         }
-        if (jl_field_isptr(jt, idx)) {
+        if (kind == JL_FIELDKIND_ISPTR) {
             setNameWithField(ctx.emission_context, addr, get_objname, jt, idx, Twine("_ptr"));
             LoadInst *Load = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, addr, Align(sizeof(void*)));
             setNameWithField(ctx.emission_context, Load, get_objname, jt, idx, Twine());
@@ -2888,7 +2907,8 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st
                 null_pointer_check(ctx, fldv, nullcheck);
             return mark_julia_type(ctx, fldv, true, jfty);
         }
-        else if (jl_is_uniontype(jfty)) {
+        else if (kind == JL_FIELDKIND_ISUNION) {
+            assert(jl_is_uniontype(jfty));
             size_t fsz = 0, al = 0;
             int union_max = jl_islayout_inline(jfty, &fsz, &al);
             bool isptr = (union_max == 0);
@@ -2908,11 +2928,13 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st
             }
             return val;
         }
-        assert(jl_is_concrete_type(jfty));
-        if (jl_field_isconst(jt, idx) && !(maybe_null && (jfty == (jl_value_t*)jl_bool_type ||
-                                            ((jl_datatype_t*)jfty)->layout->npointers))) {
-            // just compute the pointer and let user load it when necessary
-            return mark_julia_slot(addr, jfty, NULL, tbaa);
+        if (kind != JL_FIELDKIND_ISOTHER) {
+            assert(jl_is_concrete_type(jfty));
+            if (jl_field_isconst(jt, idx) && !(maybe_null && (jfty == (jl_value_t*)jl_bool_type ||
+                                                ((jl_datatype_t*)jfty)->layout->npointers))) {
+                // just compute the pointer and let user load it when necessary
+                return mark_julia_slot(addr, jfty, NULL, tbaa);
+            }
         }
         unsigned align = jl_field_align(jt, idx);
         if (needlock)

src/datatype.c

@@ -244,17 +244,17 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz,
         if (fielddesc_type == 0) {
             desc8[i].offset = desc[i].offset;
             desc8[i].size = desc[i].size;
-            desc8[i].isptr = desc[i].isptr;
+            desc8[i].kind = desc[i].kind;
         }
         else if (fielddesc_type == 1) {
             desc16[i].offset = desc[i].offset;
             desc16[i].size = desc[i].size;
-            desc16[i].isptr = desc[i].isptr;
+            desc16[i].kind = desc[i].kind;
         }
         else {
             desc32[i].offset = desc[i].offset;
             desc32[i].size = desc[i].size;
-            desc32[i].isptr = desc[i].isptr;
+            desc32[i].kind = desc[i].kind;
         }
     }
     if (first_ptr >= 0) {
@@ -686,23 +686,32 @@ void jl_compute_field_offsets(jl_datatype_t *st)
         for (i = 0; i < nfields; i++) {
             jl_value_t *fld = jl_field_type(st, i);
             int isatomic = jl_field_isatomic(st, i);
+            jl_value_t *restfld = fld;
+            if (jl_is_uniontype(fld) && jl_is_datatype_singleton(((jl_uniontype_t*)fld)->a)) {
+                restfld = ((jl_uniontype_t*)fld)->b;
+            }
             size_t fsz = 0, al = 1;
-            if (jl_islayout_inline(fld, &fsz, &al) && (!isatomic || jl_is_datatype(fld))) { // aka jl_datatype_isinlinealloc
+            if (jl_islayout_inline(restfld, &fsz, &al) && (!isatomic || jl_is_datatype(fld))) { // aka jl_datatype_isinlinealloc
                 if (__unlikely(fsz > max_size))
                     // Should never happen
                     throw_ovf(should_malloc, desc, st, fsz);
-                desc[i].isptr = 0;
-                if (jl_is_uniontype(fld)) {
+                if (!jl_is_datatype(restfld) || (restfld != fld && jl_islayout_inline(fld, &fsz, &al)))
+                    desc[i].kind = JL_FIELDKIND_ISUNION;
+                else if (((jl_datatype_t*)restfld)->layout->first_ptr >= 0)
+                    desc[i].kind = JL_FIELDKIND_ISOTHER;
+                else
+                    desc[i].kind = JL_FIELDKIND_ISBITS;
+                if (desc[i].kind == JL_FIELDKIND_ISUNION) {
                     fsz += 1; // selector byte
                     zeroinit = 1;
                     // TODO: Some unions could be bits comparable.
                     isbitsegal = 0;
                 }
                 else {
-                    uint32_t fld_npointers = ((jl_datatype_t*)fld)->layout->npointers;
-                    if (((jl_datatype_t*)fld)->layout->flags.haspadding)
+                    uint32_t fld_npointers = ((jl_datatype_t*)restfld)->layout->npointers;
+                    if (((jl_datatype_t*)restfld)->layout->flags.haspadding)
                         haspadding = 1;
-                    if (!((jl_datatype_t*)fld)->layout->flags.isbitsegal)
+                    if (!((jl_datatype_t*)restfld)->layout->flags.isbitsegal)
                         isbitsegal = 0;
                     if (i >= nfields - st->name->n_uninitialized && fld_npointers &&
                         fld_npointers * sizeof(void*) != fsz) {
@@ -719,7 +728,7 @@ void jl_compute_field_offsets(jl_datatype_t *st)
                         isbitsegal = 0;
                     }
                     if (!zeroinit)
-                        zeroinit = ((jl_datatype_t*)fld)->zeroinit;
+                        zeroinit = ((jl_datatype_t*)restfld)->zeroinit;
                     npointers += fld_npointers;
                 }
             }
@@ -728,7 +737,7 @@ void jl_compute_field_offsets(jl_datatype_t *st)
                 if (fsz > MAX_ALIGN)
                     fsz = MAX_ALIGN;
                 al = fsz;
-                desc[i].isptr = 1;
+                desc[i].kind = JL_FIELDKIND_ISPTR;
                 zeroinit = 1;
                 npointers++;
                 if (!jl_pointer_egal(fld)) {
@@ -783,9 +792,12 @@ void jl_compute_field_offsets(jl_datatype_t *st)
         for (i = 0; i < nfields; i++) {
             jl_value_t *fld = jl_field_type(st, i);
             uint32_t offset = desc[i].offset / sizeof(jl_value_t**);
-            if (desc[i].isptr)
+            if (desc[i].kind == JL_FIELDKIND_ISPTR)
                 pointers[ptr_i++] = offset;
-            else if (jl_is_datatype(fld)) {
+            else if (desc[i].kind == JL_FIELDKIND_ISOTHER) {
+                if (jl_is_uniontype(fld)) {
+                    fld = ((jl_uniontype_t*)fld)->b;
+                }
                 int j, npointers = ((jl_datatype_t*)fld)->layout->npointers;
                 for (j = 0; j < npointers; j++) {
                     pointers[ptr_i++] = offset + jl_ptr_offset((jl_datatype_t*)fld, j);
@@ -1755,18 +1767,28 @@ JL_DLLEXPORT jl_value_t *jl_get_nth_field(jl_value_t *v, size_t i)
     if (i >= jl_datatype_nfields(st))
         jl_bounds_error_int(v, i + 1);
     size_t offs = jl_field_offset(st, i);
-    if (jl_field_isptr(st, i)) {
+    enum jl_fieldkind_t kind = jl_field_kind(st, i);
+    if (kind == JL_FIELDKIND_ISPTR) {
         return jl_atomic_load_relaxed((_Atomic(jl_value_t*)*)((char*)v + offs));
     }
     jl_value_t *ty = jl_field_type_concrete(st, i);
     int isatomic = jl_field_isatomic(st, i);
-    if (jl_is_uniontype(ty)) {
+    jl_value_t *undefval = NULL;
+    if (kind != JL_FIELDKIND_ISBITS) {
         assert(!isatomic);
-        size_t fsz = jl_field_size(st, i);
-        uint8_t sel = ((uint8_t*)v)[offs + fsz - 1];
-        ty = jl_nth_union_component(ty, sel);
-        if (jl_is_datatype_singleton((jl_datatype_t*)ty))
-            return ((jl_datatype_t*)ty)->instance;
+        if (kind == JL_FIELDKIND_ISUNION) {
+            size_t fsz = jl_field_size(st, i);
+            uint8_t sel = ((uint8_t*)v)[offs + fsz - 1];
+            ty = jl_nth_union_component(ty, sel);
+            if (jl_is_datatype_singleton((jl_datatype_t*)ty))
+                return ((jl_datatype_t*)ty)->instance;
+        }
+        else { // kind == JL_FIELDKIND_ISOTHER
+            assert(jl_is_uniontype(ty));
+            jl_uniontype_t *uty = (jl_uniontype_t*)ty;
+            undefval = ((jl_datatype_t*)uty->a)->instance;
+            ty = uty->b;
+        }
     }
     jl_value_t *r;
     size_t fsz = jl_datatype_size(ty);
@@ -1785,7 +1807,9 @@ JL_DLLEXPORT jl_value_t *jl_get_nth_field(jl_value_t *v, size_t i)
         // TODO: a finalizer here could make the isunion case not quite right
         r = jl_new_bits(ty, (char*)v + offs);
     }
-    return undefref_check((jl_datatype_t*)ty, r);
+    if (kind != JL_FIELDKIND_ISOTHER)
+        return r;
+    return undefref_check((jl_datatype_t*)ty, r, undefval);
 }
 
 JL_DLLEXPORT jl_value_t *jl_get_nth_field_noalloc(jl_value_t *v JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT
@@ -1812,16 +1836,16 @@ inline void set_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t
         assert(jl_field_isptr(st, i) && *(jl_value_t**)((char*)v + offs) == NULL);
         return;
     }
-    if (jl_field_isptr(st, i)) {
+    enum jl_fieldkind_t kind = jl_field_kind(st, i);
+    if (kind == JL_FIELDKIND_ISPTR) {
         jl_atomic_store_release((_Atomic(jl_value_t*)*)((char*)v + offs), rhs);
         jl_gc_wb(v, rhs);
     }
     else {
         jl_value_t *ty = jl_field_type_concrete(st, i);
         jl_value_t *rty = jl_typeof(rhs);
-        int hasptr;
-        int isunion = jl_is_uniontype(ty);
-        if (isunion) {
+        int hasptr = (kind == JL_FIELDKIND_ISOTHER);
+        if (kind == JL_FIELDKIND_ISUNION) {
             assert(!isatomic);
             size_t fsz = jl_field_size(st, i);
             uint8_t *psel = &((uint8_t*)v)[offs + fsz - 1];
@@ -1831,10 +1855,19 @@ inline void set_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_t
             *psel = nth;
             if (jl_is_datatype_singleton((jl_datatype_t*)rty))
                 return;
-            hasptr = 0;
         }
         else {
-            hasptr = ((jl_datatype_t*)ty)->layout->first_ptr >= 0;
+            if (hasptr && jl_is_uniontype(ty)) {
+                jl_uniontype_t *uty = (jl_uniontype_t*)ty;
+                jl_value_t *undefval = ((jl_datatype_t*)uty->a)->instance;
+                ty = uty->b;
+                assert(((jl_datatype_t*)ty)->layout->first_ptr >= 0);
+                if (undefval == rhs) {
+                    jl_atomic_store_relaxed((_Atomic(jl_value_t*)*)((char*)v + offs) + ((jl_datatype_t*)ty)->layout->first_ptr, NULL);
+                    return;
+                }
+            }
+            assert(hasptr == (((jl_datatype_t*)ty)->layout->first_ptr >= 0));
         }
         size_t fsz = jl_datatype_size((jl_datatype_t*)rty); // need to shrink-wrap the final copy
         assert(!isatomic || jl_typeis(rhs, ty));
@@ -2235,9 +2268,9 @@ JL_DLLEXPORT int jl_field_isdefined(jl_value_t *v, size_t i) JL_NOTSAFEPOINT
     jl_datatype_t *st = (jl_datatype_t*)jl_typeof(v);
     size_t offs = jl_field_offset(st, i);
     _Atomic(jl_value_t*) *fld = (_Atomic(jl_value_t*)*)((char*)v + offs);
-    if (!jl_field_isptr(st, i)) {
-        jl_datatype_t *ft = (jl_datatype_t*)jl_field_type_concrete(st, i);
-        if (!jl_is_datatype(ft) || ft->layout->first_ptr < 0)
+    enum jl_fieldkind_t kind = jl_field_kind(st, i);
+    if (kind != JL_FIELDKIND_ISPTR) {
+        if (kind != JL_FIELDKIND_ISOTHER || jl_is_uniontype(jl_field_type_concrete(st, i)))
             return 2; // isbits are always defined
         fld += ft->layout->first_ptr;
     }