WIP: implement sret_union ABI for pointer-ful types

This is a combination the existing:
 - `sret`  ABI (which can stack-allocate a _single_ pointerful type)
 - `union` ABI (which can stack-allocate many _pointer-free_ types)

This provides some nice speed-ups for temporary "wrappers":
```julia
const v = Any[]
@noinline maybe_wrapped(i) = (i % 32 != 0) ? Some(v) : nothing
function foo()
    count = 0
    for i = 1:1_000_000
        count += (maybe_wrapped(i) !== nothing) ? 1 : 0
    end
    return count
end
```

On this PR this gives:
```julia
julia> @Btime foo()
  1.675 ms (0 allocations: 0 bytes)
968750
```

compared to current master:
```julia
julia> @Btime foo()
  6.877 ms (968750 allocations: 14.78 MiB)
968750
```

The most outstanding TODO here is what to do about PHI nodes. Right now,
if the incoming `Union{...}` type has a pointer-containing type then the
object is forced to be boxed, even if the object at run-time is actually
pointer-free.

But that's just a band-aid - it means we introduce new boxes where we
didn't have them before, which is a regression that almost certainly to
be fixed before landing this.

Author: topolarity

src/cgutils.cpp

@@ -948,7 +948,7 @@ static bool for_each_uniontype_small(
         allunbox &= for_each_uniontype_small(f, ((jl_uniontype_t*)ty)->b, counter);
         return allunbox;
     }
-    else if (jl_is_pointerfree(ty)) {
+    else if (!deserves_unionbox(ty)) {
         f(++counter, (jl_datatype_t*)ty);
         return true;
     }
@@ -1105,6 +1105,22 @@ static void emit_memcpy(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const &dst
     emit_memcpy_llvm(ctx, dst, dst_ai, data_pointer(ctx, src), src_ai, sz, align_dst, align_src, is_volatile);
 }
 
+static void emit_sret_roots(jl_codectx_t &ctx, bool isptr, Value *Src, Type *T, Value *Shadow, Type *ShadowT, unsigned count)
+{
+    assert(T != NULL);
+    unsigned emitted = TrackWithShadow(Src, T, isptr, Shadow, ShadowT, ctx.builder); //This comes from Late-GC-Lowering??
+    if (emitted < count) {
+        Value *ToZero = ctx.builder.CreateConstInBoundsGEP1_32(
+            ctx.types().T_prjlvalue, Shadow, emitted);
+        ctx.builder.CreateMemSet(ToZero,
+            /* Val */ ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0),
+            /* Size */ ConstantInt::get(ctx.types().T_size, (count - emitted) * sizeof(jl_value_t *)),
+            /* Align */ Align(sizeof(jl_value_t *)));
+        emitted += count - emitted;
+    }
+    assert(emitted <= count); (void)emitted; (void)count;
+}
+
 static Value *emit_tagfrom(jl_codectx_t &ctx, jl_datatype_t *dt)
 {
     if (dt->smalltag)
@@ -3317,44 +3333,55 @@ static Value *compute_tindex_unboxed(jl_codectx_t &ctx, const jl_cgval_t &val, j
 
 
 static void union_alloca_type(jl_uniontype_t *ut,
-        bool &allunbox, size_t &nbytes, size_t &align, size_t &min_align)
+        bool &allunbox, size_t &nbytes, size_t &align, size_t &min_align, size_t &return_roots)
 {
     nbytes = 0;
     align = 0;
     min_align = MAX_ALIGN;
+    return_roots = 0;
     // compute the size of the union alloca that could hold this type
     unsigned counter = 0;
     allunbox = for_each_uniontype_small(
             [&](unsigned idx, jl_datatype_t *jt) {
                 if (!jl_is_datatype_singleton(jt)) {
                     size_t nb1 = jl_datatype_size(jt);
                     size_t align1 = jl_datatype_align(jt);
+                    size_t n_roots = jl_datatype_layout(jt)->npointers;
                     if (nb1 > nbytes)
                         nbytes = nb1;
                     if (align1 > align)
                         align = align1;
                     if (align1 < min_align)
                         min_align = align1;
+                    if (n_roots > return_roots)
+                        return_roots = n_roots;
                 }
             },
             (jl_value_t*)ut,
             counter);
 }
 
-static AllocaInst *try_emit_union_alloca(jl_codectx_t &ctx, jl_uniontype_t *ut, bool &allunbox, size_t &min_align, size_t &nbytes)
+static AllocaInst *try_emit_union_alloca(jl_codectx_t &ctx, jl_uniontype_t *ut, bool &allunbox, size_t &min_align, size_t &nbytes, /* TODO: remove */bool require_pointerfree, const char *name)
 {
-    size_t align;
-    union_alloca_type(ut, allunbox, nbytes, align, min_align);
-    if (nbytes > 0) {
+    size_t align, return_roots;
+    union_alloca_type(ut, allunbox, nbytes, align, min_align, return_roots);
+    if (nbytes > 0 && !(require_pointerfree && return_roots != 0)) {
         // at least some of the values can live on the stack
         // try to pick an Integer type size such that SROA will emit reasonable code
         Type *AT = ArrayType::get(IntegerType::get(ctx.builder.getContext(), 8 * min_align), (nbytes + min_align - 1) / min_align);
         AllocaInst *lv = emit_static_alloca(ctx, AT);
-        setName(ctx.emission_context, lv, "unionalloca");
+        setName(ctx.emission_context, lv, name);
         if (align > 1)
             lv->setAlignment(Align(align));
+        if (return_roots > 0) {
+            StoreInst *SI = new StoreInst(Constant::getNullValue(AT), lv, false, Align(sizeof(void*)));
+            SI->insertAfter(ctx.topalloca);
+        }
         return lv;
     }
+    allunbox = false;
+    nbytes = 0;
+    min_align = 0;
     return NULL;
 }
 
@@ -3566,18 +3593,50 @@ static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &vinfo, bool is_promotab
 }
 
 // copy src to dest, if src is justbits. if skip is true, the value of dest is undefined
-static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, const jl_cgval_t &src, Value *skip, bool isVolatile=false)
+static void emit_unionmove(jl_codectx_t &ctx, Value *dest, jl_cgval_shadow_t *shadow,
+        MDNode *tbaa_dst, const jl_cgval_t &src, Value *skip, bool isVolatile=false)
 {
     if (AllocaInst *ai = dyn_cast<AllocaInst>(dest))
         // TODO: make this a lifetime_end & dereferenceable annotation?
         ctx.builder.CreateAlignedStore(UndefValue::get(ai->getAllocatedType()), ai, ai->getAlign());
+
     if (jl_is_concrete_type(src.typ) || src.constant) {
         jl_value_t *typ = src.constant ? jl_typeof(src.constant) : src.typ;
-        assert(skip || jl_is_pointerfree(typ));
-        if (jl_is_pointerfree(typ)) {
+        // FIXME: This assumes that the `skip` is used to skip any boxed union types, which are any !pointerfree types
+        //
+        // This should really be skipped by the caller instead (and/or emit a trap)
+        assert(skip || !deserves_unionbox(typ)); //jl_is_pointerfree(typ));
+        if (!deserves_unionbox(typ)) {
             unsigned alignment = julia_alignment(typ);
             if (!src.ispointer() || src.constant) {
-                emit_unbox_store(ctx, src, dest, tbaa_dst, alignment, isVolatile);
+                if (!src.isghost) {
+                    // inlined version of emit_unbox_store
+                    if (!src.ispointer()) {
+                        Value *unboxed = zext_struct(ctx, src.V);
+
+                        if (shadow != nullptr && !jl_is_pointerfree(typ)) {
+                            assert(jl_is_concrete_type(typ));
+                            Type *store_ty = julia_type_to_llvm(ctx, typ);
+                            emit_sret_roots(ctx, /* isptr */ false, unboxed, store_ty, shadow->value, shadow->typ, shadow->n_roots);
+                        }
+                        Type *dest_ty = unboxed->getType()->getPointerTo();
+                        if (dest->getType() != dest_ty)
+                            dest = emit_bitcast(ctx, dest, dest_ty);
+                        StoreInst *store = ctx.builder.CreateAlignedStore(unboxed, dest, Align(alignment));
+                        store->setVolatile(isVolatile);
+                        jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst);
+                        ai.decorateInst(store);
+                    } else {
+                        Value *src_ptr = data_pointer(ctx, src);
+                        if (shadow != nullptr && !jl_is_pointerfree(typ)) {
+                            Type *store_ty = julia_type_to_llvm(ctx, typ);
+                            emit_sret_roots(ctx, /* isptr */ true, src_ptr, store_ty, shadow->value, shadow->typ, shadow->n_roots);
+                        }
+                        emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), src_ptr,
+                                jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), jl_datatype_size(src.typ), alignment, julia_alignment(src.typ), isVolatile);
+                    }
+                }
+//                emit_unbox_store(ctx, src, dest, tbaa_dst, alignment, isVolatile);
             }
             else {
                 Value *src_ptr = data_pointer(ctx, src);
@@ -3586,6 +3645,11 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con
                 //   select copy dest -> dest to simulate an undef value / conditional copy
                 // if (skip) src_ptr = ctx.builder.CreateSelect(skip, dest, src_ptr);
                 auto f = [&] {
+                    if (shadow != nullptr && !jl_is_pointerfree(typ)) {
+                        assert(jl_is_concrete_type(typ));
+                        Type *store_ty = julia_type_to_llvm(ctx, typ);
+                        emit_sret_roots(ctx, /* isptr */ true, src_ptr, store_ty, shadow->value, shadow->typ, shadow->n_roots);
+                    }
                     (void)emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), src_ptr,
                                       jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), nb, alignment, alignment, isVolatile);
                     return nullptr;
@@ -3621,6 +3685,11 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con
                             ctx.builder.CreateUnreachable();
                             return;
                         } else {
+                            // Maybe assert here (TBD)
+                            if (shadow != nullptr && !jl_is_pointerfree((jl_value_t *)jt)) {
+                                Type *store_ty = julia_type_to_llvm(ctx, (jl_value_t *)jt);
+                                emit_sret_roots(ctx, /* isptr */ true, src_ptr, store_ty, shadow->value, shadow->typ, shadow->n_roots);
+                            }
                             emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), src_ptr,
                                         jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), nb, alignment, alignment, isVolatile);
                         }
@@ -3645,10 +3714,18 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con
     else {
         assert(src.isboxed && "expected boxed value for sizeof/alignment computation");
         auto f = [&] {
-            Value *datatype = emit_typeof(ctx, src, false, false);
-            Value *copy_bytes = emit_datatype_size(ctx, datatype);
-            (void)emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), data_pointer(ctx, src),
-                              jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), copy_bytes, 1, 1, isVolatile);
+            if (shadow == nullptr) {
+                Value *datatype = emit_typeof(ctx, src, false, false);
+                Value *copy_bytes = emit_datatype_size(ctx, datatype);
+                (void)emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), data_pointer(ctx, src),
+                                  jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), copy_bytes, 1, 1, isVolatile);
+            } else {
+                Function *trap_func = Intrinsic::getDeclaration(
+                        ctx.f->getParent(),
+                        Intrinsic::trap);
+                ctx.builder.CreateCall(trap_func);
+                ctx.builder.CreateUnreachable();
+            }
             return nullptr;
         };
         if (skip)
@@ -3807,7 +3884,7 @@ static jl_cgval_t union_store(jl_codectx_t &ctx,
     ai.decorateInst(ctx.builder.CreateAlignedStore(tindex, ptindex, Align(1)));
     // copy data
     if (!rhs.isghost) {
-        emit_unionmove(ctx, ptr, tbaa, rhs, nullptr);
+        emit_unionmove(ctx, ptr, /*shadow*/nullptr, tbaa, rhs, /*skip*/nullptr);
     }
     if (isreplacefield || ismodifyfield) {
         ctx.builder.CreateBr(DoneBB);
@@ -4001,7 +4078,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg
                             AllocaInst *lv = emit_static_alloca(ctx, ET);
                             setName(ctx.emission_context, lv, "unioninit");
                             lv->setOperand(0, ConstantInt::get(getInt32Ty(ctx.builder.getContext()), (fsz1 + al - 1) / al));
-                            emit_unionmove(ctx, lv, ctx.tbaa().tbaa_stack, fval_info, nullptr);
+                            emit_unionmove(ctx, lv, /* shadow */ nullptr, ctx.tbaa().tbaa_stack, fval_info, /* skip */ nullptr);
                             // emit all of the align-sized words
                             unsigned i = 0;
                             for (; i < fsz1 / al; i++) {
@@ -4031,7 +4108,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg
                         jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_unionselbyte);
                         ai.decorateInst(ctx.builder.CreateAlignedStore(tindex, ptindex, Align(1)));
                         if (!rhs_union.isghost)
-                            emit_unionmove(ctx, dest, ctx.tbaa().tbaa_stack, fval_info, nullptr);
+                            emit_unionmove(ctx, dest, /*shadow*/nullptr, ctx.tbaa().tbaa_stack, fval_info, /*skip*/nullptr);
                     }
                 }
                 else {