Merge pull request #1852 from cfallin/reftypes

Reference type support in MachInst backend and on AArch64

Author: cfallin

Cargo.lock

@@ -26,7 +26,7 @@ smallvec = { version = "1.0.0" }
 thiserror = "1.0.4"
 byteorder = { version = "1.3.2", default-features = false }
 peepmatic-runtime = { path = "../peepmatic/crates/runtime", optional = true, version = "0.2.0" }
-regalloc = "0.0.26"
+regalloc = { version = "0.0.27" }
 # It is a goal of the cranelift-codegen crate to have minimal external dependencies.
 # Please don't add any unless they are essential to the task of creating binary
 # machine code. Integration tests that need external dependencies can be

cranelift/codegen/Cargo.toml

@@ -61,3 +61,10 @@ pub fn is_constant_64bit(func: &Function, inst: Inst) -> Option<u64> {
         _ => None,
     }
 }
+
+/// Is the given instruction a safepoint (i.e., potentially causes a GC, depending on the
+/// embedding, and so requires reftyped values to be enumerated with a stackmap)?
+pub fn is_safepoint(func: &Function, inst: Inst) -> bool {
+    let op = func.dfg[inst].opcode();
+    op.is_resumable_trap() || op.is_call()
+}

cranelift/codegen/src/inst_predicates.rs

@@ -90,12 +90,13 @@
 //!   - Return v1 in memory at `[P+8]`.
 //!   - Return v0 in memory at `[P+16]`.
 
+use crate::binemit::Stackmap;
 use crate::ir;
 use crate::ir::types;
 use crate::ir::types::*;
 use crate::ir::{ArgumentExtension, StackSlot};
 use crate::isa;
-use crate::isa::aarch64::{inst::*, lower::ty_bits};
+use crate::isa::aarch64::{inst::EmitState, inst::*, lower::ty_bits};
 use crate::machinst::*;
 use crate::settings;
 use crate::{CodegenError, CodegenResult};
@@ -372,7 +373,10 @@ pub struct AArch64ABIBody {
     clobbered: Set<Writable<RealReg>>,
     /// Total number of spillslots, from regalloc.
     spillslots: Option<usize>,
-    /// Total frame size.
+    /// "Total frame size", as defined by "distance between FP and nominal-SP".
+    /// Some items are pushed below nominal SP, so the function may actually use
+    /// more stack than this would otherwise imply. It is simply the initial
+    /// frame/allocation size needed for stackslots and spillslots.
     total_frame_size: Option<u32>,
     /// The register holding the return-area pointer, if needed.
     ret_area_ptr: Option<Writable<Reg>>,
@@ -400,6 +404,8 @@ fn in_int_reg(ty: ir::Type) -> bool {
     match ty {
         types::I8 | types::I16 | types::I32 | types::I64 => true,
         types::B1 | types::B8 | types::B16 | types::B32 | types::B64 => true,
+        types::R64 => true,
+        types::R32 => panic!("Unexpected 32-bit reference on a 64-bit platform!"),
         _ => false,
     }
 }
@@ -653,12 +659,12 @@ fn load_stack(mem: MemArg, into_reg: Writable<Reg>, ty: Type) -> Inst {
             mem,
             srcloc: None,
         },
-        types::B32 | types::I32 => Inst::ULoad32 {
+        types::B32 | types::I32 | types::R32 => Inst::ULoad32 {
             rd: into_reg,
             mem,
             srcloc: None,
         },
-        types::B64 | types::I64 => Inst::ULoad64 {
+        types::B64 | types::I64 | types::R64 => Inst::ULoad64 {
             rd: into_reg,
             mem,
             srcloc: None,
@@ -689,12 +695,12 @@ fn store_stack(mem: MemArg, from_reg: Reg, ty: Type) -> Inst {
             mem,
             srcloc: None,
         },
-        types::B32 | types::I32 => Inst::Store32 {
+        types::B32 | types::I32 | types::R32 => Inst::Store32 {
             rd: from_reg,
             mem,
             srcloc: None,
         },
-        types::B64 | types::I64 => Inst::Store64 {
+        types::B64 | types::I64 | types::R64 => Inst::Store64 {
             rd: from_reg,
             mem,
             srcloc: None,
@@ -810,6 +816,35 @@ fn get_caller_saves(call_conv: isa::CallConv) -> Vec<Writable<Reg>> {
     caller_saved
 }
 
+fn gen_sp_adjust_insts<F: FnMut(Inst)>(adj: u64, is_sub: bool, mut f: F) {
+    let alu_op = if is_sub { ALUOp::Sub64 } else { ALUOp::Add64 };
+
+    if let Some(imm12) = Imm12::maybe_from_u64(adj) {
+        let adj_inst = Inst::AluRRImm12 {
+            alu_op,
+            rd: writable_stack_reg(),
+            rn: stack_reg(),
+            imm12,
+        };
+        f(adj_inst);
+    } else {
+        let tmp = writable_spilltmp_reg();
+        let const_inst = Inst::LoadConst64 {
+            rd: tmp,
+            const_data: adj,
+        };
+        let adj_inst = Inst::AluRRRExtend {
+            alu_op,
+            rd: writable_stack_reg(),
+            rn: stack_reg(),
+            rm: tmp.to_reg(),
+            extendop: ExtendOp::UXTX,
+        };
+        f(const_inst);
+        f(adj_inst);
+    }
+}
+
 impl ABIBody for AArch64ABIBody {
     type I = Inst;
 
@@ -1024,6 +1059,29 @@ impl ABIBody for AArch64ABIBody {
         store_stack(MemArg::NominalSPOffset(sp_off, ty), from_reg, ty)
     }
 
+    fn spillslots_to_stackmap(&self, slots: &[SpillSlot], state: &EmitState) -> Stackmap {
+        assert!(state.virtual_sp_offset >= 0);
+        trace!(
+            "spillslots_to_stackmap: slots = {:?}, state = {:?}",
+            slots,
+            state
+        );
+        let map_size = (state.virtual_sp_offset + state.nominal_sp_to_fp) as u32;
+        let map_words = (map_size + 7) / 8;
+        let mut bits = std::iter::repeat(false)
+            .take(map_words as usize)
+            .collect::<Vec<bool>>();
+
+        let first_spillslot_word =
+            ((self.stackslots_size + state.virtual_sp_offset as u32) / 8) as usize;
+        for &slot in slots {
+            let slot = slot.get() as usize;
+            bits[first_spillslot_word + slot] = true;
+        }
+
+        Stackmap::from_slice(&bits[..])
+    }
+
     fn gen_prologue(&mut self) -> Vec<Inst> {
         let mut insts = vec![];
         if !self.call_conv.extends_baldrdash() {
@@ -1059,6 +1117,9 @@ impl ABIBody for AArch64ABIBody {
         }
         let total_stacksize = (total_stacksize + 15) & !15; // 16-align the stack.
 
+        let mut total_sp_adjust = 0;
+        let mut nominal_sp_to_real_sp = 0;
+
         if !self.call_conv.extends_baldrdash() {
             // Leaf functions with zero stack don't need a stack check if one's
             // specified, otherwise always insert the stack check.
@@ -1069,42 +1130,29 @@ impl ABIBody for AArch64ABIBody {
                 }
             }
             if total_stacksize > 0 {
-                // sub sp, sp, #total_stacksize
-                if let Some(imm12) = Imm12::maybe_from_u64(total_stacksize as u64) {
-                    let sub_inst = Inst::AluRRImm12 {
-                        alu_op: ALUOp::Sub64,
-                        rd: writable_stack_reg(),
-                        rn: stack_reg(),
-                        imm12,
-                    };
-                    insts.push(sub_inst);
-                } else {
-                    let tmp = writable_spilltmp_reg();
-                    let const_inst = Inst::LoadConst64 {
-                        rd: tmp,
-                        const_data: total_stacksize as u64,
-                    };
-                    let sub_inst = Inst::AluRRRExtend {
-                        alu_op: ALUOp::Sub64,
-                        rd: writable_stack_reg(),
-                        rn: stack_reg(),
-                        rm: tmp.to_reg(),
-                        extendop: ExtendOp::UXTX,
-                    };
-                    insts.push(const_inst);
-                    insts.push(sub_inst);
-                }
+                total_sp_adjust += total_stacksize as u64;
             }
         }
 
-        // N.B.: "nominal SP", which we use to refer to stackslots
-        // and spillslots, is *here* (the value of SP at this program point).
+        // N.B.: "nominal SP", which we use to refer to stackslots and
+        // spillslots, is defined to be equal to the stack pointer at this point
+        // in the prologue.
+        //
         // If we push any clobbers below, we emit a virtual-SP adjustment
         // meta-instruction so that the nominal-SP references behave as if SP
         // were still at this point. See documentation for
         // [crate::isa::aarch64::abi](this module) for more details on
         // stackframe layout and nominal-SP maintenance.
 
+        if total_sp_adjust > 0 {
+            // sub sp, sp, #total_stacksize
+            gen_sp_adjust_insts(
+                total_sp_adjust,
+                /* is_sub = */ true,
+                |inst| insts.push(inst),
+            );
+        }
+
         // Save clobbered registers.
         let (clobbered_int, clobbered_vec) =
             get_callee_saves(self.call_conv, self.clobbered.to_vec());
@@ -1148,10 +1196,11 @@ impl ABIBody for AArch64ABIBody {
                 srcloc: None,
             });
         }
+        nominal_sp_to_real_sp += clobber_size as i64;
 
         if clobber_size > 0 {
             insts.push(Inst::VirtualSPOffsetAdj {
-                offset: clobber_size as i64,
+                offset: nominal_sp_to_real_sp,
             });
         }
 
@@ -1245,6 +1294,10 @@ impl ABIBody for AArch64ABIBody {
             .expect("frame size not computed before prologue generation")
     }
 
+    fn stack_args_size(&self) -> u32 {
+        self.sig.stack_arg_space as u32
+    }
+
     fn get_spillslot_size(&self, rc: RegClass, ty: Type) -> u32 {
         // We allocate in terms of 8-byte slots.
         match (rc, ty) {
@@ -1255,15 +1308,42 @@ impl ABIBody for AArch64ABIBody {
         }
     }
 
-    fn gen_spill(&self, to_slot: SpillSlot, from_reg: RealReg, ty: Type) -> Inst {
+    fn gen_spill(&self, to_slot: SpillSlot, from_reg: RealReg, ty: Option<Type>) -> Inst {
+        let ty = ty_from_ty_hint_or_reg_class(from_reg.to_reg(), ty);
         self.store_spillslot(to_slot, ty, from_reg.to_reg())
     }
 
-    fn gen_reload(&self, to_reg: Writable<RealReg>, from_slot: SpillSlot, ty: Type) -> Inst {
+    fn gen_reload(
+        &self,
+        to_reg: Writable<RealReg>,
+        from_slot: SpillSlot,
+        ty: Option<Type>,
+    ) -> Inst {
+        let ty = ty_from_ty_hint_or_reg_class(to_reg.to_reg().to_reg(), ty);
         self.load_spillslot(from_slot, ty, to_reg.map(|r| r.to_reg()))
     }
 }
 
+/// Return a type either from an optional type hint, or if not, from the default
+/// type associated with the given register's class. This is used to generate
+/// loads/spills appropriately given the type of value loaded/stored (which may
+/// be narrower than the spillslot). We usually have the type because the
+/// regalloc usually provides the vreg being spilled/reloaded, and we know every
+/// vreg's type. However, the regalloc *can* request a spill/reload without an
+/// associated vreg when needed to satisfy a safepoint (which requires all
+/// ref-typed values, even those in real registers in the original vcode, to be
+/// in spillslots).
+fn ty_from_ty_hint_or_reg_class(r: Reg, ty: Option<Type>) -> Type {
+    match (ty, r.get_class()) {
+        // If the type is provided
+        (Some(t), _) => t,
+        // If no type is provided, this should be a register spill for a
+        // safepoint, so we only expect I64 (integer) registers.
+        (None, RegClass::I64) => I64,
+        _ => panic!("Unexpected register class!"),
+    }
+}
+
 enum CallDest {
     ExtName(ir::ExternalName, RelocDistance),
     Reg(Reg),
@@ -1342,7 +1422,7 @@ impl AArch64ABICall {
     }
 }
 
-fn adjust_stack<C: LowerCtx<I = Inst>>(ctx: &mut C, amount: u64, is_sub: bool) {
+fn adjust_stack_and_nominal_sp<C: LowerCtx<I = Inst>>(ctx: &mut C, amount: u64, is_sub: bool) {
     if amount == 0 {
         return;
     }
@@ -1356,27 +1436,9 @@ fn adjust_stack<C: LowerCtx<I = Inst>>(ctx: &mut C, amount: u64, is_sub: bool) {
         offset: sp_adjustment,
     });
 
-    let alu_op = if is_sub { ALUOp::Sub64 } else { ALUOp::Add64 };
-    if let Some(imm12) = Imm12::maybe_from_u64(amount) {
-        ctx.emit(Inst::AluRRImm12 {
-            alu_op,
-            rd: writable_stack_reg(),
-            rn: stack_reg(),
-            imm12,
-        })
-    } else {
-        ctx.emit(Inst::LoadConst64 {
-            rd: writable_spilltmp_reg(),
-            const_data: amount,
-        });
-        ctx.emit(Inst::AluRRRExtend {
-            alu_op,
-            rd: writable_stack_reg(),
-            rn: stack_reg(),
-            rm: spilltmp_reg(),
-            extendop: ExtendOp::UXTX,
-        });
-    }
+    gen_sp_adjust_insts(amount, is_sub, |inst| {
+        ctx.emit(inst);
+    });
 }
 
 impl ABICall for AArch64ABICall {
@@ -1392,12 +1454,12 @@ impl ABICall for AArch64ABICall {
 
     fn emit_stack_pre_adjust<C: LowerCtx<I = Self::I>>(&self, ctx: &mut C) {
         let off = self.sig.stack_arg_space + self.sig.stack_ret_space;
-        adjust_stack(ctx, off as u64, /* is_sub = */ true)
+        adjust_stack_and_nominal_sp(ctx, off as u64, /* is_sub = */ true)
     }
 
     fn emit_stack_post_adjust<C: LowerCtx<I = Self::I>>(&self, ctx: &mut C) {
         let off = self.sig.stack_arg_space + self.sig.stack_ret_space;
-        adjust_stack(ctx, off as u64, /* is_sub = */ false)
+        adjust_stack_and_nominal_sp(ctx, off as u64, /* is_sub = */ false)
     }
 
     fn emit_copy_reg_to_arg<C: LowerCtx<I = Self::I>>(
@@ -1452,7 +1514,7 @@ impl ABICall for AArch64ABICall {
             self.emit_copy_reg_to_arg(ctx, i, rd.to_reg());
         }
         match &self.dest {
-            &CallDest::ExtName(ref name, RelocDistance::Near) => ctx.emit(Inst::Call {
+            &CallDest::ExtName(ref name, RelocDistance::Near) => ctx.emit_safepoint(Inst::Call {
                 info: Box::new(CallInfo {
                     dest: name.clone(),
                     uses,
@@ -1468,7 +1530,7 @@ impl ABICall for AArch64ABICall {
                     offset: 0,
                     srcloc: self.loc,
                 });
-                ctx.emit(Inst::CallInd {
+                ctx.emit_safepoint(Inst::CallInd {
                     info: Box::new(CallIndInfo {
                         rn: spilltmp_reg(),
                         uses,
@@ -1478,7 +1540,7 @@ impl ABICall for AArch64ABICall {
                     }),
                 });
             }
-            &CallDest::Reg(reg) => ctx.emit(Inst::CallInd {
+            &CallDest::Reg(reg) => ctx.emit_safepoint(Inst::CallInd {
                 info: Box::new(CallIndInfo {
                     rn: reg,
                     uses,