Initial blocklinking

Author: Grarak

src/jit/assembler/block_asm.rs

@@ -96,7 +96,7 @@ impl<'a> BlockAsm<'a> {
 
         instance.buf.insts.push(BlockInst::Prologue);
 
-        // First argument is write_to_host_sp: bool
+        // First argument is store_host_sp: bool
         instance.cmp(BlockReg::Fixed(Reg::R0), 0);
         instance.start_cond_block(Cond::NE);
         let host_sp_addr_reg = thread_regs_addr_reg;
@@ -160,14 +160,17 @@ impl<'a> BlockAsm<'a> {
         BlockLabel(id)
     }
 
+    alu3!(and, And, None, false);
     alu3!(sub, Sub, None, false);
     alu3!(add, Add, None, false);
     alu3!(bic, Bic, None, false);
     alu3!(orr, Orr, None, false);
 
     alu2_op1!(cmp, Cmp, Host, false);
+    alu2_op1!(tst, Tst, Host, false);
 
     alu2_op0!(mov, Mov, None, false);
+    alu2_op0!(mvn, Mvn, None, false);
 
     alu3!(ands_guest_thumb_pc_aligned, And, HostGuest, true);
     alu3!(eors_guest_thumb_pc_aligned, Eor, HostGuest, true);
@@ -513,7 +516,7 @@ impl<'a> BlockAsm<'a> {
 
             if let BlockInst::Label { label, guest_pc } = self.buf.insts[i] {
                 if let Some((p_label, p_guest_pc)) = previous_label {
-                    let replace_guest_pc = p_guest_pc.or_else(|| guest_pc);
+                    let replace_guest_pc = p_guest_pc.or(guest_pc);
                     previous_label = Some((p_label, replace_guest_pc));
                     let previous_node = BlockInstList::deref(current_node).previous;
                     let previous_i = BlockInstList::deref(previous_node).value;
@@ -549,11 +552,19 @@ impl<'a> BlockAsm<'a> {
         }
     }
 
-    fn resolve_df_ordering(already_processed: &mut NoHashSet<usize>, basic_blocks: &mut [BasicBlock], block_i: usize, ordering: &mut [usize], ordering_start: &mut usize, ordering_end: &mut usize) {
+    fn resolve_df_ordering(
+        already_processed: &mut NoHashSet<usize>,
+        completed: &mut NoHashSet<usize>,
+        basic_blocks: &mut [BasicBlock],
+        block_i: usize,
+        ordering: &mut [usize],
+        ordering_start: &mut usize,
+        ordering_end: &mut usize,
+    ) {
         already_processed.insert(block_i);
         let mut cycle = false;
         for exit_i in &basic_blocks[block_i].exit_blocks {
-            if already_processed.contains(exit_i) {
+            if already_processed.contains(exit_i) && !completed.contains(exit_i) {
                 cycle = true;
                 break;
             }
@@ -567,9 +578,10 @@ impl<'a> BlockAsm<'a> {
         }
         for exit_i in basic_blocks[block_i].exit_blocks.clone() {
             if !already_processed.contains(&exit_i) {
-                Self::resolve_df_ordering(already_processed, basic_blocks, exit_i, ordering, ordering_start, ordering_end);
+                Self::resolve_df_ordering(already_processed, completed, basic_blocks, exit_i, ordering, ordering_start, ordering_end);
             }
         }
+        completed.insert(block_i);
     }
 
     fn assemble_basic_blocks(&mut self, block_start_pc: u32, thumb: bool) -> (Vec<BasicBlock>, Vec<usize>) {
@@ -671,11 +683,9 @@ impl<'a> BlockAsm<'a> {
             let mut current_node = basic_block.block_entry_start;
             while !current_node.is_null() {
                 match &self.buf.insts[BlockInstList::deref(current_node).value] {
-                    BlockInst::Label { guest_pc, .. } => {
-                        if let Some(pc) = guest_pc {
-                            basic_block_start_pc = *pc;
-                            break;
-                        }
+                    BlockInst::Label { guest_pc: Some(pc), .. } => {
+                        basic_block_start_pc = *pc;
+                        break;
                     }
                     BlockInst::GuestPc(pc) => {
                         basic_block_start_pc = *pc;
@@ -700,10 +710,19 @@ impl<'a> BlockAsm<'a> {
         }
 
         let mut df_already_processed = NoHashSet::default();
+        let mut df_completed = NoHashSet::default();
         let mut df_ordering = vec![0; basic_blocks_len];
         let mut df_ordering_start = 0;
         let mut df_ordering_end = basic_blocks_len - 1;
-        Self::resolve_df_ordering(&mut df_already_processed, &mut basic_blocks, 0, &mut df_ordering, &mut df_ordering_start, &mut df_ordering_end);
+        Self::resolve_df_ordering(
+            &mut df_already_processed,
+            &mut df_completed,
+            &mut basic_blocks,
+            0,
+            &mut df_ordering,
+            &mut df_ordering_start,
+            &mut df_ordering_end,
+        );
 
         (basic_blocks, df_ordering)
     }
@@ -724,6 +743,7 @@ impl<'a> BlockAsm<'a> {
                         break;
                     }
                 }
+                assert!(i < end_j);
                 intervals.insert(reg, (i, end_j));
             }
             processed_regs += outputs;
@@ -753,6 +773,11 @@ impl<'a> BlockAsm<'a> {
 
         let mut reg_intervals = Self::assemble_intervals(&basic_blocks, &basic_blocks_order);
 
+        if unsafe { BLOCK_LOG } {
+            println!("reg intervals {reg_intervals:?} ");
+            println!("block ordering {basic_blocks_order:?}");
+        }
+
         self.buf.reg_allocator.global_mapping.clear();
         let mut free_regs = block_reg_allocator::ALLOCATION_REGS;
         while !free_regs.is_empty() {
@@ -785,10 +810,10 @@ impl<'a> BlockAsm<'a> {
         let mut branch_placeholders = Vec::new();
         let mut opcodes_offset = Vec::with_capacity(basic_blocks.len());
         for (i, basic_block) in basic_blocks.iter().enumerate() {
+            opcodes_offset.push(opcodes.len());
             if i != 0 && basic_block.enter_blocks.is_empty() {
                 continue;
             }
-            opcodes_offset.push(opcodes.len());
             opcodes.extend(basic_block.emit_opcodes(&mut self, &mut branch_placeholders, opcodes.len()));
         }
 

src/jit/assembler/block_inst.rs

@@ -600,8 +600,9 @@ impl BlockInst {
                 opcodes.push(inst_info.assemble());
             }
 
-            BlockInst::Prologue => opcodes.push(LdmStm::generic(Reg::SP, block_reg_allocator::ALLOCATION_REGS + Reg::LR, false, true, false, true, Cond::AL)),
-            BlockInst::Epilogue => opcodes.push(LdmStm::generic(Reg::SP, block_reg_allocator::ALLOCATION_REGS + Reg::PC, true, true, true, false, Cond::AL)),
+            // r4-r12,{lr|pc} since we need an even amount of registers for 8 byte alignment, in case the compiler decides to use neon instructions
+            BlockInst::Prologue => opcodes.push(LdmStm::generic(Reg::SP, block_reg_allocator::ALLOCATION_REGS + Reg::R12 + Reg::LR, false, true, false, true, Cond::AL)),
+            BlockInst::Epilogue => opcodes.push(LdmStm::generic(Reg::SP, block_reg_allocator::ALLOCATION_REGS + Reg::R12 + Reg::PC, true, true, true, false, Cond::AL)),
 
             BlockInst::Label { .. } | BlockInst::GuestPc(_) => {}
         }

src/jit/emitter/emit.rs

@@ -1,6 +1,7 @@
 use crate::core::CpuType;
 use crate::core::CpuType::ARM7;
 use crate::jit::assembler::block_asm::BlockAsm;
+use crate::jit::assembler::{BlockLabel, BlockReg};
 use crate::jit::inst_threag_regs_handler::{register_restore_spsr, restore_thumb_after_restore_spsr, set_pc_arm_mode};
 use crate::jit::jit_asm::{JitAsm, JitRuntimeData};
 use crate::jit::op::Op;
@@ -18,7 +19,8 @@ impl<'a, const CPU: CpuType> JitAsm<'a, CPU> {
         block_asm.start_cond_block(cond);
         match op {
             Op::B | Op::Bl => self.emit_branch_label(block_asm),
-            Op::Bx | Op::BlxReg => self.emit_branch_reg(block_asm),
+            Op::Bx => self.emit_bx(block_asm),
+            Op::BlxReg => self.emit_blx(block_asm),
             Op::Blx => self.emit_blx_label(block_asm),
             Op::Mcr | Op::Mrc => self.emit_cp15(block_asm),
             Op::MsrRc | Op::MsrIc | Op::MsrRs | Op::MsrIs => self.emit_msr(block_asm),
@@ -99,7 +101,7 @@ impl<'a, const CPU: CpuType> JitAsm<'a, CPU> {
         self._emit_branch_out_metadata(block_asm, true)
     }
 
-    pub fn emit_flush_cycles<ContinueFn: Fn(&mut Self, &mut BlockAsm), BreakoutFn: Fn(&mut Self, &mut BlockAsm)>(
+    pub fn emit_flush_cycles<ContinueFn: Fn(&mut Self, &mut BlockAsm, BlockLabel, BlockReg), BreakoutFn: Fn(&mut Self, &mut BlockAsm)>(
         &mut self,
         block_asm: &mut BlockAsm,
         target_pre_cycle_count_sum: u16,
@@ -155,7 +157,7 @@ impl<'a, const CPU: CpuType> JitAsm<'a, CPU> {
             false,
             MemoryAmount::Half,
         );
-        continue_fn(self, block_asm);
+        continue_fn(self, block_asm, breakout_label, runtime_data_addr_reg);
 
         block_asm.label(breakout_label);
         breakout_fn(self, block_asm);

src/jit/emitter/emit_branch.rs

@@ -1,11 +1,13 @@
+use crate::core::emu::get_jit;
 use crate::core::CpuType;
 use crate::core::CpuType::ARM9;
 use crate::jit::assembler::block_asm::BlockAsm;
+use crate::jit::assembler::{BlockOperand, BlockReg};
 use crate::jit::inst_info::InstInfo;
-use crate::jit::jit_asm::JitAsm;
+use crate::jit::jit_asm::{JitAsm, JitRuntimeData, BLOCK_LINK_STACK_SIZE};
 use crate::jit::op::Op;
 use crate::jit::reg::{reg_reserve, Reg, RegReserve};
-use crate::jit::Cond;
+use crate::jit::{jit_memory_map, Cond, MemoryAmount, ShiftType};
 
 pub enum JitBranchInfo {
     Idle,
@@ -80,15 +82,15 @@ impl<'a, const CPU: CpuType> JitAsm<'a, CPU> {
         let branch_info = Self::analyze_branch_label::<THUMB>(&self.jit_buf.insts, self.jit_buf.current_index, cond, self.jit_buf.current_pc, target_pc);
 
         if let JitBranchInfo::Local(target_index) = branch_info {
-            let target_pre_cycle_count_sum = if target_index == 0 { 0 } else { self.jit_buf.insts_cycle_counts[target_index] };
+            let target_pre_cycle_count_sum = self.jit_buf.insts_cycle_counts[target_index] - self.jit_buf.insts[target_index].cycle as u16;
 
             let backed_up_cpsr_reg = block_asm.new_reg();
             block_asm.mrs_cpsr(backed_up_cpsr_reg);
 
             self.emit_flush_cycles(
                 block_asm,
                 target_pre_cycle_count_sum,
-                |_, block_asm| {
+                |_, block_asm, _, _| {
                     block_asm.msr_cpsr(backed_up_cpsr_reg);
                     block_asm.guest_branch(Cond::AL, target_pc);
                 },
@@ -113,19 +115,105 @@ impl<'a, const CPU: CpuType> JitAsm<'a, CPU> {
         block_asm.epilogue();
     }
 
-    pub fn emit_branch_reg(&mut self, block_asm: &mut BlockAsm) {
+    pub fn emit_bx(&mut self, block_asm: &mut BlockAsm) {
         let inst_info = self.jit_buf.current_inst();
         let branch_to = *inst_info.operands()[0].as_reg_no_shift().unwrap();
 
-        if inst_info.op == Op::BlxReg {
-            block_asm.mov(Reg::LR, self.jit_buf.current_pc + 4);
-        }
         block_asm.mov(Reg::PC, branch_to);
         block_asm.save_context();
         self.emit_branch_out_metadata(block_asm);
         block_asm.epilogue();
     }
 
+    pub fn emit_blx(&mut self, block_asm: &mut BlockAsm) {
+        let inst_info = self.jit_buf.current_inst();
+        let target_pc_reg = *inst_info.operands()[0].as_reg_no_shift().unwrap();
+
+        block_asm.mov(Reg::LR, self.jit_buf.current_pc + 4);
+        self.emit_branch_reg_common(block_asm, target_pc_reg.into());
+    }
+
+    pub fn emit_branch_reg_common(&mut self, block_asm: &mut BlockAsm, target_pc_reg: BlockReg) {
+        block_asm.mov(Reg::PC, target_pc_reg);
+        block_asm.save_context();
+
+        self.emit_flush_cycles(
+            block_asm,
+            0,
+            |asm, block_asm, breakout_label, runtime_data_addr_reg| {
+                let block_link_ptr_reg = block_asm.new_reg();
+
+                block_asm.transfer_read(block_link_ptr_reg, runtime_data_addr_reg, JitRuntimeData::get_block_link_ptr_offset() as u32, false, MemoryAmount::Byte);
+
+                block_asm.cmp(block_link_ptr_reg, BLOCK_LINK_STACK_SIZE as u32);
+                block_asm.branch(breakout_label, Cond::EQ);
+
+                let block_link_stack_ptr_reg = block_asm.new_reg();
+                block_asm.add(block_link_stack_ptr_reg, runtime_data_addr_reg, JitRuntimeData::get_block_link_stack_offset() as u32);
+                block_asm.add(block_link_stack_ptr_reg, block_link_stack_ptr_reg, (block_link_ptr_reg.into(), ShiftType::Lsl, BlockOperand::from(3)));
+                block_asm.transfer_write(Reg::LR, block_link_stack_ptr_reg, 0, false, MemoryAmount::Word);
+
+                let return_pre_cycle_count_sum_reg = block_asm.new_reg();
+                block_asm.mov(return_pre_cycle_count_sum_reg, asm.jit_buf.insts_cycle_counts[asm.jit_buf.current_index] as u32);
+                block_asm.transfer_write(return_pre_cycle_count_sum_reg, block_link_stack_ptr_reg, 4, false, MemoryAmount::Half);
+
+                block_asm.add(block_link_ptr_reg, block_link_ptr_reg, 1);
+                block_asm.transfer_write(block_link_ptr_reg, runtime_data_addr_reg, JitRuntimeData::get_block_link_ptr_offset() as u32, false, MemoryAmount::Byte);
+
+                block_asm.free_reg(return_pre_cycle_count_sum_reg);
+                block_asm.free_reg(block_link_stack_ptr_reg);
+                block_asm.free_reg(block_link_ptr_reg);
+
+                let target_addr_reg = block_asm.new_reg();
+                let pc_mask_reg = block_asm.new_reg();
+
+                // Align pc to !1 or !3
+                block_asm.mvn(pc_mask_reg, 1);
+                block_asm.tst(target_pc_reg, 1);
+                block_asm.start_cond_block(Cond::EQ);
+                block_asm.mvn(pc_mask_reg, 3);
+                block_asm.end_cond_block();
+
+                block_asm.and(target_addr_reg, target_pc_reg, pc_mask_reg);
+
+                let map_ptr = get_jit!(asm.emu).jit_memory_map.get_map_ptr::<CPU>();
+
+                let map_ptr_reg = block_asm.new_reg();
+                let map_index_reg = block_asm.new_reg();
+                let map_entry_base_ptr_reg = block_asm.new_reg();
+
+                block_asm.mov(map_ptr_reg, map_ptr as u32);
+                block_asm.mov(map_index_reg, (target_addr_reg.into(), ShiftType::Lsr, BlockOperand::from(jit_memory_map::BLOCK_SHIFT as u32 + 1)));
+                block_asm.transfer_read(
+                    map_entry_base_ptr_reg,
+                    map_ptr_reg,
+                    (map_index_reg.into(), ShiftType::Lsl, BlockOperand::from(2)),
+                    false,
+                    MemoryAmount::Word,
+                );
+                let block_size_mask_reg = map_index_reg;
+                block_asm.mov(block_size_mask_reg, (jit_memory_map::BLOCK_SIZE as u32 - 1) << 2);
+                block_asm.and(target_addr_reg, block_size_mask_reg, (target_addr_reg.into(), ShiftType::Lsl, BlockOperand::from(1)));
+
+                let entry_fn_reg = block_asm.new_reg();
+                block_asm.transfer_read(entry_fn_reg, map_entry_base_ptr_reg, target_addr_reg, false, MemoryAmount::Word);
+
+                block_asm.call1(entry_fn_reg, 0);
+
+                block_asm.free_reg(entry_fn_reg);
+                block_asm.free_reg(map_entry_base_ptr_reg);
+                block_asm.free_reg(map_index_reg);
+                block_asm.free_reg(map_ptr_reg);
+                block_asm.free_reg(pc_mask_reg);
+                block_asm.free_reg(target_addr_reg);
+            },
+            |asm, block_asm| {
+                asm.emit_branch_out_metadata(block_asm);
+                block_asm.epilogue();
+            },
+        );
+    }
+
     pub fn emit_blx_label(&mut self, block_asm: &mut BlockAsm) {
         if CPU != ARM9 {
             return;
@@ -134,10 +222,12 @@ impl<'a, const CPU: CpuType> JitAsm<'a, CPU> {
         let relative_pc = *self.jit_buf.current_inst().operands()[0].as_imm().unwrap() as i32 + 8;
         let target_pc = (self.jit_buf.current_pc as i32 + relative_pc) as u32;
 
+        let target_pc_reg = block_asm.new_reg();
+        block_asm.mov(target_pc_reg, target_pc | 1);
+
         block_asm.mov(Reg::LR, self.jit_buf.current_pc + 4);
-        block_asm.mov(Reg::PC, target_pc | 1);
-        block_asm.save_context();
-        self.emit_branch_out_metadata(block_asm);
-        block_asm.epilogue();
+        self.emit_branch_reg_common(block_asm, target_pc_reg);
+
+        block_asm.free_reg(target_pc_reg);
     }
 }

src/jit/emitter/thumb/emit_branch_thumb.rs

@@ -63,10 +63,7 @@ impl<'a, const CPU: CpuType> JitAsm<'a, CPU> {
         }
         block_asm.mov(Reg::PC, target_pc_reg);
 
-        block_asm.save_context();
-        self.emit_branch_out_metadata(block_asm);
-        block_asm.epilogue();
-
+        self.emit_branch_reg_common(block_asm, target_pc_reg);
         block_asm.free_reg(target_pc_reg);
     }
 

src/jit/emitter/thumb/emit_thumb.rs

@@ -53,7 +53,7 @@ impl<'a, const CPU: CpuType> JitAsm<'a, CPU> {
             Op::BxRegT | Op::BlxRegT => self.emit_bx_thumb(block_asm),
 
             Op::SwiT => self.emit_swi::<true>(block_asm),
-            Op::UnkThumb => todo!("{op:?}"),
+            Op::UnkThumb => {}
             op if op.is_single_mem_transfer() => {
                 if op.mem_is_write() {
                     self.emit_str_thumb(block_asm)

src/jit/inst_mem_handler.rs

@@ -161,9 +161,10 @@ macro_rules! imm_breakout {
         $asm.runtime_data.branch_out_total_cycles = $total_cycles;
         crate::core::emu::get_regs_mut!($asm.emu, CPU).pc = $pc + if $thumb { 3 } else { 4 };
         crate::core::emu::get_mem_mut!($asm.emu).breakout_imm = false;
+        // r4-r12,pc since we need an even amount of registers for 8 byte alignment, in case the compiler decides to use neon instructions
         std::arch::asm!(
             "mov sp, {}",
-            "pop {{r4-r11,pc}}",
+            "pop {{r4-r12,pc}}",
             in(reg) $asm.runtime_data.host_sp
         );
         std::hint::unreachable_unchecked();
@@ -215,9 +216,10 @@ pub unsafe extern "C" fn inst_mem_handler_multiple<
                 get_regs_mut!((*asm).emu, CPU).pc &= !1;
             }
         }
+        // r4-r12,pc since we need an even amount of registers for 8 byte alignment, in case the compiler decides to use neon instructions
         asm!(
             "mov sp, {}",
-            "pop {{r4-r11,pc}}",
+            "pop {{r4-r12,pc}}",
             in(reg) (*asm).runtime_data.host_sp
         );
         unreachable_unchecked();