WIP

Author: Grarak

src/core/hle/power_manager_hle.rs

@@ -23,6 +23,9 @@ impl PowerManagerHle {
 
         let cmd = (self.data[0] >> 8) - 0x60;
         match cmd {
+            1 => {
+                Arm7Hle::send_ipc_fifo(0x8, 0x0300E300, 0, emu);
+            }
             3 => {
                 Arm7Hle::send_ipc_fifo(0x8, 0x0300E300, 0, emu);
             }

src/jit/assembler/basic_block.rs

@@ -1,8 +1,9 @@
+use crate::jit::assembler::arm::alu_assembler::AluShiftImm;
 use crate::jit::assembler::block_asm::{BlockAsm, BLOCK_LOG};
 use crate::jit::assembler::block_inst::{BlockAluOp, BlockAluSetCond, BlockSystemRegOp, BlockTransferOp};
 use crate::jit::assembler::block_inst_list::{BlockInstList, BlockInstListEntry};
 use crate::jit::assembler::block_reg_set::BlockRegSet;
-use crate::jit::assembler::{BlockAsmBuf, BlockInstKind};
+use crate::jit::assembler::{BlockAsmBuf, BlockInstKind, BlockLabel};
 use crate::jit::reg::{Reg, RegReserve};
 use crate::jit::{MemoryAmount, ShiftType};
 use crate::IS_DEBUG;
@@ -14,6 +15,9 @@ pub struct BasicBlock {
     pub block_entry_start: *mut BlockInstListEntry,
     pub block_entry_end: *mut BlockInstListEntry,
 
+    pub pad_label: Option<BlockLabel>,
+    pub pad_size: usize,
+
     pub guest_regs_resolved: bool,
     pub guest_regs_input_dirty: RegReserve,
     pub guest_regs_output_dirty: RegReserve,
@@ -27,6 +31,8 @@ pub struct BasicBlock {
     pub insts_link: BlockInstList,
 
     pub start_pc: u32,
+
+    pub opcodes: Vec<u32>,
 }
 
 impl BasicBlock {
@@ -37,6 +43,9 @@ impl BasicBlock {
             block_entry_start,
             block_entry_end,
 
+            pad_label: None,
+            pad_size: 0,
+
             guest_regs_resolved: false,
             guest_regs_input_dirty: RegReserve::new(),
             guest_regs_output_dirty: RegReserve::new(),
@@ -50,6 +59,8 @@ impl BasicBlock {
             insts_link: BlockInstList::new(),
 
             start_pc: 0,
+
+            opcodes: Vec::new(),
         }
     }
 
@@ -113,6 +124,7 @@ impl BasicBlock {
                         add_inst = false;
                     }
                     BlockInstKind::SaveReg { .. } | BlockInstKind::MarkRegDirty { .. } => {}
+                    BlockInstKind::PadBlock(label) => self.pad_label = Some(*label),
                     _ => {
                         let (inputs, _) = asm.buf.insts[i].get_io();
                         for guest_reg in inputs.get_guests() {
@@ -300,36 +312,48 @@ impl BasicBlock {
         }
     }
 
-    pub fn emit_opcodes(&self, asm: &mut BlockAsm, opcodes_offset: usize, used_host_regs: RegReserve) -> Vec<u32> {
-        let mut opcodes = Vec::new();
+    pub fn emit_opcodes(&mut self, asm: &mut BlockAsm, opcodes_offset: usize, block_index: usize, used_host_regs: RegReserve) {
+        if IS_DEBUG && unsafe { BLOCK_LOG } && opcodes_offset != 0 {
+            self.opcodes.clear();
+        }
+
+        if !self.opcodes.is_empty() {
+            return;
+        }
+
+        asm.buf.branch_placeholders[block_index].clear();
+
         let mut inst_opcodes = Vec::new();
         for entry in self.insts_link.iter() {
             let inst = &mut asm.buf.insts[entry.value];
             if inst.skip {
                 continue;
             }
 
-            if IS_DEBUG && unsafe { BLOCK_LOG } {
+            if IS_DEBUG && unsafe { BLOCK_LOG } && opcodes_offset != 0 {
                 match &inst.kind {
                     BlockInstKind::GuestPc(pc) => {
-                        println!("(0x{:x}, 0x{pc:x}),", opcodes.len() + opcodes_offset);
+                        println!("(0x{:x}, 0x{pc:x}),", self.opcodes.len() + opcodes_offset);
                     }
                     BlockInstKind::Label { guest_pc: Some(pc), .. } => {
-                        println!("(0x{:x}, 0x{pc:x}),", opcodes.len() + opcodes_offset);
+                        println!("(0x{:x}, 0x{pc:x}),", self.opcodes.len() + opcodes_offset);
                     }
                     _ => {}
                 }
             }
 
             inst_opcodes.clear();
             inst.kind
-                .emit_opcode(&mut inst_opcodes, opcodes.len(), &mut asm.buf.branch_placeholders, opcodes_offset, used_host_regs);
+                .emit_opcode(&mut inst_opcodes, self.opcodes.len(), &mut asm.buf.branch_placeholders[block_index], used_host_regs);
             for opcode in &mut inst_opcodes {
                 *opcode = (*opcode & !(0xF << 28)) | ((inst.cond as u32) << 28);
             }
-            opcodes.extend(&inst_opcodes);
+            self.opcodes.extend(&inst_opcodes);
+        }
+
+        for _ in self.opcodes.len()..self.pad_size {
+            self.opcodes.push(AluShiftImm::mov_al(Reg::R0, Reg::R0));
         }
-        opcodes
     }
 }
 

src/jit/assembler/block_asm.rs

@@ -722,6 +722,10 @@ impl<'a> BlockAsm<'a> {
         }
     }
 
+    pub fn pad_block(&mut self, label: BlockLabel) {
+        self.insert_inst(BlockInstKind::PadBlock(label));
+    }
+
     // Convert guest pc with labels into labels
     fn resolve_labels(&mut self, label_aliases: &mut NoHashMap<u16, u16>) {
         let mut previous_label: Option<(BlockLabel, Option<u32>)> = None;
@@ -965,6 +969,8 @@ impl<'a> BlockAsm<'a> {
             basic_block.remove_dead_code(self);
         }
 
+        self.buf.basic_block_label_mapping = basic_block_label_mapping;
+
         (basic_blocks, reachable_blocks)
     }
 
@@ -1041,45 +1047,57 @@ impl<'a> BlockAsm<'a> {
         }
 
         self.buf.opcodes.clear();
-        self.buf.block_opcode_offsets.clear();
-        self.buf.branch_placeholders.clear();
+        self.buf.block_opcode_offsets.resize(basic_blocks.len(), 0);
+        self.buf.branch_placeholders.resize(basic_blocks.len(), Vec::new());
 
-        for (i, basic_block) in basic_blocks.iter().enumerate() {
+        for i in 0..basic_blocks.len() {
+            if let Some(label) = basic_blocks[i].pad_label {
+                let block_to_pad_to = *self.buf.basic_block_label_mapping.get(&label.0).unwrap();
+                self.buf.branch_placeholders[block_to_pad_to].clear();
+                basic_blocks[block_to_pad_to].emit_opcodes(self, 0, block_to_pad_to, used_host_regs);
+                basic_blocks[i].pad_size = basic_blocks[block_to_pad_to].opcodes.len();
+            }
+        }
+
+        for (i, basic_block) in basic_blocks.iter_mut().enumerate() {
             let opcodes_len = self.buf.opcodes.len();
-            self.buf.block_opcode_offsets.push(opcodes_len);
+            self.buf.block_opcode_offsets[i] = opcodes_len;
 
             if !reachable_blocks.contains(&i) {
                 continue;
             }
 
-            let opcodes = basic_block.emit_opcodes(self, opcodes_len, used_host_regs);
-            self.buf.opcodes.extend(opcodes);
+            basic_block.emit_opcodes(self, opcodes_len, i, used_host_regs);
+            self.buf.opcodes.extend(&basic_block.opcodes);
         }
 
         self.buf.opcodes.len()
     }
 
     pub fn finalize(&mut self, jit_mem_offset: usize) -> &Vec<u32> {
-        for &branch_placeholder in &self.buf.branch_placeholders {
-            let encoding = BranchEncoding::from(self.buf.opcodes[branch_placeholder]);
-            if Cond::from(u8::from(encoding.cond())) == Cond::NV {
-                self.buf.opcodes[branch_placeholder] = AluShiftImm::mov_al(Reg::R0, Reg::R0);
-                continue;
-            }
+        for (block_index, branch_placeholders) in self.buf.branch_placeholders.iter().enumerate() {
+            for branch_placeholder in branch_placeholders {
+                let index = self.buf.block_opcode_offsets[block_index] + *branch_placeholder;
+                let encoding = BranchEncoding::from(self.buf.opcodes[index]);
+                if Cond::from(u8::from(encoding.cond())) == Cond::NV {
+                    self.buf.opcodes[index] = AluShiftImm::mov_al(Reg::R0, Reg::R0);
+                    continue;
+                }
 
-            let diff = if encoding.is_call_common() {
-                let opcode_index = (jit_mem_offset >> 2) + branch_placeholder;
-                let branch_to = u32::from(encoding.index()) >> 2;
-                branch_to as i32 - opcode_index as i32
-            } else {
-                let block_index = u32::from(encoding.index());
-                let branch_to = self.buf.block_opcode_offsets[block_index as usize];
-                branch_to as i32 - branch_placeholder as i32
-            };
-            if diff == 1 && !encoding.has_return() {
-                self.buf.opcodes[branch_placeholder] = AluShiftImm::mov_al(Reg::R0, Reg::R0);
-            } else {
-                self.buf.opcodes[branch_placeholder] = if encoding.has_return() { B::bl } else { B::b }(diff - 2, Cond::from(u8::from(encoding.cond())));
+                let diff = if encoding.is_call_common() {
+                    let opcode_index = (jit_mem_offset >> 2) + index;
+                    let branch_to = u32::from(encoding.index()) >> 2;
+                    branch_to as i32 - opcode_index as i32
+                } else {
+                    let block_index = u32::from(encoding.index());
+                    let branch_to = self.buf.block_opcode_offsets[block_index as usize];
+                    branch_to as i32 - index as i32
+                };
+                if diff == 1 && !encoding.has_return() {
+                    self.buf.opcodes[index] = AluShiftImm::mov_al(Reg::R0, Reg::R0);
+                } else {
+                    self.buf.opcodes[index] = if encoding.has_return() { B::bl } else { B::b }(diff - 2, Cond::from(u8::from(encoding.cond())));
+                }
             }
         }
 

src/jit/assembler/block_inst.rs

@@ -607,7 +607,7 @@ impl BlockInstKind {
         opcodes.push(LdrStrImm::strb_offset_al(host_reg, thread_regs_addr_reg, Reg::CPSR as u16 * 4 + 3));
     }
 
-    pub fn emit_opcode(&mut self, opcodes: &mut Vec<u32>, opcode_index: usize, branch_placeholders: &mut Vec<usize>, opcodes_offset: usize, used_host_regs: RegReserve) {
+    pub fn emit_opcode(&mut self, opcodes: &mut Vec<u32>, opcode_index: usize, branch_placeholders: &mut Vec<usize>, used_host_regs: RegReserve) {
         let alu_reg = |op: BlockAluOp, op0: BlockReg, op1: BlockReg, op2: BlockReg, shift: BlockShift, set_cond: bool| match shift.value {
             BlockOperand::Reg(shift_reg) => AluReg::generic(op as u8, op0.as_fixed(), op1.as_fixed(), op2.as_fixed(), shift.shift_type, shift_reg.as_fixed(), set_cond, Cond::AL),
             BlockOperand::Imm(shift_imm) => {
@@ -781,7 +781,7 @@ impl BlockInstKind {
                 // Encode label
                 // Branch offset can only be figured out later
                 opcodes.push(BranchEncoding::new(u26::new(*block_index as u32), false, false, u4::new(Cond::AL as u8)).into());
-                branch_placeholders.push(opcodes_offset + opcode_index);
+                branch_placeholders.push(opcode_index);
             }
 
             BlockInstKind::SaveContext { .. } => unsafe { unreachable_unchecked() },
@@ -816,7 +816,7 @@ impl BlockInstKind {
                 // Encode common offset
                 // Branch offset can only be figured out later
                 opcodes.push(BranchEncoding::new(u26::new(*mem_offset as u32), *has_return, true, u4::new(Cond::AL as u8)).into());
-                branch_placeholders.push(opcodes_offset + opcode_index);
+                branch_placeholders.push(opcode_index);
             }
             BlockInstKind::Bkpt(id) => opcodes.push(Bkpt::bkpt(*id)),
             BlockInstKind::Nop => opcodes.push(AluShiftImm::mov_al(Reg::R0, Reg::R0)),

src/jit/assembler/mod.rs

@@ -347,17 +347,19 @@ pub struct BlockLabel(u16);
 
 pub struct BlockAsmBuf {
     pub insts: Vec<BlockInst>,
+    pub basic_block_label_mapping: NoHashMap<u16, usize>,
     pub guest_branches_mapping: NoHashMap<u32, BlockLabel>,
     pub reg_allocator: BlockRegAllocator,
     pub block_opcode_offsets: Vec<usize>,
     pub opcodes: Vec<u32>,
-    pub branch_placeholders: Vec<usize>,
+    pub branch_placeholders: Vec<Vec<usize>>,
 }
 
 impl BlockAsmBuf {
     pub fn new() -> Self {
         BlockAsmBuf {
             insts: Vec::new(),
+            basic_block_label_mapping: NoHashMap::default(),
             guest_branches_mapping: NoHashMap::default(),
             reg_allocator: BlockRegAllocator::new(),
             block_opcode_offsets: Vec::new(),

src/jit/emitter/emit_transfer.rs

@@ -4,6 +4,7 @@ use crate::core::CpuType;
 use crate::jit::assembler::block_asm::BlockAsm;
 use crate::jit::assembler::{BlockOperand, BlockReg};
 use crate::jit::inst_info::Operand;
+use crate::jit::inst_jit_handler::inst_slow_mem_patch;
 use crate::jit::inst_mem_handler::{inst_mem_handler, inst_mem_handler_multiple, inst_mem_handler_swp};
 use crate::jit::jit_asm::JitAsm;
 use crate::jit::op::Op;
@@ -218,10 +219,12 @@ impl<const CPU: CpuType> JitAsm<'_, CPU> {
         let fast_read_next_addr_reg = block_asm.new_reg();
         let fast_read_addr_masked_reg = block_asm.new_reg();
 
+        let slow_read_patch_label = block_asm.new_label();
         let slow_read_label = block_asm.new_label();
         let continue_label = block_asm.new_label();
 
         block_asm.branch(slow_read_label, Cond::NV);
+        block_asm.pad_block(slow_read_label);
 
         let mmu = get_mmu!(self.emu, CPU);
         let base_ptr = mmu.get_base_tcm_ptr();
@@ -249,7 +252,12 @@ impl<const CPU: CpuType> JitAsm<'_, CPU> {
         }
 
         block_asm.branch_fallthrough(continue_label, Cond::AL);
+        block_asm.branch(slow_read_patch_label, Cond::AL);
+
+        block_asm.label_unlikely(slow_read_patch_label);
+        block_asm.call(inst_slow_mem_patch as *const ());
         block_asm.branch(slow_read_label, Cond::AL);
+
         block_asm.label_unlikely(slow_read_label);
 
         block_asm.restore_reg(op0);
@@ -322,7 +330,7 @@ impl<const CPU: CpuType> JitAsm<'_, CPU> {
 
         let cpsr_backup_reg = block_asm.new_reg();
 
-        let use_fast_mem = is_valid && !inst_info.op.mem_transfer_user() && rlist.len() < (RegReserve::gp() + Reg::LR).len() - 2;
+        let use_fast_mem = false && is_valid && !inst_info.op.mem_transfer_user() && rlist.len() < (RegReserve::gp() + Reg::LR).len() - 2;
         if use_fast_mem {
             let mut gp_regs = rlist.get_gp_regs();
             let mut free_gp_regs = if gp_regs.is_empty() {

src/jit/inst_jit_handler.rs

@@ -0,0 +1,70 @@
+use crate::jit::assembler::arm::alu_assembler::AluShiftImm;
+use crate::jit::disassembler::lookup_table::lookup_opcode;
+use crate::jit::op::Op;
+use crate::jit::reg::Reg;
+use core::slice;
+use std::arch::asm;
+use crate::mmap::flush_icache;
+
+pub unsafe extern "C" fn inst_slow_mem_patch() {
+    let mut lr: u32;
+    asm!("mov {}, lr", out(reg) lr);
+
+    let nop_opcode = AluShiftImm::mov_al(Reg::R0, Reg::R0);
+
+    let mut slow_mem_start = 0;
+    for pc_offset in (4..256).step_by(4) {
+        let ptr = (lr + pc_offset) as *const u32;
+        let opcode = ptr.read();
+        if opcode == nop_opcode {
+            slow_mem_start = ptr as usize + 4;
+            break;
+        }
+    }
+    debug_assert_ne!(slow_mem_start, 0);
+
+    let mut slow_mem_end = slow_mem_start;
+    let mut fast_mem_end = 0;
+    for pc_offset in (4..256).step_by(4) {
+        let ptr = (slow_mem_start + pc_offset) as *const u32;
+        let opcode = ptr.read();
+        let (op, func) = lookup_opcode(opcode);
+        if *op == Op::B {
+            let inst = func(opcode, *op);
+            slow_mem_end = ptr as usize;
+            let relative_pc = *inst.operands()[0].as_imm().unwrap() as i32 + 8;
+            let target_pc = slow_mem_end as i32 + relative_pc;
+            fast_mem_end = target_pc as usize - 4;
+            slow_mem_end -= 4;
+            break;
+        }
+    }
+    debug_assert_ne!(slow_mem_end, slow_mem_start);
+    debug_assert_ne!(fast_mem_end, 0);
+
+    let mut fast_mem_start = 0;
+    let mut found_non_op = false;
+    for pc_offset in (4..256).step_by(4) {
+        let ptr = (fast_mem_end - pc_offset) as *const u32;
+        let opcode = ptr.read();
+        if opcode == nop_opcode {
+            if found_non_op {
+                fast_mem_start = ptr as usize;
+                break;
+            } else {
+                found_non_op = true;
+            }
+        }
+    }
+    debug_assert_ne!(fast_mem_start, 0);
+
+    let slow_mem_size = ((slow_mem_end - slow_mem_start) >> 2) + 1;
+    let fast_mem_size = ((fast_mem_end - fast_mem_start) >> 2) + 1;
+
+    let fast_mem = slice::from_raw_parts_mut(fast_mem_start as *mut u32, fast_mem_size);
+    let slow_mem = slice::from_raw_parts(slow_mem_start as *const u32, slow_mem_size);
+    fast_mem[..slow_mem_size].copy_from_slice(slow_mem);
+    fast_mem[slow_mem_size..].fill(nop_opcode);
+
+    flush_icache(fast_mem_start as _, fast_mem_size << 2);
+}