Fix wrong exit blocks

Grarak · Grarak · commit 211b90920fd4 · 2024-10-13T10:17:57.000+02:00
diff --git a/src/bitset.rs b/src/bitset.rs
@@ -42,6 +42,10 @@ impl<const SIZE: usize> Bitset<SIZE> {
     pub const fn is_empty(&self) -> bool {
         self.len() == 0
     }
+
+    pub const fn clear(&mut self) {
+        self.0 = [0; SIZE];
+    }
 }
 
 impl<const SIZE: usize> Default for Bitset<SIZE> {
diff --git a/src/jit/assembler/block_asm.rs b/src/jit/assembler/block_asm.rs
@@ -664,6 +664,17 @@ impl<'a> BlockAsm<'a> {
         }
     }
 
+    fn resolve_actual_block_count(already_processed: &mut NoHashSet<usize>, basic_blocks: &mut [BasicBlock], block_i: usize, count: &mut usize) {
+        *count += 1;
+        already_processed.insert(block_i);
+        for i in 0..basic_blocks[block_i].exit_blocks.len() {
+            let exit_i = basic_blocks[block_i].exit_blocks[i];
+            if !already_processed.contains(&exit_i) {
+                Self::resolve_actual_block_count(already_processed, basic_blocks, exit_i, count);
+            }
+        }
+    }
+
     fn resolve_df_ordering(
         already_processed: &mut Bitset<{ 1024 / 32 }>,
         completed: &mut Bitset<{ 1024 / 32 }>,
@@ -697,7 +708,7 @@ impl<'a> BlockAsm<'a> {
         *completed += block_i;
     }
 
-    fn assemble_basic_blocks(&mut self, block_start_pc: u32, thumb: bool) -> (Vec<BasicBlock>, Vec<usize>) {
+    fn assemble_basic_blocks(&mut self, block_start_pc: u32, thumb: bool) -> (Vec<BasicBlock>, Vec<usize>, NoHashSet<usize>) {
         for i in 0..self.buf.insts.len() {
             self.insts_link.insert_end(i);
         }
@@ -737,7 +748,7 @@ impl<'a> BlockAsm<'a> {
                     }
                     last_label = None;
                 }
-                BlockInstKind::Call { has_return: false, .. } | BlockInstKind::Epilogue { .. } => {
+                BlockInstKind::Call { has_return: false, .. } | BlockInstKind::CallCommon { has_return: false, .. } | BlockInstKind::Epilogue { .. } => {
                     basic_blocks.push(BasicBlock::new(self, basic_block_start, current_node));
                     basic_block_start = BlockInstList::deref(current_node).next;
                     if let Some(last_label) = last_label {
@@ -773,16 +784,13 @@ impl<'a> BlockAsm<'a> {
                     }
                 }
                 // Don't add exit when last command in basic block is a breakout
-                BlockInstKind::Call { has_return: false, .. } | BlockInstKind::Epilogue { .. } => {
-                    if i + 1 < basic_blocks_len {
-                        basic_block.exit_blocks.push(i + 1);
-                    }
-                }
-                _ => {
-                    if i + 1 < basic_blocks_len {
+                BlockInstKind::Call { has_return: false, .. } | BlockInstKind::CallCommon { has_return: false, .. } | BlockInstKind::Epilogue { .. } => {
+                    if cond != Cond::AL && i + 1 < basic_blocks_len {
                         basic_block.exit_blocks.push(i + 1);
                     }
                 }
+                _ if i + 1 < basic_blocks_len => basic_block.exit_blocks.push(i + 1),
+                _ => {}
             }
         }
 
@@ -828,11 +836,15 @@ impl<'a> BlockAsm<'a> {
             basic_block.remove_dead_code(self);
         }
 
+        let mut reachable_blocks = NoHashSet::default();
+        let mut actual_block_count = 0;
+        Self::resolve_actual_block_count(&mut reachable_blocks, &mut basic_blocks, 0, &mut actual_block_count);
+
         let mut df_already_processed = Bitset::<{ 1024 / 32 }>::new();
         let mut df_completed = Bitset::<{ 1024 / 32 }>::new();
-        let mut df_ordering = vec![0; basic_blocks_len];
+        let mut df_ordering = vec![0; actual_block_count];
         let mut df_ordering_start = 0;
-        let mut df_ordering_end = basic_blocks_len - 1;
+        let mut df_ordering_end = actual_block_count - 1;
         Self::resolve_df_ordering(
             &mut df_already_processed,
             &mut df_completed,
@@ -843,7 +855,7 @@ impl<'a> BlockAsm<'a> {
             &mut df_ordering_end,
         );
 
-        (basic_blocks, df_ordering)
+        (basic_blocks, df_ordering, reachable_blocks)
     }
 
     fn assemble_intervals(basic_blocks: &[BasicBlock], ordering: &[usize]) -> NoHashMap<u16, (usize, usize)> {
@@ -872,7 +884,7 @@ impl<'a> BlockAsm<'a> {
     }
 
     pub fn emit_opcodes(&mut self, block_start_pc: u32, thumb: bool) -> usize {
-        let (mut basic_blocks, basic_blocks_order) = self.assemble_basic_blocks(block_start_pc, thumb);
+        let (mut basic_blocks, basic_blocks_order, reachable_blocks) = self.assemble_basic_blocks(block_start_pc, thumb);
 
         if IS_DEBUG && unsafe { BLOCK_LOG } {
             for (i, basic_block) in basic_blocks.iter().enumerate() {
@@ -919,7 +931,7 @@ impl<'a> BlockAsm<'a> {
         }
 
         for (i, basic_block) in basic_blocks.iter_mut().enumerate() {
-            if unlikely(i != 0 && basic_block.enter_blocks.is_empty()) {
+            if unlikely(i != 0 && !reachable_blocks.contains(&i)) {
                 continue;
             }
             self.buf.reg_allocator.init_inputs(*basic_block.get_required_inputs());
@@ -944,7 +956,7 @@ impl<'a> BlockAsm<'a> {
         for (i, basic_block) in basic_blocks.iter().enumerate() {
             let opcodes_len = self.buf.opcodes.len();
             self.buf.block_opcode_offsets.push(opcodes_len);
-            if unlikely(i != 0 && basic_block.enter_blocks.is_empty()) {
+            if unlikely(i != 0 && !reachable_blocks.contains(&i)) {
                 continue;
             }
             let opcodes = basic_block.emit_opcodes(self, opcodes_len, used_host_regs);
diff --git a/src/jit/emitter/emit.rs b/src/jit/emitter/emit.rs
@@ -70,10 +70,10 @@ impl<'a, const CPU: CpuType> JitAsm<'a, CPU> {
                 block_asm.load_u32(guest_pc_reg, block_asm.thread_regs_addr_reg, Reg::PC as u32 * 4);
                 self.emit_branch_return_stack_common(block_asm, guest_pc_reg);
                 block_asm.free_reg(guest_pc_reg);
+            } else {
+                self.emit_branch_out_metadata(block_asm);
+                block_asm.epilogue();
             }
-
-            self.emit_branch_out_metadata(block_asm);
-            block_asm.epilogue();
         }
 
         block_asm.end_cond_block();
diff --git a/src/jit/emitter/thumb/emit_thumb.rs b/src/jit/emitter/thumb/emit_thumb.rs
@@ -90,10 +90,10 @@ impl<'a, const CPU: CpuType> JitAsm<'a, CPU> {
                 block_asm.load_u32(guest_pc_reg, block_asm.thread_regs_addr_reg, Reg::PC as u32 * 4);
                 self.emit_branch_return_stack_common(block_asm, guest_pc_reg);
                 block_asm.free_reg(guest_pc_reg);
+            } else {
+                self.emit_branch_out_metadata(block_asm);
+                block_asm.epilogue();
             }
-
-            self.emit_branch_out_metadata(block_asm);
-            block_asm.epilogue();
         }
     }
 }
diff --git a/src/jit/jit_asm.rs b/src/jit/jit_asm.rs
@@ -186,7 +186,7 @@ fn emit_code_block_internal<const CPU: CpuType, const THUMB: bool>(asm: &mut Jit
     }
 
     let jit_entry = {
-        // unsafe { BLOCK_LOG = guest_pc == 0x206d6d8 };
+        // unsafe { BLOCK_LOG = guest_pc == 0x20027ac };
 
         let mut block_asm = asm.new_block_asm(false);
 

Original file line number	Diff line number	Diff line change
`@@ -42,6 +42,10 @@ impl<const SIZE: usize> Bitset<SIZE> {`
`42`	`42`	`pub const fn is_empty(&self) -> bool {`
`43`	`43`	`self.len() == 0`
`44`	`44`	`}`
	`45`	`+`
	`46`	`+ pub const fn clear(&mut self) {`
	`47`	`+ self.0 = [0; SIZE];`
	`48`	`+ }`
`45`	`49`	`}`
`46`	`50`
`47`	`51`	`impl<const SIZE: usize> Default for Bitset<SIZE> {`
Original file line number	Diff line number	Diff line change
`@@ -664,6 +664,17 @@ impl<'a> BlockAsm<'a> {`
`664`	`664`	`}`
`665`	`665`	`}`
`666`	`666`
	`667`	`+ fn resolve_actual_block_count(already_processed: &mut NoHashSet<usize>, basic_blocks: &mut [BasicBlock], block_i: usize, count: &mut usize) {`
	`668`	`+ *count += 1;`
	`669`	`+ already_processed.insert(block_i);`
	`670`	`+ for i in 0..basic_blocks[block_i].exit_blocks.len() {`
	`671`	`+ let exit_i = basic_blocks[block_i].exit_blocks[i];`
	`672`	`+ if !already_processed.contains(&exit_i) {`
	`673`	`+ Self::resolve_actual_block_count(already_processed, basic_blocks, exit_i, count);`
	`674`	`+ }`
	`675`	`+ }`
	`676`	`+ }`
	`677`	`+`
`667`	`678`	`fn resolve_df_ordering(`
`668`	`679`	`already_processed: &mut Bitset<{ 1024 / 32 }>,`
`669`	`680`	`completed: &mut Bitset<{ 1024 / 32 }>,`
`@@ -697,7 +708,7 @@ impl<'a> BlockAsm<'a> {`
`697`	`708`	`*completed += block_i;`
`698`	`709`	`}`
`699`	`710`
`700`		`- fn assemble_basic_blocks(&mut self, block_start_pc: u32, thumb: bool) -> (Vec<BasicBlock>, Vec<usize>) {`
	`711`	`+ fn assemble_basic_blocks(&mut self, block_start_pc: u32, thumb: bool) -> (Vec<BasicBlock>, Vec<usize>, NoHashSet<usize>) {`
`701`	`712`	`for i in 0..self.buf.insts.len() {`
`702`	`713`	`self.insts_link.insert_end(i);`
`703`	`714`	`}`
`@@ -737,7 +748,7 @@ impl<'a> BlockAsm<'a> {`
`737`	`748`	`}`
`738`	`749`	`last_label = None;`
`739`	`750`	`}`
`740`		`- BlockInstKind::Call { has_return: false, .. } \| BlockInstKind::Epilogue { .. } => {`
	`751`	`+ BlockInstKind::Call { has_return: false, .. } \| BlockInstKind::CallCommon { has_return: false, .. } \| BlockInstKind::Epilogue { .. } => {`
`741`	`752`	`basic_blocks.push(BasicBlock::new(self, basic_block_start, current_node));`
`742`	`753`	`basic_block_start = BlockInstList::deref(current_node).next;`
`743`	`754`	`if let Some(last_label) = last_label {`
`@@ -773,16 +784,13 @@ impl<'a> BlockAsm<'a> {`
`773`	`784`	`}`
`774`	`785`	`}`
`775`	`786`	`// Don't add exit when last command in basic block is a breakout`
`776`		`- BlockInstKind::Call { has_return: false, .. } \| BlockInstKind::Epilogue { .. } => {`
`777`		`- if i + 1 < basic_blocks_len {`
`778`		`- basic_block.exit_blocks.push(i + 1);`
`779`		`- }`
`780`		`- }`
`781`		`- _ => {`
`782`		`- if i + 1 < basic_blocks_len {`
	`787`	`+ BlockInstKind::Call { has_return: false, .. } \| BlockInstKind::CallCommon { has_return: false, .. } \| BlockInstKind::Epilogue { .. } => {`
	`788`	`+ if cond != Cond::AL && i + 1 < basic_blocks_len {`
`783`	`789`	`basic_block.exit_blocks.push(i + 1);`
`784`	`790`	`}`
`785`	`791`	`}`
	`792`	`+ _ if i + 1 < basic_blocks_len => basic_block.exit_blocks.push(i + 1),`
	`793`	`+ _ => {}`
`786`	`794`	`}`
`787`	`795`	`}`
`788`	`796`
`@@ -828,11 +836,15 @@ impl<'a> BlockAsm<'a> {`
`828`	`836`	`basic_block.remove_dead_code(self);`
`829`	`837`	`}`
`830`	`838`
	`839`	`+ let mut reachable_blocks = NoHashSet::default();`
	`840`	`+ let mut actual_block_count = 0;`
	`841`	`+ Self::resolve_actual_block_count(&mut reachable_blocks, &mut basic_blocks, 0, &mut actual_block_count);`
	`842`	`+`
`831`	`843`	`let mut df_already_processed = Bitset::<{ 1024 / 32 }>::new();`
`832`	`844`	`let mut df_completed = Bitset::<{ 1024 / 32 }>::new();`
`833`		`- let mut df_ordering = vec![0; basic_blocks_len];`
	`845`	`+ let mut df_ordering = vec![0; actual_block_count];`
`834`	`846`	`let mut df_ordering_start = 0;`
`835`		`- let mut df_ordering_end = basic_blocks_len - 1;`
	`847`	`+ let mut df_ordering_end = actual_block_count - 1;`
`836`	`848`	`Self::resolve_df_ordering(`
`837`	`849`	`&mut df_already_processed,`
`838`	`850`	`&mut df_completed,`
`@@ -843,7 +855,7 @@ impl<'a> BlockAsm<'a> {`
`843`	`855`	`&mut df_ordering_end,`
`844`	`856`	`);`
`845`	`857`
`846`		`- (basic_blocks, df_ordering)`
	`858`	`+ (basic_blocks, df_ordering, reachable_blocks)`
`847`	`859`	`}`
`848`	`860`
`849`	`861`	`fn assemble_intervals(basic_blocks: &[BasicBlock], ordering: &[usize]) -> NoHashMap<u16, (usize, usize)> {`
`@@ -872,7 +884,7 @@ impl<'a> BlockAsm<'a> {`
`872`	`884`	`}`
`873`	`885`
`874`	`886`	`pub fn emit_opcodes(&mut self, block_start_pc: u32, thumb: bool) -> usize {`
`875`		`- let (mut basic_blocks, basic_blocks_order) = self.assemble_basic_blocks(block_start_pc, thumb);`
	`887`	`+ let (mut basic_blocks, basic_blocks_order, reachable_blocks) = self.assemble_basic_blocks(block_start_pc, thumb);`
`876`	`888`
`877`	`889`	`if IS_DEBUG && unsafe { BLOCK_LOG } {`
`878`	`890`	`for (i, basic_block) in basic_blocks.iter().enumerate() {`
`@@ -919,7 +931,7 @@ impl<'a> BlockAsm<'a> {`
`919`	`931`	`}`
`920`	`932`
`921`	`933`	`for (i, basic_block) in basic_blocks.iter_mut().enumerate() {`
`922`		`- if unlikely(i != 0 && basic_block.enter_blocks.is_empty()) {`
	`934`	`+ if unlikely(i != 0 && !reachable_blocks.contains(&i)) {`
`923`	`935`	`continue;`
`924`	`936`	`}`
`925`	`937`	`self.buf.reg_allocator.init_inputs(*basic_block.get_required_inputs());`
`@@ -944,7 +956,7 @@ impl<'a> BlockAsm<'a> {`
`944`	`956`	`for (i, basic_block) in basic_blocks.iter().enumerate() {`
`945`	`957`	`let opcodes_len = self.buf.opcodes.len();`
`946`	`958`	`self.buf.block_opcode_offsets.push(opcodes_len);`
`947`		`- if unlikely(i != 0 && basic_block.enter_blocks.is_empty()) {`
	`959`	`+ if unlikely(i != 0 && !reachable_blocks.contains(&i)) {`
`948`	`960`	`continue;`
`949`	`961`	`}`
`950`	`962`	`let opcodes = basic_block.emit_opcodes(self, opcodes_len, used_host_regs);`
Original file line number	Diff line number	Diff line change
`@@ -70,10 +70,10 @@ impl<'a, const CPU: CpuType> JitAsm<'a, CPU> {`
`70`	`70`	`block_asm.load_u32(guest_pc_reg, block_asm.thread_regs_addr_reg, Reg::PC as u32 * 4);`
`71`	`71`	`self.emit_branch_return_stack_common(block_asm, guest_pc_reg);`
`72`	`72`	`block_asm.free_reg(guest_pc_reg);`
	`73`	`+ } else {`
	`74`	`+ self.emit_branch_out_metadata(block_asm);`
	`75`	`+ block_asm.epilogue();`
`73`	`76`	`}`
`74`		`-`
`75`		`- self.emit_branch_out_metadata(block_asm);`
`76`		`- block_asm.epilogue();`
`77`	`77`	`}`
`78`	`78`
`79`	`79`	`block_asm.end_cond_block();`
Original file line number	Diff line number	Diff line change
`@@ -90,10 +90,10 @@ impl<'a, const CPU: CpuType> JitAsm<'a, CPU> {`
`90`	`90`	`block_asm.load_u32(guest_pc_reg, block_asm.thread_regs_addr_reg, Reg::PC as u32 * 4);`
`91`	`91`	`self.emit_branch_return_stack_common(block_asm, guest_pc_reg);`
`92`	`92`	`block_asm.free_reg(guest_pc_reg);`
	`93`	`+ } else {`
	`94`	`+ self.emit_branch_out_metadata(block_asm);`
	`95`	`+ block_asm.epilogue();`
`93`	`96`	`}`
`94`		`-`
`95`		`- self.emit_branch_out_metadata(block_asm);`
`96`		`- block_asm.epilogue();`
`97`	`97`	`}`
`98`	`98`	`}`
`99`	`99`	`}`
Original file line number	Diff line number	Diff line change
`@@ -186,7 +186,7 @@ fn emit_code_block_internal<const CPU: CpuType, const THUMB: bool>(asm: &mut Jit`
`186`	`186`	`}`
`187`	`187`
`188`	`188`	`let jit_entry = {`
`189`		`- // unsafe { BLOCK_LOG = guest_pc == 0x206d6d8 };`
	`189`	`+ // unsafe { BLOCK_LOG = guest_pc == 0x20027ac };`
`190`	`190`
`191`	`191`	`let mut block_asm = asm.new_block_asm(false);`
`192`	`192`