Enable MIR inlining of very small functions in debug builds

saethlin · saethlin · commit 55eefdec1ede · 2023-10-14T13:31:31.000-04:00
diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs
@@ -10,7 +10,6 @@ use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TypeVisitableExt;
 use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt};
-use rustc_session::config::OptLevel;
 use rustc_target::abi::FieldIdx;
 use rustc_target::spec::abi::Abi;
 
@@ -47,12 +46,8 @@ impl<'tcx> MirPass<'tcx> for Inline {
         }
 
         match sess.mir_opt_level() {
-            0 | 1 => false,
-            2 => {
-                (sess.opts.optimize == OptLevel::Default
-                    || sess.opts.optimize == OptLevel::Aggressive)
-                    && sess.opts.incremental == None
-            }
+            0 => false,
+            1 | 2 => sess.opts.incremental.is_none(),
             _ => true,
         }
     }
@@ -471,7 +466,15 @@ impl<'tcx> Inliner<'tcx> {
         if callee_body.basic_blocks.len() <= 3 {
             threshold += threshold / 4;
         }
-        debug!("    final inline threshold = {}", threshold);
+
+        if tcx.sess.mir_opt_level() == 1 {
+            threshold = 2 * INSTR_COST;
+
+            if callee_body.basic_blocks.len() > 1 {
+                return Err("cost above threshold");
+            }
+        }
+        debug!("final inline threshold = {}", threshold);
 
         // FIXME: Give a bonus to functions with only a single caller
 
@@ -481,6 +484,7 @@ impl<'tcx> Inliner<'tcx> {
             instance: callsite.callee,
             callee_body,
             cost: 0,
+            threshold,
         };
 
         // Traverse the MIR manually so we can account for the effects of inlining on the CFG.
@@ -493,6 +497,10 @@ impl<'tcx> Inliner<'tcx> {
 
             let blk = &callee_body.basic_blocks[bb];
             checker.visit_basic_block_data(bb, blk);
+            if callee_attrs.inline != InlineAttr::Always && checker.cost > threshold {
+                debug!("checker cost exceeded threshold partway through");
+                return Err("cost above threshold");
+            }
 
             let term = blk.terminator();
             if let TerminatorKind::Drop { ref place, target, unwind, replace: _ } = term.kind {
@@ -805,12 +813,17 @@ struct CostChecker<'b, 'tcx> {
     tcx: TyCtxt<'tcx>,
     param_env: ParamEnv<'tcx>,
     cost: usize,
+    threshold: usize,
     callee_body: &'b Body<'tcx>,
     instance: ty::Instance<'tcx>,
 }
 
 impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
     fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) {
+        if self.cost > self.threshold {
+            return;
+        }
+
         // Don't count StorageLive/StorageDead in the inlining cost.
         match statement.kind {
             StatementKind::StorageLive(_)
diff --git a/tests/codegen-units/polymorphization/unused_type_parameters.rs b/tests/codegen-units/polymorphization/unused_type_parameters.rs
@@ -1,4 +1,4 @@
-// compile-flags:-Zpolymorphize=on -Zprint-mono-items=lazy -Copt-level=1
+// compile-flags:-Zpolymorphize=on -Zprint-mono-items=lazy -Copt-level=1 -Zmir-opt-level=0
 
 #![crate_type = "rlib"]
 
diff --git a/tests/debuginfo/function-names.rs b/tests/debuginfo/function-names.rs
@@ -1,7 +1,7 @@
 // Function names are formatted differently in old versions of GDB
 // min-gdb-version: 10.1
 
-// compile-flags:-g
+// compile-flags: -g -Zmir-opt-level=0
 
 // === GDB TESTS ===================================================================================
 
diff --git a/tests/mir-opt/inline/trivial_fn_debug_build.rs b/tests/mir-opt/inline/trivial_fn_debug_build.rs
@@ -0,0 +1,22 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+// compile-flags: -Copt-level=0 -Zmir-opt-level=1
+#![crate_type = "lib"]
+
+// Test that we still inline trivial functions even in a debug build
+
+pub struct Thing {
+    inner: u8,
+}
+
+impl Thing {
+    #[inline]
+    pub fn get(&self) -> u8 {
+        self.inner
+    }
+}
+
+// EMIT_MIR trivial_fn_debug_build.wrapper.Inline.diff
+// EMIT_MIR trivial_fn_debug_build.wrapper.PreCodegen.after.mir
+pub fn wrapper(t: &Thing) -> u8 {
+    t.get()
+}
diff --git a/tests/mir-opt/inline/trivial_fn_debug_build.wrapper.Inline.panic-abort.diff b/tests/mir-opt/inline/trivial_fn_debug_build.wrapper.Inline.panic-abort.diff
@@ -0,0 +1,24 @@
+- // MIR for `wrapper` before Inline
++ // MIR for `wrapper` after Inline
+  
+  fn wrapper(_1: &Thing) -> u8 {
+      debug t => _1;
+      let mut _0: u8;
+      let mut _2: &Thing;
++     scope 1 (inlined Thing::get) {
++         debug self => _2;
++     }
+  
+      bb0: {
+          StorageLive(_2);
+          _2 = &(*_1);
+-         _0 = Thing::get(move _2) -> [return: bb1, unwind unreachable];
+-     }
+- 
+-     bb1: {
++         _0 = ((*_2).0: u8);
+          StorageDead(_2);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/trivial_fn_debug_build.wrapper.Inline.panic-unwind.diff b/tests/mir-opt/inline/trivial_fn_debug_build.wrapper.Inline.panic-unwind.diff
@@ -0,0 +1,24 @@
+- // MIR for `wrapper` before Inline
++ // MIR for `wrapper` after Inline
+  
+  fn wrapper(_1: &Thing) -> u8 {
+      debug t => _1;
+      let mut _0: u8;
+      let mut _2: &Thing;
++     scope 1 (inlined Thing::get) {
++         debug self => _2;
++     }
+  
+      bb0: {
+          StorageLive(_2);
+          _2 = &(*_1);
+-         _0 = Thing::get(move _2) -> [return: bb1, unwind continue];
+-     }
+- 
+-     bb1: {
++         _0 = ((*_2).0: u8);
+          StorageDead(_2);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/trivial_fn_debug_build.wrapper.PreCodegen.after.panic-abort.mir b/tests/mir-opt/inline/trivial_fn_debug_build.wrapper.PreCodegen.after.panic-abort.mir
@@ -0,0 +1,14 @@
+// MIR for `wrapper` after PreCodegen
+
+fn wrapper(_1: &Thing) -> u8 {
+    debug t => _1;
+    let mut _0: u8;
+    scope 1 (inlined Thing::get) {
+        debug self => _1;
+    }
+
+    bb0: {
+        _0 = ((*_1).0: u8);
+        return;
+    }
+}
diff --git a/tests/mir-opt/inline/trivial_fn_debug_build.wrapper.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/inline/trivial_fn_debug_build.wrapper.PreCodegen.after.panic-unwind.mir
@@ -0,0 +1,14 @@
+// MIR for `wrapper` after PreCodegen
+
+fn wrapper(_1: &Thing) -> u8 {
+    debug t => _1;
+    let mut _0: u8;
+    scope 1 (inlined Thing::get) {
+        debug self => _1;
+    }
+
+    bb0: {
+        _0 = ((*_1).0: u8);
+        return;
+    }
+}
diff --git a/tests/run-make/sepcomp-cci-copies/Makefile b/tests/run-make/sepcomp-cci-copies/Makefile
@@ -8,5 +8,5 @@ include ../tools.mk
 all:
 	$(RUSTC) cci_lib.rs
 	$(RUSTC) foo.rs --emit=llvm-ir -C codegen-units=6 \
-		-Z inline-in-all-cgus
+		-Z inline-in-all-cgus -Zmir-opt-level=0
 	[ "$$(cat "$(TMPDIR)"/foo.*.ll | grep -c define\ .*cci_fn)" -eq "2" ]
diff --git a/tests/run-make/sepcomp-inlining/Makefile b/tests/run-make/sepcomp-inlining/Makefile
@@ -8,7 +8,7 @@ include ../tools.mk
 
 all:
 	$(RUSTC) foo.rs --emit=llvm-ir -C codegen-units=3 \
-		-Z inline-in-all-cgus
+		-Z inline-in-all-cgus -Zmir-opt-level=0
 	[ "$$(cat "$(TMPDIR)"/foo.*.ll | grep -c define\ i32\ .*inlined)" -eq "0" ]
 	[ "$$(cat "$(TMPDIR)"/foo.*.ll | grep -c define\ internal\ i32\ .*inlined)" -eq "2" ]
 	[ "$$(cat "$(TMPDIR)"/foo.*.ll | grep -c define\ hidden\ i32\ .*normal)" -eq "1" ]
diff --git a/tests/run-make/symbol-visibility/Makefile b/tests/run-make/symbol-visibility/Makefile
@@ -32,12 +32,12 @@ endif
 RE_ANY_RUST_SYMBOL="_ZN.*h.*E\|_R[a-zA-Z0-9_]+"
 
 all:
-	$(RUSTC) -Zshare-generics=no an_rlib.rs
-	$(RUSTC) -Zshare-generics=no a_cdylib.rs
-	$(RUSTC) -Zshare-generics=no a_rust_dylib.rs
-	$(RUSTC) -Zshare-generics=no a_proc_macro.rs
-	$(RUSTC) -Zshare-generics=no an_executable.rs
-	$(RUSTC) -Zshare-generics=no a_cdylib.rs --crate-name combined_rlib_dylib --crate-type=rlib,cdylib
+	$(RUSTC) -Zshare-generics=no -Zmir-opt-level=0 an_rlib.rs
+	$(RUSTC) -Zshare-generics=no -Zmir-opt-level=0 a_cdylib.rs
+	$(RUSTC) -Zshare-generics=no -Zmir-opt-level=0 a_rust_dylib.rs
+	$(RUSTC) -Zshare-generics=no -Zmir-opt-level=0 a_proc_macro.rs
+	$(RUSTC) -Zshare-generics=no -Zmir-opt-level=0 an_executable.rs
+	$(RUSTC) -Zshare-generics=no -Zmir-opt-level=0 a_cdylib.rs --crate-name combined_rlib_dylib --crate-type=rlib,cdylib
 
 	# Check that a cdylib exports its public #[no_mangle] functions
 	[ "$$($(NM) $(TMPDIR)/$(CDYLIB_NAME) | grep -v __imp_ | grep -c public_c_function_from_cdylib)" -eq "1" ]
@@ -85,11 +85,11 @@ endif
 	[ "$$($(NM) $(TMPDIR)/$(COMBINED_CDYLIB_NAME) | grep -v __imp_ | grep -c $(RE_ANY_RUST_SYMBOL))" -eq "0" ]
 
 
-	$(RUSTC) -Zshare-generics=yes an_rlib.rs
-	$(RUSTC) -Zshare-generics=yes a_cdylib.rs
-	$(RUSTC) -Zshare-generics=yes a_rust_dylib.rs
-	$(RUSTC) -Zshare-generics=yes a_proc_macro.rs
-	$(RUSTC) -Zshare-generics=yes an_executable.rs
+	$(RUSTC) -Zshare-generics=yes -Zmir-opt-level=0 an_rlib.rs
+	$(RUSTC) -Zshare-generics=yes -Zmir-opt-level=0 a_cdylib.rs
+	$(RUSTC) -Zshare-generics=yes -Zmir-opt-level=0 a_rust_dylib.rs
+	$(RUSTC) -Zshare-generics=yes -Zmir-opt-level=0 a_proc_macro.rs
+	$(RUSTC) -Zshare-generics=yes -Zmir-opt-level=0 an_executable.rs
 
 	# Check that a cdylib exports its public #[no_mangle] functions
 	[ "$$($(NM) $(TMPDIR)/$(CDYLIB_NAME) | grep -v __imp_ | grep -c public_c_function_from_cdylib)" -eq "1" ]
diff --git a/tests/ui-fulldeps/stable-mir/crate-info.rs b/tests/ui-fulldeps/stable-mir/crate-info.rs
@@ -59,7 +59,7 @@ fn test_stable_mir(_tcx: TyCtxt<'_>) -> ControlFlow<()> {
     let foo_bar = get_item(&items, (DefKind::Fn, "foo_bar")).unwrap();
     let body = foo_bar.body();
     assert_eq!(body.locals.len(), 5);
-    assert_eq!(body.blocks.len(), 4);
+    assert_eq!(body.blocks.len(), 3);
     let block = &body.blocks[0];
     match &block.terminator.kind {
         stable_mir::mir::TerminatorKind::Call { .. } => {}
diff --git a/tests/ui/consts/const-eval/index-out-of-bounds-never-type.rs b/tests/ui/consts/const-eval/index-out-of-bounds-never-type.rs
@@ -1,4 +1,5 @@
 // build-fail
+// compile-flags: -Zmir-opt-level=0
 
 // Regression test for #66975
 #![warn(unconditional_panic)]
diff --git a/tests/ui/consts/const-eval/index-out-of-bounds-never-type.stderr b/tests/ui/consts/const-eval/index-out-of-bounds-never-type.stderr
@@ -1,11 +1,11 @@
 error[E0080]: evaluation of `PrintName::<()>::VOID` failed
-  --> $DIR/index-out-of-bounds-never-type.rs:10:61
+  --> $DIR/index-out-of-bounds-never-type.rs:11:61
    |
 LL |     const VOID: ! = { let x = 0 * std::mem::size_of::<T>(); [][x] };
    |                                                             ^^^^^ index out of bounds: the length is 0 but the index is 0
 
 note: the above error was encountered while instantiating `fn f::<()>`
-  --> $DIR/index-out-of-bounds-never-type.rs:20:5
+  --> $DIR/index-out-of-bounds-never-type.rs:21:5
    |
 LL |     f::<()>();
    |     ^^^^^^^^^
diff --git a/tests/ui/consts/const-eval/issue-50814-2.rs b/tests/ui/consts/const-eval/issue-50814-2.rs
@@ -1,4 +1,5 @@
 // build-fail
+// compile-flags: -Zmir-opt-level=0
 
 trait C {
     const BOO: usize;
diff --git a/tests/ui/consts/const-eval/issue-50814-2.stderr b/tests/ui/consts/const-eval/issue-50814-2.stderr
@@ -1,17 +1,17 @@
 error[E0080]: evaluation of `<A<()> as Foo<()>>::BAR` failed
-  --> $DIR/issue-50814-2.rs:14:24
+  --> $DIR/issue-50814-2.rs:15:24
    |
 LL |     const BAR: usize = [5, 6, 7][T::BOO];
    |                        ^^^^^^^^^^^^^^^^^ index out of bounds: the length is 3 but the index is 42
 
 note: erroneous constant encountered
-  --> $DIR/issue-50814-2.rs:18:6
+  --> $DIR/issue-50814-2.rs:19:6
    |
 LL |     &<A<T> as Foo<T>>::BAR
    |      ^^^^^^^^^^^^^^^^^^^^^
 
 note: the above error was encountered while instantiating `fn foo::<()>`
-  --> $DIR/issue-50814-2.rs:30:22
+  --> $DIR/issue-50814-2.rs:31:22
    |
 LL |     println!("{:x}", foo::<()>() as *const usize as usize);
    |                      ^^^^^^^^^^^
diff --git a/tests/ui/consts/const-eval/issue-85155.rs b/tests/ui/consts/const-eval/issue-85155.rs
@@ -11,6 +11,7 @@
 // aux-build: post_monomorphization_error.rs
 // build-fail: this is a post-monomorphization error, it passes check runs and requires building
 //             to actually fail.
+// compile-flags: -Zmir-opt-level=0
 
 extern crate post_monomorphization_error;
 
diff --git a/tests/ui/consts/const-eval/issue-85155.stderr b/tests/ui/consts/const-eval/issue-85155.stderr
@@ -5,7 +5,7 @@ LL |         let _ = 1 / ((IMM >= MIN && IMM <= MAX) as usize);
    |                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ attempt to divide `1_usize` by zero
 
 note: the above error was encountered while instantiating `fn post_monomorphization_error::stdarch_intrinsic::<2>`
-  --> $DIR/issue-85155.rs:19:5
+  --> $DIR/issue-85155.rs:20:5
    |
 LL |     post_monomorphization_error::stdarch_intrinsic::<2>();
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/tests/ui/generics/post_monomorphization_error_backtrace.rs b/tests/ui/generics/post_monomorphization_error_backtrace.rs
@@ -1,4 +1,5 @@
 // build-fail
+// compile-flags: -Zmir-opt-level=0
 
 fn assert_zst<T>() {
     struct F<T>(T);
diff --git a/tests/ui/generics/post_monomorphization_error_backtrace.stderr b/tests/ui/generics/post_monomorphization_error_backtrace.stderr
@@ -1,27 +1,27 @@
 error[E0080]: evaluation of `assert_zst::F::<u32>::V` failed
-  --> $DIR/post_monomorphization_error_backtrace.rs:6:23
+  --> $DIR/post_monomorphization_error_backtrace.rs:7:23
    |
 LL |         const V: () = assert!(std::mem::size_of::<T>() == 0);
-   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/post_monomorphization_error_backtrace.rs:6:23
+   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/post_monomorphization_error_backtrace.rs:7:23
    |
    = note: this error originates in the macro `assert` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 note: the above error was encountered while instantiating `fn assert_zst::<u32>`
-  --> $DIR/post_monomorphization_error_backtrace.rs:18:5
+  --> $DIR/post_monomorphization_error_backtrace.rs:19:5
    |
 LL |     assert_zst::<U>()
    |     ^^^^^^^^^^^^^^^^^
 
 error[E0080]: evaluation of `assert_zst::F::<i32>::V` failed
-  --> $DIR/post_monomorphization_error_backtrace.rs:6:23
+  --> $DIR/post_monomorphization_error_backtrace.rs:7:23
    |
 LL |         const V: () = assert!(std::mem::size_of::<T>() == 0);
-   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/post_monomorphization_error_backtrace.rs:6:23
+   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/post_monomorphization_error_backtrace.rs:7:23
    |
    = note: this error originates in the macro `assert` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 note: the above error was encountered while instantiating `fn assert_zst::<i32>`
-  --> $DIR/post_monomorphization_error_backtrace.rs:18:5
+  --> $DIR/post_monomorphization_error_backtrace.rs:19:5
    |
 LL |     assert_zst::<U>()
    |     ^^^^^^^^^^^^^^^^^
diff --git a/tests/ui/inline-const/const-expr-generic-err.rs b/tests/ui/inline-const/const-expr-generic-err.rs
@@ -1,4 +1,5 @@
 // build-fail
+// compile-flags: -Zmir-opt-level=0
 #![feature(inline_const)]
 
 fn foo<T>() {
diff --git a/tests/ui/inline-const/const-expr-generic-err.stderr b/tests/ui/inline-const/const-expr-generic-err.stderr
@@ -1,25 +1,25 @@
 error[E0080]: evaluation of `foo::<i32>::{constant#0}` failed
-  --> $DIR/const-expr-generic-err.rs:5:13
+  --> $DIR/const-expr-generic-err.rs:6:13
    |
 LL |     const { assert!(std::mem::size_of::<T>() == 0); }
-   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/const-expr-generic-err.rs:5:13
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/const-expr-generic-err.rs:6:13
    |
    = note: this error originates in the macro `assert` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 note: the above error was encountered while instantiating `fn foo::<i32>`
-  --> $DIR/const-expr-generic-err.rs:13:5
+  --> $DIR/const-expr-generic-err.rs:14:5
    |
 LL |     foo::<i32>();
    |     ^^^^^^^^^^^^
 
 error[E0080]: evaluation of `bar::<0>::{constant#0}` failed
-  --> $DIR/const-expr-generic-err.rs:9:13
+  --> $DIR/const-expr-generic-err.rs:10:13
    |
 LL |     const { N - 1 }
    |             ^^^^^ attempt to compute `0_usize - 1_usize`, which would overflow
 
 note: the above error was encountered while instantiating `fn bar::<0>`
-  --> $DIR/const-expr-generic-err.rs:14:5
+  --> $DIR/const-expr-generic-err.rs:15:5
    |
 LL |     bar::<0>();
    |     ^^^^^^^^^^
diff --git a/tests/ui/polymorphization/const_parameters/functions.rs b/tests/ui/polymorphization/const_parameters/functions.rs
diff --git a/tests/ui/polymorphization/type_parameters/functions.rs b/tests/ui/polymorphization/type_parameters/functions.rs
diff --git a/tests/ui/type_length_limit.rs b/tests/ui/type_length_limit.rs

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-// compile-flags:-Zpolymorphize=on -Zprint-mono-items=lazy -Copt-level=1`
	`1`	`+// compile-flags:-Zpolymorphize=on -Zprint-mono-items=lazy -Copt-level=1 -Zmir-opt-level=0`
`2`	`2`
`3`	`3`	`#![crate_type = "rlib"]`
`4`	`4`