Rollup merge of #143478 - RalfJung:miri-sync, r=RalfJung

Miri subtree update

r? `@ghost`

Author: matthiaskrgr

Cargo.lock

@@ -2394,6 +2394,7 @@ dependencies = [
  "regex",
  "rustc_version",
  "serde",
+ "serde_json",
  "smallvec",
  "tempfile",
  "tikv-jemalloc-sys",

src/tools/miri/.github/workflows/ci.yml

@@ -13,35 +13,82 @@ defaults:
     shell: bash
 
 jobs:
-  build:
+  test:
+    name: test (${{ matrix.host_target }})
     strategy:
       fail-fast: false
       matrix:
         include:
-          - os: ubuntu-latest
-            host_target: x86_64-unknown-linux-gnu
-          - os: macos-14
-            host_target: aarch64-apple-darwin
-          - os: windows-latest
-            host_target: i686-pc-windows-msvc
+          - host_target: x86_64-unknown-linux-gnu
+            os: ubuntu-latest
+          - host_target: i686-unknown-linux-gnu
+            os: ubuntu-latest
+            multiarch: i386
+            gcc_cross: i686-linux-gnu
+          - host_target: aarch64-unknown-linux-gnu
+            os: ubuntu-24.04-arm
+          - host_target: armv7-unknown-linux-gnueabihf
+            os: ubuntu-24.04-arm
+            multiarch: armhf
+            gcc_cross: arm-linux-gnueabihf
+          - host_target: riscv64gc-unknown-linux-gnu
+            os: ubuntu-latest
+            multiarch: riscv64
+            gcc_cross: riscv64-linux-gnu
+            qemu: true
+          - host_target: s390x-unknown-linux-gnu
+            os: ubuntu-latest
+            multiarch: s390x
+            gcc_cross: s390x-linux-gnu
+            qemu: true
+          - host_target: aarch64-apple-darwin
+            os: macos-latest
+          - host_target: i686-pc-windows-msvc
+            os: windows-latest
     runs-on: ${{ matrix.os }}
     env:
       HOST_TARGET: ${{ matrix.host_target }}
     steps:
       - uses: actions/checkout@v4
+      - name: install qemu
+        if: ${{ matrix.qemu }}
+        run: sudo apt install qemu-user qemu-user-binfmt
+      - name: install multiarch
+        if: ${{ matrix.multiarch != '' }}
+        run: |
+          # s390x, ppc64el need Ubuntu Ports to be in the mirror list
+          sudo bash -c "echo 'https://ports.ubuntu.com/	priority:4' >> /etc/apt/apt-mirrors.txt"
+          # Add architecture
+          sudo dpkg --add-architecture ${{ matrix.multiarch }}
+          sudo apt update
+          # Install needed packages
+          sudo apt install $(echo "libatomic1: zlib1g-dev:" | sed 's/:/:${{ matrix.multiarch }}/g')
       - uses: ./.github/workflows/setup
         with:
           toolchain_flags: "--host ${{ matrix.host_target }}"
 
-      # The `style` job only runs on Linux; this makes sure the Windows-host-specific
-      # code is also covered by clippy.
-      - name: Check clippy
-        if: ${{ matrix.os == 'windows-latest' }}
-        run: ./miri clippy -- -D warnings
+      # We set up the cross-compiler *after* the basic setup as setting CC would otherwise
+      # cause confusion.
+      - name: install gcc-cross
+        if: ${{ matrix.gcc_cross != '' }}
+        run: |
+          sudo apt install gcc-${{ matrix.gcc_cross }}
+          echo "Setting environment variables:"
+          echo "CC_${{ matrix.host_target }}=${{ matrix.gcc_cross }}-gcc" | tee -a $GITHUB_ENV
+          TARGET_UPPERCASE=$(echo ${{ matrix.host_target }} | tr '[:lower:]-' '[:upper:]_')
+          echo "CARGO_TARGET_${TARGET_UPPERCASE}_LINKER=${{ matrix.gcc_cross }}-gcc" | tee -a $GITHUB_ENV
 
-      - name: Test Miri
+      # The main test job! We don't run this in qemu as that is quite slow,
+      # so those targets only get the clippy check below.
+      - name: test Miri
+        if: ${{ !matrix.qemu }}
         run: ./ci/ci.sh
 
+      # The `style` job only runs on Linux; this makes sure the host-specific
+      # code is also covered by clippy.
+      - name: clippy
+        run: ./miri clippy -- -D warnings
+
   style:
     name: style checks
     runs-on: ubuntu-latest
@@ -51,8 +98,6 @@ jobs:
 
       - name: rustfmt
         run: ./miri fmt --check
-      - name: clippy
-        run: ./miri clippy -- -D warnings
       - name: clippy (no features)
         run: ./miri clippy --no-default-features -- -D warnings
       - name: clippy (all features)
@@ -73,7 +118,7 @@ jobs:
   # ALL THE PREVIOUS JOBS NEED TO BE ADDED TO THE `needs` SECTION OF THIS JOB!
   # And they should be added below in `cron-fail-notify` as well.
   conclusion:
-    needs: [build, style, coverage]
+    needs: [test, style, coverage]
     # We need to ensure this job does *not* get skipped if its dependencies fail,
     # because a skipped job is considered a success by GitHub. So we have to
     # overwrite `if:`. We use `!cancelled()` to ensure the job does still not get run
@@ -135,7 +180,7 @@ jobs:
   cron-fail-notify:
     name: cronjob failure notification
     runs-on: ubuntu-latest
-    needs: [build, style, coverage]
+    needs: [test, style, coverage]
     if: ${{ github.event_name == 'schedule' && failure() }}
     steps:
       # Send a Zulip notification

src/tools/miri/.github/workflows/setup/action.yml

@@ -2,6 +2,7 @@ name: "Miri CI setup"
 description: "Sets up Miri CI"
 inputs:
   toolchain_flags:
+    description: extra flags to pass to rustup-toolchain-install-master
     required: false
     default: ''
 runs:
@@ -31,18 +32,15 @@ runs:
             ~/.cargo/bin
             ~/.cargo/.crates.toml
             ~/.cargo/.crates2.json
-          key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.lock', '.github/workflows/**/*.yml') }}
-          restore-keys: cargo-${{ runner.os }}
+          # Bump the version when something here changes that needs a cache reset.
+          key: cargo-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/Cargo.lock') }}-v1
+          restore-keys: cargo-${{ runner.os }}-${{ runner.arch }}
 
-      - name: Install rustup-toolchain-install-master
+      - name: Install the tools we need
         if: steps.cache.outputs.cache-hit != 'true'
         run: cargo install -f rustup-toolchain-install-master hyperfine
         shell: bash
 
-      - name: Install nightly toolchain
-        run: rustup toolchain install nightly --profile minimal
-        shell: bash
-
       - name: Install "master" toolchain
         run: |
           if [[ ${{ github.event_name }} == 'schedule' ]]; then

src/tools/miri/CONTRIBUTING.md

@@ -158,6 +158,15 @@ compiler that has `debug=true` set in `bootstrap.toml`.
 You can set `MIRI_BACKTRACE=1` to get a backtrace of where an
 evaluation error was originally raised.
 
+#### Tracing
+
+You can generate a Chrome trace file from a Miri execution by passing `--features=tracing` during the
+build and then setting `MIRI_TRACING=1` when running Miri. This will generate a `.json` file that
+you can visualize in [Perfetto](https://ui.perfetto.dev/). For example:
+
+```sh
+MIRI_TRACING=1 ./miri run --features=tracing tests/pass/hello.rs
+```
 
 ### UI testing
 

src/tools/miri/Cargo.lock

@@ -627,6 +627,7 @@ dependencies = [
  "regex",
  "rustc_version",
  "serde",
+ "serde_json",
  "smallvec",
  "tempfile",
  "tikv-jemalloc-sys",

src/tools/miri/Cargo.toml

@@ -27,6 +27,7 @@ chrono = { version = "0.4.38", default-features = false }
 chrono-tz = "0.10"
 directories = "6"
 bitflags = "2.6"
+serde_json = { version = "1.0", optional = true }
 
 # Copied from `compiler/rustc/Cargo.toml`.
 # But only for some targets, it fails for others. Rustc configures this in its CI, but we can't
@@ -67,6 +68,7 @@ default = ["stack-cache"]
 genmc = []
 stack-cache = []
 stack-cache-consistency-check = ["stack-cache"]
+tracing = ["serde_json"]
 
 [lints.rust.unexpected_cfgs]
 level = "warn"

src/tools/miri/README.md

@@ -11,12 +11,12 @@ instance:
 * Not sufficiently aligned memory accesses and references
 * Violation of basic type invariants (a `bool` that is not 0 or 1, for example,
   or an invalid enum discriminant)
+* Data races and emulation of *some* weak memory effects, i.e.,
+  atomic reads can return outdated values
 * **Experimental**: Violations of the [Stacked Borrows] rules governing aliasing
   for reference types
 * **Experimental**: Violations of the [Tree Borrows] aliasing rules, as an optional
   alternative to [Stacked Borrows]
-* **Experimental**: Data races and emulation of weak memory effects, i.e.,
-  atomic reads can return outdated values.
 
 On top of that, Miri will also tell you about memory leaks: when there is memory
 still allocated at the end of the execution, and that memory is not reachable

src/tools/miri/ci/ci.sh

@@ -143,42 +143,53 @@ case $HOST_TARGET in
     GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 CARGO_MIRI_ENV=1 run_tests
     # Extra tier 1
     MANY_SEEDS=64 TEST_TARGET=i686-unknown-linux-gnu run_tests
-    MANY_SEEDS=64 TEST_TARGET=aarch64-unknown-linux-gnu run_tests
     MANY_SEEDS=64 TEST_TARGET=x86_64-apple-darwin run_tests
     MANY_SEEDS=64 TEST_TARGET=x86_64-pc-windows-gnu run_tests
+    ;;
+  i686-unknown-linux-gnu)
+    # Host
+    # Without GC_STRESS as this is a slow runner.
+    MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 CARGO_MIRI_ENV=1 run_tests
+    # Partially supported targets (tier 2)
+    BASIC="empty_main integer heap_alloc libc-mem vec string btreemap" # ensures we have the basics: pre-main code, system allocator
+    UNIX="hello panic/panic panic/unwind concurrency/simple atomic libc-mem libc-misc libc-random env num_cpus" # the things that are very similar across all Unixes, and hence easily supported there
+    TEST_TARGET=aarch64-linux-android  run_tests_minimal $BASIC $UNIX time hashmap random thread sync concurrency epoll eventfd
+    TEST_TARGET=wasm32-wasip2          run_tests_minimal $BASIC wasm
+    TEST_TARGET=wasm32-unknown-unknown run_tests_minimal no_std empty_main wasm # this target doesn't really have std
+    TEST_TARGET=thumbv7em-none-eabihf  run_tests_minimal no_std
+    ;;
+  aarch64-unknown-linux-gnu)
+    # Host
+    GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 CARGO_MIRI_ENV=1 run_tests
     # Extra tier 1 candidate
     MANY_SEEDS=64 TEST_TARGET=aarch64-pc-windows-msvc run_tests
+    # Extra tier 2
+    MANY_SEEDS=16 TEST_TARGET=arm-unknown-linux-gnueabi run_tests # 32bit ARM
+    MANY_SEEDS=16 TEST_TARGET=aarch64-pc-windows-gnullvm run_tests # gnullvm ABI
+    MANY_SEEDS=16 TEST_TARGET=s390x-unknown-linux-gnu run_tests # big-endian architecture of choice
+    # Custom target JSON file
+    TEST_TARGET=tests/x86_64-unknown-kernel.json MIRI_NO_STD=1 run_tests_minimal no_std
+    ;;
+  armv7-unknown-linux-gnueabihf)
+    # Host
+    GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 CARGO_MIRI_ENV=1 run_tests
     ;;
   aarch64-apple-darwin)
     # Host
     GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 CARGO_MIRI_ENV=1 run_tests
     # Extra tier 1
     MANY_SEEDS=64 TEST_TARGET=i686-pc-windows-gnu run_tests
     MANY_SEEDS=64 TEST_TARGET=x86_64-pc-windows-msvc CARGO_MIRI_ENV=1 run_tests
-    # Extra tier 2
-    MANY_SEEDS=16 TEST_TARGET=arm-unknown-linux-gnueabi run_tests # 32bit ARM
-    MANY_SEEDS=16 TEST_TARGET=aarch64-pc-windows-gnullvm run_tests # gnullvm ABI
-    MANY_SEEDS=16 TEST_TARGET=s390x-unknown-linux-gnu run_tests # big-endian architecture of choice
     # Not officially supported tier 2
     MANY_SEEDS=16 TEST_TARGET=mips-unknown-linux-gnu run_tests # a 32bit big-endian target, and also a target without 64bit atomics
     MANY_SEEDS=16 TEST_TARGET=x86_64-unknown-illumos run_tests
     MANY_SEEDS=16 TEST_TARGET=x86_64-pc-solaris run_tests
     MANY_SEEDS=16 TEST_TARGET=x86_64-unknown-freebsd run_tests
     MANY_SEEDS=16 TEST_TARGET=i686-unknown-freebsd run_tests
-    # Partially supported targets (tier 2)
-    BASIC="empty_main integer heap_alloc libc-mem vec string btreemap" # ensures we have the basics: pre-main code, system allocator
-    UNIX="hello panic/panic panic/unwind concurrency/simple atomic libc-mem libc-misc libc-random env num_cpus" # the things that are very similar across all Unixes, and hence easily supported there
-    TEST_TARGET=aarch64-linux-android  run_tests_minimal $BASIC $UNIX time hashmap random thread sync concurrency epoll eventfd
-    TEST_TARGET=wasm32-wasip2          run_tests_minimal $BASIC wasm
-    TEST_TARGET=wasm32-unknown-unknown run_tests_minimal no_std empty_main wasm # this target doesn't really have std
-    TEST_TARGET=thumbv7em-none-eabihf  run_tests_minimal no_std
-    # Custom target JSON file
-    TEST_TARGET=tests/x86_64-unknown-kernel.json MIRI_NO_STD=1 run_tests_minimal no_std
     ;;
   i686-pc-windows-msvc)
     # Host
-    # Without GC_STRESS and with reduced many-seeds count as this is the slowest runner.
-    # (The macOS runner checks windows-msvc with full many-seeds count.)
+    # Without GC_STRESS as this is the slowest runner.
     MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 run_tests
     # Extra tier 1
     # We really want to ensure a Linux target works on a Windows host,

src/tools/miri/etc/rust_analyzer_helix.toml

@@ -9,23 +9,21 @@ linkedProjects = [
 ]
 
 [language-server.rust-analyzer.config.check]
-invocationLocation = "root"
 invocationStrategy = "once"
 overrideCommand = [
-    "env",
-    "MIRI_AUTO_OPS=no",
     "./miri",
     "clippy", # make this `check` when working with a locally built rustc
     "--message-format=json",
 ]
 
+[language-server.rust-analyzer.config.cargo.extraEnv]
+MIRI_AUTO_OPS = "no"
+MIRI_IN_RA = "1"
+
 # Contrary to what the name suggests, this also affects proc macros.
-[language-server.rust-analyzer.config.buildScripts]
-invocationLocation = "root"
+[language-server.rust-analyzer.config.cargo.buildScripts]
 invocationStrategy = "once"
 overrideCommand = [
-    "env",
-    "MIRI_AUTO_OPS=no",
     "./miri",
     "check",
     "--message-format=json",

src/tools/miri/miri

@@ -2,20 +2,24 @@
 set -e
 # We want to call the binary directly, so we need to know where it ends up.
 ROOT_DIR="$(dirname "$0")"
-MIRI_SCRIPT_TARGET_DIR="$ROOT_DIR"/miri-script/target
-TOOLCHAIN="+nightly"
+TARGET_DIR="$ROOT_DIR"/miri-script/target
+# Prepare cargo invocation.
+# We have to overwrite the toolchain since `+miri` might be activated and not installed.
+TOOLCHAIN="+stable"
+CARGO_FLAGS=("-q")
 # If we are being invoked for RA, use JSON output and the default toolchain (to make proc-macros
 # work in RA). This needs a different target dir to avoid mixing up the builds.
+# Also set `-Zroot-dir` so that RA can identify where the error occurred.
 if [ -n "$MIRI_IN_RA" ]; then
-  MESSAGE_FORMAT="--message-format=json"
   TOOLCHAIN=""
-  MIRI_SCRIPT_TARGET_DIR="$MIRI_SCRIPT_TARGET_DIR"/ra
+  CARGO_FLAGS+=("--message-format=json" "-Zroot-dir=$ROOT_DIR")
+  TARGET_DIR="$ROOT_DIR"/target
 fi
-# We need a nightly toolchain, for `-Zroot-dir`.
-cargo $TOOLCHAIN build $CARGO_EXTRA_FLAGS --manifest-path "$ROOT_DIR"/miri-script/Cargo.toml \
-  -Zroot-dir="$ROOT_DIR" \
-  -q --target-dir "$MIRI_SCRIPT_TARGET_DIR" $MESSAGE_FORMAT || \
-  ( echo "Failed to build miri-script. Is the 'nightly' toolchain installed?"; exit 1 )
-# Instead of doing just `cargo run --manifest-path .. $@`, we invoke miri-script binary directly. Invoking `cargo run` goes through
-# rustup (that sets it's own environmental variables), which is undesirable.
-"$MIRI_SCRIPT_TARGET_DIR"/debug/miri-script "$@"
+# Run cargo.
+cargo $TOOLCHAIN build --manifest-path "$ROOT_DIR"/miri-script/Cargo.toml \
+  --target-dir "$TARGET_DIR" "${CARGO_FLAGS[@]}" || \
+  ( echo "Failed to build miri-script. Is the 'stable' toolchain installed?"; exit 1 )
+# Instead of doing just `cargo run --manifest-path .. $@`, we invoke miri-script binary directly.
+# Invoking `cargo run` goes through rustup (that sets it's own environmental variables), which is
+# undesirable.
+"$TARGET_DIR"/debug/miri-script "$@"

src/tools/miri/miri-script/src/commands.rs

@@ -231,11 +231,6 @@ impl Command {
         cmd!(sh, "rustup override set miri").run()?;
         // Cleanup.
         cmd!(sh, "cargo clean").run()?;
-        // Call `cargo metadata` on the sources in case that changes the lockfile
-        // (which fails under some setups when it is done from inside vscode).
-        let sysroot = cmd!(sh, "rustc --print sysroot").read()?;
-        let sysroot = sysroot.trim();
-        cmd!(sh, "cargo metadata --format-version 1 --manifest-path {sysroot}/lib/rustlib/rustc-src/rust/compiler/rustc/Cargo.toml").ignore_stdout().run()?;
         Ok(())
     }
 
@@ -707,9 +702,12 @@ impl Command {
         let mut early_flags = Vec::<OsString>::new();
 
         // In `dep` mode, the target is already passed via `MIRI_TEST_TARGET`
-        if !dep && let Some(target) = &target {
-            early_flags.push("--target".into());
-            early_flags.push(target.into());
+        #[expect(clippy::collapsible_if)] // we need to wait until this is stable
+        if !dep {
+            if let Some(target) = &target {
+                early_flags.push("--target".into());
+                early_flags.push(target.into());
+            }
         }
         early_flags.push("--edition".into());
         early_flags.push(edition.as_deref().unwrap_or("2021").into());
@@ -737,8 +735,11 @@ impl Command {
         // Add Miri flags
         let mut cmd = cmd.args(&miri_flags).args(&early_flags).args(&flags);
         // For `--dep` we also need to set the target in the env var.
-        if dep && let Some(target) = &target {
-            cmd = cmd.env("MIRI_TEST_TARGET", target);
+        #[expect(clippy::collapsible_if)] // we need to wait until this is stable
+        if dep {
+            if let Some(target) = &target {
+                cmd = cmd.env("MIRI_TEST_TARGET", target);
+            }
         }
         // Finally, run the thing.
         Ok(cmd.run()?)