[NativeAOT-LLVM] Add linux-arm64 build and testing (#3115)

* Reorganize how the testing is switched on

The goal is to get rid of switches on specific platforms.

* Add linux-arm64 build and testing

* Delete a leftover file

Author: SingleAccretion

eng/liveBuilds.targets

@@ -32,7 +32,8 @@
          does not scale for local development because it requires you to build the same host tools for each target configuration.
          E. g. you need to build the same Debug compiler 3 times if you want to compile code in Debug, Checked and Release. -->
     <IlcConfig Condition="'$(IlcConfig)' == ''">$(CoreCLRConfiguration)</IlcConfig>
-    <CoreCLRCrossILCompilerDir Condition="'$(CoreCLRCrossILCompilerDir)' != '' and '$(TargetsWasm)' == 'true'">$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'coreclr', '$(HostOS).$(BuildArchitecture).$(IlcConfig)', 'ilc'))</CoreCLRCrossILCompilerDir>
+    <IlcHostArch Condition="'$(IlcHostArch)' == ''">$(BuildArchitecture)</IlcHostArch>
+    <CoreCLRCrossILCompilerDir Condition="'$(CoreCLRCrossILCompilerDir)' != '' and '$(TargetsWasm)' == 'true'">$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'artifacts', 'bin', 'coreclr', '$(HostOS).$(IlcHostArch).$(IlcConfig)', 'ilc'))</CoreCLRCrossILCompilerDir>
 
     <Crossgen2Dir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'crossgen2-published'))</Crossgen2Dir>
     <Crossgen2InBuildDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', '$(BuildArchitecture)', 'crossgen2'))</Crossgen2InBuildDir>

eng/native/gen-buildsys.sh

@@ -97,6 +97,7 @@ fi
 cmake_extra_defines_wasm=()
 if [[ "$host_arch" == "wasm" ]]; then
     if [[ "$target_os" == "browser" ]]; then
+        export EMSDK_QUIET=1 && source $EMSDK/emsdk_env.sh
         cmake_command="emcmake $cmake_command"
     elif [[ "$target_os" == "wasi" ]]; then
         if [[ -z $WASI_SDK_PATH ]]; then

eng/pipelines/common/global-build-job.yml

@@ -207,19 +207,19 @@ jobs:
       - ${{ if ne(parameters.hostedOs, 'windows') }}:
         - script: /tmp/docker exec -t -u 0 naot-llvm-ci-container sh -c "tdnf install -y sudo"
           displayName: Install sudo
-        - script: $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-build-and-test-tools.sh $(Build.SourcesDirectory)/wasm-tools
+        - script: $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-build-and-test-tools.sh
           displayName: Install NativeAOT-LLVM build and test tools
       - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-emscripten.ps1 $(Build.SourcesDirectory)/wasm-tools -CI
         displayName: Install/activate emscripten
       - ${{ if eq(parameters.hostedOs, 'windows') }}:
         - script: call $(Build.SourcesDirectory)/eng/pipelines/runtimelab/set-cmake-path.cmd
           displayName: Set CMake path
-      - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-llvm.ps1 -CloneDir $(Build.SourcesDirectory)/wasm-tools -Configs ${{ parameters.buildConfig }} -CI
+      - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-llvm.ps1 -CloneDir $(Build.SourcesDirectory)/wasm-tools -Configs $(_BuildConfig) -Arch $(hostedTargetArch) -CI
         displayName: Install/build LLVM
       - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-nodejs.ps1 $(Build.SourcesDirectory)/wasm-tools
         displayName: Install NodeJS
 
-    - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), or(eq(parameters.platform, 'wasi_wasm_win'), eq(parameters.platform, 'wasi_wasm_linux_x64_naot_llvm'))) }}:
+    - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), eq(parameters.osGroup, 'wasi')) }}:
       # Install Wasi Wasm dependencies: wasi-sdk, wasmtime
       - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-wasi-sdk.ps1 -CI -InstallDir $(Build.SourcesDirectory)/wasm-tools
         displayName: Install wasi-sdk
@@ -228,7 +228,7 @@ jobs:
       - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-jco.ps1 $(Build.SourcesDirectory)
         displayName: Install Jco
 
-    - ${{ if or(eq(parameters.platform, 'browser_wasm_win'), and(eq(parameters.platform, 'wasi_wasm_win'), not(eq(parameters.runtimeFlavor, 'coreclr')))) }}:
+    - ${{ if and(eq(parameters.hostedOs, 'windows'), eq(parameters.archType, 'wasm')) }}:
       # Update machine certs
       - task: PowerShell@2
         displayName: Update machine certs

eng/pipelines/common/platform-matrix.yml

@@ -500,10 +500,10 @@ jobs:
         buildConfig: ${{ parameters.buildConfig }}
         ${{ insert }}: ${{ parameters.jobParameters }}
 
-# Browser WebAssembly Linux X64 for NAOT-LLVM
+# WebAssembly Linux for NAOT-LLVM
 # Use a different name to differentiate from upstream as we need the -sanitizer image to get build tools that are not present in the stock browser_wasm image.
 
-- ${{ if containsValue(parameters.platforms, 'Browser_wasm_linux_x64_naot_llvm') }}:
+- ${{ if containsValue(parameters.platforms, 'browser_wasm_linux_x64_naot_llvm') }}:
   - template: xplat-setup.yml
     parameters:
       jobTemplate: ${{ parameters.jobTemplate }}
@@ -512,19 +512,16 @@ jobs:
       osGroup: browser
       archType: wasm
       targetRid: browser-wasm
-      platform: Browser_wasm_linux_x64_naot_llvm
+      platform: browser_wasm_linux_x64_naot_llvm
       shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
-      container: linux_x64_sanitizer_naot_llvm
+      container: linux_x64_naot_llvm
       jobParameters:
         hostedOs: linux
         runtimeFlavor: ${{ parameters.runtimeFlavor }}
         stagedBuild: ${{ parameters.stagedBuild }}
         buildConfig: ${{ parameters.buildConfig }}
         ${{ insert }}: ${{ parameters.jobParameters }}
 
-# WASI WebAssembly Linux X64 for NAOT-LLVM
-# Use a different name to differentiate from upstream as we need the -sanitizer image to get build tools that are not present in the stock wasi-wasm image.
-
 - ${{ if containsValue(parameters.platforms, 'wasi_wasm_linux_x64_naot_llvm') }}:
   - template: xplat-setup.yml
     parameters:
@@ -536,9 +533,30 @@ jobs:
       targetRid: wasi-wasm
       platform: wasi_wasm_linux_x64_naot_llvm
       shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
-      container: linux_x64_sanitizer_naot_llvm
+      container: linux_x64_naot_llvm
+      jobParameters:
+        hostedOs: linux
+        runtimeFlavor: ${{ parameters.runtimeFlavor }}
+        stagedBuild: ${{ parameters.stagedBuild }}
+        buildConfig: ${{ parameters.buildConfig }}
+        ${{ insert }}: ${{ parameters.jobParameters }}
+
+- ${{ if containsValue(parameters.platforms, 'browser_wasm_linux_arm64_naot_llvm') }}:
+  - template: xplat-setup.yml
+    parameters:
+      jobTemplate: ${{ parameters.jobTemplate }}
+      helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }}
+      variables: ${{ parameters.variables }}
+      osGroup: browser
+      archType: wasm
+      targetRid: browser-wasm
+      platform: browser_wasm_linux_arm64_naot_llvm
+      shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
+      container: linux_arm64_naot_llvm
       jobParameters:
         hostedOs: linux
+        hostedTargetArch: arm64
+        nameSuffix: arm64
         runtimeFlavor: ${{ parameters.runtimeFlavor }}
         stagedBuild: ${{ parameters.stagedBuild }}
         buildConfig: ${{ parameters.buildConfig }}

eng/pipelines/common/templates/pipeline-with-resources.yml

@@ -82,12 +82,19 @@ extends:
       # We need to be able to use 'tdnf install'. We can't because of https://github.com/dotnet/dotnet-docker/issues/788.
       # This is the hackaround: https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-489692810.
       # TODO-LLVM-Upstream: replace with an image that has our prerequisites (LLDB) already installed.
-      linux_x64_sanitizer_naot_llvm:
-        image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net9.0-cross-amd64-sanitizer
+      linux_x64_naot_llvm:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net10.0-cross-amd64-sanitizer
         options: '--name naot-llvm-ci-container -v /usr/bin/docker:/tmp/docker:ro'
         env:
           ROOTFS_DIR: /crossrootfs/x64
 
+      # For arm64, we need to install QEMU.
+      linux_arm64_naot_llvm:
+        image: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net10.0-cross-arm64
+        options: '--name naot-llvm-ci-container -v /usr/bin/docker:/tmp/docker:ro'
+        env:
+          ROOTFS_DIR: /crossrootfs/arm64
+
       # We use a CentOS Stream 8 image here to test building from source on CentOS Stream 9.
       SourceBuild_centos_x64:
         image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9

eng/pipelines/common/xplat-setup.yml

@@ -52,6 +52,12 @@ jobs:
       - name: osSubgroup
         value: ${{ parameters.osSubgroup }}
 
+      - name: hostedTargetArch
+        ${{ if ne(parameters.jobParameters.hostedTargetArch, '') }}:
+          value: ${{ parameters.jobParameters.hostedTargetArch }}
+        ${{ else }}:
+          value: x64
+
       - name: _runSmokeTestsOnlyArg
         value: '/p:RunSmokeTestsOnly=$(isRunSmokeTestsOnly)'
 

eng/pipelines/runtimelab-official.yml

@@ -38,7 +38,7 @@ extends:
           platforms:
           - browser_wasm_win
           - wasi_wasm_win
-          - Browser_wasm_linux_x64_naot_llvm
+          - browser_wasm_linux_x64_naot_llvm
           jobParameters:
             templatePath: 'templates-official'
             timeoutInMinutes: 300

eng/pipelines/runtimelab.yml

@@ -61,7 +61,7 @@ extends:
             - windows_x64
             - browser_wasm_win
             - wasi_wasm_win
-            - Browser_wasm_linux_x64_naot_llvm
+            - browser_wasm_linux_x64_naot_llvm
             - wasi_wasm_linux_x64_naot_llvm
             jobParameters:
               timeoutInMinutes: 300
@@ -101,10 +101,11 @@ extends:
             - linux_x64
             - osx_x64
             - windows_x64
-            - Browser_wasm_win
+            - browser_wasm_win
             - wasi_wasm_win
+            - browser_wasm_linux_x64_naot_llvm
             - wasi_wasm_linux_x64_naot_llvm
-            - Browser_wasm_linux_x64_naot_llvm
+            - browser_wasm_linux_arm64_naot_llvm
             jobParameters:
               timeoutInMinutes: 300
               buildArgs: -s clr.aot+libs -c $(_BuildConfig)

eng/pipelines/runtimelab/install-build-and-test-tools.sh

@@ -1,5 +1,10 @@
 #!/usr/bin/env bash
 set -e
-dir=$(dirname "$0")
-echo; $dir/install-python.sh
-echo; $dir/install-lldb.sh
+
+echo Setting EMSDK_PYTHON to /usr/bin/python3
+echo '##vso[task.setvariable variable=EMSDK_PYTHON]'/usr/bin/python3
+echo
+
+echo Installing LLDB, QEMU
+sudo tdnf install -y lldb python3-lldb qemu-user
+echo