Merge remote-tracking branch 'origin/sycl' into upstream

Author: ziranzha

.ci/generate-buildkite-pipeline-premerge

@@ -128,9 +128,8 @@ if [[ "${windows_projects}" != "" ]]; then
         limit: 2
   timeout_in_minutes: 150
   env:
-    CC: 'cl'
-    CXX: 'cl'
-    LD: 'link'
+    MAX_PARALLEL_COMPILE_JOBS: '16'
+    MAX_PARALLEL_LINK_JOBS: '4'
   commands:
   - 'C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat -arch=amd64 -host_arch=amd64'
   - 'bash .ci/monolithic-windows.sh "$(echo ${windows_projects} | tr ' ' ';')" "$(echo ${windows_check_targets})"'

.ci/metrics/metrics.py

@@ -130,34 +130,6 @@ def get_per_workflow_metrics(
         workflow_jobs = workflow_run.jobs()
         if workflow_jobs.totalCount == 0:
             continue
-        if workflow_jobs.totalCount > 1:
-            raise ValueError(
-                f"Encountered an unexpected number of jobs: {workflow_jobs.totalCount}"
-            )
-
-        created_at = workflow_jobs[0].created_at
-        started_at = workflow_jobs[0].started_at
-        completed_at = workflow_jobs[0].completed_at
-
-        job_result = int(workflow_jobs[0].conclusion == "success")
-        if job_result:
-            # We still might want to mark the job as a failure if one of the steps
-            # failed. This is required due to use setting continue-on-error in
-            # the premerge pipeline to prevent sending emails while we are
-            # testing the infrastructure.
-            # TODO(boomanaiden154): Remove this once the premerge pipeline is no
-            # longer in a testing state and we can directly assert the workflow
-            # result.
-            for step in workflow_jobs[0].steps:
-                if step.conclusion != "success":
-                    job_result = 0
-                    break
-
-        queue_time = started_at - created_at
-        run_time = completed_at - started_at
-
-        if run_time.seconds == 0:
-            continue
 
         if (
             workflows_to_track[workflow_run.name] is None
@@ -170,20 +142,45 @@ def get_per_workflow_metrics(
         ):
             break
 
-        # The timestamp associated with the event is expected by Grafana to be
-        # in nanoseconds.
-        created_at_ns = int(created_at.timestamp()) * 10**9
-
-        workflow_metrics.append(
-            JobMetrics(
-                workflow_run.name,
-                queue_time.seconds,
-                run_time.seconds,
-                job_result,
-                created_at_ns,
-                workflow_run.id,
+        for workflow_job in workflow_jobs:
+            created_at = workflow_job.created_at
+            started_at = workflow_job.started_at
+            completed_at = workflow_job.completed_at
+
+            job_result = int(workflow_job.conclusion == "success")
+            if job_result:
+                # We still might want to mark the job as a failure if one of the steps
+                # failed. This is required due to use setting continue-on-error in
+                # the premerge pipeline to prevent sending emails while we are
+                # testing the infrastructure.
+                # TODO(boomanaiden154): Remove this once the premerge pipeline is no
+                # longer in a testing state and we can directly assert the workflow
+                # result.
+                for step in workflow_job.steps:
+                    if step.conclusion != "success":
+                        job_result = 0
+                        break
+
+            queue_time = started_at - created_at
+            run_time = completed_at - started_at
+
+            if run_time.seconds == 0:
+                continue
+
+            # The timestamp associated with the event is expected by Grafana to be
+            # in nanoseconds.
+            created_at_ns = int(created_at.timestamp()) * 10**9
+
+            workflow_metrics.append(
+                JobMetrics(
+                    workflow_run.name + "-" + workflow_job.name,
+                    queue_time.seconds,
+                    run_time.seconds,
+                    job_result,
+                    created_at_ns,
+                    workflow_run.id,
+                )
             )
-        )
 
     return workflow_metrics
 

.ci/monolithic-windows.sh

@@ -50,6 +50,10 @@ echo "--- cmake"
 pip install -q -r "${MONOREPO_ROOT}"/mlir/python/requirements.txt
 pip install -q -r "${MONOREPO_ROOT}"/.ci/requirements.txt
 
+export CC=cl
+export CXX=cl
+export LD=link
+
 # The CMAKE_*_LINKER_FLAGS to disable the manifest come from research
 # on fixing a build reliability issue on the build server, please
 # see https://github.com/llvm/llvm-project/pull/82393 and
@@ -72,8 +76,8 @@ cmake -S "${MONOREPO_ROOT}"/llvm -B "${BUILD_DIR}" \
       -D CMAKE_EXE_LINKER_FLAGS="/MANIFEST:NO" \
       -D CMAKE_MODULE_LINKER_FLAGS="/MANIFEST:NO" \
       -D CMAKE_SHARED_LINKER_FLAGS="/MANIFEST:NO" \
-      -D LLVM_PARALLEL_COMPILE_JOBS=16 \
-      -D LLVM_PARALLEL_LINK_JOBS=4
+      -D LLVM_PARALLEL_COMPILE_JOBS=${MAX_PARALLEL_COMPILE_JOBS} \
+      -D LLVM_PARALLEL_LINK_JOBS=${MAX_PARALLEL_LINK_JOBS}
 
 echo "--- ninja"
 # Targets are not escaped as they are passed as separate arguments.

.github/workflows/containers/github-action-ci-windows/Dockerfile

@@ -0,0 +1,118 @@
+# Agent image for LLVM org cluster.
+# .net 4.8 is required by chocolately package manager.
+FROM mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-ltsc2019
+
+# Restore the default Windows shell for correct batch processing.
+SHELL ["cmd", "/S", "/C"]
+
+# Download the Build Tools bootstrapper.
+ADD https://aka.ms/vs/16/release/vs_buildtools.exe /TEMP/vs_buildtools.exe
+
+RUN powershell -Command Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
+
+# Download channel for fixed install.
+ARG CHANNEL_URL=https://aka.ms/vs/16/release/channel
+ADD ${CHANNEL_URL} /TEMP/VisualStudio.chman
+
+# Install Build Tools with C++ workload.
+#   - Documentation for docker installation
+#     https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container?view=vs-2019
+#   - Documentation on workloads
+#     https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-build-tools?view=vs-2019#c-build-tools
+#   - Documentation on flags
+#     https://docs.microsoft.com/en-us/visualstudio/install/use-command-line-parameters-to-install-visual-studio?view=vs-2019
+RUN /TEMP/vs_buildtools.exe --quiet --wait --norestart --nocache \
+    --channelUri C:\TEMP\VisualStudio.chman \
+    --installChannelUri C:\TEMP\VisualStudio.chman \
+    --installPath C:\BuildTools \
+    --add Microsoft.VisualStudio.Workload.VCTools \
+    --add Microsoft.VisualStudio.Component.VC.ATL \
+    --includeRecommended \
+    || IF "%ERRORLEVEL%"=="3010" EXIT 0
+
+# Register DIA dll (Debug Interface Access) so it can be used to symbolize
+# the stack traces. Register dll for 32 and 64 bit.
+# see https://developercommunity.visualstudio.com/content/problem/290674/msdia140dll-is-not-registered-on-vs2017-hosts.html
+
+RUN regsvr32 /S "C:\BuildTools\DIA SDK\bin\amd64\msdia140.dll" & \
+    regsvr32 /S "C:\BuildTools\DIA SDK\bin\msdia140.dll"
+
+# install tools as described in https://llvm.org/docs/GettingStartedVS.html
+# and a few more that were not documented...
+RUN choco install -y ninja git
+# Pin an older version of Python; the current Python 3.10 fails when
+# doing "pip install" for the other dependencies, as it fails to find libxml
+# while compiling some package.
+RUN choco install -y python3 --version 3.9.7
+
+# ActivePerl is currently not installable via Chocolatey, see
+# http://disq.us/p/2ipditb. Install StrawberryPerl instead. Unfortunately,
+# StrawberryPerl not only installs Perl, but also a redundant C/C++ compiler
+# toolchain, and a copy of pkg-config which can cause misdetections for other
+# built products, see
+# https://github.com/StrawberryPerl/Perl-Dist-Strawberry/issues/11 for further
+# details. Remove the redundant and unnecessary parts of the StrawberryPerl
+# install.
+RUN choco install -y strawberryperl && \
+    rmdir /q /s c:\strawberry\c && \
+    del /q c:\strawberry\perl\bin\pkg-config*
+
+# libcxx requires clang(-cl) to be available
+RUN choco install -y sccache llvm
+RUN pip install psutil
+
+RUN curl -LO https://github.com/mstorsjo/llvm-mingw/releases/download/20230320/llvm-mingw-20230320-ucrt-x86_64.zip && \
+    powershell Expand-Archive llvm-mingw-*-ucrt-x86_64.zip -DestinationPath . && \
+    del llvm-mingw-*-ucrt-x86_64.zip && \
+    ren llvm-mingw-20230320-ucrt-x86_64 llvm-mingw
+
+# configure Python encoding
+ENV PYTHONIOENCODING=UTF-8
+
+# update the path variable
+# C:\Program Files\Git\usr\bin contains a usable bash and other unix tools.
+# C:\llvm-mingw\bin contains Clang configured for mingw targets and
+#     corresponding sysroots. Both the 'llvm' package (with Clang defaulting
+#     to MSVC targets) and this directory contains executables named
+#     'clang.exe' - add this last to let the other one have precedence.
+#     To use these compilers, use the triple prefixed form, e.g.
+#     x86_64-w64-mingw32-clang.
+# C:\buildtools and SDK paths are ones that are set by c:\BuildTools\Common7\Tools\VsDevCmd.bat -arch=amd64 -host_arch=amd64
+RUN powershell -Command \
+    [System.Environment]::SetEnvironmentVariable('PATH', \
+    [System.Environment]::GetEnvironmentVariable('PATH', 'machine') + ';C:\Program Files\Git\usr\bin;C:\llvm-mingw\bin' \
+    + ';C:\BuildTools\Common7\IDE\' \
+    + ';C:\BuildTools\Common7\IDE\CommonExt ensions\Microsoft\TeamFoundation\Team Explorer' \
+    + ';C:\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin' \
+    + ';C:\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja' \
+    + ';C:\BuildTools\Common7\IDE\CommonExtensions\Microsoft\TeamFoundation\Team Explorer' \
+    + ';C:\BuildTools\Common7\IDE\CommonExtensions\Microsoft\TestWindow' \
+    + ';C:\BuildTools\Common7\IDE\VC\VCPackages' \
+    + ';C:\BuildTools\Common7\Tools\' \
+    + ';C:\BuildTools\Common7\Tools\devinit' \
+    + ';C:\BuildTools\MSBuild\Current\Bin' \
+    + ';C:\BuildTools\MSBuild\Current\bin\Roslyn' \
+    + ';C:\BuildTools\VC\Tools\MSVC\14.29.30133\bin\HostX64\x64' \
+    + ';C:\Program Files (x86)\Microsoft SDKs\Windows\v10.0A\bin\NETFX 4.8 Tools\x64\' \
+    + ';C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x64' \
+    + ';C:\Program Files (x86)\Windows Kits\10\bin\x64' \
+    + ';C:\Windows\Microsoft.NET\Framework64\v4.0.30319' \
+    ,'machine')
+
+# support long file names during git checkout
+RUN git config --system core.longpaths true & \
+    git config --global core.autocrlf false
+
+# handle for debugging of files beeing locked by some processes.
+RUN choco install -y handle
+
+RUN pip3 install pywin32 buildbot-worker==2.8.4
+
+ARG RUNNER_VERSION=2.322.0
+ENV RUNNER_VERSION=$RUNNER_VERSION
+
+RUN powershell -Command \
+    Invoke-WebRequest -Uri https://github.com/actions/runner/releases/download/v${env:RUNNER_VERSION}/actions-runner-win-x64-${env:RUNNER_VERSION}.zip -OutFile actions-runner-win.zip ; \
+    Add-Type -AssemblyName System.IO.Compression.FileSystem ; \
+    [System.IO.Compression.ZipFile]::ExtractToDirectory('actions-runner-win.zip', $PWD) ;\
+    rm actions-runner-win.zip

.github/workflows/containers/github-action-ci/Dockerfile

@@ -0,0 +1,106 @@
+FROM docker.io/library/ubuntu:22.04 as base
+ENV LLVM_SYSROOT=/opt/llvm
+
+FROM base as stage1-toolchain
+ENV LLVM_VERSION=19.1.5
+
+RUN apt-get update && \
+    apt-get install -y \
+    wget \
+    gcc \
+    g++ \
+    cmake \
+    ninja-build \
+    python3 \
+    git \
+    curl \
+    zlib1g-dev
+
+RUN curl -O -L https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-$LLVM_VERSION.tar.gz && tar -xf llvmorg-$LLVM_VERSION.tar.gz
+
+WORKDIR /llvm-project-llvmorg-$LLVM_VERSION
+
+# Patch to enable better PGO profile data.
+# TODO: Remove this for llvm 20
+ADD https://github.com/llvm/llvm-project/commit/738250989ce516f02f809bdfde474a039c77e81f.patch .
+
+RUN patch -p1 < 738250989ce516f02f809bdfde474a039c77e81f.patch
+
+RUN cmake -B ./build -G Ninja ./llvm \
+  -C ./clang/cmake/caches/BOLT-PGO.cmake \
+  -DBOOTSTRAP_LLVM_ENABLE_LLD=ON \
+  -DBOOTSTRAP_BOOTSTRAP_LLVM_ENABLE_LLD=ON \
+  -DPGO_INSTRUMENT_LTO=Thin \
+  -DLLVM_ENABLE_RUNTIMES="compiler-rt" \
+  -DCMAKE_INSTALL_PREFIX="$LLVM_SYSROOT" \
+  -DLLVM_ENABLE_PROJECTS="bolt;clang;lld;clang-tools-extra" \
+  -DLLVM_DISTRIBUTION_COMPONENTS="lld;compiler-rt;clang-format;scan-build" \
+  -DCLANG_DEFAULT_LINKER="lld"
+
+RUN ninja -C ./build stage2-clang-bolt stage2-install-distribution && ninja -C ./build install-distribution
+
+FROM base as ci-container
+    
+COPY --from=stage1-toolchain $LLVM_SYSROOT $LLVM_SYSROOT
+
+# Need to install curl for hendrikmuhs/ccache-action
+# Need nodejs for some of the GitHub actions.
+# Need perl-modules for clang analyzer tests.
+# Need git for SPIRV-Tools tests.
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y \
+    binutils \ 
+    cmake \
+    curl \
+    git \
+    libstdc++-11-dev \
+    ninja-build \
+    nodejs \
+    perl-modules \
+    python3-psutil \
+    sudo \
+
+    # These are needed by the premerge pipeline. Pip is used to install
+    # dependent python packages and ccache is used for build caching. File and
+    # tzdata are used for tests.
+    python3-pip \
+    ccache \
+    file \
+    tzdata
+
+# Install sccache as it is needed by most of the project test workflows and
+# cannot be installed by the ccache action when executing as a non-root user.
+# TODO(boomanaiden154): This should be switched to being installed with apt
+# once we bump to Ubuntu 24.04.
+RUN curl -L 'https://github.com/mozilla/sccache/releases/download/v0.7.6/sccache-v0.7.6-x86_64-unknown-linux-musl.tar.gz' > /tmp/sccache.tar.gz && \
+    echo "2902a5e44c3342132f07b62e70cca75d9b23252922faf3b924f449808cc1ae58 /tmp/sccache.tar.gz" | sha256sum -c && \
+    tar xzf /tmp/sccache.tar.gz -O --wildcards '*/sccache' > '/usr/local/bin/sccache' && \
+    rm /tmp/sccache.tar.gz && \
+    chmod +x /usr/local/bin/sccache
+
+ENV LLVM_SYSROOT=$LLVM_SYSROOT
+ENV PATH=${LLVM_SYSROOT}/bin:${PATH}
+
+# Create a new user to avoid test failures related to a lack of expected
+# permissions issues in some tests. Set the user id to 1001 as that is the
+# user id that Github Actions uses to perform the checkout action.
+RUN useradd gha -u 1001 -m -s /bin/bash
+
+# Also add the user to passwordless sudoers so that we can install software
+# later on without having to rebuild the container.
+RUN adduser gha sudo
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+
+USER gha
+WORKDIR /home/gha
+
+FROM ci-container as ci-container-agent
+
+ENV GITHUB_RUNNER_VERSION=2.322.0
+
+RUN mkdir actions-runner && \
+    cd actions-runner && \
+    curl -O -L https://github.com/actions/runner/releases/download/v$GITHUB_RUNNER_VERSION/actions-runner-linux-x64-$GITHUB_RUNNER_VERSION.tar.gz && \
+    tar xzf ./actions-runner-linux-x64-$GITHUB_RUNNER_VERSION.tar.gz && \
+    rm ./actions-runner-linux-x64-$GITHUB_RUNNER_VERSION.tar.gz
+