Merge branch 'main' into typetracker-decorators

Author: RasmusWL

.devcontainer/swift/Dockerfile

@@ -0,0 +1,9 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.236.0/containers/cpp/.devcontainer/base.Dockerfile
+
+# [Choice] Debian / Ubuntu version (use Debian 11, Ubuntu 18.04/22.04 on local arm64/Apple Silicon): debian-11, debian-10, ubuntu-22.04, ubuntu-20.04, ubuntu-18.04
+FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-ubuntu-22.04
+
+USER root
+ADD root.sh /tmp/root.sh
+ADD update-codeql.sh /usr/local/bin/update-codeql
+RUN bash /tmp/root.sh && rm /tmp/root.sh

.devcontainer/swift/devcontainer.json

@@ -0,0 +1,25 @@
+{
+	"extensions": [
+		"github.vscode-codeql",
+		"hbenl.vscode-test-explorer",
+		"ms-vscode.test-adapter-converter",
+		"slevesque.vscode-zipexplorer",
+		"ms-vscode.cpptools"
+	],
+	"settings": {
+		"files.watcherExclude": {
+			"**/target/**": true
+		},
+		"codeQL.runningQueries.memory": 2048
+	},
+	"build": {
+		"dockerfile": "Dockerfile",
+	},
+	"runArgs": [
+		"--cap-add=SYS_PTRACE",
+		"--security-opt",
+		"seccomp=unconfined"
+	],
+	"remoteUser": "vscode",
+	"onCreateCommand": ".devcontainer/swift/user.sh"
+}

.devcontainer/swift/root.sh

@@ -0,0 +1,22 @@
+set -xe
+
+BAZELISK_VERSION=v1.12.0
+BAZELISK_DOWNLOAD_SHA=6b0bcb2ea15bca16fffabe6fda75803440375354c085480fe361d2cbf32501db
+
+apt-get update
+export DEBIAN_FRONTEND=noninteractive
+apt-get -y install --no-install-recommends \
+    zlib1g-dev \
+    uuid-dev \
+    python3-distutils \
+    python3-pip \
+    bash-completion
+
+# Install Bazel
+curl -fSsL -o /usr/local/bin/bazelisk https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-linux-amd64
+echo "${BAZELISK_DOWNLOAD_SHA} */usr/local/bin/bazelisk" | sha256sum --check -
+chmod 0755 /usr/local/bin/bazelisk
+ln -s bazelisk /usr/local/bin/bazel
+
+# install latest codeql
+update-codeql

.devcontainer/swift/update-codeql.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -e
+
+URL=https://github.com/github/codeql-cli-binaries/releases
+LATEST_VERSION=$(curl -L -s -H 'Accept: application/json' $URL/latest | sed -e 's/.*"tag_name":"\([^"]*\)".*/\1/')
+CURRENT_VERSION=v$(codeql version 2>/dev/null | sed -ne 's/.*release \([0-9.]*\)\./\1/p')
+if [[ $CURRENT_VERSION != $LATEST_VERSION ]]; then
+  if [[ $UID != 0 ]]; then
+    echo "update required, please run this script with sudo:"
+    echo "  sudo $0"
+    exit 1
+  fi
+  ZIP=$(mktemp codeql.XXXX.zip)
+  curl -fSqL -o $ZIP $URL/download/$LATEST_VERSION/codeql-linux64.zip
+  unzip -q $ZIP -d /opt
+  rm $ZIP
+  ln -sf /opt/codeql/codeql /usr/local/bin/codeql
+  echo installed version $LATEST_VERSION
+else
+  echo current version $CURRENT_VERSION is up-to-date
+fi

.devcontainer/swift/user.sh

@@ -0,0 +1,13 @@
+set -xe
+
+# add the workspace to the codeql search path
+mkdir -p /home/vscode/.config/codeql
+echo "--search-path /workspaces/codeql" > /home/vscode/.config/codeql/config
+
+# create a swift extractor pack with the current state
+cd /workspaces/codeql
+bazel run swift/create-extractor-pack
+
+#install and set up pre-commit
+python3 -m pip install pre-commit --no-warn-script-location
+$HOME/.local/bin/pre-commit install

.github/actions/fetch-codeql/action.yml

@@ -3,22 +3,12 @@ description: Fetches the latest version of CodeQL
 runs:
   using: composite
   steps:
-    - name: Select platform - Linux
-      if: runner.os == 'Linux'
-      shell: bash
-      run: echo "GA_CODEQL_CLI_PLATFORM=linux64" >> $GITHUB_ENV
-
-    - name: Select platform - MacOS
-      if: runner.os == 'MacOS'
-      shell: bash
-      run: echo "GA_CODEQL_CLI_PLATFORM=osx64" >> $GITHUB_ENV
-
     - name: Fetch CodeQL
       shell: bash
       run: |
-        LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
-        gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-$GA_CODEQL_CLI_PLATFORM.zip "$LATEST"
-        unzip -q -d "${RUNNER_TEMP}" codeql-$GA_CODEQL_CLI_PLATFORM.zip
-        echo "${RUNNER_TEMP}/codeql" >> "${GITHUB_PATH}"
+        gh extension install github/gh-codeql
+        gh codeql set-channel nightly
+        gh codeql version
+        gh codeql version --format=json | jq -r .unpackedLocation >> "${GITHUB_PATH}"
       env:
         GITHUB_TOKEN: ${{ github.token }}

.github/workflows/check-change-note.yml

@@ -10,6 +10,7 @@ on:
       - "*/ql/lib/**/*.qll"
       - "!**/experimental/**"
       - "!ql/**"
+      - "!swift/**"
       - ".github/workflows/check-change-note.yml"
 
 jobs:

.github/workflows/check-qldoc.yml

@@ -5,6 +5,7 @@ on:
     paths:
       - "*/ql/lib/**"
       - .github/workflows/check-qldoc.yml
+      - .github/actions/fetch-codeql/action.yml
     branches:
       - main
       - "rc/*"
@@ -14,18 +15,13 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Install CodeQL
-        run: |
-          gh extension install github/gh-codeql
-          gh codeql set-channel nightly
-          gh codeql version
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-
       - uses: actions/checkout@v3
         with:
           fetch-depth: 2
 
+      - name: Install CodeQL
+        uses: ./.github/actions/fetch-codeql
+
       - name: Check QLdoc coverage
         shell: bash
         run: |
@@ -34,15 +30,15 @@ jobs:
           changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -Po '^(?!swift)[a-z]*/ql/lib' || true; } | sort -u)"
           for pack_dir in ${changed_lib_packs}; do
             lang="${pack_dir%/ql/lib}"
-            gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"
+            codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"
           done
           git checkout HEAD^
           for pack_dir in ${changed_lib_packs}; do
             # When we add a new language, pack_dir would not exist in HEAD^.
             # In this case the right thing to do is to skip the check.
             [[ ! -d "${pack_dir}" ]] && continue
             lang="${pack_dir%/ql/lib}"
-            gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-baseline.txt" --dir="${pack_dir}"
+            codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-baseline.txt" --dir="${pack_dir}"
             awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-current.txt" | sort -u > "${RUNNER_TEMP}/current-undocumented.txt"
             awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-baseline.txt" | sort -u > "${RUNNER_TEMP}/baseline-undocumented.txt"
             UNDOCUMENTED="$(grep -f <(comm -13 "${RUNNER_TEMP}/baseline-undocumented.txt" "${RUNNER_TEMP}/current-undocumented.txt") "${RUNNER_TEMP}/${lang}-current.txt" || true)"

.github/workflows/csv-coverage-metrics.yml

@@ -12,6 +12,7 @@ on:
       - main
     paths:
       - ".github/workflows/csv-coverage-metrics.yml"
+      - ".github/actions/fetch-codeql/action.yml"
 
 jobs:
   publish-java:

.github/workflows/csv-coverage-pr-artifacts.yml

@@ -3,18 +3,20 @@ name: Check framework coverage changes
 on:
   pull_request:
     paths:
-      - '.github/workflows/csv-coverage-pr-comment.yml'
-      - '*/ql/src/**/*.ql'
-      - '*/ql/src/**/*.qll'
-      - '*/ql/lib/**/*.ql'
-      - '*/ql/lib/**/*.qll'
-      - 'misc/scripts/library-coverage/*.py'
+      - ".github/workflows/csv-coverage-pr-comment.yml"
+      - ".github/workflows/csv-coverage-pr-artifacts.yml"
+      - ".github/actions/fetch-codeql/action.yml"
+      - "*/ql/src/**/*.ql"
+      - "*/ql/src/**/*.qll"
+      - "*/ql/lib/**/*.ql"
+      - "*/ql/lib/**/*.qll"
+      - "misc/scripts/library-coverage/*.py"
       # input data files
-      - '*/documentation/library-coverage/cwe-sink.csv'
-      - '*/documentation/library-coverage/frameworks.csv'
+      - "*/documentation/library-coverage/cwe-sink.csv"
+      - "*/documentation/library-coverage/frameworks.csv"
     branches:
       - main
-      - 'rc/*'
+      - "rc/*"
 
 jobs:
   generate:
@@ -23,77 +25,72 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - name: Dump GitHub context
-      env:
-        GITHUB_CONTEXT: ${{ toJSON(github.event) }}
-      run: echo "$GITHUB_CONTEXT"
-    - name: Clone self (github/codeql) - MERGE
-      uses: actions/checkout@v3
-      with:
-        path: merge
-    - name: Clone self (github/codeql) - BASE
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 2
-        path: base
-    - run: |
-        git checkout HEAD^1
-        git log -1 --format='%H'
-      working-directory: base
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.8
-    - name: Download CodeQL CLI
-      env:
-         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
-    - name: Unzip CodeQL CLI
-      run: unzip -d codeql-cli codeql-linux64.zip
-    - name: Generate CSV files on merge commit of the PR
-      run: |
-        echo "Running generator on merge"
-        PATH="$PATH:codeql-cli/codeql" python merge/misc/scripts/library-coverage/generate-report.py ci merge merge
-        mkdir out_merge
-        cp framework-coverage-*.csv out_merge/
-        cp framework-coverage-*.rst out_merge/
-    - name: Generate CSV files on base commit of the PR
-      run: |
-        echo "Running generator on base"
-        PATH="$PATH:codeql-cli/codeql" python base/misc/scripts/library-coverage/generate-report.py ci base base
-        mkdir out_base
-        cp framework-coverage-*.csv out_base/
-        cp framework-coverage-*.rst out_base/
-    - name: Generate diff of coverage reports
-      run: |
-        python base/misc/scripts/library-coverage/compare-folders.py out_base out_merge comparison.md
-    - name: Upload CSV package list
-      uses: actions/upload-artifact@v3
-      with:
-        name: csv-framework-coverage-merge
-        path: |
-          out_merge/framework-coverage-*.csv
-          out_merge/framework-coverage-*.rst
-    - name: Upload CSV package list
-      uses: actions/upload-artifact@v3
-      with:
-        name: csv-framework-coverage-base
-        path: |
-          out_base/framework-coverage-*.csv
-          out_base/framework-coverage-*.rst
-    - name: Upload comparison results
-      uses: actions/upload-artifact@v3
-      with:
-        name: comparison
-        path: |
-          comparison.md
-    - name: Save PR number
-      run: |
-        mkdir -p pr
-        echo ${{ github.event.pull_request.number }} > pr/NR
-    - name: Upload PR number
-      uses: actions/upload-artifact@v3
-      with:
-        name: pr
-        path: pr/
+      - name: Dump GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJSON(github.event) }}
+        run: echo "$GITHUB_CONTEXT"
+      - name: Clone self (github/codeql) - MERGE
+        uses: actions/checkout@v3
+        with:
+          path: merge
+      - name: Clone self (github/codeql) - BASE
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+          path: base
+      - run: |
+          git checkout HEAD^1
+          git log -1 --format='%H'
+        working-directory: base
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Download CodeQL CLI
+        uses: ./merge/.github/actions/fetch-codeql
+      - name: Generate CSV files on merge commit of the PR
+        run: |
+          echo "Running generator on merge"
+          python merge/misc/scripts/library-coverage/generate-report.py ci merge merge
+          mkdir out_merge
+          cp framework-coverage-*.csv out_merge/
+          cp framework-coverage-*.rst out_merge/
+      - name: Generate CSV files on base commit of the PR
+        run: |
+          echo "Running generator on base"
+          python base/misc/scripts/library-coverage/generate-report.py ci base base
+          mkdir out_base
+          cp framework-coverage-*.csv out_base/
+          cp framework-coverage-*.rst out_base/
+      - name: Generate diff of coverage reports
+        run: |
+          python base/misc/scripts/library-coverage/compare-folders.py out_base out_merge comparison.md
+      - name: Upload CSV package list
+        uses: actions/upload-artifact@v3
+        with:
+          name: csv-framework-coverage-merge
+          path: |
+            out_merge/framework-coverage-*.csv
+            out_merge/framework-coverage-*.rst
+      - name: Upload CSV package list
+        uses: actions/upload-artifact@v3
+        with:
+          name: csv-framework-coverage-base
+          path: |
+            out_base/framework-coverage-*.csv
+            out_base/framework-coverage-*.rst
+      - name: Upload comparison results
+        uses: actions/upload-artifact@v3
+        with:
+          name: comparison
+          path: |
+            comparison.md
+      - name: Save PR number
+        run: |
+          mkdir -p pr
+          echo ${{ github.event.pull_request.number }} > pr/NR
+      - name: Upload PR number
+        uses: actions/upload-artifact@v3
+        with:
+          name: pr
+          path: pr/