Skip to content

Commit 9440a45

Browse files
hvitvedhvitved
authored
Merge branch 'main' into post-release-prep/codeql-cli-2.8.0
2 parents b4ab86c + 3206384 commit 9440a45

File tree

767 files changed

+224204
-41394
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

767 files changed

+224204
-41394
lines changed

‎.codeqlmanifest.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
"*/ql/examples/qlpack.yml",
77
"cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
88
"javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
9+
"javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml",
910
"javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
1011
"csharp/ql/campaigns/Solorigate/lib/qlpack.yml",
1112
"csharp/ql/campaigns/Solorigate/src/qlpack.yml",

‎.gitattributes

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,9 @@
5050
*.pdb -text
5151

5252
java/ql/test/stubs/**/*.java linguist-generated=true
53-
java/ql/test/experimental/stubs/**/*.java linguist-generated=true
53+
java/ql/test/experimental/stubs/**/*.java linguist-generated=true
54+
55+
# Generated test files - these are synced from the standard JavaScript libraries using
56+
# `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
57+
javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
58+
javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge

‎.github/workflows/codeql-analysis.yml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@ jobs:
2727
pull-requests: read
2828

2929
steps:
30+
- name: Setup dotnet
31+
uses: actions/setup-dotnet@v1
32+
with:
33+
dotnet-version: 6.0.101
34+
3035
- name: Checkout repository
3136
uses: actions/checkout@v2
3237

@@ -51,7 +56,7 @@ jobs:
5156
# uses a compiled language
5257

5358
- run: |
54-
dotnet build csharp
59+
dotnet build csharp /p:UseSharedCompilation=false
5560
5661
- name: Perform CodeQL Analysis
5762
uses: github/codeql-action/analyze@main

‎.github/workflows/csv-coverage-metrics.yml

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
name: "Publish framework coverage as metrics"
2+
3+
on:
4+
schedule:
5+
- cron: '5 0 * * *'
6+
push:
7+
branches:
8+
- main
9+
workflow_dispatch:
10+
pull_request:
11+
branches:
12+
- main
13+
paths:
14+
- ".github/workflows/csv-coverage-metrics.yml"
15+
16+
jobs:
17+
publish:
18+
runs-on: ubuntu-latest
19+
steps:
20+
- name: Checkout repository
21+
uses: actions/checkout@v2
22+
- name: Setup CodeQL
23+
uses: ./.github/actions/fetch-codeql
24+
- name: Create empty database
25+
run: |
26+
DATABASE="${{ runner.temp }}/java-database"
27+
PROJECT="${{ runner.temp }}/java-project"
28+
mkdir -p "$PROJECT/src/tmp/empty"
29+
echo "class Empty {}" >> "$PROJECT/src/tmp/empty/Empty.java"
30+
codeql database create "$DATABASE" --language=java --source-root="$PROJECT" --command 'javac src/tmp/empty/Empty.java'
31+
- name: Capture coverage information
32+
run: |
33+
DATABASE="${{ runner.temp }}/java-database"
34+
codeql database analyze --format=sarif-latest --output=metrics.sarif -- "$DATABASE" ./java/ql/src/Metrics/Summaries/FrameworkCoverage.ql
35+
- uses: actions/upload-artifact@v2
36+
with:
37+
name: metrics.sarif
38+
path: metrics.sarif
39+
retention-days: 20
40+
- name: Upload SARIF file
41+
uses: github/codeql-action/upload-sarif@v1
42+
with:
43+
sarif_file: metrics.sarif

‎.github/workflows/js-ml-tests.yml

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
name: JS ML-powered queries tests
2+
3+
on:
4+
push:
5+
paths:
6+
- "javascript/ql/experimental/adaptivethreatmodeling/**"
7+
- .github/workflows/js-ml-tests.yml
8+
branches:
9+
- main
10+
- "rc/*"
11+
pull_request:
12+
paths:
13+
- "javascript/ql/experimental/adaptivethreatmodeling/**"
14+
- .github/workflows/js-ml-tests.yml
15+
16+
defaults:
17+
run:
18+
working-directory: javascript/ql/experimental/adaptivethreatmodeling
19+
20+
jobs:
21+
qlformat:
22+
name: Check QL formatting
23+
runs-on: ubuntu-latest
24+
steps:
25+
- uses: actions/checkout@v2
26+
27+
- uses: ./.github/actions/fetch-codeql
28+
29+
- name: Check QL formatting
30+
run: |
31+
find . "(" -name "*.ql" -or -name "*.qll" ")" -print0 | \
32+
xargs -0 codeql query format --check-only
33+
34+
qlcompile:
35+
name: Check QL compilation
36+
runs-on: ubuntu-latest
37+
steps:
38+
- uses: actions/checkout@v2
39+
40+
- uses: ./.github/actions/fetch-codeql
41+
42+
- name: Check QL compilation
43+
run: |
44+
codeql query compile \
45+
--check-only \
46+
--ram 5120 \
47+
--additional-packs "${{ github.workspace }}" \
48+
--threads=0 \
49+
-- \
50+
lib modelbuilding src
51+
52+
qltest:
53+
name: Run QL tests
54+
runs-on: ubuntu-latest
55+
steps:
56+
- uses: actions/checkout@v2
57+
58+
- uses: ./.github/actions/fetch-codeql
59+
60+
- name: Run QL tests
61+
run: |
62+
codeql test run \
63+
--threads=0 \
64+
--ram 5120 \
65+
--additional-packs "${{ github.workspace }}" \
66+
-- \
67+
test

‎.github/workflows/mad_modelDiff.yml

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
name: Models as Data - Diff
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
projects:
7+
description: "The projects to generate models for"
8+
required: true
9+
default: '["netty/netty"]'
10+
pull_request:
11+
branches:
12+
- main
13+
paths:
14+
- "java/ql/src/utils/model-generator/**/*.*"
15+
- ".github/workflows/mad_modelDiff.yml"
16+
17+
permissions:
18+
contents: read
19+
20+
jobs:
21+
model-diff:
22+
name: Model Difference
23+
runs-on: ubuntu-latest
24+
if: github.repository == 'github/codeql'
25+
strategy:
26+
matrix:
27+
slug: ${{fromJson(github.event.inputs.projects || '["apache/commons-codec", "apache/commons-io", "apache/commons-beanutils", "apache/commons-logging", "apache/commons-fileupload", "apache/commons-lang", "apache/commons-validator", "apache/commons-csv", "apache/dubbo"]' )}}
28+
steps:
29+
- name: Clone github/codeql from PR
30+
uses: actions/checkout@v2
31+
if: github.event.pull_request
32+
with:
33+
path: codeql-pr
34+
- name: Clone github/codeql from main
35+
uses: actions/checkout@v2
36+
with:
37+
path: codeql-main
38+
ref: main
39+
- uses: ./codeql-main/.github/actions/fetch-codeql
40+
- name: Download database
41+
env:
42+
SLUG: ${{ matrix.slug }}
43+
run: |
44+
set -x
45+
mkdir lib-dbs
46+
SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
47+
projectId=`curl -s https://lgtm.com/api/v1.0/projects/g/${SLUG} | jq .id`
48+
curl -L "https://lgtm.com/api/v1.0/snapshots/$projectId/java" -o "$SHORTNAME.zip"
49+
unzip -q -d "${SHORTNAME}-db" "${SHORTNAME}.zip"
50+
mkdir "lib-dbs/$SHORTNAME/"
51+
mv "${SHORTNAME}-db/"$(ls -1 "${SHORTNAME}"-db)/* "lib-dbs/${SHORTNAME}/"
52+
- name: Generate Models (PR and main)
53+
run: |
54+
set -x
55+
mkdir tmp-models
56+
MODELS=`pwd`/tmp-models
57+
DATABASES=`pwd`/lib-dbs
58+
59+
analyzeDatabaseWithCheckout() {
60+
QL_VARIANT=$1
61+
DATABASE=$2
62+
cd codeql-$QL_VARIANT
63+
SHORTNAME=`basename $DATABASE`
64+
python java/ql/src/utils/model-generator/GenerateFlowModel.py $DATABASE $MODELS/${SHORTNAME}.qll
65+
mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.qll
66+
cd ..
67+
}
68+
69+
for d in $DATABASES/*/ ; do
70+
ls -1 "$d"
71+
72+
analyzeDatabaseWithCheckout "main" $d
73+
if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]
74+
then
75+
analyzeDatabaseWithCheckout "pr" $d
76+
fi
77+
done
78+
- name: Install diff2html
79+
if: github.event.pull_request
80+
run: |
81+
npm install -g diff2html-cli
82+
- name: Generate Model Diff
83+
if: github.event.pull_request
84+
run: |
85+
set -x
86+
MODELS=`pwd`/tmp-models
87+
ls -1 tmp-models/
88+
for m in $MODELS/*_main.qll ; do
89+
t="${m/main/"pr"}"
90+
basename=`basename $m`
91+
name="diff_${basename/_main.qll/""}"
92+
(diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
93+
done
94+
- uses: actions/upload-artifact@v2
95+
with:
96+
name: models
97+
path: tmp-models/*.qll
98+
retention-days: 20
99+
- uses: actions/upload-artifact@v2
100+
with:
101+
name: diffs
102+
path: tmp-models/*.html
103+
retention-days: 20

‎.github/workflows/mad_regenerate-models.yml

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
name: Regenerate framework models
2+
3+
on:
4+
workflow_dispatch:
5+
schedule:
6+
- cron: "30 2 * * *"
7+
pull_request:
8+
branches:
9+
- main
10+
paths:
11+
- ".github/workflows/mad_regenerate-models.yml"
12+
13+
jobs:
14+
regenerate-models:
15+
runs-on: ubuntu-latest
16+
strategy:
17+
matrix:
18+
# placeholder required for each axis, excluded below, replaced by the actual combinations (see include)
19+
slug: ["placeholder"]
20+
ref: ["placeholder"]
21+
include:
22+
- slug: "apache/commons-io"
23+
ref: "8985de8fe74f6622a419b37a6eed0dbc484dc128"
24+
exclude:
25+
- slug: "placeholder"
26+
ref: "placeholder"
27+
steps:
28+
- name: Clone self (github/codeql)
29+
uses: actions/checkout@v2
30+
- name: Setup CodeQL binaries
31+
uses: ./.github/actions/fetch-codeql
32+
- name: Clone repositories
33+
uses: actions/checkout@v2
34+
with:
35+
path: repos/${{ matrix.ref }}
36+
ref: ${{ matrix.ref }}
37+
repository: ${{ matrix.slug }}
38+
- name: Build database
39+
env:
40+
SLUG: ${{ matrix.slug }}
41+
REF: ${{ matrix.ref }}
42+
run: |
43+
mkdir dbs
44+
cd repos/${REF}
45+
SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
46+
codeql database create --language=java ../../dbs/${SHORTNAME}
47+
- name: Regenerate models in-place
48+
env:
49+
SLUG: ${{ matrix.slug }}
50+
run: |
51+
SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
52+
java/ql/src/utils/model-generator/RegenerateModels.py "${SLUG}" dbs/${SHORTNAME}
53+
- name: Stage changes
54+
run: |
55+
find java -name "*.qll" -print0 | xargs -0 git add
56+
git status
57+
git diff --cached > models.patch
58+
- uses: actions/upload-artifact@v2
59+
with:
60+
name: patch
61+
path: models.patch
62+
retention-days: 7

‎.github/workflows/ql-for-ql-build.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,4 +189,11 @@ jobs:
189189
uses: github/codeql-action/analyze@erik-krogh/ql
190190
with:
191191
category: "ql-for-ql-${{ matrix.folder }}"
192+
- name: Copy sarif file to CWD
193+
run: cp ../results/ql.sarif ./${{ matrix.folder }}.sarif
194+
- name: Sarif as artifact
195+
uses: actions/upload-artifact@v2
196+
with:
197+
name: ${{ matrix.folder }}.sarif
198+
path: ${{ matrix.folder }}.sarif
192199

‎.github/workflows/ql-for-ql-dataset_measure.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
1818
strategy:
1919
matrix:
20-
repo:
20+
repo:
2121
- github/codeql
2222
- github/codeql-go
2323
runs-on: ubuntu-latest
@@ -35,7 +35,7 @@ jobs:
3535
~/.cargo/registry
3636
~/.cargo/git
3737
ql/target
38-
key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
38+
key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
3939
- name: Build Extractor
4040
run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./create-extractor-pack.sh
4141
env:

‎.github/workflows/ql-for-ql-tests.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,24 +29,24 @@ jobs:
2929
~/.cargo/registry
3030
~/.cargo/git
3131
ql/target
32-
key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
32+
key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
3333
- name: Build extractor
3434
run: |
3535
cd ql;
3636
codeqlpath=$(dirname ${{ steps.find-codeql.outputs.codeql-path }});
3737
env "PATH=$PATH:$codeqlpath" ./create-extractor-pack.sh
3838
- name: Run QL tests
39-
run: |
39+
run: |
4040
"${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries ql/ql/test
4141
env:
4242
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
4343
- name: Check QL formatting
44-
run: |
44+
run: |
4545
find ql/ql "(" -name "*.ql" -or -name "*.qll" ")" -print0 | xargs -0 "${CODEQL}" query format --check-only
4646
env:
4747
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
4848
- name: Check QL compilation
49-
run: |
49+
run: |
5050
"${CODEQL}" query compile --check-only --threads=4 --warnings=error --search-path "${{ github.workspace }}/ql/extractor-pack" "ql/ql/src" "ql/ql/examples"
5151
env:
5252
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}

0 commit comments

Comments
 (0)