Skip to content

Commit dd383f9

Browse files
committed
Merge remote-tracking branch 'origin/main' into ruby/clear-text-logging
2 parents 33f4fff + a448db1 commit dd383f9

File tree

1,113 files changed

+253167
-44674
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,113 files changed

+253167
-44674
lines changed

.codeqlmanifest.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,17 @@
44
"*/ql/lib/qlpack.yml",
55
"*/ql/test/qlpack.yml",
66
"*/ql/examples/qlpack.yml",
7+
"*/ql/consistency-queries/qlpack.yml",
78
"cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
89
"javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
10+
"javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml",
911
"javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
1012
"csharp/ql/campaigns/Solorigate/lib/qlpack.yml",
1113
"csharp/ql/campaigns/Solorigate/src/qlpack.yml",
1214
"csharp/ql/campaigns/Solorigate/test/qlpack.yml",
1315
"misc/legacy-support/*/qlpack.yml",
1416
"misc/suite-helpers/qlpack.yml",
1517
"ruby/extractor-pack/codeql-extractor.yml",
16-
"ruby/ql/consistency-queries/qlpack.yml",
17-
"ql/ql/consistency-queries/qlpack.yml",
1818
"ql/extractor-pack/codeql-extractor.yml"
1919
],
2020
"versionPolicies": {

.gitattributes

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,15 @@
5050
*.pdb -text
5151

5252
java/ql/test/stubs/**/*.java linguist-generated=true
53-
java/ql/test/experimental/stubs/**/*.java linguist-generated=true
53+
java/ql/test/experimental/stubs/**/*.java linguist-generated=true
54+
55+
# For some languages, upgrade script testing references really old dbscheme
56+
# files from legacy upgrades that have CRLF line endings. Since upgrade
57+
# resolution relies on object hashes, we must suppress line ending conversion
58+
# for those testing dbscheme files.
59+
*/ql/lib/upgrades/initial/*.dbscheme -text
60+
61+
# Generated test files - these are synced from the standard JavaScript libraries using
62+
# `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
63+
javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
64+
javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge

.github/workflows/check-change-note.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,11 @@ on:
66
paths:
77
- "*/ql/src/**/*.ql"
88
- "*/ql/src/**/*.qll"
9+
- "*/ql/lib/**/*.ql"
10+
- "*/ql/lib/**/*.qll"
911
- "!**/experimental/**"
1012
- "!ql/**"
13+
- ".github/workflows/check-change-note.yml"
1114

1215
jobs:
1316
check-change-note:

.github/workflows/codeql-analysis.yml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@ jobs:
2727
pull-requests: read
2828

2929
steps:
30+
- name: Setup dotnet
31+
uses: actions/setup-dotnet@v1
32+
with:
33+
dotnet-version: 6.0.101
34+
3035
- name: Checkout repository
3136
uses: actions/checkout@v2
3237

@@ -51,7 +56,7 @@ jobs:
5156
# uses a compiled language
5257

5358
- run: |
54-
dotnet build csharp
59+
dotnet build csharp /p:UseSharedCompilation=false
5560
5661
- name: Perform CodeQL Analysis
5762
uses: github/codeql-action/analyze@main
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
name: "Publish framework coverage as metrics"
2+
3+
on:
4+
schedule:
5+
- cron: '5 0 * * *'
6+
push:
7+
branches:
8+
- main
9+
workflow_dispatch:
10+
pull_request:
11+
branches:
12+
- main
13+
paths:
14+
- ".github/workflows/csv-coverage-metrics.yml"
15+
16+
jobs:
17+
publish:
18+
runs-on: ubuntu-latest
19+
steps:
20+
- name: Checkout repository
21+
uses: actions/checkout@v2
22+
- name: Setup CodeQL
23+
uses: ./.github/actions/fetch-codeql
24+
- name: Create empty database
25+
run: |
26+
DATABASE="${{ runner.temp }}/java-database"
27+
PROJECT="${{ runner.temp }}/java-project"
28+
mkdir -p "$PROJECT/src/tmp/empty"
29+
echo "class Empty {}" >> "$PROJECT/src/tmp/empty/Empty.java"
30+
codeql database create "$DATABASE" --language=java --source-root="$PROJECT" --command 'javac src/tmp/empty/Empty.java'
31+
- name: Capture coverage information
32+
run: |
33+
DATABASE="${{ runner.temp }}/java-database"
34+
codeql database analyze --format=sarif-latest --output=metrics.sarif -- "$DATABASE" ./java/ql/src/Metrics/Summaries/FrameworkCoverage.ql
35+
- uses: actions/upload-artifact@v2
36+
with:
37+
name: metrics.sarif
38+
path: metrics.sarif
39+
retention-days: 20
40+
- name: Upload SARIF file
41+
uses: github/codeql-action/upload-sarif@v1
42+
with:
43+
sarif_file: metrics.sarif

.github/workflows/js-ml-tests.yml

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
name: JS ML-powered queries tests
2+
3+
on:
4+
push:
5+
paths:
6+
- "javascript/ql/experimental/adaptivethreatmodeling/**"
7+
- .github/workflows/js-ml-tests.yml
8+
branches:
9+
- main
10+
- "rc/*"
11+
pull_request:
12+
paths:
13+
- "javascript/ql/experimental/adaptivethreatmodeling/**"
14+
- .github/workflows/js-ml-tests.yml
15+
16+
defaults:
17+
run:
18+
working-directory: javascript/ql/experimental/adaptivethreatmodeling
19+
20+
jobs:
21+
qlformat:
22+
name: Check QL formatting
23+
runs-on: ubuntu-latest
24+
steps:
25+
- uses: actions/checkout@v2
26+
27+
- uses: ./.github/actions/fetch-codeql
28+
29+
- name: Check QL formatting
30+
run: |
31+
find . "(" -name "*.ql" -or -name "*.qll" ")" -print0 | \
32+
xargs -0 codeql query format --check-only
33+
34+
qlcompile:
35+
name: Check QL compilation
36+
runs-on: ubuntu-latest
37+
steps:
38+
- uses: actions/checkout@v2
39+
40+
- uses: ./.github/actions/fetch-codeql
41+
42+
- name: Check QL compilation
43+
run: |
44+
codeql query compile \
45+
--check-only \
46+
--ram 5120 \
47+
--additional-packs "${{ github.workspace }}" \
48+
--threads=0 \
49+
-- \
50+
lib modelbuilding src
51+
52+
qltest:
53+
name: Run QL tests
54+
runs-on: ubuntu-latest
55+
steps:
56+
- uses: actions/checkout@v2
57+
58+
- uses: ./.github/actions/fetch-codeql
59+
60+
- name: Run QL tests
61+
run: |
62+
codeql test run \
63+
--threads=0 \
64+
--ram 5120 \
65+
--additional-packs "${{ github.workspace }}" \
66+
-- \
67+
test

.github/workflows/mad_modelDiff.yml

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
name: Models as Data - Diff
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
projects:
7+
description: "The projects to generate models for"
8+
required: true
9+
default: '["netty/netty"]'
10+
pull_request:
11+
branches:
12+
- main
13+
paths:
14+
- "java/ql/src/utils/model-generator/**/*.*"
15+
- ".github/workflows/mad_modelDiff.yml"
16+
17+
permissions:
18+
contents: read
19+
20+
jobs:
21+
model-diff:
22+
name: Model Difference
23+
runs-on: ubuntu-latest
24+
if: github.repository == 'github/codeql'
25+
strategy:
26+
matrix:
27+
slug: ${{fromJson(github.event.inputs.projects || '["apache/commons-codec", "apache/commons-io", "apache/commons-beanutils", "apache/commons-logging", "apache/commons-fileupload", "apache/commons-lang", "apache/commons-validator", "apache/commons-csv", "apache/dubbo"]' )}}
28+
steps:
29+
- name: Clone github/codeql from PR
30+
uses: actions/checkout@v2
31+
if: github.event.pull_request
32+
with:
33+
path: codeql-pr
34+
- name: Clone github/codeql from main
35+
uses: actions/checkout@v2
36+
with:
37+
path: codeql-main
38+
ref: main
39+
- uses: ./codeql-main/.github/actions/fetch-codeql
40+
- name: Download database
41+
env:
42+
SLUG: ${{ matrix.slug }}
43+
run: |
44+
set -x
45+
mkdir lib-dbs
46+
SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
47+
projectId=`curl -s https://lgtm.com/api/v1.0/projects/g/${SLUG} | jq .id`
48+
curl -L "https://lgtm.com/api/v1.0/snapshots/$projectId/java" -o "$SHORTNAME.zip"
49+
unzip -q -d "${SHORTNAME}-db" "${SHORTNAME}.zip"
50+
mkdir "lib-dbs/$SHORTNAME/"
51+
mv "${SHORTNAME}-db/"$(ls -1 "${SHORTNAME}"-db)/* "lib-dbs/${SHORTNAME}/"
52+
- name: Generate Models (PR and main)
53+
run: |
54+
set -x
55+
mkdir tmp-models
56+
MODELS=`pwd`/tmp-models
57+
DATABASES=`pwd`/lib-dbs
58+
59+
analyzeDatabaseWithCheckout() {
60+
QL_VARIANT=$1
61+
DATABASE=$2
62+
cd codeql-$QL_VARIANT
63+
SHORTNAME=`basename $DATABASE`
64+
python java/ql/src/utils/model-generator/GenerateFlowModel.py $DATABASE $MODELS/${SHORTNAME}.qll
65+
mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.qll
66+
cd ..
67+
}
68+
69+
for d in $DATABASES/*/ ; do
70+
ls -1 "$d"
71+
72+
analyzeDatabaseWithCheckout "main" $d
73+
if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]
74+
then
75+
analyzeDatabaseWithCheckout "pr" $d
76+
fi
77+
done
78+
- name: Install diff2html
79+
if: github.event.pull_request
80+
run: |
81+
npm install -g diff2html-cli
82+
- name: Generate Model Diff
83+
if: github.event.pull_request
84+
run: |
85+
set -x
86+
MODELS=`pwd`/tmp-models
87+
ls -1 tmp-models/
88+
for m in $MODELS/*_main.qll ; do
89+
t="${m/main/"pr"}"
90+
basename=`basename $m`
91+
name="diff_${basename/_main.qll/""}"
92+
(diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
93+
done
94+
- uses: actions/upload-artifact@v2
95+
with:
96+
name: models
97+
path: tmp-models/*.qll
98+
retention-days: 20
99+
- uses: actions/upload-artifact@v2
100+
with:
101+
name: diffs
102+
path: tmp-models/*.html
103+
retention-days: 20
Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
name: Regenerate framework models
2+
3+
on:
4+
workflow_dispatch:
5+
schedule:
6+
- cron: "30 2 * * *"
7+
pull_request:
8+
branches:
9+
- main
10+
paths:
11+
- ".github/workflows/mad_regenerate-models.yml"
12+
13+
jobs:
14+
regenerate-models:
15+
runs-on: ubuntu-latest
16+
strategy:
17+
matrix:
18+
# placeholder required for each axis, excluded below, replaced by the actual combinations (see include)
19+
slug: ["placeholder"]
20+
ref: ["placeholder"]
21+
include:
22+
- slug: "apache/commons-io"
23+
ref: "8985de8fe74f6622a419b37a6eed0dbc484dc128"
24+
exclude:
25+
- slug: "placeholder"
26+
ref: "placeholder"
27+
steps:
28+
- name: Clone self (github/codeql)
29+
uses: actions/checkout@v2
30+
- name: Setup CodeQL binaries
31+
uses: ./.github/actions/fetch-codeql
32+
- name: Clone repositories
33+
uses: actions/checkout@v2
34+
with:
35+
path: repos/${{ matrix.ref }}
36+
ref: ${{ matrix.ref }}
37+
repository: ${{ matrix.slug }}
38+
- name: Build database
39+
env:
40+
SLUG: ${{ matrix.slug }}
41+
REF: ${{ matrix.ref }}
42+
run: |
43+
mkdir dbs
44+
cd repos/${REF}
45+
SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
46+
codeql database create --language=java ../../dbs/${SHORTNAME}
47+
- name: Regenerate models in-place
48+
env:
49+
SLUG: ${{ matrix.slug }}
50+
run: |
51+
SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
52+
java/ql/src/utils/model-generator/RegenerateModels.py "${SLUG}" dbs/${SHORTNAME}
53+
- name: Stage changes
54+
run: |
55+
find java -name "*.qll" -print0 | xargs -0 git add
56+
git status
57+
git diff --cached > models.patch
58+
- uses: actions/upload-artifact@v2
59+
with:
60+
name: patch
61+
path: models.patch
62+
retention-days: 7

.github/workflows/ql-for-ql-build.yml

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,13 @@ jobs:
3131
uses: actions/cache@v2
3232
with:
3333
path: ${{ runner.temp }}/query-pack.zip
34-
key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}
34+
key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}
3535
- name: Build query pack
3636
if: steps.cache-queries.outputs.cache-hit != 'true'
3737
run: |
3838
cd ql/ql/src
3939
"${CODEQL}" pack create
40-
cd .codeql/pack/codeql/ql-all/0.0.0
40+
cd .codeql/pack/codeql/ql/0.0.0
4141
zip "${PACKZIP}" -r .
4242
env:
4343
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
@@ -189,4 +189,11 @@ jobs:
189189
uses: github/codeql-action/analyze@erik-krogh/ql
190190
with:
191191
category: "ql-for-ql-${{ matrix.folder }}"
192+
- name: Copy sarif file to CWD
193+
run: cp ../results/ql.sarif ./${{ matrix.folder }}.sarif
194+
- name: Sarif as artifact
195+
uses: actions/upload-artifact@v2
196+
with:
197+
name: ${{ matrix.folder }}.sarif
198+
path: ${{ matrix.folder }}.sarif
192199

.github/workflows/ql-for-ql-dataset_measure.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
1818
strategy:
1919
matrix:
20-
repo:
20+
repo:
2121
- github/codeql
2222
- github/codeql-go
2323
runs-on: ubuntu-latest
@@ -35,7 +35,7 @@ jobs:
3535
~/.cargo/registry
3636
~/.cargo/git
3737
ql/target
38-
key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('**/Cargo.lock') }}
38+
key: ${{ runner.os }}-qltest-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
3939
- name: Build Extractor
4040
run: cd ql; env "PATH=$PATH:`dirname ${CODEQL}`" ./create-extractor-pack.sh
4141
env:

0 commit comments

Comments
 (0)