Skip to content

Commit 62553ab

Browse files
author
Dave Bartolomeo
committed
Merge remote-tracking branch 'upstream/main' into semantic-scratch
2 parents f22c91b + b45f56a commit 62553ab

File tree

2,689 files changed

+364363
-135484
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

2,689 files changed

+364363
-135484
lines changed

‎.codeqlmanifest.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,17 @@
44
"*/ql/lib/qlpack.yml",
55
"*/ql/test/qlpack.yml",
66
"*/ql/examples/qlpack.yml",
7+
"*/ql/consistency-queries/qlpack.yml",
78
"cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
89
"javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
10+
"javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml",
911
"javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
1012
"csharp/ql/campaigns/Solorigate/lib/qlpack.yml",
1113
"csharp/ql/campaigns/Solorigate/src/qlpack.yml",
1214
"csharp/ql/campaigns/Solorigate/test/qlpack.yml",
1315
"misc/legacy-support/*/qlpack.yml",
1416
"misc/suite-helpers/qlpack.yml",
1517
"ruby/extractor-pack/codeql-extractor.yml",
16-
"ruby/ql/consistency-queries/qlpack.yml",
17-
"ql/ql/consistency-queries/qlpack.yml",
1818
"ql/extractor-pack/codeql-extractor.yml"
1919
],
2020
"versionPolicies": {

‎.gitattributes

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,15 @@
5050
*.pdb -text
5151

5252
java/ql/test/stubs/**/*.java linguist-generated=true
53-
java/ql/test/experimental/stubs/**/*.java linguist-generated=true
53+
java/ql/test/experimental/stubs/**/*.java linguist-generated=true
54+
55+
# For some languages, upgrade script testing references really old dbscheme
56+
# files from legacy upgrades that have CRLF line endings. Since upgrade
57+
# resolution relies on object hashes, we must suppress line ending conversion
58+
# for those testing dbscheme files.
59+
*/ql/lib/upgrades/initial/*.dbscheme -text
60+
61+
# Generated test files - these are synced from the standard JavaScript libraries using
62+
# `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
63+
javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
64+
javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge

‎.github/workflows/check-change-note.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,11 @@ on:
66
paths:
77
- "*/ql/src/**/*.ql"
88
- "*/ql/src/**/*.qll"
9+
- "*/ql/lib/**/*.ql"
10+
- "*/ql/lib/**/*.qll"
911
- "!**/experimental/**"
1012
- "!ql/**"
13+
- ".github/workflows/check-change-note.yml"
1114

1215
jobs:
1316
check-change-note:

‎.github/workflows/check-qldoc.yml

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
name: "Check QLdoc coverage"
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- "*/ql/lib/**"
7+
- .github/workflows/check-qldoc.yml
8+
branches:
9+
- main
10+
- "rc/*"
11+
12+
jobs:
13+
qldoc:
14+
runs-on: ubuntu-latest
15+
16+
steps:
17+
- name: Install CodeQL
18+
run: |
19+
gh extension install github/gh-codeql
20+
gh codeql set-channel nightly
21+
gh codeql version
22+
env:
23+
GITHUB_TOKEN: ${{ github.token }}
24+
25+
- uses: actions/checkout@v2
26+
with:
27+
fetch-depth: 2
28+
29+
- name: Check QLdoc coverage
30+
shell: bash
31+
run: |
32+
EXIT_CODE=0
33+
changed_lib_packs="$(git diff --name-only --diff-filter=ACMRT HEAD^ HEAD | { grep -o '^[a-z]*/ql/lib' || true; } | sort -u)"
34+
for pack_dir in ${changed_lib_packs}; do
35+
lang="${pack_dir%/ql/lib}"
36+
gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-current.txt" --dir="${pack_dir}"
37+
done
38+
git checkout HEAD^
39+
for pack_dir in ${changed_lib_packs}; do
40+
lang="${pack_dir%/ql/lib}"
41+
gh codeql generate library-doc-coverage --output="${RUNNER_TEMP}/${lang}-baseline.txt" --dir="${pack_dir}"
42+
awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-current.txt" | sort -u > "${RUNNER_TEMP}/current-undocumented.txt"
43+
awk -F, '{gsub(/"/,""); if ($4==0 && $6=="public") print "\""$3"\"" }' "${RUNNER_TEMP}/${lang}-baseline.txt" | sort -u > "${RUNNER_TEMP}/baseline-undocumented.txt"
44+
UNDOCUMENTED="$(grep -f <(comm -13 "${RUNNER_TEMP}/baseline-undocumented.txt" "${RUNNER_TEMP}/current-undocumented.txt") "${RUNNER_TEMP}/${lang}-current.txt" || true)"
45+
if [ -n "$UNDOCUMENTED" ]; then
46+
echo "$UNDOCUMENTED" | awk -F, '{gsub(/"/,""); print "::warning file='"${pack_dir}"'/"$1",line="$2"::Missing QLdoc for "$5, $3 }'
47+
EXIT_CODE=1
48+
fi
49+
done
50+
exit "${EXIT_CODE}"

‎.github/workflows/codeql-analysis.yml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@ jobs:
2727
pull-requests: read
2828

2929
steps:
30+
- name: Setup dotnet
31+
uses: actions/setup-dotnet@v1
32+
with:
33+
dotnet-version: 6.0.101
34+
3035
- name: Checkout repository
3136
uses: actions/checkout@v2
3237

@@ -51,7 +56,7 @@ jobs:
5156
# uses a compiled language
5257

5358
- run: |
54-
dotnet build csharp
59+
dotnet build csharp /p:UseSharedCompilation=false
5560
5661
- name: Perform CodeQL Analysis
5762
uses: github/codeql-action/analyze@main

‎.github/workflows/csv-coverage-metrics.yml

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
name: "Publish framework coverage as metrics"
2+
3+
on:
4+
schedule:
5+
- cron: '5 0 * * *'
6+
push:
7+
branches:
8+
- main
9+
workflow_dispatch:
10+
pull_request:
11+
branches:
12+
- main
13+
paths:
14+
- ".github/workflows/csv-coverage-metrics.yml"
15+
16+
jobs:
17+
publish:
18+
runs-on: ubuntu-latest
19+
steps:
20+
- name: Checkout repository
21+
uses: actions/checkout@v2
22+
- name: Setup CodeQL
23+
uses: ./.github/actions/fetch-codeql
24+
- name: Create empty database
25+
run: |
26+
DATABASE="${{ runner.temp }}/java-database"
27+
PROJECT="${{ runner.temp }}/java-project"
28+
mkdir -p "$PROJECT/src/tmp/empty"
29+
echo "class Empty {}" >> "$PROJECT/src/tmp/empty/Empty.java"
30+
codeql database create "$DATABASE" --language=java --source-root="$PROJECT" --command 'javac src/tmp/empty/Empty.java'
31+
- name: Capture coverage information
32+
run: |
33+
DATABASE="${{ runner.temp }}/java-database"
34+
codeql database analyze --format=sarif-latest --output=metrics.sarif -- "$DATABASE" ./java/ql/src/Metrics/Summaries/FrameworkCoverage.ql
35+
- uses: actions/upload-artifact@v2
36+
with:
37+
name: metrics.sarif
38+
path: metrics.sarif
39+
retention-days: 20
40+
- name: Upload SARIF file
41+
uses: github/codeql-action/upload-sarif@v1
42+
with:
43+
sarif_file: metrics.sarif

‎.github/workflows/js-ml-tests.yml

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
name: JS ML-powered queries tests
2+
3+
on:
4+
push:
5+
paths:
6+
- "javascript/ql/experimental/adaptivethreatmodeling/**"
7+
- .github/workflows/js-ml-tests.yml
8+
branches:
9+
- main
10+
- "rc/*"
11+
pull_request:
12+
paths:
13+
- "javascript/ql/experimental/adaptivethreatmodeling/**"
14+
- .github/workflows/js-ml-tests.yml
15+
16+
defaults:
17+
run:
18+
working-directory: javascript/ql/experimental/adaptivethreatmodeling
19+
20+
jobs:
21+
qlformat:
22+
name: Check QL formatting
23+
runs-on: ubuntu-latest
24+
steps:
25+
- uses: actions/checkout@v2
26+
27+
- uses: ./.github/actions/fetch-codeql
28+
29+
- name: Check QL formatting
30+
run: |
31+
find . "(" -name "*.ql" -or -name "*.qll" ")" -print0 | \
32+
xargs -0 codeql query format --check-only
33+
34+
qlcompile:
35+
name: Check QL compilation
36+
runs-on: ubuntu-latest
37+
steps:
38+
- uses: actions/checkout@v2
39+
40+
- uses: ./.github/actions/fetch-codeql
41+
42+
- name: Install pack dependencies
43+
run: |
44+
for pack in modelbuilding src; do
45+
codeql pack install --mode verify -- "${pack}"
46+
done
47+
48+
- name: Check QL compilation
49+
run: |
50+
codeql query compile \
51+
--check-only \
52+
--ram 5120 \
53+
--additional-packs "${{ github.workspace }}" \
54+
--threads=0 \
55+
-- \
56+
lib modelbuilding src
57+
58+
qltest:
59+
name: Run QL tests
60+
runs-on: ubuntu-latest
61+
steps:
62+
- uses: actions/checkout@v2
63+
64+
- uses: ./.github/actions/fetch-codeql
65+
66+
- name: Install pack dependencies
67+
run: codeql pack install -- test
68+
69+
- name: Run QL tests
70+
run: |
71+
codeql test run \
72+
--threads=0 \
73+
--ram 5120 \
74+
--additional-packs "${{ github.workspace }}" \
75+
-- \
76+
test

‎.github/workflows/mad_modelDiff.yml

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
name: Models as Data - Diff
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
projects:
7+
description: "The projects to generate models for"
8+
required: true
9+
default: '["netty/netty"]'
10+
pull_request:
11+
branches:
12+
- main
13+
paths:
14+
- "java/ql/src/utils/model-generator/**/*.*"
15+
- ".github/workflows/mad_modelDiff.yml"
16+
17+
permissions:
18+
contents: read
19+
20+
jobs:
21+
model-diff:
22+
name: Model Difference
23+
runs-on: ubuntu-latest
24+
if: github.repository == 'github/codeql'
25+
strategy:
26+
matrix:
27+
slug: ${{fromJson(github.event.inputs.projects || '["apache/commons-codec", "apache/commons-io", "apache/commons-beanutils", "apache/commons-logging", "apache/commons-fileupload", "apache/commons-lang", "apache/commons-validator", "apache/commons-csv", "apache/dubbo"]' )}}
28+
steps:
29+
- name: Clone github/codeql from PR
30+
uses: actions/checkout@v2
31+
if: github.event.pull_request
32+
with:
33+
path: codeql-pr
34+
- name: Clone github/codeql from main
35+
uses: actions/checkout@v2
36+
with:
37+
path: codeql-main
38+
ref: main
39+
- uses: ./codeql-main/.github/actions/fetch-codeql
40+
- name: Download database
41+
env:
42+
SLUG: ${{ matrix.slug }}
43+
run: |
44+
set -x
45+
mkdir lib-dbs
46+
SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
47+
projectId=`curl -s https://lgtm.com/api/v1.0/projects/g/${SLUG} | jq .id`
48+
curl -L "https://lgtm.com/api/v1.0/snapshots/$projectId/java" -o "$SHORTNAME.zip"
49+
unzip -q -d "${SHORTNAME}-db" "${SHORTNAME}.zip"
50+
mkdir "lib-dbs/$SHORTNAME/"
51+
mv "${SHORTNAME}-db/"$(ls -1 "${SHORTNAME}"-db)/* "lib-dbs/${SHORTNAME}/"
52+
- name: Generate Models (PR and main)
53+
run: |
54+
set -x
55+
mkdir tmp-models
56+
MODELS=`pwd`/tmp-models
57+
DATABASES=`pwd`/lib-dbs
58+
59+
analyzeDatabaseWithCheckout() {
60+
QL_VARIANT=$1
61+
DATABASE=$2
62+
cd codeql-$QL_VARIANT
63+
SHORTNAME=`basename $DATABASE`
64+
python java/ql/src/utils/model-generator/GenerateFlowModel.py $DATABASE $MODELS/${SHORTNAME}.qll
65+
mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.qll
66+
cd ..
67+
}
68+
69+
for d in $DATABASES/*/ ; do
70+
ls -1 "$d"
71+
72+
analyzeDatabaseWithCheckout "main" $d
73+
if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]
74+
then
75+
analyzeDatabaseWithCheckout "pr" $d
76+
fi
77+
done
78+
- name: Install diff2html
79+
if: github.event.pull_request
80+
run: |
81+
npm install -g diff2html-cli
82+
- name: Generate Model Diff
83+
if: github.event.pull_request
84+
run: |
85+
set -x
86+
MODELS=`pwd`/tmp-models
87+
ls -1 tmp-models/
88+
for m in $MODELS/*_main.qll ; do
89+
t="${m/main/"pr"}"
90+
basename=`basename $m`
91+
name="diff_${basename/_main.qll/""}"
92+
(diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
93+
done
94+
- uses: actions/upload-artifact@v2
95+
with:
96+
name: models
97+
path: tmp-models/*.qll
98+
retention-days: 20
99+
- uses: actions/upload-artifact@v2
100+
with:
101+
name: diffs
102+
path: tmp-models/*.html
103+
retention-days: 20

0 commit comments

Comments
 (0)