Skip to content

Commit 3d1f4d5

Browse files
ihsinmeihsinme
authored
Merge pull request #1 from github/main
up to head
2 parents 5709365 + ab3cad7 commit 3d1f4d5

File tree

4,979 files changed

+566214
-143333
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

4,979 files changed

+566214
-143333
lines changed

‎.codeqlmanifest.json

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,27 @@
1-
{ "provide": [ "ruby/.codeqlmanifest.json",
2-
"*/ql/src/qlpack.yml",
3-
"*/ql/lib/qlpack.yml",
4-
"*/ql/test/qlpack.yml",
5-
"cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
6-
"*/ql/examples/qlpack.yml",
7-
"*/upgrades/qlpack.yml",
8-
"javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
9-
"javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
10-
"misc/legacy-support/*/qlpack.yml",
11-
"misc/suite-helpers/qlpack.yml" ] }
1+
{
2+
"provide": [
3+
"*/ql/src/qlpack.yml",
4+
"*/ql/lib/qlpack.yml",
5+
"*/ql/test/qlpack.yml",
6+
"*/ql/examples/qlpack.yml",
7+
"*/ql/consistency-queries/qlpack.yml",
8+
"cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",
9+
"javascript/ql/experimental/adaptivethreatmodeling/lib/qlpack.yml",
10+
"javascript/ql/experimental/adaptivethreatmodeling/modelbuilding/qlpack.yml",
11+
"javascript/ql/experimental/adaptivethreatmodeling/src/qlpack.yml",
12+
"csharp/ql/campaigns/Solorigate/lib/qlpack.yml",
13+
"csharp/ql/campaigns/Solorigate/src/qlpack.yml",
14+
"csharp/ql/campaigns/Solorigate/test/qlpack.yml",
15+
"misc/legacy-support/*/qlpack.yml",
16+
"misc/suite-helpers/qlpack.yml",
17+
"ruby/extractor-pack/codeql-extractor.yml",
18+
"ql/extractor-pack/codeql-extractor.yml"
19+
],
20+
"versionPolicies": {
21+
"default": {
22+
"requireChangeNotes": true,
23+
"committedPrereleaseSuffix": "dev",
24+
"committedVersion": "nextPatchRelease"
25+
}
26+
}
27+
}

‎.gitattributes

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,15 @@
5050
*.pdb -text
5151

5252
java/ql/test/stubs/**/*.java linguist-generated=true
53-
java/ql/test/experimental/stubs/**/*.java linguist-generated=true
53+
java/ql/test/experimental/stubs/**/*.java linguist-generated=true
54+
55+
# For some languages, upgrade script testing references really old dbscheme
56+
# files from legacy upgrades that have CRLF line endings. Since upgrade
57+
# resolution relies on object hashes, we must suppress line ending conversion
58+
# for those testing dbscheme files.
59+
*/ql/lib/upgrades/initial/*.dbscheme -text
60+
61+
# Generated test files - these are synced from the standard JavaScript libraries using
62+
# `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
63+
javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
64+
javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge

‎.github/actions/fetch-codeql/action.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ runs:
88
run: |
99
LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
1010
gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$LATEST"
11-
unzip -q codeql-linux64.zip
12-
echo "${{ github.workspace }}/codeql" >> $GITHUB_PATH
11+
unzip -q -d "${RUNNER_TEMP}" codeql-linux64.zip
12+
echo "${RUNNER_TEMP}/codeql" >> "${GITHUB_PATH}"
1313
env:
1414
GITHUB_TOKEN: ${{ github.token }}

‎.github/labeler.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,14 @@ Python:
1818
- python/**/*
1919
- change-notes/**/*python*
2020

21+
Ruby:
22+
- ruby/**/*
23+
- change-notes/**/*ruby*
24+
2125
documentation:
2226
- "**/*.qhelp"
2327
- "**/*.md"
2428
- docs/**/*
29+
30+
"QL-for-QL":
31+
- ql/**/*

‎.github/workflows/check-change-note.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,11 @@ on:
66
paths:
77
- "*/ql/src/**/*.ql"
88
- "*/ql/src/**/*.qll"
9+
- "*/ql/lib/**/*.ql"
10+
- "*/ql/lib/**/*.qll"
911
- "!**/experimental/**"
12+
- "!ql/**"
13+
- ".github/workflows/check-change-note.yml"
1014

1115
jobs:
1216
check-change-note:

‎.github/workflows/codeql-analysis.yml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@ jobs:
2727
pull-requests: read
2828

2929
steps:
30+
- name: Setup dotnet
31+
uses: actions/setup-dotnet@v1
32+
with:
33+
dotnet-version: 6.0.101
34+
3035
- name: Checkout repository
3136
uses: actions/checkout@v2
3237

@@ -51,7 +56,7 @@ jobs:
5156
# uses a compiled language
5257

5358
- run: |
54-
dotnet build csharp
59+
dotnet build csharp /p:UseSharedCompilation=false
5560
5661
- name: Perform CodeQL Analysis
5762
uses: github/codeql-action/analyze@main

‎.github/workflows/csv-coverage-metrics.yml

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
name: "Publish framework coverage as metrics"
2+
3+
on:
4+
schedule:
5+
- cron: '5 0 * * *'
6+
push:
7+
branches:
8+
- main
9+
workflow_dispatch:
10+
pull_request:
11+
branches:
12+
- main
13+
paths:
14+
- ".github/workflows/csv-coverage-metrics.yml"
15+
16+
jobs:
17+
publish:
18+
runs-on: ubuntu-latest
19+
steps:
20+
- name: Checkout repository
21+
uses: actions/checkout@v2
22+
- name: Setup CodeQL
23+
uses: ./.github/actions/fetch-codeql
24+
- name: Create empty database
25+
run: |
26+
DATABASE="${{ runner.temp }}/java-database"
27+
PROJECT="${{ runner.temp }}/java-project"
28+
mkdir -p "$PROJECT/src/tmp/empty"
29+
echo "class Empty {}" >> "$PROJECT/src/tmp/empty/Empty.java"
30+
codeql database create "$DATABASE" --language=java --source-root="$PROJECT" --command 'javac src/tmp/empty/Empty.java'
31+
- name: Capture coverage information
32+
run: |
33+
DATABASE="${{ runner.temp }}/java-database"
34+
codeql database analyze --format=sarif-latest --output=metrics.sarif -- "$DATABASE" ./java/ql/src/Metrics/Summaries/FrameworkCoverage.ql
35+
- uses: actions/upload-artifact@v2
36+
with:
37+
name: metrics.sarif
38+
path: metrics.sarif
39+
retention-days: 20
40+
- name: Upload SARIF file
41+
uses: github/codeql-action/upload-sarif@v1
42+
with:
43+
sarif_file: metrics.sarif

‎.github/workflows/js-ml-tests.yml

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
name: JS ML-powered queries tests
2+
3+
on:
4+
push:
5+
paths:
6+
- "javascript/ql/experimental/adaptivethreatmodeling/**"
7+
- .github/workflows/js-ml-tests.yml
8+
branches:
9+
- main
10+
- "rc/*"
11+
pull_request:
12+
paths:
13+
- "javascript/ql/experimental/adaptivethreatmodeling/**"
14+
- .github/workflows/js-ml-tests.yml
15+
16+
defaults:
17+
run:
18+
working-directory: javascript/ql/experimental/adaptivethreatmodeling
19+
20+
jobs:
21+
qlformat:
22+
name: Check QL formatting
23+
runs-on: ubuntu-latest
24+
steps:
25+
- uses: actions/checkout@v2
26+
27+
- uses: ./.github/actions/fetch-codeql
28+
29+
- name: Check QL formatting
30+
run: |
31+
find . "(" -name "*.ql" -or -name "*.qll" ")" -print0 | \
32+
xargs -0 codeql query format --check-only
33+
34+
qlcompile:
35+
name: Check QL compilation
36+
runs-on: ubuntu-latest
37+
steps:
38+
- uses: actions/checkout@v2
39+
40+
- uses: ./.github/actions/fetch-codeql
41+
42+
- name: Install pack dependencies
43+
run: |
44+
for pack in modelbuilding src; do
45+
codeql pack install --mode verify -- "${pack}"
46+
done
47+
48+
- name: Check QL compilation
49+
run: |
50+
codeql query compile \
51+
--check-only \
52+
--ram 5120 \
53+
--additional-packs "${{ github.workspace }}" \
54+
--threads=0 \
55+
-- \
56+
lib modelbuilding src
57+
58+
qltest:
59+
name: Run QL tests
60+
runs-on: ubuntu-latest
61+
steps:
62+
- uses: actions/checkout@v2
63+
64+
- uses: ./.github/actions/fetch-codeql
65+
66+
- name: Install pack dependencies
67+
run: codeql pack install -- test
68+
69+
- name: Run QL tests
70+
run: |
71+
codeql test run \
72+
--threads=0 \
73+
--ram 5120 \
74+
--additional-packs "${{ github.workspace }}" \
75+
-- \
76+
test

‎.github/workflows/mad_modelDiff.yml

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
name: Models as Data - Diff
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
projects:
7+
description: "The projects to generate models for"
8+
required: true
9+
default: '["netty/netty"]'
10+
pull_request:
11+
branches:
12+
- main
13+
paths:
14+
- "java/ql/src/utils/model-generator/**/*.*"
15+
- ".github/workflows/mad_modelDiff.yml"
16+
17+
permissions:
18+
contents: read
19+
20+
jobs:
21+
model-diff:
22+
name: Model Difference
23+
runs-on: ubuntu-latest
24+
if: github.repository == 'github/codeql'
25+
strategy:
26+
matrix:
27+
slug: ${{fromJson(github.event.inputs.projects || '["apache/commons-codec", "apache/commons-io", "apache/commons-beanutils", "apache/commons-logging", "apache/commons-fileupload", "apache/commons-lang", "apache/commons-validator", "apache/commons-csv", "apache/dubbo"]' )}}
28+
steps:
29+
- name: Clone github/codeql from PR
30+
uses: actions/checkout@v2
31+
if: github.event.pull_request
32+
with:
33+
path: codeql-pr
34+
- name: Clone github/codeql from main
35+
uses: actions/checkout@v2
36+
with:
37+
path: codeql-main
38+
ref: main
39+
- uses: ./codeql-main/.github/actions/fetch-codeql
40+
- name: Download database
41+
env:
42+
SLUG: ${{ matrix.slug }}
43+
run: |
44+
set -x
45+
mkdir lib-dbs
46+
SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
47+
projectId=`curl -s https://lgtm.com/api/v1.0/projects/g/${SLUG} | jq .id`
48+
curl -L "https://lgtm.com/api/v1.0/snapshots/$projectId/java" -o "$SHORTNAME.zip"
49+
unzip -q -d "${SHORTNAME}-db" "${SHORTNAME}.zip"
50+
mkdir "lib-dbs/$SHORTNAME/"
51+
mv "${SHORTNAME}-db/"$(ls -1 "${SHORTNAME}"-db)/* "lib-dbs/${SHORTNAME}/"
52+
- name: Generate Models (PR and main)
53+
run: |
54+
set -x
55+
mkdir tmp-models
56+
MODELS=`pwd`/tmp-models
57+
DATABASES=`pwd`/lib-dbs
58+
59+
analyzeDatabaseWithCheckout() {
60+
QL_VARIANT=$1
61+
DATABASE=$2
62+
cd codeql-$QL_VARIANT
63+
SHORTNAME=`basename $DATABASE`
64+
python java/ql/src/utils/model-generator/GenerateFlowModel.py $DATABASE $MODELS/${SHORTNAME}.qll
65+
mv $MODELS/${SHORTNAME}.qll $MODELS/${SHORTNAME}Generated_${QL_VARIANT}.qll
66+
cd ..
67+
}
68+
69+
for d in $DATABASES/*/ ; do
70+
ls -1 "$d"
71+
72+
analyzeDatabaseWithCheckout "main" $d
73+
if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]
74+
then
75+
analyzeDatabaseWithCheckout "pr" $d
76+
fi
77+
done
78+
- name: Install diff2html
79+
if: github.event.pull_request
80+
run: |
81+
npm install -g diff2html-cli
82+
- name: Generate Model Diff
83+
if: github.event.pull_request
84+
run: |
85+
set -x
86+
MODELS=`pwd`/tmp-models
87+
ls -1 tmp-models/
88+
for m in $MODELS/*_main.qll ; do
89+
t="${m/main/"pr"}"
90+
basename=`basename $m`
91+
name="diff_${basename/_main.qll/""}"
92+
(diff -w -u $m $t | diff2html -i stdin -F $MODELS/$name.html) || true
93+
done
94+
- uses: actions/upload-artifact@v2
95+
with:
96+
name: models
97+
path: tmp-models/*.qll
98+
retention-days: 20
99+
- uses: actions/upload-artifact@v2
100+
with:
101+
name: diffs
102+
path: tmp-models/*.html
103+
retention-days: 20

0 commit comments

Comments
 (0)