Skip to content

Commit afc7867

Browse files
aibaarsaibaars
authored
Merge pull request #6942 from github/aibaars/patch-10
Merge codeql-ruby into codeql
2 parents 7648815 + 4f79398 commit afc7867

File tree

560 files changed

+138836
-12
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

560 files changed

+138836
-12
lines changed

‎.codeqlmanifest.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
{ "provide": [ "*/ql/src/qlpack.yml",
1+
{ "provide": [ "ruby/.codeqlmanifest.json",
2+
"*/ql/src/qlpack.yml",
23
"*/ql/lib/qlpack.yml",
34
"*/ql/test/qlpack.yml",
45
"cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml",

‎.devcontainer/devcontainer.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,14 @@
11
{
22
"extensions": [
3+
"rust-lang.rust",
4+
"bungcip.better-toml",
35
"github.vscode-codeql",
46
"slevesque.vscode-zipexplorer"
57
],
68
"settings": {
9+
"files.watcherExclude": {
10+
"**/target/**": true
11+
},
712
"codeQL.runningQueries.memory": 2048
813
}
914
}

‎.github/actions/fetch-codeql/action.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
name: Fetch CodeQL
2+
description: Fetches the latest version of CodeQL
3+
runs:
4+
using: composite
5+
steps:
6+
- name: Fetch CodeQL
7+
shell: bash
8+
run: |
9+
LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
10+
gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$LATEST"
11+
unzip -q codeql-linux64.zip
12+
echo "${{ github.workspace }}/codeql" >> $GITHUB_PATH
13+
env:
14+
GITHUB_TOKEN: ${{ github.token }}

‎.github/dependabot.yml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
version: 2
2+
updates:
3+
- package-ecosystem: "cargo"
4+
directory: "ruby/node-types"
5+
schedule:
6+
interval: "daily"
7+
- package-ecosystem: "cargo"
8+
directory: "ruby/generator"
9+
schedule:
10+
interval: "daily"
11+
- package-ecosystem: "cargo"
12+
directory: "ruby/extractor"
13+
schedule:
14+
interval: "daily"
15+
- package-ecosystem: "cargo"
16+
directory: "ruby/autobuilder"
17+
schedule:
18+
interval: "daily"

‎.github/workflows/qhelp-pr-preview.yml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: Query help preview
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- main
7+
- 'rc/*'
8+
paths:
9+
- "ruby/**/*.qhelp"
10+
11+
jobs:
12+
qhelp:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- uses: actions/checkout@v2
16+
with:
17+
fetch-depth: 2
18+
- name: Determine changed files
19+
id: changes
20+
run: |
21+
echo -n "::set-output name=qhelp_files::"
22+
(git diff --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep .qhelp$ | grep -v .inc.qhelp;
23+
git diff --name-only --diff-filter=ACMRT HEAD~1 HEAD | grep .inc.qhelp$ | xargs -d '\n' -rn1 basename | xargs -d '\n' -rn1 git grep -l) |
24+
sort -u | xargs -d '\n' -n1 printf "'%s' "
25+
26+
- uses: ./.github/actions/fetch-codeql
27+
28+
- name: QHelp preview
29+
if: ${{ steps.changes.outputs.qhelp_files }}
30+
run: |
31+
( echo "QHelp previews:";
32+
for path in ${{ steps.changes.outputs.qhelp_files }} ; do
33+
echo "<details> <summary>${path}</summary>"
34+
echo
35+
codeql generate query-help --format=markdown ${path}
36+
echo "</details>"
37+
done) | gh pr comment "${{ github.event.pull_request.number }}" -F -
38+
env:
39+
GITHUB_TOKEN: ${{ github.token }}

‎.github/workflows/ruby-build.yml

Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
name: "Ruby: Build"
2+
3+
on:
4+
push:
5+
paths:
6+
- 'ruby/**'
7+
branches:
8+
- main
9+
- 'rc/*'
10+
pull_request:
11+
paths:
12+
- 'ruby/**'
13+
branches:
14+
- main
15+
- 'rc/*'
16+
workflow_dispatch:
17+
inputs:
18+
tag:
19+
description: "Version tag to create"
20+
required: false
21+
22+
env:
23+
CARGO_TERM_COLOR: always
24+
25+
defaults:
26+
run:
27+
working-directory: ruby
28+
29+
jobs:
30+
build:
31+
strategy:
32+
fail-fast: false
33+
matrix:
34+
os: [ubuntu-latest, macos-latest, windows-latest]
35+
36+
runs-on: ${{ matrix.os }}
37+
38+
steps:
39+
- uses: actions/checkout@v2
40+
- name: Install GNU tar
41+
if: runner.os == 'macOS'
42+
run: |
43+
brew install gnu-tar
44+
echo "/usr/local/opt/gnu-tar/libexec/gnubin" >> $GITHUB_PATH
45+
- uses: actions/cache@v2
46+
with:
47+
path: |
48+
~/.cargo/registry
49+
~/.cargo/git
50+
ruby/target
51+
key: ${{ runner.os }}-rust-cargo-${{ hashFiles('**/Cargo.lock') }}
52+
- name: Check formatting
53+
run: cargo fmt --all -- --check
54+
- name: Build
55+
run: cargo build --verbose
56+
- name: Run tests
57+
run: cargo test --verbose
58+
- name: Release build
59+
run: cargo build --release
60+
- name: Generate dbscheme
61+
if: ${{ matrix.os == 'ubuntu-latest' }}
62+
run: target/release/ruby-generator --dbscheme ql/lib/ruby.dbscheme --library ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
63+
- uses: actions/upload-artifact@v2
64+
if: ${{ matrix.os == 'ubuntu-latest' }}
65+
with:
66+
name: ruby.dbscheme
67+
path: ruby/ql/lib/ruby.dbscheme
68+
- uses: actions/upload-artifact@v2
69+
if: ${{ matrix.os == 'ubuntu-latest' }}
70+
with:
71+
name: TreeSitter.qll
72+
path: ruby/ql/lib/codeql/ruby/ast/internal/TreeSitter.qll
73+
- uses: actions/upload-artifact@v2
74+
with:
75+
name: extractor-${{ matrix.os }}
76+
path: |
77+
ruby/target/release/ruby-autobuilder
78+
ruby/target/release/ruby-autobuilder.exe
79+
ruby/target/release/ruby-extractor
80+
ruby/target/release/ruby-extractor.exe
81+
retention-days: 1
82+
compile-queries:
83+
runs-on: ubuntu-latest
84+
env:
85+
CODEQL_THREADS: 4 # TODO: remove this once it's set by the CLI
86+
steps:
87+
- uses: actions/checkout@v2
88+
- name: Fetch CodeQL
89+
run: |
90+
LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
91+
gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$LATEST"
92+
unzip -q codeql-linux64.zip
93+
env:
94+
GITHUB_TOKEN: ${{ github.token }}
95+
- name: Build Query Pack
96+
run: |
97+
codeql/codeql pack create ql/lib --output target/packs
98+
codeql/codeql pack install ql/src
99+
codeql/codeql pack create ql/src --output target/packs
100+
PACK_FOLDER=$(readlink -f target/packs/codeql/ruby-queries/*)
101+
codeql/codeql generate query-help --format=sarifv2.1.0 --output="${PACK_FOLDER}/rules.sarif" ql/src
102+
(cd ql/src; find queries \( -name '*.qhelp' -o -name '*.rb' -o -name '*.erb' \) -exec bash -c 'mkdir -p "'"${PACK_FOLDER}"'/$(dirname "{}")"' \; -exec cp "{}" "${PACK_FOLDER}/{}" \;)
103+
- name: Compile with previous CodeQL versions
104+
run: |
105+
for version in $(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | sort --version-sort | tail -3 | head -2); do
106+
rm -f codeql-linux64.zip
107+
gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip "$version"
108+
rm -rf codeql; unzip -q codeql-linux64.zip
109+
codeql/codeql query compile target/packs/*
110+
done
111+
env:
112+
GITHUB_TOKEN: ${{ github.token }}
113+
- uses: actions/upload-artifact@v2
114+
with:
115+
name: codeql-ruby-queries
116+
path: |
117+
ruby/target/packs/*
118+
retention-days: 1
119+
120+
package:
121+
runs-on: ubuntu-latest
122+
needs: [build, compile-queries]
123+
steps:
124+
- uses: actions/checkout@v2
125+
- uses: actions/download-artifact@v2
126+
with:
127+
name: ruby.dbscheme
128+
path: ruby/ruby
129+
- uses: actions/download-artifact@v2
130+
with:
131+
name: extractor-ubuntu-latest
132+
path: ruby/linux64
133+
- uses: actions/download-artifact@v2
134+
with:
135+
name: extractor-windows-latest
136+
path: ruby/win64
137+
- uses: actions/download-artifact@v2
138+
with:
139+
name: extractor-macos-latest
140+
path: ruby/osx64
141+
- run: |
142+
mkdir -p ruby
143+
cp -r codeql-extractor.yml tools ql/lib/ruby.dbscheme.stats ruby/
144+
mkdir -p ruby/tools/{linux64,osx64,win64}
145+
cp linux64/ruby-autobuilder ruby/tools/linux64/autobuilder
146+
cp osx64/ruby-autobuilder ruby/tools/osx64/autobuilder
147+
cp win64/ruby-autobuilder.exe ruby/tools/win64/autobuilder.exe
148+
cp linux64/ruby-extractor ruby/tools/linux64/extractor
149+
cp osx64/ruby-extractor ruby/tools/osx64/extractor
150+
cp win64/ruby-extractor.exe ruby/tools/win64/extractor.exe
151+
chmod +x ruby/tools/{linux64,osx64}/{autobuilder,extractor}
152+
zip -rq codeql-ruby.zip ruby
153+
- uses: actions/upload-artifact@v2
154+
with:
155+
name: codeql-ruby-pack
156+
path: ruby/codeql-ruby.zip
157+
retention-days: 1
158+
- uses: actions/download-artifact@v2
159+
with:
160+
name: codeql-ruby-queries
161+
path: ruby/qlpacks
162+
- run: |
163+
echo '{
164+
"provide": [
165+
"ruby/codeql-extractor.yml",
166+
"qlpacks/*/*/*/qlpack.yml"
167+
]
168+
}' > .codeqlmanifest.json
169+
zip -rq codeql-ruby-bundle.zip .codeqlmanifest.json ruby qlpacks
170+
- uses: actions/upload-artifact@v2
171+
with:
172+
name: codeql-ruby-bundle
173+
path: ruby/codeql-ruby-bundle.zip
174+
retention-days: 1
175+
176+
test:
177+
defaults:
178+
run:
179+
working-directory: ${{ github.workspace }}
180+
strategy:
181+
fail-fast: false
182+
matrix:
183+
os: [ubuntu-latest, macos-latest, windows-latest]
184+
185+
runs-on: ${{ matrix.os }}
186+
needs: [package]
187+
steps:
188+
- uses: actions/checkout@v2
189+
with:
190+
repository: Shopify/example-ruby-app
191+
ref: 67a0decc5eb550f3a9228eda53925c3afd40dfe9
192+
- name: Fetch CodeQL
193+
shell: bash
194+
run: |
195+
LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | grep -v beta | sort --version-sort | tail -1)
196+
gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql.zip "$LATEST"
197+
unzip -q codeql.zip
198+
env:
199+
GITHUB_TOKEN: ${{ github.token }}
200+
working-directory: ${{ runner.temp }}
201+
- name: Download Ruby bundle
202+
uses: actions/download-artifact@v2
203+
with:
204+
name: codeql-ruby-bundle
205+
path: ${{ runner.temp }}
206+
- name: Unzip Ruby bundle
207+
shell: bash
208+
run: unzip -q -d "${{ runner.temp }}/ruby-bundle" "${{ runner.temp }}/codeql-ruby-bundle.zip"
209+
- name: Prepare test files
210+
shell: bash
211+
run: |
212+
echo "import ruby select count(File f)" > "test.ql"
213+
echo "| 4 |" > "test.expected"
214+
echo 'name: sample-tests
215+
version: 0.0.0
216+
dependencies:
217+
codeql/ruby-all: 0.0.1
218+
extractor: ruby
219+
tests: .
220+
' > qlpack.yml
221+
- name: Run QL test
222+
shell: bash
223+
run: |
224+
"${{ runner.temp }}/codeql/codeql" test run --search-path "${{ runner.temp }}/ruby-bundle" --additional-packs "${{ runner.temp }}/ruby-bundle" .
225+
- name: Create database
226+
shell: bash
227+
run: |
228+
"${{ runner.temp }}/codeql/codeql" database create --search-path "${{ runner.temp }}/ruby-bundle" --language ruby --source-root . ../database
229+
- name: Analyze database
230+
shell: bash
231+
run: |
232+
"${{ runner.temp }}/codeql/codeql" database analyze --search-path "${{ runner.temp }}/ruby-bundle" --format=sarifv2.1.0 --output=out.sarif ../database ruby-code-scanning.qls

0 commit comments

Comments
 (0)