Skip to content

Commit b7e2692

Browse files
alancai98alancai98
authored
Adds conformance report generation and report comparison; publishes generated reports (#126)
1 parent 1bc00ce commit b7e2692

File tree

8 files changed

+492
-11
lines changed

8 files changed

+492
-11
lines changed

‎.github/workflows/ci_build_test.yml

Lines changed: 91 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,6 @@ jobs:
66
build:
77
name: Build and Test
88
runs-on: ${{ matrix.os }}
9-
# We want to run on external PRs, but not on internal ones as push automatically builds
10-
# H/T: https://github.com/Dart-Code/Dart-Code/commit/612732d5879730608baa9622bf7f5e5b7b51ae65
11-
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != 'partiql/partiql-lang-rust'
129
strategy:
1310
matrix:
1411
os: [ubuntu-latest, windows-latest, macos-latest]
@@ -77,3 +74,94 @@ jobs:
7774
# Adding comments to the PR requires the GITHUB_TOKEN secret.
7875
token: ${{ secrets.GITHUB_TOKEN }}
7976
args: --all-features
77+
# Cache the `cargo build` so future jobs can reuse build
78+
- name: Cache cargo build
79+
if: matrix.os == 'ubuntu-latest'
80+
uses: actions/cache@v2
81+
id: restore-build
82+
with:
83+
path: ./*
84+
key: ${{ github.sha }}
85+
# Conformance report generation and comparison report generation job will run only after the `Build and Test` job
86+
# succeeds.
87+
conformance:
88+
name: Check conformance
89+
runs-on: ubuntu-latest
90+
needs: [build]
91+
steps:
92+
# Pull down the cached `partiql-lang-rust` build from the `Build and Test` job. This allows us to reuse without
93+
# needing to rebuild. If pulling the build fails, the subsequent `cargo test` will rebuild.
94+
- uses: actions/checkout@v2
95+
with:
96+
submodules: recursive
97+
- name: Rust Toolchain
98+
uses: actions-rs/toolchain@v1
99+
with:
100+
profile: minimal
101+
toolchain: stable
102+
override: true
103+
- uses: actions/cache@v2
104+
id: restore-build
105+
with:
106+
path: ./*
107+
key: ${{ github.sha }}
108+
# Run the conformance tests (i.e. `cargo test`) and save to a json file. This uses the "format" unstable compile
109+
# option to save as json. In the future, may want to use the `cargo_metadata` crate (https://crates.io/crates/cargo_metadata)
110+
# to format and use the output.
111+
- name: Cargo Test of the conformance tests (can fail) and save to json file
112+
continue-on-error: true
113+
run: cargo test --verbose --package partiql-conformance-tests --features "conformance_test" -- -Z unstable-options --format json > cargo_test_results.json
114+
# Create a conformance report from the `cargo test` json file
115+
- run: ./target/debug/generate_cts_report cargo_test_results.json ${GITHUB_SHA} cts_report.json
116+
# Upload conformance report for comparison with future runs
117+
- name: Upload cts_report.json
118+
uses: actions/upload-artifact@v3
119+
with:
120+
path: cts_report.json
121+
# Download conformance report from `main` to create comparison report. If `main` has no report, use a backup
122+
# report (stored in partiql-conformance-tests/backup_conformance_report.json). Alternatively, we could consider
123+
# - pulling `main` branch and rerun the tests
124+
# - pulling latest release and rerun the tests
125+
# - compare to an empty file
126+
# - don't perform comparisons
127+
- name: Download cts_report.json from `main` branch
128+
uses: dawidd6/action-download-artifact@v2
129+
id: download-report
130+
continue-on-error: true
131+
with:
132+
workflow: ci_build_test.yml
133+
branch: main
134+
- name: backup case if download fails
135+
if: ${{ steps.download-report.outcome == 'failure' }}
136+
run: mkdir -p main-branch-report && cp -r ./partiql-conformance-tests/backup_conformance_report.json ./main-branch-report/cts_report.json
137+
# Run conformance report comparison. Generates cts-comparison-report.md
138+
- run: ./target/debug/generate_comparison_report ./main-branch-report/cts_report.json cts_report.json cts-comparison-report.md
139+
# Print conformance report to GitHub actions workflow summary page
140+
- name: print markdown in run
141+
run: cat cts-comparison-report.md >> $GITHUB_STEP_SUMMARY
142+
# Find comment w/ conformance comparison if previous comment published
143+
- name: Find Comment
144+
uses: peter-evans/find-comment@v2
145+
id: fc
146+
if: github.event_name == 'pull_request'
147+
with:
148+
issue-number: ${{ github.event.pull_request.number }}
149+
comment-author: 'github-actions[bot]'
150+
body-includes: Conformance
151+
# Convert the markdown comparison report to a readable form for GitHub PR comments
152+
- id: get-comment-body
153+
run: |
154+
body="$(cat ./cts-comparison-report.md)"
155+
body="${body//'%'/'%25'}"
156+
body="${body//$'\n'/'%0A'}"
157+
body="${body//$'\r'/'%0D'}"
158+
echo "::set-output name=body::$body"
159+
# Create or update (if previous comment exists) with markdown version of comparison report
160+
- name: Create or update comment
161+
uses: peter-evans/create-or-update-comment@v2
162+
if: github.event_name == 'pull_request'
163+
with:
164+
comment-id: ${{ steps.fc.outputs.comment-id }}
165+
issue-number: ${{ github.event.pull_request.number }}
166+
body: ${{ steps.get-comment-body.outputs.body }}
167+
edit-mode: replace

‎partiql-conformance-test-generator/src/generator.rs

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use crate::schema::TestCaseKind::Parse;
1+
use crate::schema::TestCaseKind::{NotYetImplemented, Parse};
22
use crate::schema::{Namespace, ParseAssertions, ParseTestCase, TestCase, TestDocument};
33
use codegen::{Function, Module, Scope};
44

@@ -40,10 +40,10 @@ fn namespace_to_module(namespace: &Namespace) -> Module {
4040

4141
/// Converts a test case into a testing `Function`
4242
fn test_case_to_function(test_case: &TestCase) -> Function {
43+
let mut test_fn: Function = Function::new(&test_case.test_name);
44+
test_fn.attr("test");
4345
match &test_case.test_kind {
4446
Parse(ParseTestCase { parse_assertions }) => {
45-
let mut test_fn: Function = Function::new(&test_case.test_name);
46-
test_fn.attr("test");
4747
test_fn.line(format!("let query = r#\"{}\"#;", &test_case.statement));
4848
test_fn.line("let res = partiql_parser::parse_partiql(query);");
4949
match parse_assertions {
@@ -52,9 +52,15 @@ fn test_case_to_function(test_case: &TestCase) -> Function {
5252
ParseAssertions::ParseFail => test_fn
5353
.line(r#"assert!(res.is_err(), "For `{}`, expected `Err(_)`, but was `{:#?}`", query, res);"#),
5454
};
55-
test_fn
5655
}
57-
}
56+
NotYetImplemented => {
57+
// for `NotYetImplemented` test cases, just output the statement and add the 'ignore'
58+
// annotation
59+
test_fn.attr("ignore");
60+
test_fn.line(format!("let _statement = r#\"{}\"#;", &test_case.statement));
61+
}
62+
};
63+
test_fn
5864
}
5965

6066
#[cfg(test)]

‎partiql-conformance-test-generator/src/lib.rs

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ pub mod generator;
22
mod schema;
33
pub mod util;
44

5-
use crate::schema::TestCaseKind::Parse;
5+
use crate::schema::TestCaseKind::{NotYetImplemented, Parse};
66
use crate::schema::{
77
Namespace, Namespaces, ParseAssertions, ParseTestCase, TestCase, TestCases, TestDocument,
88
};
@@ -111,6 +111,8 @@ pub fn test_namespace(element: &OwnedElement) -> Namespace {
111111
/// Parses the given IonStruct to a `TestCase`. Requires for there to be an annotation indicating
112112
/// the test case category (currently limited to just 'parse'). The IonStruct requires two string
113113
/// fields with the 'name' and 'statement'.
114+
///
115+
/// For test case categories that are not supported, will create a `NotYetImplemented` test case.
114116
fn test_case(element: &OwnedElement) -> TestCase {
115117
let annot: Vec<_> = element.annotations().map(|a| a.text().expect("")).collect();
116118

@@ -136,7 +138,11 @@ fn test_case(element: &OwnedElement) -> TestCase {
136138
test_kind: Parse(parse_test_case(element)),
137139
}
138140
} else {
139-
panic!("Invalid test category annotation provided")
141+
TestCase {
142+
test_name,
143+
statement,
144+
test_kind: NotYetImplemented,
145+
}
140146
}
141147
}
142148

‎partiql-conformance-test-generator/src/schema.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,11 @@ pub struct TestCase {
2727
/// Test case kind
2828
///
2929
/// Currently, just supports `Parse` test cases. In the future, other test case variants will be
30-
/// added (e.g. evaluation, type-checking)
30+
/// added (e.g. evaluation, type-checking). For now, the other test case variants will be
31+
/// `NotYetImplemented`.
3132
pub enum TestCaseKind {
3233
Parse(ParseTestCase),
34+
NotYetImplemented,
3335
}
3436

3537
/// Test case to test the parsing behavior of a PartiQL statement

‎partiql-conformance-tests/Cargo.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ partiql-conformance-test-generator = { path = "../partiql-conformance-test-gener
2525

2626
[dependencies]
2727
partiql-parser = { path = "../partiql-parser" }
28+
serde_json = "1.0"
29+
serde = { version = "1.0", features = ["derive"] }
2830

2931
[features]
3032
conformance_test=[]

0 commit comments

Comments
 (0)