Skip to content

Commit c284a45

Browse files
jsingletjsinglet
committed
fixes
1 parent 91f21a7 commit c284a45

File tree

4 files changed

+126
-21
lines changed

4 files changed

+126
-21
lines changed

‎.github/workflows/DEMO-run-codeql-unit-tests-cpp.yml

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -109,18 +109,17 @@ jobs:
109109
needs: [run-test-suites]
110110
runs-on: ubuntu-latest
111111
steps:
112+
113+
- name: Install QLT
114+
id: install-qlt
115+
uses: ./.github/actions/install-qlt
116+
with:
117+
qlt-version: 'latest'
118+
add-to-path: true
119+
112120
- name: Collect test results
113121
uses: actions/download-artifact@v2
114122

115123
- name: Validate test results
116124
run: |
117-
for json_report in test-results-*/test_report_*
118-
do
119-
jq --raw-output '"PASS \(map(select(.pass == true)) | length)/\(length)'" $json_report\"" "$json_report"
120-
done
121-
FAILING_TESTS=$(jq --raw-output '.[] | select(.pass == false)' test-results-*/test_report_*.json)
122-
if [[ ! -z "$FAILING_TESTS" ]]; then
123-
echo "ERROR: The following tests failed:"
124-
echo $FAILING_TESTS | jq .
125-
exit 1
126-
fi
125+
qlt test run validate-unit-test-results --results-directory .

‎src/CodeQLToolkit.Features.Test/Commands/Targets/ValidateUnitTestsCommand.cs

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
using Newtonsoft.Json;
2+
using System;
3+
using System.Collections.Generic;
4+
using System.Linq;
5+
using System.Text;
6+
using System.Threading.Tasks;
7+
8+
namespace CodeQLToolkit.Features.Test.Commands.Targets
9+
{
10+
11+
12+
public class UnitTestResult
13+
{
14+
public string test { get; set; }
15+
public bool pass { get; set; }
16+
public string failureStage { get; set; }
17+
public string failureDescription { get; set; }
18+
public object[] messages { get; set; }
19+
public int compilationMs { get; set; }
20+
public int evaluationMs { get; set; }
21+
public string expected { get; set; }
22+
public string actual { get; set; }
23+
public string[] diff { get; set; }
24+
}
25+
26+
public class ValidateUnitTestsCommand : CommandTarget
27+
{
28+
public string ResultsDirectory { get; set; }
29+
30+
31+
public override void Run()
32+
{
33+
Log<ValidateUnitTestsCommand>.G().LogInformation($"Validating unit tests in {ResultsDirectory}");
34+
35+
string[] results = Directory.GetFiles(ResultsDirectory, "test_report_*", SearchOption.AllDirectories);
36+
37+
List<UnitTestResult> failures = new List<UnitTestResult>();
38+
39+
foreach (string result in results)
40+
{
41+
using (StreamReader r = new StreamReader(result))
42+
{
43+
string json = r.ReadToEnd();
44+
List<UnitTestResult> items = JsonConvert.DeserializeObject<List<UnitTestResult>>(json);
45+
46+
foreach(var item in items)
47+
{
48+
if (item.pass == false)
49+
{
50+
failures.Add(item);
51+
}
52+
}
53+
}
54+
}
55+
56+
if(failures.Count > 0 )
57+
{
58+
Log<ValidateUnitTestsCommand>.G().LogError($"One or more unit tests failed. Details below:");
59+
Log<ValidateUnitTestsCommand>.G().LogError($"---------------------------------------------");
60+
61+
foreach (var item in failures)
62+
{
63+
64+
if(item.failureStage == "RESULT")
65+
{
66+
Log<ValidateUnitTestsCommand>.G().LogError($"Test: {item.test}");
67+
Log<ValidateUnitTestsCommand>.G().LogError($"--------------------------------------------------");
68+
Log<ValidateUnitTestsCommand>.G().LogError($"Diff of Expected vs Actual");
69+
Log<ValidateUnitTestsCommand>.G().LogError($"--------------------------------------------------");
70+
71+
foreach(var diff in item.diff)
72+
{
73+
Log<ValidateUnitTestsCommand>.G().LogError($"{diff}");
74+
}
75+
76+
Log<ValidateUnitTestsCommand>.G().LogError($"\n\n\n\n");
77+
78+
}
79+
else
80+
{
81+
Log<ValidateUnitTestsCommand>.G().LogError($"Non-test content related failure:");
82+
Log<ValidateUnitTestsCommand>.G().LogError($"Test: {item.test}");
83+
Log<ValidateUnitTestsCommand>.G().LogError($"Failure Description: {item.failureDescription}");
84+
}
85+
}
86+
Log<ValidateUnitTestsCommand>.G().LogError($"---------------------------------------------");
87+
88+
DieWithError("One or more failures during run unit tests.");
89+
}
90+
91+
}
92+
}
93+
}

‎src/CodeQLToolkit.Features.Test/Commands/TestCommandFeature.cs

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
using CodeQLToolkit.Features.Test.Lifecycle;
1+
using CodeQLToolkit.Features.Test.Commands.Targets;
2+
using CodeQLToolkit.Features.Test.Lifecycle;
23
using CodeQLToolkit.Shared.Types;
34
using Microsoft.VisualBasic;
45
using System;
@@ -100,6 +101,19 @@ public void Register(Command parentCommand)
100101
featureTarget.Run();
101102

102103
}, Globals.BasePathOption, Globals.AutomationTypeOption, numThreadsOption, workDirectoryOption, languageOption, runnerOSOption, cliVersionOption, stdLibIdentOption);
104+
105+
106+
validateUnitTestsCommand.SetHandler((resultsDirectory) =>
107+
{
108+
Log<TestCommandFeature>.G().LogInformation("Executing validate-unit-tests command...");
109+
110+
new ValidateUnitTestsCommand()
111+
{
112+
ResultsDirectory = resultsDirectory
113+
}.Run();
114+
115+
116+
}, resultsDirectoryOption);
103117
}
104118

105119
public int Run()

‎src/CodeQLToolkit.Features.Test/Templates/Test/Actions/run-unit-tests.liquid

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -108,19 +108,18 @@ jobs:
108108
needs: [run-test-suites]
109109
runs-on: ubuntu-latest
110110
steps:
111+
112+
- name: Install QLT
113+
id: install-qlt
114+
uses: ./.github/actions/install-qlt
115+
with:
116+
qlt-version: 'latest'
117+
add-to-path: true
118+
111119
- name: Collect test results
112120
uses: actions/download-artifact@v2
113121
114122
- name: Validate test results
115123
run: |
116-
for json_report in test-results-*/test_report_*
117-
do
118-
jq --raw-output '"PASS \(map(select(.pass == true)) | length)/\(length)'" $json_report\"" "$json_report"
119-
done
120-
FAILING_TESTS=$(jq --raw-output '.[] | select(.pass == false)' test-results-*/test_report_*.json)
121-
if [[ ! -z "$FAILING_TESTS" ]]; then
122-
echo "ERROR: The following tests failed:"
123-
echo $FAILING_TESTS | jq .
124-
exit 1
125-
fi
124+
qlt test run validate-unit-test-results --results-directory .
126125
{% endraw %}

0 commit comments

Comments
 (0)