Merge pull request #5 from code-dot-org/molly/improve-validation-output

Return validation results in a standard format

Author: molly-moen

packages/unittest_runner/pyproject.toml

@@ -4,5 +4,5 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "unittest_runner"
-version = "0.0.1"
+version = "0.1.0"
 dependencies = ["unittest"]

packages/unittest_runner/unittest_runner/run_tests.py

@@ -1,18 +1,36 @@
 import unittest
 from .validation_runner import ValidationTestResult
 
-# Run the tests in the given file pattern and display the results to the user.
-# Validation tests use the ValidationTestResult class to display only the short description of the test.
 def run_validation_tests(file_pattern):
-  run_tests(file_pattern, ValidationTestResult)
+  """
+  Run the tests matching the given file pattern and display the results to the user.
+  Validation tests use the ValidationTestResult class to display only the short description of the test.
+
+  Args:
+    file_pattern (str): A glob pattern to match test files.
+
+  Returns:
+      List[Dict[str, str]]: A simplified list of test results with each entry containing:
+          - 'name': The name of the test.
+          - 'result': The outcome, which is one of the following:
+            'PASS', 'FAIL', 'ERROR', 'SKIP', 'EXPECTED_FAILURE', 'UNEXPECTED_SUCCESS'.
+  """
+  result = run_tests(file_pattern, ValidationTestResult)
+  return result.simplified_results
+
 
-# Run the tests in the given file pattern and display the results to the user.
-# Student tests use the standard TextTestResult class to display the test name and short description.
 def run_student_tests(file_pattern):
+  """
+  Run the tests in the given file pattern and display the results to the user.
+  Student tests use the standard TextTestResult class to display the test name and short description.
+  
+  Args:
+    file_pattern (str): A glob pattern to match test files.
+  """
   run_tests(file_pattern, unittest.TextTestResult)
 
 def run_tests(file_pattern, resultclass):
   loader = unittest.TestLoader()
   test_suite = loader.discover('.', file_pattern)
   runner = unittest.TextTestRunner(verbosity=2, resultclass=resultclass)
-  result = runner.run(test_suite)
+  return runner.run(test_suite)

packages/unittest_runner/unittest_runner/validation_runner.py

@@ -2,6 +2,10 @@
 from unittest import TextTestResult
 
 class ValidationTestResult(TextTestResult):
+  def __init__(self, stream, descriptions, verbosity):
+    super(ValidationTestResult, self).__init__(stream, descriptions, verbosity)
+    self.simplified_results = []
+
   # The default behavior is to display the test name and short description.
   # We want to display only the short description if it exists, otherwise just the test name.
   # Students don't see the tests, so we don't need to display the test name.
@@ -12,3 +16,27 @@ def getDescription(self, test):
       return doc_first_line
     else:
       return str(test)
+
+  def addSuccess(self, test):
+      super(ValidationTestResult, self).addSuccess(test)
+      self.simplified_results.append({'name': self.getDescription(test), 'result': "PASS"})
+
+  def addError(self, test, err):
+      super(ValidationTestResult, self).addError(test, err)
+      self.simplified_results.append({'name': self.getDescription(test), 'result': "ERROR"})
+
+  def addFailure(self, test, err):
+      super(ValidationTestResult, self).addFailure(test, err)
+      self.simplified_results.append({'name': self.getDescription(test), 'result': "FAIL"})
+
+  def addSkip(self, test, reason):
+      super(ValidationTestResult, self).addSkip(test, reason)
+      self.simplified_results.append({'name': self.getDescription(test), 'result': "SKIP"})
+
+  def addExpectedFailure(self, test, err):
+      super(ValidationTestResult, self).addExpectedFailure(test, err)
+      self.simplified_results.append({'name': self.getDescription(test), 'result': "EXPECTED_FAILURE"})
+
+  def addUnexpectedSuccess(self, test):
+      super(ValidationTestResult, self).addUnexpectedSuccess(test)
+      self.simplified_results.append({'name': self.getDescription(test), 'result': "UNEXPECTED_SUCCESS"})