Update host-check rc and summary string for case of no errors/failures but warnings (#36)

* Added tests for new error codes

* updated changelog

* .

* autoformatted

* comment updates

* NamedTuple

* formatting

* Same exit code

* review comments

* changed changelog

* changed changelog

* changed changelog

* date

Author: ebeaty-cisco

CHANGELOG.md

@@ -5,6 +5,10 @@
 
 The v1 release supports Cisco IOS-XR release versions from 7.7.1 to 7.10.1.
 
+### v1.1.14 (2023-11-27)
+
+- `host-check` now summarizes failures and warnings separately. When testing both XR platforms, a failure for either is now treated as a failure overall.
+
 ### v1.1.13 (2023-10-11)
 
 - Modify Linux kernel version check to require kernel version 4.6 or higher.

scripts/host-check

@@ -67,6 +67,12 @@ from typing import (
 # Globals
 # -----------------------------------------------------------------------------
 
+
+# Exit codes
+EXIT_SUCCESS = 0
+EXIT_ERROR = 1
+
+
 INOTIFY_RECOMMENDED = 64000
 INOTIFY_PER_CONTAINER = 4000
 
@@ -319,12 +325,13 @@ class CheckState(enum.Enum):
     # 'success' check states
     SUCCESS = "success"
     NEUTRAL = "neutral"
-    # 'failure' check states
+    # 'warning' check states
     WARNING = "warning"
+    # 'failure' check states
     SKIPPED = "skipped"
     FAILED = "failed"
     # Check state for when the check fails to run/errors
-    # Note: this is neither 'success' nor 'failure'
+    # Note: this is not 'success', 'warning' or 'failure'
     ERROR = "error"
 
     def is_failure(self) -> bool:
@@ -335,7 +342,6 @@ class CheckState(enum.Enum):
         return self in [
             CheckState.FAILED,
             CheckState.SKIPPED,
-            CheckState.WARNING,
         ]
 
 
@@ -1597,20 +1603,29 @@ def perform_checks(checks: List[Check]) -> Mapping[str, CheckState]:
     return results
 
 
+class ExtraResults(NamedTuple):
+    """The results of any extra checks."""
+
+    supported: Set[str]
+    not_supported: Set[str]
+    errored: Set[str]
+    warned: Set[str]
+
+
 def run_if_extra_checks(
     extra_checks: Optional[List[str]] = None,
-) -> Tuple[Set[str], Set[str], Set[str]]:
+) -> ExtraResults:
     """
-    Run the specified extra checks, printing the results and returning the
-    checks that succeeded, failed and errored.
+    Run the specified extra checks, printing the results.
     :param extra_checks:
         List of extra checks to perform.
     :returns:
-        Tuple of the checks that: succeeded, failed and errored.
+        NamedTuple of sets of checks that succeeded, failed, errored and warned.
     """
     extras_supported = set()
     extras_not_supported = set()
     extras_errored = set()
+    extras_warned = set()
 
     if extra_checks:
         print("")  # insert a newline
@@ -1623,40 +1638,49 @@ def run_if_extra_checks(
                 extras_not_supported.add(extra_check)
             elif any(c is CheckState.ERROR for c in extra_results.values()):
                 extras_errored.add(extra_check)
+            elif any(c is CheckState.WARNING for c in extra_results.values()):
+                extras_warned.add(extra_check)
             else:
                 extras_supported.add(extra_check)
 
-    return extras_supported, extras_not_supported, extras_errored
+    return ExtraResults(
+        extras_supported,
+        extras_not_supported,
+        extras_errored,
+        extras_warned,
+    )
 
 
-def print_extra_checks_summary(
-    extras_supported: Set[str],
-    extras_not_supported: Set[str],
-    extras_errored: Set[str],
-) -> None:
+def print_extra_checks_summary(extra_results: ExtraResults) -> None:
     """
     Print a summary of the extra checks that passed and failed.
-    :param extras_supported:
-        The checks that were successful.
-    :param extras_not_supported:
-        The checks that were not successful.
-    :param extras_errored:
-        The checks that errored/failed to run.
+    :param extras_results:
+        The results of the extra checks.
     """
     print(f"{DASHED_LINE}")
-    if extras_supported:
-        print("Extra checks passed: " + ", ".join(sorted(extras_supported)))
-    if extras_not_supported:
+    if extra_results.supported:
+        print(
+            "Extra checks passed: "
+            + ", ".join(sorted(extra_results.supported))
+        )
+    if extra_results.not_supported:
+        print(
+            "Extra checks failed: "
+            + ", ".join(sorted(extra_results.not_supported))
+        )
+    if extra_results.errored:
         print(
-            "Extra checks failed: " + ", ".join(sorted(extras_not_supported))
+            "Extra checks errored: " + ", ".join(sorted(extra_results.errored))
+        )
+    if extra_results.warned:
+        print(
+            "Extra checks warned: " + ", ".join(sorted(extra_results.warned))
         )
-    if extras_errored:
-        print("Extra checks errored: " + ", ".join(sorted(extras_errored)))
 
 
 def run_checks_specific_platform(
     platform: str, extra_checks: List[str]
-) -> bool:
+) -> int:
     """
     Run checks for the given platform (including the base checks that are
     required for all platforms).
@@ -1666,19 +1690,18 @@ def run_checks_specific_platform(
     :param extra_checks:
         A list of the extra checks.
     :return:
-        Whether all checks that were performed succeeded.
+        EXIT_SUCCESS if all checks were successful
+        EXIT_ERROR if there are any failures, warnings or errors (including
+            extra checks)
     """
 
     print_heading(f"Platform checks - {platform}")
     plat_results = perform_checks(BASE_CHECKS + PLATFORM_CHECKS_MAP[platform])
     plat_failure = any(check.is_failure() for check in plat_results.values())
     plat_error = any(c is CheckState.ERROR for c in plat_results.values())
+    plat_warning = any(c is CheckState.WARNING for c in plat_results.values())
 
-    (
-        extras_supported,
-        extras_not_supported,
-        extras_errored,
-    ) = run_if_extra_checks(extra_checks)
+    extra_results = run_if_extra_checks(extra_checks)
 
     # Print a summary of what is and is not supported
     print(f"\n{DOUBLE_DASHED_LINE}")
@@ -1688,29 +1711,44 @@ def run_checks_specific_platform(
         print(
             "!! One or more platform checks could not be performed, see errors above !!"
         )
+    elif plat_warning:
+        print(
+            "!! One or more platform checks resulted in a warning, see warnings above !!"
+        )
     else:
         print(f"Host environment set up correctly for {platform}")
     if extra_checks:
-        print_extra_checks_summary(
-            extras_supported, extras_not_supported, extras_errored
-        )
+        print_extra_checks_summary(extra_results)
     print(f"{DOUBLE_DASHED_LINE}")
 
-    return not (
-        plat_failure or plat_error or extras_not_supported or extras_errored
-    )
+    if (
+        plat_failure
+        or plat_error
+        or extra_results.not_supported
+        or extra_results.errored
+        or plat_warning
+        or extra_results.warned
+    ):
+        rc = EXIT_ERROR
+    else:
+        rc = EXIT_SUCCESS
+
+    return rc
 
 
-def run_checks_all_platforms(extra_checks: List[str]) -> bool:
+def run_checks_all_platforms(extra_checks: List[str]) -> int:
     """
     Run checks for all platforms and report which platforms are supported.
 
     :return:
-        Whether at least one XR platform is supported.
+        EXIT_SUCCESS if all checks were successful
+        EXIT_ERROR if there are any failures, warnings or errors (including
+            extra tests)
     """
     platforms_supported: Set[str] = set()
     platforms_not_supported: Set[str] = set()
     platforms_errored: Set[str] = set()
+    platforms_warned: Set[str] = set()
 
     print_heading("Platform checks")
     print_subheading("base checks")
@@ -1727,14 +1765,14 @@ def run_checks_all_platforms(extra_checks: List[str]) -> bool:
             result is CheckState.ERROR for result in base_and_plat_results
         ):
             platforms_errored.add(platform)
+        elif any(
+            result is CheckState.WARNING for result in base_and_plat_results
+        ):
+            platforms_warned.add(platform)
         else:
             platforms_supported.add(platform)
 
-    (
-        extras_supported,
-        extras_not_supported,
-        extras_errored,
-    ) = run_if_extra_checks(extra_checks)
+    extra_results = run_if_extra_checks(extra_checks)
 
     # Print a summary of what is and is not supported
     print(f"\n{DOUBLE_DASHED_LINE}")
@@ -1744,21 +1782,43 @@ def run_checks_all_platforms(extra_checks: List[str]) -> bool:
         )
     if platforms_not_supported:
         print(
-            "XR platforms NOT supported: "
+            "!! XR platforms NOT supported: "
             + ", ".join(sorted(platforms_not_supported))
+            + " !!"
         )
     if platforms_errored:
         print(
-            "!! One or more platform checks could not be performed, see errors above !!"
+            "!! One or more platform checks could not be performed for XR platforms: !!"
+            + "\n      "
+            + ",\n      ".join(sorted(platforms_errored))
+            + "\n   See errors above."
+        )
+    if platforms_warned:
+        print(
+            "!! One or more platform checks resulted in a warning for XR platforms: !!"
+            + "\n      "
+            + ",\n      ".join(sorted(platforms_warned))
+            + "\n   See warnings above."
         )
+
     # Print the result of any extra checks
     if extra_checks:
-        print_extra_checks_summary(
-            extras_supported, extras_not_supported, extras_errored
-        )
+        print_extra_checks_summary(extra_results)
     print(f"{DOUBLE_DASHED_LINE}")
 
-    return len(platforms_supported) > 0
+    if (
+        platforms_not_supported
+        or platforms_errored
+        or extra_results.not_supported
+        or extra_results.errored
+        or platforms_warned
+        or extra_results.warned
+    ):
+        rc = EXIT_ERROR
+    else:
+        rc = EXIT_SUCCESS
+
+    return rc
 
 
 def parse_args(argv: List[str]) -> argparse.Namespace:
@@ -1786,12 +1846,10 @@ def parse_args(argv: List[str]) -> argparse.Namespace:
 def main(argv: List[str]) -> NoReturn:
     args = parse_args(argv)
     if args.platform:
-        success = run_checks_specific_platform(
-            args.platform, args.extra_checks
-        )
+        rc = run_checks_specific_platform(args.platform, args.extra_checks)
     else:
-        success = run_checks_all_platforms(args.extra_checks)
-    sys.exit(0 if success else 1)
+        rc = run_checks_all_platforms(args.extra_checks)
+    sys.exit(rc)
 
 
 if __name__ == "__main__":