Generate solacc reports (#2757)

## Changes
Currently, `solacc` outputs advices to the console, and does not keep
track of linting time.
This PR:
 - dumps advices to an 'advices.txt' file
- collects stats and dumps them to a 'stats.json' file (in the form of a
json-like file, with 1 json object per line)
 - uploads the above 

### Linked issues
None

### Functionality
None

### Tests
- [x] manually tested:

Sample solacc run:
![Screenshot 2024-09-27 at 16 44
22](https://github.com/user-attachments/assets/a0f508e4-4095-45b8-be38-51d08824b15e)

Sample stats.json (expanded):
```
{
  "run_id": "1",
  "name": "ab-testing",
  "start_timestamp": "2024-09-27 10:16:02.512363+00:00",
  "end_timestamp": "2024-09-27 10:17:07.622161+00:00",
  "files_count": 6,
  "files_size": 34934
}
{
  "run_id": "1",
  "name": "adverse-drug-events",
  "start_timestamp": "2024-09-27 10:17:08.669225+00:00",
  "end_timestamp": "2024-09-27 10:17:09.399495+00:00",
  "files_count": 5,
  "files_size": 48743
}
{
  "run_id": "1",
  "name": "als-recommender",
  "start_timestamp": "2024-09-27 10:17:09.565942+00:00",
  "end_timestamp": "2024-09-27 10:17:12.039422+00:00",
  "files_count": 6,
  "files_size": 62750
}
```


Sample advices.txt:
```
./dist/ab-testing/4. Real time inference.py:76:0: [legacy-context-in-shared-clusters] sc is not supported on UC Shared Clusters. Rewrite it using spark
./dist/ab-testing/4. Real time inference.py:139:0: [jvm-access-in-shared-clusters] Cannot access Spark Driver JVM on UC Shared Clusters
./dist/ab-testing/4. Real time inference.py:139:0: [legacy-context-in-shared-clusters] sc is not supported on UC Shared Clusters. Rewrite it using spark
./dist/ab-testing/RUNME.py:1:1: [library-install-failed] Unsupported 'pip' command: DBTITLE
./dist/ab-testing/5. AB testing metrics.py:50:0: [jvm-access-in-shared-clusters] Cannot access Spark Driver JVM on UC Shared Clusters
./dist/ab-testing/5. AB testing metrics.py:50:0: [legacy-context-in-shared-clusters] sc is not supported on UC Shared Clusters. Rewrite it using spark
./dist/ab-testing/5. AB testing metrics.py:51:0: [jvm-access-in-shared-clusters] Cannot access Spark Driver JVM on UC Shared Clusters
./dist/ab-testing/5. AB testing metrics.py:51:0: [legacy-context-in-shared-clusters] sc is not supported on UC Shared Clusters. Rewrite it using spark
./dist/ab-testing/5. AB testing metrics.py:54:0: [jvm-access-in-shared-clusters] Cannot access Spark Driver JVM on UC Shared Clusters
./dist/ab-testing/5. AB testing metrics.py:54:0: [legacy-context-in-shared-clusters] sc is not supported on UC Shared Clusters. Rewrite it using spark
./dist/ab-testing/5. AB testing metrics.py:58:0: [jvm-access-in-shared-clusters] Cannot access Spark Driver JVM on UC Shared Clusters
./dist/ab-testing/5. AB testing metrics.py:58:0: [legacy-context-in-shared-clusters] sc is not supported on UC Shared Clusters. Rewrite it using spark
./dist/ab-testing/2. Model training.py:34:0: [jvm-access-in-shared-clusters] Cannot access Spark Driver JVM on UC Shared Clusters
./dist/ab-testing/2. Model training.py:34:0: [legacy-context-in-shared-clusters] sc is not supported on UC Shared Clusters. Rewrite it using spark
./dist/ab-testing/2. Model training.py:36:0: [legacy-context-in-shared-clusters] sc is not supported on UC Shared Clusters. Rewrite it using spark
./dist/ab-testing/2. Model training.py:37:0: [legacy-context-in-shared-clusters] sc is not supported on UC Shared Clusters. Rewrite it using spark
./dist/ab-testing/2. Model training.py:37:0: [rdd-in-shared-clusters] RDD APIs are not supported on UC Shared Clusters. Rewrite it using DataFrame API
./dist/ab-testing/2. Model training.py:38:0: [jvm-access-in-shared-clusters] Cannot access Spark Driver JVM on UC Shared Clusters
./dist/ab-testing/2. Model training.py:41:0: [jvm-access-in-shared-clusters] Cannot access Spark Driver JVM on UC Shared Clusters
./dist/ab-testing/2. Model training.py:41:0: [legacy-context-in-shared-clusters] sc is not supported on UC Shared Clusters. Rewrite it using spark
./dist/ab-testing/5. AB testing metrics.py:22:0: [default-format-changed-in-dbr8] The default format changed in Databricks Runtime 8.0, from Parquet to Delta
./dist/ab-testing/5. AB testing metrics.py:36:27: [default-format-changed-in-dbr8] The default format changed in Databricks Runtime 8.0, from Parquet to Delta
./dist/ab-testing/5. AB testing metrics.py:90:2: [default-format-changed-in-dbr8] The default format changed in Databricks Runtime 8.0, from Parquet to Delta
./dist/ab-testing/5. AB testing metrics.py:97:2: [default-format-changed-in-dbr8] The default format changed in Databricks Runtime 8.0, from Parquet to Delta
./dist/ab-testing/risk_demo.dbdash:1:0: [unknown-language] Cannot detect language for /home/runner/work/ucx/ucx/dist/ab-testing/risk_demo.dbdash
./dist/ab-testing/1. Introduction.py:72:2: [direct-filesystem-access] The use of direct filesystem references is deprecated: /tmp/german_credit_data.csv
./dist/ab-testing/4. Real time inference.py:10:0: [direct-filesystem-access] The use of direct filesystem references is deprecated: /FileStore/tmp/streaming_ckpnt_risk_demo
./dist/ab-testing/4. Real time inference.py:10:14: [direct-filesystem-access] The use of direct filesystem references is deprecated: /FileStore/tmp/streaming_ckpnt_risk_demo
./dist/ab-testing/4. Real time inference.py:29:5: [default-format-changed-in-dbr8] The default format changed in Databricks Runtime 8.0, from Parquet to Delta
./dist/ab-testing/4. Real time inference.py:214:32: [direct-filesystem-access] The use of direct filesystem references is deprecated: /FileStore/tmp/streaming_ckpnt_risk_demo
./dist/ab-testing/4. Real time inference.py:236:2: [default-format-changed-in-dbr8] The default format changed in Databricks Runtime 8.0, from Parquet to Delta
./dist/ab-testing/4. Real time inference.py:254:27: [default-format-changed-in-dbr8] The default format changed in Databricks Runtime 8.0, from Parquet to Delta
./dist/ab-testing/2. Model training.py:53:22: [direct-filesystem-access] The use of direct filesystem references is deprecated: /Users/Uninferable/german_credit_experiment
./dist/ab-testing/2. Model training.py:66:5: [default-format-changed-in-dbr8] The default format changed in Databricks Runtime 8.0, from Parquet to Delta
./dist/ab-testing/2. Model training.py:197:67: [direct-filesystem-access] The use of direct filesystem references is deprecated: /tmp/pr-curve-model-a.png
./dist/ab-testing/2. Model training.py:229:67: [direct-filesystem-access] The use of direct filesystem references is deprecated: /tmp/pr-curve-model-b.png
./dist/adverse-drug-events/RUNME.py:1:1: [library-install-failed] Unsupported 'pip' command: DBTITLE
./dist/adverse-drug-events/01-ade-extraction.py:23:0: [jvm-access-in-shared-clusters] Cannot access Spark Driver JVM on UC Shared Clusters
./dist/adverse-drug-events/01-ade-extraction.py:23:0: [legacy-context-in-shared-clusters] sc is not supported on UC Shared Clusters. Rewrite it using spark
./dist/adverse-drug-events/02-ade-analysis.py:13:0: [jvm-access-in-shared-clusters] Cannot access Spark Driver JVM on UC Shared Clusters
./dist/adverse-drug-events/02-ade-analysis.py:13:0: [legacy-context-in-shared-clusters] sc is not supported on UC Shared Clusters. Rewrite it using spark
./dist/adverse-drug-events/02-ade-analysis.py:14:0: [jvm-access-in-shared-clusters] Cannot access Spark Driver JVM on UC Shared Clusters

```

---------

Co-authored-by: Eric Vergnaud <eric.vergnaud@databricks.com>

Author: ericvergnaud

.github/workflows/solacc.yml

@@ -26,3 +26,10 @@ jobs:
 
       - name: Verify linters on solution accelerators
         run: make solacc
+
+      - name: Upload reports
+        uses: actions/upload-artifact@v4
+        with:
+          name: report
+          path: build/
+          if-no-files-found: error

src/databricks/labs/ucx/source_code/base.py

@@ -96,7 +96,10 @@ def message_relative_to(self, base: Path, *, default: Path | None = None) -> str
             logger.debug(f'THIS IS A BUG! {advice.code}:{advice.message} has unknown path')
         if default is not None:
             path = default
-        path = path.relative_to(base)
+        try:
+            path = path.relative_to(base)
+        except ValueError:
+            logger.debug(f'Not a relative path: {path} to base: {base}')
         # increment start_line because it is 0-based whereas IDEs are usually 1-based
         return f"./{path.as_posix()}:{advice.start_line+1}:{advice.start_col}: [{advice.code}] {advice.message}"
 

src/databricks/labs/ucx/source_code/notebooks/sources.py

@@ -163,7 +163,13 @@ def __init__(
         self._python_trees: dict[PythonCell, Tree] = {}  # the original trees to be linted
 
     def lint(self) -> Iterable[Advice]:
-        yield from self._load_tree_from_notebook(self._notebook, True)
+        has_failure = False
+        for advice in self._load_tree_from_notebook(self._notebook, True):
+            if isinstance(advice, Failure):  # happens when a cell is unparseable
+                has_failure = True
+            yield advice
+        if has_failure:
+            return
         for cell in self._notebook.cells:
             if not self._context.is_supported(cell.language.language):
                 continue

tests/integration/source_code/solacc.py

@@ -1,8 +1,11 @@
+import dataclasses
+import json
 import logging
 import os
 import shutil
 import sys
 from dataclasses import dataclass, field
+from datetime import datetime, timezone
 from pathlib import Path
 
 import requests
@@ -20,6 +23,8 @@
 
 this_file = Path(__file__)
 dist = (this_file / '../../../../dist').resolve().absolute()
+build = dist.parent / "build"
+build.mkdir(exist_ok=True)
 
 
 def _get_repos_to_clone() -> dict[str, str]:
@@ -72,23 +77,41 @@ def _collect_uninferrable_count(advices: list[LocatedAdvice]):
 
 
 def _collect_unparseable(advices: list[LocatedAdvice]):
-    return set(located_advice for located_advice in advices if located_advice.advice.code == 'parse-error')
+    return list(located_advice for located_advice in advices if located_advice.advice.code == 'parse-error')
 
 
 def _print_advices(advices: list[LocatedAdvice]):
-    for located_advice in advices:
-        message = located_advice.message_relative_to(dist.parent)
-        sys.stdout.write(f"{message}\n")
+    messages = list(
+        located_advice.message_relative_to(dist.parent).replace('\n', ' ') + '\n' for located_advice in advices
+    )
+    if os.getenv("CI"):
+        advices_path = build / "advices.txt"
+        with advices_path.open("a") as advices_file:
+            advices_file.writelines(messages)
+    else:
+        for message in messages:
+            sys.stdout.write(message)
+
+
+@dataclass
+class _SolaccStats:
+    run_id: str
+    name: str
+    start_timestamp: datetime
+    end_timestamp: datetime
+    files_count: int
+    files_size: int
 
 
 @dataclass
 class _SolaccContext:
     unparsed_files_path: Path | None = None
-    files_to_skip: set[str] | None = None
+    files_to_skip: set[Path] | None = None
     total_count = 0
     parseable_count = 0
     uninferrable_count = 0
     missing_imports: dict[str, dict[str, int]] = field(default_factory=dict)
+    stats: list[_SolaccStats] = field(default_factory=list)
 
     @classmethod
     def create(cls, for_all_dirs: bool):
@@ -98,11 +121,11 @@ def create(cls, for_all_dirs: bool):
             unparsed_path = Path(Path(__file__).parent, "solacc-unparsed.txt")
             if unparsed_path.exists():
                 os.remove(unparsed_path)
-        files_to_skip: set[str] | None = None
+        files_to_skip: set[Path] | None = None
         malformed = Path(__file__).parent / "solacc-malformed.txt"
         if for_all_dirs and malformed.exists():
             lines = malformed.read_text(encoding="utf-8").split("\n")
-            files_to_skip = set(line for line in lines if len(line) > 0 and not line.startswith("#"))
+            files_to_skip = set(dist / line for line in lines if len(line) > 0 and not line.startswith("#"))
         return _SolaccContext(unparsed_files_path=unparsed_path, files_to_skip=files_to_skip)
 
     def register_missing_import(self, missing_import: str):
@@ -153,7 +176,19 @@ def _lint_dir(solacc: _SolaccContext, soldir: Path):
     files_to_skip = set(solacc.files_to_skip) if solacc.files_to_skip else set()
     linted_files = set(files_to_skip)
     # lint solution
+    start_timestamp = datetime.now(timezone.utc)
     advices = list(ctx.local_code_linter.lint_path(soldir, linted_files))
+    end_timestamp = datetime.now(timezone.utc)
+    # record stats
+    stats = _SolaccStats(
+        run_id=os.getenv("GITHUB_RUN_ATTEMPT") or "local",
+        start_timestamp=start_timestamp,
+        end_timestamp=end_timestamp,
+        name=soldir.name,
+        files_count=len(all_files),
+        files_size=sum(path.stat().st_size for path in [soldir / filename for filename in all_files]),
+    )
+    solacc.stats.append(stats)
     # collect unparseable files
     unparseables = _collect_unparseable(advices)
     solacc.parseable_count += len(linted_files) - len(files_to_skip) - len(set(advice.path for advice in unparseables))
@@ -162,7 +197,11 @@ def _lint_dir(solacc: _SolaccContext, soldir: Path):
             logger.error(f"Error during parsing of {unparseable.path}: {unparseable.advice.message}".replace("\n", " "))
             # populate solacc-unparsed.txt
             with solacc.unparsed_files_path.open(mode="a", encoding="utf-8") as f:
-                f.write(unparseable.path.relative_to(dist).as_posix())
+                try:
+                    path = unparseable.path.relative_to(dist)
+                except ValueError:
+                    path = unparseable.path
+                f.write(path.as_posix())
                 f.write("\n")
     # collect missing imports
     for missing_import in _collect_missing_imports(advices):
@@ -178,8 +217,8 @@ def _lint_dir(solacc: _SolaccContext, soldir: Path):
 def _lint_repos(clone_urls, sol_to_lint: str | None):
     solacc = _SolaccContext.create(sol_to_lint is not None)
     if sol_to_lint:
-        # don't clone if linting just one file, assumption is we're troubleshooting
-        _lint_dir(solacc, dist / sol_to_lint)
+        sol_dir = _clone_repo(clone_urls[sol_to_lint], sol_to_lint)
+        _lint_dir(solacc, sol_dir)
     else:
         names: list[str] = list(clone_urls.keys())
         for name in sorted(names, key=str.casefold):
@@ -199,6 +238,12 @@ def _lint_repos(clone_urls, sol_to_lint: str | None):
         f"not computed: {solacc.uninferrable_count}"
     )
     solacc.log_missing_imports()
+    # log stats
+    stats_path = build / "stats.json"
+    with stats_path.open("a") as stats_file:
+        for stats in solacc.stats:
+            message = json.dumps(dataclasses.asdict(stats), default=str)
+            stats_file.writelines([message])
     # fail the job if files are unparseable
     if parseable_pct < 100:
         sys.exit(1)