Refactor linting related resources (#3633)

## Changes
Refactor linting related resources by moving them into the `linters/`
submodule **with the exception of lsp resources as I did not want to
touch them**. This refactor PR should make the work on the linter easier
as i) it is easier to find the linter resources, ii) those PR required
small refactors because the resources were entangled.

- Moved linting resources to the `linters` submodule.
- Introduced `files.py` modules containing file related resources
(files, notebooks, imports)
- Introduced `folders.py` modules containing folder related resource
(combining files and notebooks)
- Introduced a `linters/base.py` for linter base resources.

High level linter resources:
- `WorkflowLinter` in `jobs.py` (kept `jobs.py` as that is consistent
with other parts of the code base)
- `LocalCodeLinter` in `folders.py`, lints a directory of files

Mid level linter resources:
- `FileLinter` and `FileMigrator` (will be merged later)
- `NotebookLinter` and `NotebookMigrator` (will be merged later)

Low level linter resources
- `PythonAnalyzer`
- All the linter and collectors that parse a Python tree

Author: JCZuurmond

src/databricks/labs/ucx/cli.py

@@ -19,7 +19,7 @@
 from databricks.labs.ucx.contexts.workspace_cli import WorkspaceContext, LocalCheckoutContext
 from databricks.labs.ucx.hive_metastore.tables import What
 from databricks.labs.ucx.install import AccountInstaller
-from databricks.labs.ucx.source_code.linters.files import LocalCodeLinter
+from databricks.labs.ucx.source_code.linters.folders import LocalCodeLinter
 from databricks.labs.ucx.workspace_access.groups import AccountGroupLookup
 
 ucx = App(__file__)

src/databricks/labs/ucx/contexts/application.py

@@ -65,16 +65,17 @@
 from databricks.labs.ucx.installer.workflows import DeployedWorkflows
 from databricks.labs.ucx.progress.install import VerifyProgressTracking
 from databricks.labs.ucx.source_code.graph import DependencyResolver
-from databricks.labs.ucx.source_code.jobs import WorkflowLinter
+from databricks.labs.ucx.source_code.linters.jobs import WorkflowLinter
 from databricks.labs.ucx.source_code.known import KnownList
-from databricks.labs.ucx.source_code.linters.files import FileLoader, FolderLoader, ImportFileResolver
+from databricks.labs.ucx.source_code.folders import FolderLoader
+from databricks.labs.ucx.source_code.files import FileLoader, ImportFileResolver
 from databricks.labs.ucx.source_code.notebooks.loaders import (
     NotebookResolver,
     NotebookLoader,
 )
 from databricks.labs.ucx.source_code.path_lookup import PathLookup
-from databricks.labs.ucx.source_code.queries import QueryLinter
-from databricks.labs.ucx.source_code.redash import Redash
+from databricks.labs.ucx.source_code.linters.queries import QueryLinter
+from databricks.labs.ucx.source_code.linters.redash import Redash
 from databricks.labs.ucx.workspace_access import generic, redash
 from databricks.labs.ucx.workspace_access.groups import GroupManager
 from databricks.labs.ucx.workspace_access.manager import PermissionManager

src/databricks/labs/ucx/contexts/workspace_cli.py

@@ -23,7 +23,7 @@
 from databricks.labs.ucx.progress.install import ProgressTrackingInstallation
 from databricks.labs.ucx.source_code.base import CurrentSessionState
 from databricks.labs.ucx.source_code.linters.context import LinterContext
-from databricks.labs.ucx.source_code.linters.files import LocalFileMigrator, LocalCodeLinter
+from databricks.labs.ucx.source_code.linters.folders import LocalCodeLinter, LocalFileMigrator
 from databricks.labs.ucx.source_code.notebooks.loaders import NotebookLoader
 from databricks.labs.ucx.workspace_access.clusters import ClusterAccess
 

src/databricks/labs/ucx/install.py

@@ -74,7 +74,7 @@
 from databricks.labs.ucx.source_code.base import UsedTable
 from databricks.labs.ucx.source_code.directfs_access import DirectFsAccess
 from databricks.labs.ucx.source_code.jobs import JobProblem
-from databricks.labs.ucx.source_code.queries import QueryProblem
+from databricks.labs.ucx.source_code.linters.queries import QueryProblem
 from databricks.labs.ucx.workspace_access.base import Permissions
 from databricks.labs.ucx.workspace_access.generic import WorkspaceObjectInfo
 from databricks.labs.ucx.workspace_access.groups import AccountGroupLookup, ConfigureGroups, MigratedGroup

src/databricks/labs/ucx/progress/dashboards.py

@@ -11,7 +11,7 @@
 from databricks.labs.ucx.progress.history import ProgressEncoder
 from databricks.labs.ucx.progress.install import Historical
 from databricks.labs.ucx.source_code.base import UsedTable
-from databricks.labs.ucx.source_code.queries import QueryProblem
+from databricks.labs.ucx.source_code.linters.queries import QueryProblem
 from databricks.labs.ucx.source_code.used_table import UsedTablesCrawler
 
 

src/databricks/labs/ucx/source_code/base.py

@@ -13,8 +13,6 @@
 from typing import Any, BinaryIO, TextIO, TypeVar
 
 from astroid import NodeNG  # type: ignore
-from sqlglot import Expression, parse as parse_sql
-from sqlglot.errors import SqlglotError
 
 from databricks.sdk.service import compute
 from databricks.sdk.service.workspace import Language
@@ -125,56 +123,6 @@ class Convention(Advice):
     """A suggestion for a better way to write the code."""
 
 
-class Linter:
-    @abstractmethod
-    def lint(self, code: str) -> Iterable[Advice]: ...
-
-
-class SqlLinter(Linter):
-
-    def lint(self, code: str) -> Iterable[Advice]:
-        try:
-            # TODO: unify with SqlParser.walk_expressions(...)
-            expressions = parse_sql(code, read='databricks')
-            for expression in expressions:
-                if not expression:
-                    continue
-                yield from self.lint_expression(expression)
-        except SqlglotError as e:
-            logger.debug(f"Failed to parse SQL: {code}", exc_info=e)
-            yield self.sql_parse_failure(code)
-
-    @staticmethod
-    def sql_parse_failure(code: str) -> Failure:
-        return Failure(
-            code='sql-parse-error',
-            message=f"SQL expression is not supported yet: {code}",
-            # SQLGlot does not propagate tokens yet. See https://github.com/tobymao/sqlglot/issues/3159
-            start_line=0,
-            start_col=0,
-            end_line=0,
-            end_col=1024,
-        )
-
-    @abstractmethod
-    def lint_expression(self, expression: Expression) -> Iterable[Advice]: ...
-
-
-class Fixer(ABC):
-
-    @property
-    @abstractmethod
-    def diagnostic_code(self) -> str:
-        """The diagnostic code that this fixer fixes."""
-
-    def is_supported(self, diagnostic_code: str) -> bool:
-        """Indicate if the diagnostic code is supported by this fixer."""
-        return self.diagnostic_code is not None and diagnostic_code == self.diagnostic_code
-
-    @abstractmethod
-    def apply(self, code: str) -> str: ...
-
-
 @dataclass
 class LineageAtom:
 
@@ -366,31 +314,6 @@ def parse_security_mode(mode_str: str | None) -> compute.DataSecurityMode | None
             return None
 
 
-class SqlSequentialLinter(SqlLinter, DfsaCollector, TableCollector):
-
-    def __init__(
-        self,
-        linters: list[SqlLinter],
-        dfsa_collectors: list[DfsaSqlCollector],
-        table_collectors: list[TableSqlCollector],
-    ):
-        self._linters = linters
-        self._dfsa_collectors = dfsa_collectors
-        self._table_collectors = table_collectors
-
-    def lint_expression(self, expression: Expression) -> Iterable[Advice]:
-        for linter in self._linters:
-            yield from linter.lint_expression(expression)
-
-    def collect_dfsas(self, source_code: str) -> Iterable[DirectFsAccess]:
-        for collector in self._dfsa_collectors:
-            yield from collector.collect_dfsas(source_code)
-
-    def collect_tables(self, source_code: str) -> Iterable[UsedTable]:
-        for collector in self._table_collectors:
-            yield from collector.collect_tables(source_code)
-
-
 SUPPORTED_EXTENSION_LANGUAGES = {
     '.py': Language.PYTHON,
     '.sql': Language.SQL,

src/databricks/labs/ucx/source_code/files.py

@@ -0,0 +1,169 @@
+from __future__ import annotations
+
+import dataclasses
+import logging
+from pathlib import Path
+
+from databricks.sdk.service.workspace import Language
+
+from databricks.labs.ucx.source_code.base import file_language
+from databricks.labs.ucx.source_code.graph import (
+    SourceContainer,
+    DependencyGraph,
+    DependencyProblem,
+    InheritedContext,
+    DependencyLoader,
+    Dependency,
+    BaseImportResolver,
+    BaseFileResolver,
+    MaybeDependency,
+)
+from databricks.labs.ucx.source_code.known import KnownList
+from databricks.labs.ucx.source_code.notebooks.cells import CellLanguage
+from databricks.labs.ucx.source_code.path_lookup import PathLookup
+from databricks.labs.ucx.source_code.linters.python import PythonCodeAnalyzer
+
+
+logger = logging.getLogger(__name__)
+
+
+class LocalFile(SourceContainer):
+
+    def __init__(self, path: Path, source: str, language: Language):
+        self._path = path
+        self._original_code = source
+        # using CellLanguage so we can reuse the facilities it provides
+        self._language = CellLanguage.of_language(language)
+
+    @property
+    def path(self) -> Path:
+        return self._path
+
+    def build_dependency_graph(self, parent: DependencyGraph) -> list[DependencyProblem]:
+        if self._language is CellLanguage.PYTHON:
+            context = parent.new_dependency_graph_context()
+            analyzer = PythonCodeAnalyzer(context, self._original_code)
+            problems = analyzer.build_graph()
+            for idx, problem in enumerate(problems):
+                if problem.has_missing_path():
+                    problems[idx] = dataclasses.replace(problem, source_path=self._path)
+            return problems
+        # supported language that does not generate dependencies
+        if self._language is CellLanguage.SQL:
+            return []
+        logger.warning(f"Unsupported language: {self._language.language}")
+        return []
+
+    def build_inherited_context(self, graph: DependencyGraph, child_path: Path) -> InheritedContext:
+        if self._language is CellLanguage.PYTHON:
+            context = graph.new_dependency_graph_context()
+            analyzer = PythonCodeAnalyzer(context, self._original_code)
+            inherited = analyzer.build_inherited_context(child_path)
+            problems = list(inherited.problems)
+            for idx, problem in enumerate(problems):
+                if problem.has_missing_path():
+                    problems[idx] = dataclasses.replace(problem, source_path=self._path)
+            return dataclasses.replace(inherited, problems=problems)
+        return InheritedContext(None, False, [])
+
+    def __repr__(self):
+        return f"<LocalFile {self._path}>"
+
+
+class StubContainer(SourceContainer):
+
+    def __init__(self, path: Path):
+        super().__init__()
+        self._path = path
+
+    def build_dependency_graph(self, parent: DependencyGraph) -> list[DependencyProblem]:
+        return []
+
+
+class FileLoader(DependencyLoader):
+    def load_dependency(self, path_lookup: PathLookup, dependency: Dependency) -> SourceContainer | None:
+        absolute_path = path_lookup.resolve(dependency.path)
+        if not absolute_path:
+            return None
+        language = file_language(absolute_path)
+        if not language:
+            return StubContainer(absolute_path)
+        for encoding in ("utf-8", "ascii"):
+            try:
+                code = absolute_path.read_text(encoding)
+                return LocalFile(absolute_path, code, language)
+            except UnicodeDecodeError:
+                pass
+        return None
+
+    def exists(self, path: Path) -> bool:
+        return path.exists()
+
+    def __repr__(self):
+        return "FileLoader()"
+
+
+class ImportFileResolver(BaseImportResolver, BaseFileResolver):
+
+    def __init__(self, file_loader: FileLoader, allow_list: KnownList):
+        super().__init__()
+        self._allow_list = allow_list
+        self._file_loader = file_loader
+
+    def resolve_file(self, path_lookup, path: Path) -> MaybeDependency:
+        absolute_path = path_lookup.resolve(path)
+        if absolute_path:
+            return MaybeDependency(Dependency(self._file_loader, absolute_path), [])
+        problem = DependencyProblem("file-not-found", f"File not found: {path.as_posix()}")
+        return MaybeDependency(None, [problem])
+
+    def resolve_import(self, path_lookup: PathLookup, name: str) -> MaybeDependency:
+        maybe = self._resolve_allow_list(name)
+        if maybe is not None:
+            return maybe
+        maybe = self._resolve_import(path_lookup, name)
+        if maybe is not None:
+            return maybe
+        return self._fail('import-not-found', f"Could not locate import: {name}")
+
+    def _resolve_allow_list(self, name: str) -> MaybeDependency | None:
+        compatibility = self._allow_list.module_compatibility(name)
+        if not compatibility.known:
+            logger.debug(f"Resolving unknown import: {name}")
+            return None
+        if not compatibility.problems:
+            return MaybeDependency(None, [])
+        # TODO move to linter, see https://github.com/databrickslabs/ucx/issues/1527
+        return MaybeDependency(None, compatibility.problems)
+
+    def _resolve_import(self, path_lookup: PathLookup, name: str) -> MaybeDependency | None:
+        if not name:
+            return MaybeDependency(None, [DependencyProblem("ucx-bug", "Import name is empty")])
+        parts = []
+        # Relative imports use leading dots. A single leading dot indicates a relative import, starting with
+        # the current package. Two or more leading dots indicate a relative import to the parent(s) of the current
+        # package, one level per dot after the first.
+        # see https://docs.python.org/3/reference/import.html#package-relative-imports
+        for i, rune in enumerate(name):
+            if not i and rune == '.':  # leading single dot
+                parts.append(path_lookup.cwd.as_posix())
+                continue
+            if rune != '.':
+                parts.append(name[i:].replace('.', '/'))
+                break
+            parts.append("..")
+        for candidate in (f'{"/".join(parts)}.py', f'{"/".join(parts)}/__init__.py'):
+            relative_path = Path(candidate)
+            absolute_path = path_lookup.resolve(relative_path)
+            if not absolute_path:
+                continue
+            dependency = Dependency(self._file_loader, absolute_path, False)
+            return MaybeDependency(dependency, [])
+        return None
+
+    @staticmethod
+    def _fail(code: str, message: str) -> MaybeDependency:
+        return MaybeDependency(None, [DependencyProblem(code, message)])
+
+    def __repr__(self):
+        return "ImportFileResolver()"

src/databricks/labs/ucx/source_code/folders.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from pathlib import Path
+from collections.abc import Iterable
+
+from databricks.labs.ucx.source_code.base import is_a_notebook
+from databricks.labs.ucx.source_code.files import FileLoader
+from databricks.labs.ucx.source_code.graph import SourceContainer, DependencyGraph, DependencyProblem, Dependency
+from databricks.labs.ucx.source_code.notebooks.loaders import NotebookLoader
+from databricks.labs.ucx.source_code.path_lookup import PathLookup
+
+
+class Folder(SourceContainer):
+
+    def __init__(
+        self,
+        path: Path,
+        notebook_loader: NotebookLoader,
+        file_loader: FileLoader,
+        folder_loader: FolderLoader,
+    ):
+        self._path = path
+        self._notebook_loader = notebook_loader
+        self._file_loader = file_loader
+        self._folder_loader = folder_loader
+
+    @property
+    def path(self) -> Path:
+        return self._path
+
+    def build_dependency_graph(self, parent: DependencyGraph) -> list[DependencyProblem]:
+        # don't directly scan non-source directories, let it be done for relevant imports only
+        if self._path.name in {"__pycache__", ".git", ".github", ".venv", ".mypy_cache", "site-packages"}:
+            return []
+        return list(self._build_dependency_graph(parent))
+
+    def _build_dependency_graph(self, parent: DependencyGraph) -> Iterable[DependencyProblem]:
+        for child_path in self._path.iterdir():
+            is_file = child_path.is_file()
+            is_notebook = is_a_notebook(child_path)
+            loader = self._notebook_loader if is_notebook else self._file_loader if is_file else self._folder_loader
+            dependency = Dependency(loader, child_path, inherits_context=is_notebook)
+            yield from parent.register_dependency(dependency).problems
+
+    def __repr__(self):
+        return f"<Folder {self._path}>"
+
+
+class FolderLoader(FileLoader):
+
+    def __init__(self, notebook_loader: NotebookLoader, file_loader: FileLoader):
+        self._notebook_loader = notebook_loader
+        self._file_loader = file_loader
+
+    def load_dependency(self, path_lookup: PathLookup, dependency: Dependency) -> SourceContainer | None:
+        absolute_path = path_lookup.resolve(dependency.path)
+        if not absolute_path:
+            return None
+        return Folder(absolute_path, self._notebook_loader, self._file_loader, self)