|
1 | 1 | from __future__ import annotations |
2 | 2 |
|
3 | 3 | import logging |
4 | | -import sys |
5 | 4 | from collections.abc import Callable, Iterable |
6 | 5 | from pathlib import Path |
7 | | -from typing import TextIO |
8 | 6 |
|
9 | | -from databricks.labs.blueprint.tui import Prompts |
10 | | -from databricks.sdk.service.workspace import Language |
11 | | - |
12 | | -from databricks.labs.ucx.source_code.base import CurrentSessionState, LocatedAdvice, is_a_notebook |
13 | | -from databricks.labs.ucx.source_code.graph import DependencyResolver, DependencyLoader, Dependency, DependencyGraph |
14 | | -from databricks.labs.ucx.source_code.linters.context import LinterContext |
15 | | -from databricks.labs.ucx.source_code.folders import FolderLoader |
| 7 | +from databricks.labs.ucx.source_code.base import LocatedAdvice, is_a_notebook |
16 | 8 | from databricks.labs.ucx.source_code.files import FileLoader |
17 | | -from databricks.labs.ucx.source_code.linters.graph_walkers import LintingWalker |
| 9 | +from databricks.labs.ucx.source_code.folders import FolderLoader |
| 10 | +from databricks.labs.ucx.source_code.graph import ( |
| 11 | + Dependency, |
| 12 | + DependencyGraph, |
| 13 | + DependencyLoader, |
| 14 | + DependencyProblem, |
| 15 | + DependencyResolver, |
| 16 | + MaybeGraph, |
| 17 | +) |
| 18 | +from databricks.labs.ucx.source_code.linters.context import LinterContext |
| 19 | +from databricks.labs.ucx.source_code.linters.graph_walkers import FixerWalker, LinterWalker |
18 | 20 | from databricks.labs.ucx.source_code.notebooks.loaders import NotebookLoader |
19 | 21 | from databricks.labs.ucx.source_code.path_lookup import PathLookup |
20 | 22 |
|
|
23 | 25 |
|
24 | 26 |
|
25 | 27 | class LocalCodeLinter: |
| 28 | + """Lint local code to become Unity Catalog compatible.""" |
26 | 29 |
|
27 | 30 | def __init__( |
28 | 31 | self, |
29 | 32 | notebook_loader: NotebookLoader, |
30 | 33 | file_loader: FileLoader, |
31 | 34 | folder_loader: FolderLoader, |
32 | 35 | path_lookup: PathLookup, |
33 | | - session_state: CurrentSessionState, |
34 | 36 | dependency_resolver: DependencyResolver, |
35 | 37 | context_factory: Callable[[], LinterContext], |
36 | 38 | ) -> None: |
37 | 39 | self._notebook_loader = notebook_loader |
38 | 40 | self._file_loader = file_loader |
39 | 41 | self._folder_loader = folder_loader |
40 | 42 | self._path_lookup = path_lookup |
41 | | - self._session_state = session_state |
42 | 43 | self._dependency_resolver = dependency_resolver |
43 | | - self._extensions = {".py": Language.PYTHON, ".sql": Language.SQL} |
44 | 44 | self._context_factory = context_factory |
45 | 45 |
|
46 | | - def lint( |
47 | | - self, |
48 | | - prompts: Prompts, |
49 | | - path: Path | None, |
50 | | - stdout: TextIO = sys.stdout, |
51 | | - ) -> list[LocatedAdvice]: |
52 | | - """Lint local code files looking for problems in notebooks and python files.""" |
53 | | - if path is None: |
54 | | - response = prompts.question( |
55 | | - "Which file or directory do you want to lint?", |
56 | | - default=Path.cwd().as_posix(), |
57 | | - validate=lambda p_: Path(p_).exists(), |
58 | | - ) |
59 | | - path = Path(response) |
60 | | - located_advices = list(self.lint_path(path)) |
61 | | - for located_advice in located_advices: |
62 | | - stdout.write(f"{located_advice}\n") |
63 | | - return located_advices |
| 46 | + def lint(self, path: Path) -> Iterable[LocatedAdvice]: |
| 47 | + """Lint local code generating advices on becoming Unity Catalog compatible. |
64 | 48 |
|
65 | | - def lint_path(self, path: Path) -> Iterable[LocatedAdvice]: |
66 | | - is_dir = path.is_dir() |
67 | | - loader: DependencyLoader |
68 | | - if is_a_notebook(path): |
69 | | - loader = self._notebook_loader |
70 | | - elif path.is_dir(): |
71 | | - loader = self._folder_loader |
72 | | - else: |
73 | | - loader = self._file_loader |
74 | | - path_lookup = self._path_lookup.change_directory(path if is_dir else path.parent) |
75 | | - root_dependency = Dependency(loader, path, not is_dir) # don't inherit context when traversing folders |
76 | | - graph = DependencyGraph(root_dependency, None, self._dependency_resolver, path_lookup, self._session_state) |
77 | | - container = root_dependency.load(path_lookup) |
78 | | - assert container is not None # because we just created it |
79 | | - problems = container.build_dependency_graph(graph) |
80 | | - for problem in problems: |
81 | | - yield problem.as_located_advice() |
| 49 | + Parameters : |
| 50 | + path (Path) : The path to the resource(s) to lint. If the path is a directory, then all files within the |
| 51 | + directory and subdirectories are linted. |
| 52 | + """ |
| 53 | + maybe_graph = self._build_dependency_graph_from_path(path) |
| 54 | + if maybe_graph.problems: |
| 55 | + for problem in maybe_graph.problems: |
| 56 | + yield problem.as_located_advice() |
82 | 57 | return |
83 | | - walker = LintingWalker(graph, self._path_lookup, self._context_factory) |
| 58 | + assert maybe_graph.graph |
| 59 | + walker = LinterWalker(maybe_graph.graph, self._path_lookup, self._context_factory) |
84 | 60 | yield from walker |
85 | 61 |
|
| 62 | + def apply(self, path: Path) -> Iterable[LocatedAdvice]: |
| 63 | + """Apply local code fixes to become Unity Catalog compatible. |
86 | 64 |
|
87 | | -class LocalFileMigrator: |
88 | | - """The LocalFileMigrator class is responsible for fixing code files based on their language.""" |
| 65 | + Parameters : |
| 66 | + path (Path) : The path to the resource(s) to lint. If the path is a directory, then all files within the |
| 67 | + directory and subdirectories are linted. |
| 68 | + """ |
| 69 | + maybe_graph = self._build_dependency_graph_from_path(path) |
| 70 | + if maybe_graph.problems: |
| 71 | + for problem in maybe_graph.problems: |
| 72 | + yield problem.as_located_advice() |
| 73 | + return |
| 74 | + assert maybe_graph.graph |
| 75 | + walker = FixerWalker(maybe_graph.graph, self._path_lookup, self._context_factory) |
| 76 | + list(walker) # Nothing to yield |
89 | 77 |
|
90 | | - def __init__(self, context_factory: Callable[[], LinterContext]): |
91 | | - self._extensions = {".py": Language.PYTHON, ".sql": Language.SQL} |
92 | | - self._context_factory = context_factory |
| 78 | + def _build_dependency_graph_from_path(self, path: Path) -> MaybeGraph: |
| 79 | + """Build a dependency graph from the path. |
93 | 80 |
|
94 | | - def apply(self, path: Path) -> bool: |
95 | | - if path.is_dir(): |
96 | | - for child_path in path.iterdir(): |
97 | | - self.apply(child_path) |
98 | | - return True |
99 | | - return self._apply_file_fix(path) |
| 81 | + It tries to load the path as a directory, file or notebook. |
100 | 82 |
|
101 | | - def _apply_file_fix(self, path: Path): |
102 | | - """ |
103 | | - The fix method reads a file, lints it, applies fixes, and writes the fixed code back to the file. |
| 83 | + Returns : |
| 84 | + MaybeGraph : If the loading fails, the returned maybe graph contains a problem. Otherwise, returned maybe |
| 85 | + graph contains the graph. |
104 | 86 | """ |
105 | | - # Check if the file extension is in the list of supported extensions |
106 | | - if path.suffix not in self._extensions: |
107 | | - return False |
108 | | - # Get the language corresponding to the file extension |
109 | | - language = self._extensions[path.suffix] |
110 | | - # If the language is not supported, return |
111 | | - if not language: |
112 | | - return False |
113 | | - logger.info(f"Analysing {path}") |
114 | | - # Get the linter for the language |
115 | | - context = self._context_factory() |
116 | | - linter = context.linter(language) |
117 | | - # Open the file and read the code |
118 | | - with path.open("r") as f: |
119 | | - try: |
120 | | - code = f.read() |
121 | | - except UnicodeDecodeError as e: |
122 | | - logger.warning(f"Could not decode file {path}: {e}") |
123 | | - return False |
124 | | - applied = False |
125 | | - # Lint the code and apply fixes |
126 | | - for advice in linter.lint(code): |
127 | | - logger.info(f"Found: {advice}") |
128 | | - fixer = context.fixer(language, advice.code) |
129 | | - if not fixer: |
130 | | - continue |
131 | | - logger.info(f"Applying fix for {advice}") |
132 | | - code = fixer.apply(code) |
133 | | - applied = True |
134 | | - if not applied: |
135 | | - return False |
136 | | - # Write the fixed code back to the file |
137 | | - with path.open("w") as f: |
138 | | - logger.info(f"Overwriting {path}") |
139 | | - f.write(code) |
140 | | - return True |
| 87 | + resolved_path = self._path_lookup.resolve(path) |
| 88 | + if not resolved_path: |
| 89 | + problem = DependencyProblem("path-not-found", "Path not found", source_path=path) |
| 90 | + return MaybeGraph(None, [problem]) |
| 91 | + is_dir = resolved_path.is_dir() |
| 92 | + loader: DependencyLoader |
| 93 | + if is_a_notebook(resolved_path): |
| 94 | + loader = self._notebook_loader |
| 95 | + elif is_dir: |
| 96 | + loader = self._folder_loader |
| 97 | + else: |
| 98 | + loader = self._file_loader |
| 99 | + root_dependency = Dependency(loader, resolved_path, not is_dir) # don't inherit context when traversing folders |
| 100 | + container = root_dependency.load(self._path_lookup) |
| 101 | + if container is None: |
| 102 | + problem = DependencyProblem("dependency-not-found", "Dependency not found", source_path=path) |
| 103 | + return MaybeGraph(None, [problem]) |
| 104 | + session_state = self._context_factory().session_state |
| 105 | + graph = DependencyGraph(root_dependency, None, self._dependency_resolver, self._path_lookup, session_state) |
| 106 | + problems = list(container.build_dependency_graph(graph)) |
| 107 | + if problems: |
| 108 | + return MaybeGraph(None, problems) |
| 109 | + return MaybeGraph(graph, []) |
0 commit comments