Merge branch 'main' into more-workflow-history-snapshots

Author: asnare

src/databricks/labs/ucx/source_code/jobs.py

@@ -17,7 +17,7 @@
 from databricks.labs.blueprint.paths import DBFSPath
 from databricks.labs.lsql.backends import SqlBackend
 from databricks.sdk import WorkspaceClient
-from databricks.sdk.errors import NotFound, ResourceDoesNotExist, BadRequest, InvalidParameterValue
+from databricks.sdk.errors import NotFound, ResourceDoesNotExist, BadRequest, InvalidParameterValue, DatabricksError
 from databricks.sdk.service import compute, jobs
 from databricks.sdk.service.compute import DataSecurityMode
 from databricks.sdk.service.jobs import Source
@@ -165,11 +165,15 @@ def _as_path(self, path: str) -> Path:
     @classmethod
     @contextmanager
     def _temporary_copy(cls, path: Path) -> Generator[Path, None, None]:
-        with tempfile.TemporaryDirectory() as directory:
-            temporary_path = Path(directory) / path.name
-            with path.open("rb") as src, temporary_path.open("wb") as dst:
-                shutil.copyfileobj(src, dst)
-            yield temporary_path
+        try:
+            with tempfile.TemporaryDirectory() as directory:
+                temporary_path = Path(directory) / path.name
+                with path.open("rb") as src, temporary_path.open("wb") as dst:
+                    shutil.copyfileobj(src, dst)
+                yield temporary_path
+        except DatabricksError as e:
+            # Cover cases like `ResourceDoesNotExist: Path (/Volumes/...-py3-none-any.whl) doesn't exist.`
+            raise InvalidPath(f"Cannot load file: {path}") from e
 
     def _register_library(self, graph: DependencyGraph, library: compute.Library) -> Iterable[DependencyProblem]:
         if library.pypi:

src/databricks/labs/ucx/source_code/known.py

@@ -166,7 +166,7 @@ def _analyze_dist_info(cls, dist_info_folder, known_distributions, library_root)
             try:
                 cls._analyze_file(known_distributions, library_root, dist_info, module_path)
             except RecursionError:
-                logger.error(f"Recursion error in {module_path}")
+                logger.warning(f"Recursion error in {module_path}")
                 continue
 
     @classmethod

src/databricks/labs/ucx/source_code/linters/files.py

@@ -228,7 +228,7 @@ def _apply_file_fix(self, path: Path):
             try:
                 code = f.read()
             except UnicodeDecodeError as e:
-                logger.error(f"Could not decode file {path}: {e}")
+                logger.warning(f"Could not decode file {path}: {e}")
                 return False
             applied = False
             # Lint the code and apply fixes

src/databricks/labs/ucx/source_code/linters/from_table.py

@@ -97,7 +97,7 @@ def apply(self, code: str) -> str:
             for old_table in self._dependent_tables(statement):
                 src_schema = old_table.db if old_table.db else self._session_state.schema
                 if not src_schema:
-                    logger.error(f"Could not determine schema for table {old_table.name}")
+                    logger.warning(f"Could not determine schema for table {old_table.name}")
                     continue
                 dst = self._index.get(src_schema, old_table.name)
                 if not dst:

src/databricks/labs/ucx/source_code/redash.py

@@ -49,7 +49,7 @@ def _list_dashboards(self, dashboard_id: str | None) -> list[Dashboard]:
                 return list(self._ws.dashboards.list())
             return [self._ws.dashboards.get(dashboard_id)]
         except DatabricksError as e:
-            logger.error(f"Cannot list dashboards: {e}")
+            logger.warning(f"Cannot list dashboards: {e}")
             return []
 
     def _fix_query(self, query: LegacyQuery) -> None:

src/databricks/labs/ucx/source_code/sql/sql_parser.py

@@ -52,7 +52,7 @@ def _collect_table_info(
         # Sqlglot uses db instead of schema, watch out for that
         src_schema = table.db if table.db else session_state.schema
         if not src_schema:
-            logger.error(f"Could not determine schema for table {table.name}")
+            logger.warning(f"Could not determine schema for table {table.name}")
             return None
         return UsedTable(
             catalog_name=catalog_name,