diff --git a/src/databricks/labs/ucx/assessment/workflows.py b/src/databricks/labs/ucx/assessment/workflows.py index 3c0efbd7af..3ed1853e19 100644 --- a/src/databricks/labs/ucx/assessment/workflows.py +++ b/src/databricks/labs/ucx/assessment/workflows.py @@ -207,7 +207,7 @@ def assess_dashboards(self, ctx: RuntimeContext): """ ctx.query_linter.refresh_report() - @job_task(depends_on=[assess_jobs]) + @job_task(depends_on=[crawl_tables, assess_jobs]) def assess_workflows(self, ctx: RuntimeContext): """Scans all jobs for migration issues in notebooks jobs. diff --git a/src/databricks/labs/ucx/contexts/application.py b/src/databricks/labs/ucx/contexts/application.py index 0965ffb583..cbff1e9d2e 100644 --- a/src/databricks/labs/ucx/contexts/application.py +++ b/src/databricks/labs/ucx/contexts/application.py @@ -11,6 +11,9 @@ from databricks.labs.blueprint.tui import Prompts from databricks.labs.blueprint.wheels import ProductInfo, WheelsV2 from databricks.labs.lsql.backends import SqlBackend +from databricks.sdk import AccountClient, WorkspaceClient, core +from databricks.sdk.errors import DatabricksError +from databricks.sdk.service import sql from databricks.labs.ucx.assessment.dashboards import DashboardOwnership from databricks.labs.ucx.assessment.jobs import JobsCrawler @@ -24,9 +27,6 @@ from databricks.labs.ucx.source_code.directfs_access import DirectFsAccessCrawler, DirectFsAccessOwnership from databricks.labs.ucx.source_code.python_libraries import PythonLibraryResolver from databricks.labs.ucx.source_code.used_table import UsedTablesCrawler -from databricks.sdk import AccountClient, WorkspaceClient, core -from databricks.sdk.service import sql - from databricks.labs.ucx.account.workspaces import WorkspaceInfo from databricks.labs.ucx.assessment.azure import AzureServicePrincipalCrawler from databricks.labs.ucx.assessment.dashboards import LakeviewDashboardCrawler, RedashDashboardCrawler @@ -48,7 +48,6 @@ PrincipalACL, ) from databricks.labs.ucx.hive_metastore.mapping import TableMapping -from databricks.labs.ucx.hive_metastore.table_migration_status import TableMigrationIndex from databricks.labs.ucx.hive_metastore.ownership import ( TableMigrationOwnership, TableOwnership, @@ -59,6 +58,7 @@ TableMigrationStatusRefresher, TablesMigrator, ) +from databricks.labs.ucx.hive_metastore.table_migration_status import TableMigrationIndex from databricks.labs.ucx.hive_metastore.table_move import TableMove from databricks.labs.ucx.hive_metastore.udfs import UdfsCrawler, UdfOwnership from databricks.labs.ucx.hive_metastore.verification import VerifyHasCatalog, VerifyHasMetastore @@ -585,6 +585,15 @@ def dependency_resolver(self) -> DependencyResolver: self.pip_resolver, self.notebook_resolver, self.file_resolver, self.file_resolver, self.path_lookup ) + @cached_property + def table_migration_index(self) -> TableMigrationIndex: + try: + index = self.tables_migrator.index() + except DatabricksError as e: + logger.warning("Table migration cannot be loaded. Initializing empty index.", exc_info=e) + index = TableMigrationIndex([]) + return index + @cached_property def workflow_linter(self) -> WorkflowLinter: return WorkflowLinter( @@ -592,7 +601,7 @@ def workflow_linter(self) -> WorkflowLinter: self.jobs_crawler, self.dependency_resolver, self.path_lookup, - TableMigrationIndex([]), # TODO: bring back self.tables_migrator.index() + self.table_migration_index, self.directfs_access_crawler_for_paths, self.used_tables_crawler_for_paths, ) diff --git a/src/databricks/labs/ucx/contexts/workspace_cli.py b/src/databricks/labs/ucx/contexts/workspace_cli.py index 065b480b5f..6ce1224889 100644 --- a/src/databricks/labs/ucx/contexts/workspace_cli.py +++ b/src/databricks/labs/ucx/contexts/workspace_cli.py @@ -6,7 +6,6 @@ from databricks.labs.lsql.backends import SqlBackend, StatementExecutionBackend from databricks.sdk import WorkspaceClient -from databricks.sdk.errors import NotFound from databricks.labs.ucx.assessment.aws import AWSResources from databricks.labs.ucx.framework.utils import run_command @@ -19,7 +18,6 @@ from databricks.labs.ucx.azure.resources import AzureAPIClient, AzureResources from databricks.labs.ucx.contexts.application import CliContext from databricks.labs.ucx.hive_metastore.federation import HiveMetastoreFederation, HiveMetastoreFederationEnabler -from databricks.labs.ucx.hive_metastore.table_migration_status import TableMigrationIndex from databricks.labs.ucx.progress.install import ProgressTrackingInstallation from databricks.labs.ucx.source_code.linters.context import LinterContext from databricks.labs.ucx.source_code.linters.folders import LocalCodeLinter @@ -212,15 +210,6 @@ class LocalCheckoutContext(WorkspaceContext): """Local context extends Workspace context to provide extra properties for running local operations.""" - @cached_property - def _migration_index(self) -> TableMigrationIndex: - try: - index = self.tables_migrator.index() - except NotFound: - logger.warning("Metastore does not seem to exist yet. Skipping loading of migration status.") - index = TableMigrationIndex([]) - return index - @cached_property def local_code_linter(self) -> LocalCodeLinter: return LocalCodeLinter( @@ -229,5 +218,5 @@ def local_code_linter(self) -> LocalCodeLinter: self.folder_loader, self.path_lookup, self.dependency_resolver, - lambda: LinterContext(self._migration_index), + lambda: LinterContext(self.table_migration_index), ) diff --git a/src/databricks/labs/ucx/hive_metastore/table_migration_status.py b/src/databricks/labs/ucx/hive_metastore/table_migration_status.py index 8b766755fe..13de171072 100644 --- a/src/databricks/labs/ucx/hive_metastore/table_migration_status.py +++ b/src/databricks/labs/ucx/hive_metastore/table_migration_status.py @@ -166,8 +166,8 @@ def _iter_catalogs(self) -> Iterable[CatalogInfo]: if catalog.securable_kind in self._skip_catalog_securable_kinds: continue yield catalog - except DatabricksError as e: - logger.error("Cannot list catalogs", exc_info=e) + except DatabricksError: + logger.warning("Cannot list catalogs; skipping iterating catalogs") def _iter_schemas(self) -> Iterable[SchemaInfo]: for catalog in self._iter_catalogs():