7
7
from databricks .labs .lsql .backends import SqlBackend
8
8
from databricks .sdk import WorkspaceClient
9
9
from databricks .sdk .errors import DatabricksError , NotFound
10
- from databricks .sdk .service .catalog import CatalogInfo
10
+ from databricks .sdk .service .catalog import CatalogInfo , CatalogInfoSecurableKind , SchemaInfo
11
11
12
12
from databricks .labs .ucx .framework .crawlers import CrawlerBase
13
13
from databricks .labs .ucx .framework .utils import escape_sql_identifier
@@ -79,6 +79,11 @@ class TableMigrationStatusRefresher(CrawlerBase[TableMigrationStatus]):
79
79
properties for the presence of the marker.
80
80
"""
81
81
82
+ _skip_catalog_securable_kinds = [
83
+ CatalogInfoSecurableKind .CATALOG_INTERNAL ,
84
+ CatalogInfoSecurableKind .CATALOG_SYSTEM ,
85
+ ]
86
+
82
87
def __init__ (self , ws : WorkspaceClient , sql_backend : SqlBackend , schema , tables_crawler : TablesCrawler ):
83
88
super ().__init__ (sql_backend , "hive_metastore" , schema , "migration_status" , TableMigrationStatus )
84
89
self ._ws = ws
@@ -90,6 +95,8 @@ def index(self, *, force_refresh: bool = False) -> TableMigrationIndex:
90
95
def get_seen_tables (self ) -> dict [str , str ]:
91
96
seen_tables : dict [str , str ] = {}
92
97
for schema in self ._iter_schemas ():
98
+ if schema .catalog_name is None or schema .name is None :
99
+ continue
93
100
try :
94
101
# ws.tables.list returns Iterator[TableInfo], so we need to convert it to a list in order to catch the exception
95
102
tables = list (self ._ws .tables .list (catalog_name = schema .catalog_name , schema_name = schema .name ))
@@ -136,9 +143,7 @@ def _crawl(self) -> Iterable[TableMigrationStatus]:
136
143
src_schema = table .database .lower ()
137
144
src_table = table .name .lower ()
138
145
table_migration_status = TableMigrationStatus (
139
- src_schema = src_schema ,
140
- src_table = src_table ,
141
- update_ts = str (timestamp ),
146
+ src_schema = src_schema , src_table = src_table , update_ts = str (timestamp )
142
147
)
143
148
if table .key in reverse_seen and self .is_migrated (src_schema , src_table ):
144
149
target_table = reverse_seen [table .key ]
@@ -157,12 +162,17 @@ def _try_fetch(self) -> Iterable[TableMigrationStatus]:
157
162
158
163
def _iter_catalogs (self ) -> Iterable [CatalogInfo ]:
159
164
try :
160
- yield from self ._ws .catalogs .list ()
165
+ for catalog in self ._ws .catalogs .list ():
166
+ if catalog .securable_kind in self ._skip_catalog_securable_kinds :
167
+ continue
168
+ yield catalog
161
169
except DatabricksError as e :
162
170
logger .error ("Cannot list catalogs" , exc_info = e )
163
171
164
- def _iter_schemas (self ):
172
+ def _iter_schemas (self ) -> Iterable [ SchemaInfo ] :
165
173
for catalog in self ._iter_catalogs ():
174
+ if catalog .name is None :
175
+ continue
166
176
try :
167
177
yield from self ._ws .schemas .list (catalog_name = catalog .name )
168
178
except NotFound :
0 commit comments