Fixed TokenError in assessment workflow (#2896)

Previously we were catching parse errors only, but badly tokenized
queries were causing failures still. `SqlglotError` is the common
ancestor of both `ParseError` and `TokenError`, so we're more safe (in
ignorance) now.

Fix #2811

Author: nfx

src/databricks/labs/ucx/source_code/sql/sql_parser.py

@@ -2,7 +2,8 @@
 from collections.abc import Callable, Iterable, Iterator
 from typing import TypeVar
 
-from sqlglot import parse, ParseError
+from sqlglot import parse
+from sqlglot.errors import SqlglotError
 from sqlglot.expressions import Create, Delete, Drop, Expression, Select, Table, Update, Use
 
 from databricks.labs.ucx.source_code.base import UsedTable, CurrentSessionState
@@ -72,6 +73,6 @@ def walk_expressions(cls, sql_code: str, callback: Callable[[SqlExpression], Ite
                 if not expression:
                     continue
                 yield from callback(SqlExpression(expression))
-        except ParseError as e:
+        except SqlglotError as e:
             logger.debug(f"Failed to parse SQL: {sql_code}", exc_info=e)
             raise e