Merge branch 'main' into feature/llm-transpile

Author: hiroyukinakazato-db

.github/workflows/acceptance.yml

@@ -38,15 +38,15 @@ jobs:
           python-version: '3.10'
 
       - name: Install hatch
-        run: pip install hatch==1.14.1 'click<8.3.0' # https://github.com/pallets/click/issues/3065
+        run: pip install hatch==1.14.2
 
       - name: Install MSSQL ODBC Driver
         run: |
           chmod +x $GITHUB_WORKSPACE/.github/scripts/setup_mssql_odbc.sh
           $GITHUB_WORKSPACE/.github/scripts/setup_mssql_odbc.sh
 
       - name: Run integration tests
-        uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.2
+        uses: databrickslabs/sandbox/acceptance@acceptance/v0.4.4
         with:
           vault_uri: ${{ secrets.VAULT_URI }}
           directory: ${{ github.workspace }}

.github/workflows/push.yml

@@ -15,7 +15,7 @@ on:
       - main
 
 env:
-  HATCH_VERSION: 1.14.1
+  HATCH_VERSION: 1.14.2
 
 jobs:
   test-python:
@@ -34,7 +34,7 @@ jobs:
           python-version: '3.10'
 
       - name: Install hatch
-        run: pip install hatch==$HATCH_VERSION 'click<8.3.0' # https://github.com/pallets/click/issues/3065
+        run: pip install hatch==$HATCH_VERSION
 
       - name: Run unit tests
         run: hatch run test
@@ -61,7 +61,7 @@ jobs:
           python-version: '3.10'
 
       - name: Install hatch
-        run: pip install hatch==$HATCH_VERSION 'click<8.3.0' # https://github.com/pallets/click/issues/3065
+        run: pip install hatch==$HATCH_VERSION
 
       - name: Setup Spark Remote
         run: |
@@ -94,7 +94,7 @@ jobs:
           python-version: 3.10.x
 
       - name: Install hatch
-        run: pip install hatch==$HATCH_VERSION 'click<8.3.0' # https://github.com/pallets/click/issues/3065
+        run: pip install hatch==$HATCH_VERSION
 
       - name: Reformat code
         run: make fmt
@@ -147,7 +147,7 @@ jobs:
           python-version: 3.10.x
 
       - name: Install hatch
-        run: pip install hatch==$HATCH_VERSION 'click<8.3.0' # https://github.com/pallets/click/issues/3065
+        run: pip install hatch==$HATCH_VERSION
 
       - name: Install Databricks CLI
         uses: databricks/setup-cli@main
@@ -199,11 +199,11 @@ jobs:
 
     - name: Install hatch (Windows)
       if: runner.os == 'Windows'
-      run: pip install hatch==$env:HATCH_VERSION 'click<8.3.0' # https://github.com/pallets/click/issues/3065
+      run: pip install hatch==$env:HATCH_VERSION
 
     - name: Install hatch (Non-Windows)
       if: runner.os != 'Windows'
-      run: pip install hatch==$HATCH_VERSION 'click<8.3.0' # https://github.com/pallets/click/issues/3065
+      run: pip install hatch==$HATCH_VERSION
 
     - name: Install Databricks CLI
       uses: databricks/setup-cli@main

.github/workflows/release.yml

@@ -27,7 +27,7 @@ jobs:
 
       - name: Build wheels
         run: |
-          pip install hatch==1.14.1 'click<8.3.0' # https://github.com/pallets/click/issues/3065
+          pip install hatch==1.14.2
           hatch build
 
       - name: Draft release
@@ -41,7 +41,7 @@ jobs:
         name: Publish package distributions to PyPI
 
       - name: Sign artifacts with Sigstore
-        uses: sigstore/gh-action-sigstore-python@v3.0.0
+        uses: sigstore/gh-action-sigstore-python@v3.0.1
         with:
           inputs: |
             dist/databricks_*.whl

CHANGELOG.md

@@ -1,5 +1,78 @@
 # Version changelog
 
+## # Lakebridge v0.10.11 Release Notes
+
+## Analyzer
+No updates in this release
+
+## Converters
+
+### General
+- Fixed special character handling in filenames by introducing from_uri() helper function for safer URI handling
+- Ensured SQL converter returns UTF-8 encoded files for proper character encoding
+- Fixed filename to correctly output databricks_conversion_supplements.py supplemental file
+- Fixed broken splitter URL by updating directory naming conventions from "Downloads" to "downloads"
+- Improved handling of encoding-related errors by catching UnicodeDecodeError and LookupError exceptions during file processing, creating TranspileError with specific encoding-error codes instead of stopping
+
+### Morpheus
+
+#### Snowflake
+- Added support for TRUNCATE TABLE statements with proper IR and translation support
+- Correctly support  and  system variables
+- Refactored and extended grammar and AST support for SQL procedure creation with improved handling of raw string literals
+- Enhanced schema reconciliation functionality to support Snowflake arrays, addressing the corner case where Databricks arrays are typed and Snowflake arrays are untyped
+
+#### TSQL
+- Added support for TRUNCATE TABLE statements with proper IR and translation support
+- Support full CREATE and ALTER INDEX statements in TSQL parsing, rejecting INDEX CREATE/ALTER statements sensibly instead of raising syntax errors
+- Fixed implementation of IF scripting blocks with improvements to SQL parser, grammar enhancements, and enhanced scripting grammar for more robust handling of block statements and conditional branches
+- Allow CLUSTERED to be an identifier to improve CREATE TABLE syntax as a CONSTRAINT qualifier
+- Support percentage expressions in TSQL options (e.g., OPT = 42%) instead of raising parsing errors
+- Added support for REVOKE statements, similar to existing GRANT statement implementation
+- Ensure that ROWS and OBJECTS can be used as identifiers even with Jinja templates
+- Correctly support  and  system variables
+
+#### General (Multiple Dialects)
+- Support comments on column declarations when generating SQL and renamed legacy builders for consistency
+- Refactored IR around CREATE FUNCTION and CREATE PROCEDURE, unifying all ways to create stored procedures under a single CreateStoredProcedure IR node and all ways to create user defined functions under a single CreateUDF IR node
+- Implemented grammar and IR placeholders for named windows, introducing initial support for the SQL standard WINDOW clause in parser grammar
+
+### BladeBridge
+
+#### Oracle
+- Removed unsupported Oracle DDL constraints (add/create constraint unique) and extraneous TBLPROPERTIES from converted output
+
+#### MSSQL
+- Added handle_xml_nodes function for MS SQL processing
+- Fixed multiple MSSQL issues including CTEs in views/stored procedures, ADD CONSTRAINT problems, DEFAULT value handling, and parameter data types
+
+#### Synapse
+- Fixed multiple Synapse issues including CTEs in views/stored procedures, ADD CONSTRAINT problems, DEFAULT value handling, parameter data types, error handling in stored procedures, and Synapse-specific features (e.g., table distribution)
+
+#### Teradata
+- Added Teradata function mappings including ZEROIFNULL, TEMPORAL_TIMESTAMP, TRYCAST, ANY, FIRST, NULLIFZERO, DECODE with different parameter counts, and HASHAMP
+- Removed collect statistics and lock table statements
+
+#### DataStage
+- Implemented DataStage Checksum component translation to SparkSQL equivalent and fixed Pyspark checksum translation to use MD5() instead of SHA2()
+
+## Reconcile
+- Added handling for special characters in reconcile aggregate, enhancing the library to handle special characters in column names by properly delimiting identifiers in SQL queries
+- Fixed deploy reconcile jobs by updating wheel file handling, simplifying deployment process to use single wheel path, and fixing broken documentation links
+
+## Documentation
+- Fixed download link in docs (reconcile automation) by replacing broken markdown link with JSX link utilizing useBaseUrl hook
+
+## General
+- Implemented new describe-transpile CLI subcommand that describes installed transpilers, including their versions, configuration paths, and supported source dialects
+- Switched from urllib to requests library for making HTTP calls to PyPI and Maven Central, with default 60-second timeout and improved error handling
+- Work around DATABRICKS_HOST normalization issue during install and uninstall by introducing new Lakebridge subclass with appropriate workspace client
+
+# Dependency updates
+* Bump Databricks SDK Version to 0.67.0 by @goodwillpunning in https://github.com/databrickslabs/lakebridge/pull/2062
+* Bump sigstore/gh-action-sigstore-python from 3.0.0 to 3.0.1 by @dependabot[bot] in https://github.com/databrickslabs/lakebridge/pull/1753
+
+Special thanks to @BrianDeacon for his contribution to fix https://github.com/databrickslabs/lakebridge/issues/1858
 ## 0.10.10
 
 ## Analyzer

pyproject.toml

@@ -15,14 +15,16 @@ classifiers = [
     "Programming Language :: Python",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Programming Language :: Python :: Implementation :: CPython",
     "Environment :: Console",
     "Framework :: Pytest",
     "Intended Audience :: Developers",
     "Intended Audience :: System Administrators",
     "Operating System :: MacOS",
     "Operating System :: Microsoft :: Windows",
-    "Topic :: Software Development :: Libraries",
+    "Operating System :: POSIX :: Linux",
     "Topic :: Utilities",
 ]
 
@@ -31,9 +33,9 @@ dependencies = [
   "standard-distutils~=3.11.9; python_version>='3.11'",
   "databricks-bb-analyzer~=0.1.9",
   "sqlglot==26.1.3",
-  "databricks-labs-blueprint[yaml]>=0.11.3,<0.12.0",
+  "databricks-labs-blueprint[yaml]>=0.11.4,<0.12.0",
   "databricks-labs-lsql==0.16.0",
-  "cryptography>=44.0.2,<45.1.0",
+  "cryptography>=44.0.2,<46.1.0",
   "pyodbc~=5.2.0",
   "SQLAlchemy~=2.0.40",
   "pygls~=2.0.0a2",
@@ -66,26 +68,25 @@ path = ".venv"
 dependencies = [
   "pylint~=3.2.2",
   "pylint-pytest==2.0.0a0",
-  "coverage[toml]~=7.8.0",
-  "pytest~=8.3.5",
-  "pytest-cov>=5.0.0,<6.0.0",
-  "pytest-asyncio~=0.26.0",
-  "pytest-xdist~=3.5.0",
+  "coverage[toml]~=7.10.7",
+  "pytest~=8.4.2",
+  "pytest-cov~=7.0.0",
+  "pytest-asyncio~=1.2.0",
+  "pytest-xdist~=3.8.0",
   "pytest-timeout~=2.4.0",
-  "black~=25.1.0",
-  "ruff~=0.11.6",
+  "black~=25.9.0",
+  "ruff~=0.13.2",
   "databricks-connect==15.1",
   "types-requests>=2.28.1,<3",  # Matches the 'requests' above.
   "types-pyYAML~=6.0.12",
   "types-pytz~=2025.2",
   "databricks-labs-pylint~=0.4.0",
   "databricks-labs-pytester>=0.3.0",
-  "mypy~=1.10.0",
+  "mypy~=1.18.2",
   "numpy~=1.26.4",
   "pandas~=2.3.1",
   "pandas-stubs~=2.3.0.250703",
   "cattrs>=25.2.0",
-  "click<8.3.0", # https://github.com/pallets/click/issues/3065 for ruff
   "faker"
 ]
 
@@ -151,7 +152,6 @@ lint.ignore = [
   # Ignore Exception must not use a string literal, assign to variable first
   "EM101",
   "PLR2004",
-  "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)`
 ]
 extend-exclude = [
   "notebooks/*.py"
@@ -459,7 +459,7 @@ max-bool-expr = 5
 max-branches = 20
 
 # Maximum number of locals for function / method body.
-max-locals = 19
+max-locals = 20
 
 # Maximum number of parents for a class (see R0901).
 max-parents = 7

src/databricks/labs/lakebridge/__about__.py

@@ -1,2 +1,2 @@
 # DO NOT MODIFY THIS FILE
-__version__ = "0.10.10"
+__version__ = "0.10.11"

src/databricks/labs/lakebridge/assessments/__init__.py

@@ -0,0 +1,15 @@
+from ._constants import (
+    PRODUCT_NAME,
+    PRODUCT_PATH_PREFIX,
+    PROFILER_SOURCE_SYSTEM,
+    PLATFORM_TO_SOURCE_TECHNOLOGY,
+    CONNECTOR_REQUIRED,
+)
+
+__all__ = [
+    "PRODUCT_NAME",
+    "PRODUCT_PATH_PREFIX",
+    "PROFILER_SOURCE_SYSTEM",
+    "PLATFORM_TO_SOURCE_TECHNOLOGY",
+    "CONNECTOR_REQUIRED",
+]

src/databricks/labs/lakebridge/assessments/_constants.py

@@ -0,0 +1,20 @@
+from pathlib import Path
+
+PRODUCT_NAME = "lakebridge"
+PRODUCT_PATH_PREFIX = Path.home() / ".databricks" / "labs" / PRODUCT_NAME / "lib"
+
+PLATFORM_TO_SOURCE_TECHNOLOGY = {
+    "synapse": "src/databricks/labs/lakebridge/resources/assessments/synapse/pipeline_config.yml",
+}
+
+# TODO modify this PLATFORM_TO_SOURCE_TECHNOLOGY.keys() once all platforms are supported
+PROFILER_SOURCE_SYSTEM = ["mssql", "synapse"]
+
+# This flag indicates whether a connector is required for the source system when pipeline is trigger
+# For example in the case of synapse no connector is required and the python scripts
+# manage the connection by directly reading the credentials files
+# Revisit this when more source systems are added to standardize the approach
+CONNECTOR_REQUIRED = {
+    "synapse": False,
+    "mssql": True,
+}