Bumped up the version to 4.0.0.b3 and also changed the structure to have pyarrow as optional

Author: jprakash-db

CHANGELOG.md

@@ -1,5 +1,20 @@
 # Release History
 
+# 3.6.0 (2024-10-25)
+
+- Support encryption headers in the cloud fetch request (https://github.com/databricks/databricks-sql-python/pull/460 by @jackyhu-db)
+
+# 3.5.0 (2024-10-18)
+
+- Create a non pyarrow flow to handle small results for the column set (databricks/databricks-sql-python#440 by @jprakash-db)
+- Fix: On non-retryable error, ensure PySQL includes useful information in error (databricks/databricks-sql-python#447 by @shivam2680)
+
+# 3.4.0 (2024-08-27)
+
+- Unpin pandas to support v2.2.2 (databricks/databricks-sql-python#416 by @kfollesdal)
+- Make OAuth as the default authenticator if no authentication setting is provided (databricks/databricks-sql-python#419 by @jackyhu-db)
+- Fix (regression): use SSL options with HTTPS connection pool (databricks/databricks-sql-python#425 by @kravets-levko)
+
 # 3.3.0 (2024-07-18)
 
 - Don't retry requests that fail with HTTP code 401 (databricks/databricks-sql-python#408 by @Hodnebo)

CONTRIBUTING.md

@@ -85,18 +85,18 @@ We use [Pytest](https://docs.pytest.org/en/7.1.x/) as our test runner. Invoke it
 Unit tests do not require a Databricks account.
 
 ```bash
-poetry run python -m pytest databricks_sql_connector_core/tests/unit
+poetry run python -m pytest tests/unit
 ```
 #### Only a specific test file
 
 ```bash
-poetry run python -m pytest databricks_sql_connector_core/tests/unit/tests.py
+poetry run python -m pytest tests/unit/tests.py
 ```
 
 #### Only a specific method
 
 ```bash
-poetry run python -m pytest databricks_sql_connector_core/tests/unit/tests.py::ClientTestSuite::test_closing_connection_closes_commands
+poetry run python -m pytest tests/unit/tests.py::ClientTestSuite::test_closing_connection_closes_commands
 ```
 
 #### e2e Tests
@@ -133,7 +133,7 @@ There are several e2e test suites available:
 To execute the core test suite:
 
 ```bash
-poetry run python -m pytest databricks_sql_connector_core/tests/e2e/driver_tests.py::PySQLCoreTestSuite
+poetry run python -m pytest tests/e2e/driver_tests.py::PySQLCoreTestSuite
 ```
 
 The `PySQLCoreTestSuite` namespace contains tests for all of the connector's basic features and behaviours. This is the default namespace where tests should be written unless they require specially configured clusters or take an especially long-time to execute by design.

README.md

@@ -3,7 +3,7 @@
 [![PyPI](https://img.shields.io/pypi/v/databricks-sql-connector?style=flat-square)](https://pypi.org/project/databricks-sql-connector/)
 [![Downloads](https://pepy.tech/badge/databricks-sql-connector)](https://pepy.tech/project/databricks-sql-connector)
 
-The Databricks SQL Connector for Python allows you to develop Python applications that connect to Databricks clusters and SQL warehouses. It is a Thrift-based client with no dependencies on ODBC or JDBC. It conforms to the [Python DB API 2.0 specification](https://www.python.org/dev/peps/pep-0249/) and exposes a [SQLAlchemy](https://www.sqlalchemy.org/) dialect for use with tools like `pandas` and `alembic` which use SQLAlchemy to execute DDL. Use `pip install databricks-sql-connector[databricks-sqlalchemy]` to install with SQLAlchemy's dependencies. `pip install databricks-sql-connector[alembic]` will install alembic's dependencies.
+The Databricks SQL Connector for Python allows you to develop Python applications that connect to Databricks clusters and SQL warehouses. It is a Thrift-based client with no dependencies on ODBC or JDBC. It conforms to the [Python DB API 2.0 specification](https://www.python.org/dev/peps/pep-0249/) and exposes a [SQLAlchemy](https://www.sqlalchemy.org/) dialect for use with tools like `pandas` and `alembic` which use SQLAlchemy to execute DDL. Use `pip install databricks-sql-connector[sqlalchemy]` to install with SQLAlchemy's dependencies. `pip install databricks-sql-connector[alembic]` will install alembic's dependencies.
 
 This connector uses Arrow as the data-exchange format, and supports APIs to directly fetch Arrow tables. Arrow tables are wrapped in the `ArrowQueue` class to provide a natural API to get several rows at a time.
 

databricks_sql_connector_core/tests/conftest.py renamed to conftest.py

@@ -4,23 +4,27 @@
 from databricks.sdk.oauth import OAuthClient
 import os
 
-oauth_client = OAuthClient(host=os.getenv("DATABRICKS_SERVER_HOSTNAME"),
-                           client_id=os.getenv("DATABRICKS_CLIENT_ID"),
-                           client_secret=os.getenv("DATABRICKS_CLIENT_SECRET"),
-                           redirect_url=os.getenv("APP_REDIRECT_URL"),
-                           scopes=['all-apis', 'offline_access'])
+oauth_client = OAuthClient(
+    host=os.getenv("DATABRICKS_SERVER_HOSTNAME"),
+    client_id=os.getenv("DATABRICKS_CLIENT_ID"),
+    client_secret=os.getenv("DATABRICKS_CLIENT_SECRET"),
+    redirect_url=os.getenv("APP_REDIRECT_URL"),
+    scopes=["all-apis", "offline_access"],
+)
 
 consent = oauth_client.initiate_consent()
 
 creds = consent.launch_external_browser()
 
-with sql.connect(server_hostname = os.getenv("DATABRICKS_SERVER_HOSTNAME"),
-                 http_path       = os.getenv("DATABRICKS_HTTP_PATH"),
-                 credentials_provider=creds) as connection:
+with sql.connect(
+    server_hostname=os.getenv("DATABRICKS_SERVER_HOSTNAME"),
+    http_path=os.getenv("DATABRICKS_HTTP_PATH"),
+    credentials_provider=creds,
+) as connection:
 
     for x in range(1, 5):
         cursor = connection.cursor()
-        cursor.execute('SELECT 1+1')
+        cursor.execute("SELECT 1+1")
         result = cursor.fetchall()
         for row in result:
             print(row)

databricks_sql_connector/pyproject.toml

@@ -1,21 +1,23 @@
 from databricks import sql
 import os
 
-with sql.connect(server_hostname = os.getenv("DATABRICKS_SERVER_HOSTNAME"),
-                 http_path       = os.getenv("DATABRICKS_HTTP_PATH"),
-                 access_token    = os.getenv("DATABRICKS_TOKEN")) as connection:
+with sql.connect(
+    server_hostname=os.getenv("DATABRICKS_SERVER_HOSTNAME"),
+    http_path=os.getenv("DATABRICKS_HTTP_PATH"),
+    access_token=os.getenv("DATABRICKS_TOKEN"),
+) as connection:
 
-  with connection.cursor() as cursor:
-    cursor.execute("CREATE TABLE IF NOT EXISTS squares (x int, x_squared int)")
+    with connection.cursor() as cursor:
+        cursor.execute("CREATE TABLE IF NOT EXISTS squares (x int, x_squared int)")
 
-    squares = [(i, i * i) for i in range(100)]
-    values = ",".join([f"({x}, {y})" for (x, y) in squares])
+        squares = [(i, i * i) for i in range(100)]
+        values = ",".join([f"({x}, {y})" for (x, y) in squares])
 
-    cursor.execute(f"INSERT INTO squares VALUES {values}")
+        cursor.execute(f"INSERT INTO squares VALUES {values}")
 
-    cursor.execute("SELECT * FROM squares LIMIT 10")
+        cursor.execute("SELECT * FROM squares LIMIT 10")
 
-    result = cursor.fetchall()
+        result = cursor.fetchall()
 
-    for row in result:
-      print(row)
+        for row in result:
+            print(row)

databricks_sql_connector_core/tests/unit/__init__.py

@@ -13,12 +13,14 @@
 token across script executions.
 """
 
-with sql.connect(server_hostname = os.getenv("DATABRICKS_SERVER_HOSTNAME"),
-                 http_path       = os.getenv("DATABRICKS_HTTP_PATH")) as connection:
+with sql.connect(
+    server_hostname=os.getenv("DATABRICKS_SERVER_HOSTNAME"),
+    http_path=os.getenv("DATABRICKS_HTTP_PATH"),
+) as connection:
 
     for x in range(1, 100):
         cursor = connection.cursor()
-        cursor.execute('SELECT 1+1')
+        cursor.execute("SELECT 1+1")
         result = cursor.fetchall()
         for row in result:
             print(row)

examples/custom_cred_provider.py

@@ -22,17 +22,19 @@ def credential_provider():
         # Service Principal UUID
         client_id=os.getenv("DATABRICKS_CLIENT_ID"),
         # Service Principal Secret
-        client_secret=os.getenv("DATABRICKS_CLIENT_SECRET"))
+        client_secret=os.getenv("DATABRICKS_CLIENT_SECRET"),
+    )
     return oauth_service_principal(config)
 
 
 with sql.connect(
-        server_hostname=server_hostname,
-        http_path=os.getenv("DATABRICKS_HTTP_PATH"),
-        credentials_provider=credential_provider) as connection:
+    server_hostname=server_hostname,
+    http_path=os.getenv("DATABRICKS_HTTP_PATH"),
+    credentials_provider=credential_provider,
+) as connection:
     for x in range(1, 100):
         cursor = connection.cursor()
-        cursor.execute('SELECT 1+1')
+        cursor.execute("SELECT 1+1")
         result = cursor.fetchall()
         for row in result:
             print(row)