Merge pull request #8 from edgarrmondragon/object-schema-and-paths

Use YAML objects instead of colon-separated strings

Author: codeforkjeff

README.md

@@ -67,7 +67,9 @@ dbt_sqlite:
       schema: 'main'
 
       # connect schemas to paths: at least one of these must be 'main'
-      schemas_and_paths: 'main=/my_project/data/etl.db;dataset=/my_project/data/dataset_v1.db'
+      schemas_and_paths:
+        main: '/my_project/data/etl.db'
+        dataset: '/my_project/data/dataset_v1.db'
 
       # directory where all *.db files are attached as schema, using base filename
       # as schema name, and where new schema are created. this can overlap with the dirs of
@@ -76,7 +78,8 @@ dbt_sqlite:
 
       # optional: semi-colon separated list of file paths for SQLite extensions to load.
       # crypto.so is needed to provide for snapshots to work; see README
-      extensions: "/path/to/sqlean/crypto.so"
+      extensions:
+        - "/path/to/sqlean/crypto.so"
 
 ```
 

dbt/adapters/sqlite/connections.py

@@ -1,10 +1,10 @@
 
 from contextlib import contextmanager
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 import glob
 import os.path
 import sqlite3
-from typing import Optional, Tuple, Any
+from typing import Optional, Tuple, Any, Dict, List
 
 
 from dbt.adapters.base import Credentials
@@ -23,9 +23,9 @@
 class SQLiteCredentials(Credentials):
     """ Required connections for a SQLite connection"""
 
-    schemas_and_paths: str
+    schemas_and_paths: Dict[str, str]
     schema_directory: str
-    extensions: Optional[str] = None
+    extensions: List[str] = field(default_factory=list)
 
     @property
     def type(self):
@@ -40,17 +40,16 @@ class SQLiteConnectionManager(SQLConnectionManager):
     TYPE = "sqlite"
 
     @classmethod
-    def open(cls, connection):
+    def open(cls, connection: Connection):
         if connection.state == "open":
             logger.debug("Connection is already open, skipping open.")
             return connection
 
-        credentials = connection.credentials
+        credentials: SQLiteCredentials = connection.credentials
 
         schemas_and_paths = {}
-        for path_entry in credentials.schemas_and_paths.split(";"):
-            schema, path = path_entry.split("=", 1)
-            # store abs path so we can tell if we've attached the file already
+        for schema, path in credentials.schemas_and_paths.items():
+            # Make .db file path absolute
             schemas_and_paths[schema] = os.path.abspath(path)
 
         try:
@@ -59,12 +58,10 @@ def open(cls, connection):
             else:
                 raise FailedToConnectException("at least one schema must be called 'main'")
 
-            extensions = [e for e in (connection.credentials.extensions or "").split(";") if e]
-
-            if len(extensions) > 0:
+            if len(credentials.extensions) > 0:
                 handle.enable_load_extension(True)
 
-            for ext_path in extensions:
+            for ext_path in credentials.extensions:
                 handle.load_extension(ext_path)
 
             cursor = handle.cursor()

dbt/include/sqlite/sample_profiles.yml

@@ -6,17 +6,21 @@ default:
       threads: 1
       database: <database name>
       schema: 'main'
-      schemas_and_paths: 'main=/my_project/data/etl.db'
+      schemas_and_paths:
+        main: '/my_project/data/etl.db'
       schema_directory: '/my_project/data'
-      extensions: '/path/to/sqlite-digest/digest.so'
+      extensions:
+        - '/path/to/sqlite-digest/digest.so'
 
     prod:
       type: sqlite
       threads: 1
       database: <database name>
       schema: 'main'
-      schemas_and_paths: 'main=/my_project/data/etl.db'
+      schemas_and_paths:
+        main: '/my_project/data/etl.db'
       schema_directory: '/my_project/data'
-      extensions: '/path/to/sqlite-digest/digest.so'
+      extensions:
+        - '/path/to/sqlite-digest/digest.so'
 
   target: dev

test/sqlite.dbtspec

@@ -3,9 +3,11 @@ target:
   type: sqlite
   database: adapter_test
   schema: 'main'
-  schemas_and_paths: "main=/tmp/dbt-sqlite-tests/adapter_test.db"
+  schemas_and_paths:
+    main: '/tmp/dbt-sqlite-tests/adapter_test.db'
   schema_directory: '/tmp/dbt-sqlite-tests'
-  extensions: "/home/jeff/git/sqlite-digest/digest.so"
+  extensions:
+    - "/home/spoton/edgar/sqlite-digest/digest.so"
   threads: 1
 sequences:
   test_dbt_empty: empty