feat(sqlx.toml): support SQLite extensions in macros and sqlx-cli (#3917)

* feat: create `sqlx.toml` format

* feat: add support for ignored_chars config to sqlx_core::migrate

* chore: test ignored_chars with `U+FEFF` (ZWNBSP/BOM)

https://en.wikipedia.org/wiki/Byte_order_mark

* refactor: make `Config` always compiled

simplifies usage while still making parsing optional for less generated code

* refactor: add origin information to `Column`

* feat(macros): implement `type_override` and `column_override` from `sqlx.toml`

* refactor(sqlx.toml): make all keys kebab-case, create `macros.preferred-crates`

* feat: make macros aware of `macros.preferred-crates`

* feat: make `sqlx-cli` aware of `database-url-var`

* feat: teach macros about `migrate.table-name`, `migrations-dir`

* feat: teach macros about `migrate.ignored-chars`

* feat: teach `sqlx-cli` about `migrate.defaults`

* feat: teach `sqlx-cli` about `migrate.migrations-dir`

* feat: teach `sqlx-cli` about `migrate.table-name`

* feat: introduce `migrate.create-schemas`

* WIP feat: create multi-tenant database example

* SQLite extension loading via sqlx.toml for CLI and query macros

* fix: allow start_database to function when the SQLite database file does not already exist

* Added example demonstrating migration and compile-time checking with SQLite extensions

* remove accidentally included db file

* Update sqlx-core/src/config/common.rs

Doc formatting tweak

Co-authored-by: Josh McKinney <joshka@users.noreply.github.com>

* feat: create `sqlx.toml` format

* feat: add support for ignored_chars config to sqlx_core::migrate

* chore: test ignored_chars with `U+FEFF` (ZWNBSP/BOM)

https://en.wikipedia.org/wiki/Byte_order_mark

* refactor: make `Config` always compiled

simplifies usage while still making parsing optional for less generated code

* refactor: add origin information to `Column`

* feat(macros): implement `type_override` and `column_override` from `sqlx.toml`

* refactor(sqlx.toml): make all keys kebab-case, create `macros.preferred-crates`

* feat: make macros aware of `macros.preferred-crates`

* feat: make `sqlx-cli` aware of `database-url-var`

* feat: teach macros about `migrate.table-name`, `migrations-dir`

* feat: teach macros about `migrate.ignored-chars`

* feat: teach `sqlx-cli` about `migrate.defaults`

* feat: teach `sqlx-cli` about `migrate.migrations-dir`

* feat: teach `sqlx-cli` about `migrate.table-name`

* feat: introduce `migrate.create-schemas`

* fix(postgres): don't fetch `ColumnOrigin` for transparently-prepared statements

* feat: progress on axum-multi-tenant example

* feat(config): better errors for mislabeled fields

* WIP feat: filling out axum-multi-tenant example

* feat: multi-tenant example

No longer Axum-based because filling out the request routes would have distracted from the purpose of the example.

* chore(ci): test multi-tenant example

* fixup after merge

* fix: CI, README for `multi-tenant`

* fix: clippy warnings

* fix: multi-tenant README

* fix: sequential versioning inference for migrations

* fix: migration versioning with explicit overrides

* fix: only warn on ambiguous crates if the invocation relies on it

* fix: remove unused imports

* fix: `sqlx mig add` behavior and tests

* fix: restore original type-checking order

* fix: deprecation warning in `tests/postgres/macros.rs`

* feat: create postgres/multi-database example

* fix: examples/postgres/multi-database

* fix: cargo fmt

* chore: add tests for config `migrate.defaults`

* fix: sqlx-cli/tests/add.rs

* feat(cli): add `--config` override to all relevant commands

* chore: run `sqlx mig add` test with `RUST_BACKTRACE=1`

* fix: properly canonicalize config path for `sqlx mig add` test

* fix: get `sqlx mig add` test passing

* fix(cli): test `migrate.ignored-chars`, fix bugs

* feat: create `macros.preferred-crates` example

* fix(examples): use workspace `sqlx`

* fix: examples

* fix: run `cargo fmt`

* fix: more example fixes

* fix(ci): preferred-crates setup

* fix: axum-multi-tenant example locked to specific sqlx version

* import anyhow::Context trait in sqlx-cli/src/lib.rs since it was being used and causing a compile error

* rebased on upstream/main

* make cargo fmt happy

* make clippy happy

* make clippy happier still

* fix: improved error reporting, added parsing test, removed sqlx-toml flag use

* switched to kebab-case for the config key

* switched to kebab-case for the config key

---------

Co-authored-by: Austin Bonander <austin.bonander@gmail.com>
Co-authored-by: Josh McKinney <joshka@users.noreply.github.com>

Author: 3 people

Cargo.lock

@@ -23,6 +23,7 @@ members = [
     "examples/postgres/todos",
     "examples/postgres/transaction",
     "examples/sqlite/todos",
+    "examples/sqlite/extension",
 ]
 
 [workspace.package]
@@ -65,7 +66,7 @@ macros = ["derive", "sqlx-macros/macros"]
 migrate = ["sqlx-core/migrate", "sqlx-macros?/migrate", "sqlx-mysql?/migrate", "sqlx-postgres?/migrate", "sqlx-sqlite?/migrate"]
 
 # Enable parsing of `sqlx.toml` for configuring macros and migrations.
-sqlx-toml = ["sqlx-core/sqlx-toml", "sqlx-macros?/sqlx-toml"]
+sqlx-toml = ["sqlx-core/sqlx-toml", "sqlx-macros?/sqlx-toml", "sqlx-sqlite?/sqlx-toml"]
 
 # intended mainly for CI and docs
 all-databases = ["mysql", "sqlite", "postgres", "any"]

Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "sqlx-example-sqlite-extension"
+version = "0.1.0"
+license.workspace = true
+edition.workspace = true
+repository.workspace = true
+keywords.workspace = true
+categories.workspace = true
+authors.workspace = true
+
+[dependencies]
+sqlx = { path = "../../../", features = [ "sqlite", "runtime-tokio", "tls-native-tls", "sqlx-toml"] }
+tokio = { version = "1.20.0", features = ["rt", "macros"]}
+anyhow = "1.0"
+
+[lints]
+workspace = true

examples/sqlite/extension/Cargo.toml

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# This grabs a pre-compiled version of the extension used in this
+# example, and stores it in a temporary directory. That's a bit
+# unusual. Normally, any extensions you need will be installed into a
+# directory on the library search path, either by using the system
+# package manager or by compiling and installing it yourself.
+
+mkdir /tmp/sqlite3-lib && wget -O /tmp/sqlite3-lib/ipaddr.so https://github.com/nalgeon/sqlean/releases/download/0.15.2/ipaddr.so

examples/sqlite/extension/download-extension.sh

@@ -0,0 +1,25 @@
+create table addresses (address text, family integer);
+
+-- The `ipfamily` function is provided by the
+-- [ipaddr](https://github.com/nalgeon/sqlean/blob/main/docs/ipaddr.md)
+-- sqlite extension, and so this migration can not run if that
+-- extension is not loaded.
+insert into addresses (address, family) values
+  ('fd04:3d29:9f41::1', ipfamily('fd04:3d29:9f41::1')),
+  ('10.0.0.1', ipfamily('10.0.0.1')),
+  ('10.0.0.2', ipfamily('10.0.0.2')),
+  ('fd04:3d29:9f41::2', ipfamily('fd04:3d29:9f41::2')),
+  ('fd04:3d29:9f41::3', ipfamily('fd04:3d29:9f41::3')),
+  ('10.0.0.3', ipfamily('10.0.0.3')),
+  ('fd04:3d29:9f41::4', ipfamily('fd04:3d29:9f41::4')),
+  ('fd04:3d29:9f41::5', ipfamily('fd04:3d29:9f41::5')),
+  ('fd04:3d29:9f41::6', ipfamily('fd04:3d29:9f41::6')),
+  ('10.0.0.4', ipfamily('10.0.0.4')),
+  ('10.0.0.5', ipfamily('10.0.0.5')),
+  ('10.0.0.6', ipfamily('10.0.0.6')),
+  ('10.0.0.7', ipfamily('10.0.0.7')),
+  ('fd04:3d29:9f41::7', ipfamily('fd04:3d29:9f41::7')),
+  ('fd04:3d29:9f41::8', ipfamily('fd04:3d29:9f41::8')),
+  ('10.0.0.8', ipfamily('10.0.0.8')),
+  ('fd04:3d29:9f41::9', ipfamily('fd04:3d29:9f41::9')),
+  ('10.0.0.9', ipfamily('10.0.0.9'));

examples/sqlite/extension/migrations/20250203094951_addresses.sql

@@ -0,0 +1,12 @@
+[common.drivers.sqlite]
+# Including the full path to the extension is somewhat unusual,
+# because normally an extension will be installed in a standard
+# directory which is part of the library search path. If that were the
+# case here, the load-extensions value could just be `["ipaddr"]`
+#
+# When the extension file is installed in a non-standard location, as
+# in this example, there are two options:
+# * Provide the full path the the extension, as seen below.
+# * Add the non-standard location to the library search path, which on
+#   Linux means adding it to the LD_LIBRARY_PATH environment variable.
+load-extensions = ["/tmp/sqlite3-lib/ipaddr"]

examples/sqlite/extension/sqlx.toml

@@ -0,0 +1,47 @@
+use std::str::FromStr;
+
+use sqlx::{
+    query,
+    sqlite::{SqliteConnectOptions, SqlitePool},
+};
+
+#[tokio::main(flavor = "current_thread")]
+async fn main() -> anyhow::Result<()> {
+    let opts = SqliteConnectOptions::from_str(&std::env::var("DATABASE_URL")?)?
+        // The sqlx.toml file controls loading extensions for the CLI
+        // and for the query checking macros, *not* for the
+        // application while it's running. Thus, if we want the
+        // extension to be available during program execution, we need
+        // to load it.
+        //
+        // Note that while in this case the extension path is the same
+        // when checking the program (sqlx.toml) and when running it
+        // (here), this is not required. The runtime environment can
+        // be entirely different from the development one.
+        //
+        // The extension can be described with a full path, as seen
+        // here, but in many cases that will not be necessary. As long
+        // as the extension is installed in a directory on the library
+        // search path, it is sufficient to just provide the extension
+        // name, like "ipaddr"
+        .extension("/tmp/sqlite3-lib/ipaddr");
+
+    let db = SqlitePool::connect_with(opts).await?;
+
+    // We're not running the migrations here, for the sake of brevity
+    // and to confirm that the needed extension was loaded during the
+    // CLI migrate operation. It would not be unusual to run the
+    // migrations here as well, though, using the database connection
+    // we just configured.
+
+    query!(
+        "insert into addresses (address, family) values (?1, ipfamily(?1))",
+        "10.0.0.10"
+    )
+    .execute(&db)
+    .await?;
+
+    println!("Query which requires the extension was successfully executed.");
+
+    Ok(())
+}

examples/sqlite/extension/src/main.rs

@@ -85,3 +85,4 @@ def project(name, database=None, driver=None):
 project("mysql/todos", driver="mysql_8", database="todos")
 project("postgres/todos", driver="postgres_12", database="todos")
 project("sqlite/todos", driver="sqlite", database="todos.db")
+project("sqlite/extension", driver="sqlite", database="extension.db")

examples/x.py

@@ -4,7 +4,7 @@ use std::time::Duration;
 
 use futures_util::TryFutureExt;
 
-use sqlx::{AnyConnection, Connection};
+use sqlx::AnyConnection;
 use tokio::{select, signal};
 
 use crate::opt::{Command, ConnectOpts, DatabaseCommand, MigrateCommand};
@@ -189,7 +189,28 @@ async fn do_run(opt: Opt) -> anyhow::Result<()> {
 
 /// Attempt to connect to the database server, retrying up to `ops.connect_timeout`.
 async fn connect(opts: &ConnectOpts) -> anyhow::Result<AnyConnection> {
-    retry_connect_errors(opts, AnyConnection::connect).await
+    retry_connect_errors(opts, move |url| {
+        // This only handles the default case. For good support of
+        // the new command line options, we need to work out some
+        // way to make the appropriate ConfigOpt available here. I
+        // suspect that that infrastructure would be useful for
+        // other things in the future, as well, but it also seems
+        // like an extensive and intrusive change.
+        //
+        // On the other hand, the compile-time checking macros
+        // can't be configured to use a different config file at
+        // all, so I believe this is okay for the time being.
+        let config = Some(std::path::PathBuf::from("sqlx.toml")).and_then(|p| {
+            if p.exists() {
+                Some(p)
+            } else {
+                None
+            }
+        });
+
+        async move { AnyConnection::connect_with_config(url, config.clone()).await }
+    })
+    .await
 }
 
 /// Attempt an operation that may return errors like `ConnectionRefused`,

sqlx-cli/src/lib.rs

@@ -41,6 +41,24 @@ impl AnyConnection {
         })
     }
 
+    /// UNSTABLE: for use with `sqlx-cli`
+    ///
+    /// Connect to the database, and instruct the nested driver to
+    /// read options from the sqlx.toml file as appropriate.
+    #[cfg(feature = "sqlx-toml")]
+    #[doc(hidden)]
+    pub fn connect_with_config(
+        url: &str,
+        path: Option<std::path::PathBuf>,
+    ) -> BoxFuture<'static, Result<Self, Error>>
+    where
+        Self: Sized,
+    {
+        let options: Result<AnyConnectOptions, Error> = url.parse();
+
+        Box::pin(async move { Self::connect_with(&options?.with_config_file(path)).await })
+    }
+
     pub(crate) fn connect_with_db<DB: Database>(
         options: &AnyConnectOptions,
     ) -> BoxFuture<'_, crate::Result<Self>>

sqlx-core/src/any/connection/mod.rs

@@ -19,6 +19,7 @@ use url::Url;
 pub struct AnyConnectOptions {
     pub database_url: Url,
     pub log_settings: LogSettings,
+    pub enable_config: Option<std::path::PathBuf>,
 }
 impl FromStr for AnyConnectOptions {
     type Err = Error;
@@ -29,6 +30,7 @@ impl FromStr for AnyConnectOptions {
                 .parse::<Url>()
                 .map_err(|e| Error::Configuration(e.into()))?,
             log_settings: LogSettings::default(),
+            enable_config: None,
         })
     }
 }
@@ -40,6 +42,7 @@ impl ConnectOptions for AnyConnectOptions {
         Ok(AnyConnectOptions {
             database_url: url.clone(),
             log_settings: LogSettings::default(),
+            enable_config: None,
         })
     }
 
@@ -63,3 +66,15 @@ impl ConnectOptions for AnyConnectOptions {
         self
     }
 }
+
+impl AnyConnectOptions {
+    /// UNSTABLE: for use with `sqlx-cli`
+    ///
+    /// Allow nested drivers to extract configuration information from
+    /// the sqlx.toml file.
+    #[doc(hidden)]
+    pub fn with_config_file(mut self, path: Option<impl Into<std::path::PathBuf>>) -> Self {
+        self.enable_config = path.map(|p| p.into());
+        self
+    }
+}

sqlx-core/src/any/options.rs

@@ -40,10 +40,49 @@ pub struct Config {
     /// The query macros used in `foo` will use `FOO_DATABASE_URL`,
     /// and the ones used in `bar` will use `BAR_DATABASE_URL`.
     pub database_url_var: Option<String>,
+
+    /// Settings for specific database drivers.
+    ///
+    /// These settings apply when checking queries, or when applying
+    /// migrations via `sqlx-cli`. These settings *do not* apply when
+    /// applying migrations via the macro, as that uses the run-time
+    /// database connection configured by the application.
+    pub drivers: Drivers,
 }
 
 impl Config {
     pub fn database_url_var(&self) -> &str {
         self.database_url_var.as_deref().unwrap_or("DATABASE_URL")
     }
 }
+
+/// Configuration for specific database drivers.
+#[derive(Debug, Default)]
+#[cfg_attr(
+    feature = "sqlx-toml",
+    derive(serde::Deserialize),
+    serde(default, rename_all = "kebab-case", deny_unknown_fields)
+)]
+pub struct Drivers {
+    /// Specify options for the SQLite driver.
+    pub sqlite: SQLite,
+}
+
+/// Configuration for the SQLite database driver.
+#[derive(Debug, Default)]
+#[cfg_attr(
+    feature = "sqlx-toml",
+    derive(serde::Deserialize),
+    serde(default, rename_all = "kebab-case", deny_unknown_fields)
+)]
+pub struct SQLite {
+    /// Specify extensions to load.
+    ///
+    /// # Example: Load the "uuid" and "vsv" extensions
+    /// `sqlx.toml`:
+    /// ```toml
+    /// [common.drivers.sqlite]
+    /// load-extensions = ["uuid", "vsv"]
+    /// ```
+    pub load_extensions: Vec<String>,
+}

sqlx-core/src/config/common.rs

@@ -15,6 +15,12 @@
 # If not specified, defaults to `DATABASE_URL`
 database-url-var = "FOO_DATABASE_URL"
 
+[common.drivers.sqlite]
+# Load extensions into SQLite when running macros or migrations
+#
+# Defaults to an empty list, which has no effect.
+load-extensions = ["uuid", "vsv"]
+
 ###############################################################################################
 
 # Configuration for the `query!()` family of macros.

sqlx-core/src/config/reference.toml

@@ -14,6 +14,7 @@ fn reference_parses_as_config() {
 
 fn assert_common_config(config: &config::common::Config) {
     assert_eq!(config.database_url_var.as_deref(), Some("FOO_DATABASE_URL"));
+    assert_eq!(config.drivers.sqlite.load_extensions[1].as_str(), "vsv");
 }
 
 fn assert_macros_config(config: &config::macros::Config) {

sqlx-core/src/config/tests.rs

@@ -30,6 +30,10 @@ pub struct UnexpectedNullError;
 #[derive(Debug, thiserror::Error)]
 #[non_exhaustive]
 pub enum Error {
+    /// Error occurred while reading configuration file
+    #[error("error reading configuration file: {0}")]
+    ConfigFile(#[source] crate::config::ConfigError),
+
     /// Error occurred while parsing a connection string.
     #[error("error with configuration: {0}")]
     Configuration(#[source] BoxDynError),

sqlx-core/src/error.rs

@@ -27,7 +27,7 @@ derive = []
 macros = []
 migrate = ["sqlx-core/migrate"]
 
-sqlx-toml = ["sqlx-core/sqlx-toml"]
+sqlx-toml = ["sqlx-core/sqlx-toml", "sqlx-sqlite?/sqlx-toml"]
 
 # database
 mysql = ["sqlx-mysql"]

sqlx-macros-core/Cargo.toml

@@ -27,6 +27,8 @@ preupdate-hook = ["libsqlite3-sys/preupdate_hook"]
 bundled = ["libsqlite3-sys/bundled"]
 unbundled = ["libsqlite3-sys/buildtime_bindgen"]
 
+sqlx-toml = ["sqlx-core/sqlx-toml"]
+
 # Note: currently unused, only to satisfy "unexpected `cfg` condition" lint
 bigdecimal = []
 rust_decimal = []

sqlx-sqlite/Cargo.toml

@@ -198,6 +198,23 @@ impl<'a> TryFrom<&'a AnyConnectOptions> for SqliteConnectOptions {
     fn try_from(opts: &'a AnyConnectOptions) -> Result<Self, Self::Error> {
         let mut opts_out = SqliteConnectOptions::from_url(&opts.database_url)?;
         opts_out.log_settings = opts.log_settings.clone();
+
+        if let Some(ref path) = opts.enable_config {
+            if path.exists() {
+                let config = match sqlx_core::config::Config::try_from_path(path.to_path_buf()) {
+                    Ok(cfg) => cfg,
+                    Err(sqlx_core::config::ConfigError::NotFound { path: _ }) => {
+                        return Ok(opts_out)
+                    }
+                    Err(err) => return Err(Self::Error::ConfigFile(err)),
+                };
+
+                for extension in config.common.drivers.sqlite.load_extensions.iter() {
+                    opts_out = opts_out.extension(extension.to_owned());
+                }
+            }
+        }
+
         Ok(opts_out)
     }
 }