diff --git a/Cargo.lock b/Cargo.lock index 3c0d9eccb5..fe56e255f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3818,6 +3818,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "sqlx-example-sqlite-extension" +version = "0.1.0" +dependencies = [ + "anyhow", + "sqlx", + "tokio", +] + [[package]] name = "sqlx-example-sqlite-todos" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index b72ec9afbd..3521ee1a1b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ members = [ "examples/postgres/todos", "examples/postgres/transaction", "examples/sqlite/todos", + "examples/sqlite/extension", ] [workspace.package] @@ -65,7 +66,7 @@ macros = ["derive", "sqlx-macros/macros"] migrate = ["sqlx-core/migrate", "sqlx-macros?/migrate", "sqlx-mysql?/migrate", "sqlx-postgres?/migrate", "sqlx-sqlite?/migrate"] # Enable parsing of `sqlx.toml` for configuring macros and migrations. -sqlx-toml = ["sqlx-core/sqlx-toml", "sqlx-macros?/sqlx-toml"] +sqlx-toml = ["sqlx-core/sqlx-toml", "sqlx-macros?/sqlx-toml", "sqlx-sqlite?/sqlx-toml"] # intended mainly for CI and docs all-databases = ["mysql", "sqlite", "postgres", "any"] diff --git a/examples/sqlite/extension/Cargo.toml b/examples/sqlite/extension/Cargo.toml new file mode 100644 index 0000000000..fa2042e343 --- /dev/null +++ b/examples/sqlite/extension/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "sqlx-example-sqlite-extension" +version = "0.1.0" +license.workspace = true +edition.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true +authors.workspace = true + +[dependencies] +sqlx = { path = "../../../", features = [ "sqlite", "runtime-tokio", "tls-native-tls", "sqlx-toml"] } +tokio = { version = "1.20.0", features = ["rt", "macros"]} +anyhow = "1.0" + +[lints] +workspace = true diff --git a/examples/sqlite/extension/download-extension.sh b/examples/sqlite/extension/download-extension.sh new file mode 100755 index 0000000000..ce7f23a486 --- /dev/null +++ b/examples/sqlite/extension/download-extension.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# This grabs a pre-compiled version of the extension used in this +# example, and stores it in a temporary directory. That's a bit +# unusual. Normally, any extensions you need will be installed into a +# directory on the library search path, either by using the system +# package manager or by compiling and installing it yourself. + +mkdir /tmp/sqlite3-lib && wget -O /tmp/sqlite3-lib/ipaddr.so https://github.com/nalgeon/sqlean/releases/download/0.15.2/ipaddr.so diff --git a/examples/sqlite/extension/migrations/20250203094951_addresses.sql b/examples/sqlite/extension/migrations/20250203094951_addresses.sql new file mode 100644 index 0000000000..af38213d5f --- /dev/null +++ b/examples/sqlite/extension/migrations/20250203094951_addresses.sql @@ -0,0 +1,25 @@ +create table addresses (address text, family integer); + +-- The `ipfamily` function is provided by the +-- [ipaddr](https://github.com/nalgeon/sqlean/blob/main/docs/ipaddr.md) +-- sqlite extension, and so this migration can not run if that +-- extension is not loaded. +insert into addresses (address, family) values + ('fd04:3d29:9f41::1', ipfamily('fd04:3d29:9f41::1')), + ('10.0.0.1', ipfamily('10.0.0.1')), + ('10.0.0.2', ipfamily('10.0.0.2')), + ('fd04:3d29:9f41::2', ipfamily('fd04:3d29:9f41::2')), + ('fd04:3d29:9f41::3', ipfamily('fd04:3d29:9f41::3')), + ('10.0.0.3', ipfamily('10.0.0.3')), + ('fd04:3d29:9f41::4', ipfamily('fd04:3d29:9f41::4')), + ('fd04:3d29:9f41::5', ipfamily('fd04:3d29:9f41::5')), + ('fd04:3d29:9f41::6', ipfamily('fd04:3d29:9f41::6')), + ('10.0.0.4', ipfamily('10.0.0.4')), + ('10.0.0.5', ipfamily('10.0.0.5')), + ('10.0.0.6', ipfamily('10.0.0.6')), + ('10.0.0.7', ipfamily('10.0.0.7')), + ('fd04:3d29:9f41::7', ipfamily('fd04:3d29:9f41::7')), + ('fd04:3d29:9f41::8', ipfamily('fd04:3d29:9f41::8')), + ('10.0.0.8', ipfamily('10.0.0.8')), + ('fd04:3d29:9f41::9', ipfamily('fd04:3d29:9f41::9')), + ('10.0.0.9', ipfamily('10.0.0.9')); diff --git a/examples/sqlite/extension/sqlx.toml b/examples/sqlite/extension/sqlx.toml new file mode 100644 index 0000000000..77f844642f --- /dev/null +++ b/examples/sqlite/extension/sqlx.toml @@ -0,0 +1,12 @@ +[common.drivers.sqlite] +# Including the full path to the extension is somewhat unusual, +# because normally an extension will be installed in a standard +# directory which is part of the library search path. If that were the +# case here, the load-extensions value could just be `["ipaddr"]` +# +# When the extension file is installed in a non-standard location, as +# in this example, there are two options: +# * Provide the full path the the extension, as seen below. +# * Add the non-standard location to the library search path, which on +# Linux means adding it to the LD_LIBRARY_PATH environment variable. +load-extensions = ["/tmp/sqlite3-lib/ipaddr"] \ No newline at end of file diff --git a/examples/sqlite/extension/src/main.rs b/examples/sqlite/extension/src/main.rs new file mode 100644 index 0000000000..ee859c55b8 --- /dev/null +++ b/examples/sqlite/extension/src/main.rs @@ -0,0 +1,47 @@ +use std::str::FromStr; + +use sqlx::{ + query, + sqlite::{SqliteConnectOptions, SqlitePool}, +}; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> anyhow::Result<()> { + let opts = SqliteConnectOptions::from_str(&std::env::var("DATABASE_URL")?)? + // The sqlx.toml file controls loading extensions for the CLI + // and for the query checking macros, *not* for the + // application while it's running. Thus, if we want the + // extension to be available during program execution, we need + // to load it. + // + // Note that while in this case the extension path is the same + // when checking the program (sqlx.toml) and when running it + // (here), this is not required. The runtime environment can + // be entirely different from the development one. + // + // The extension can be described with a full path, as seen + // here, but in many cases that will not be necessary. As long + // as the extension is installed in a directory on the library + // search path, it is sufficient to just provide the extension + // name, like "ipaddr" + .extension("/tmp/sqlite3-lib/ipaddr"); + + let db = SqlitePool::connect_with(opts).await?; + + // We're not running the migrations here, for the sake of brevity + // and to confirm that the needed extension was loaded during the + // CLI migrate operation. It would not be unusual to run the + // migrations here as well, though, using the database connection + // we just configured. + + query!( + "insert into addresses (address, family) values (?1, ipfamily(?1))", + "10.0.0.10" + ) + .execute(&db) + .await?; + + println!("Query which requires the extension was successfully executed."); + + Ok(()) +} diff --git a/examples/x.py b/examples/x.py index 79f6fda1ba..aaf4170c77 100755 --- a/examples/x.py +++ b/examples/x.py @@ -85,3 +85,4 @@ def project(name, database=None, driver=None): project("mysql/todos", driver="mysql_8", database="todos") project("postgres/todos", driver="postgres_12", database="todos") project("sqlite/todos", driver="sqlite", database="todos.db") +project("sqlite/extension", driver="sqlite", database="extension.db") diff --git a/sqlx-cli/src/lib.rs b/sqlx-cli/src/lib.rs index adcc6a1306..f0db083b1d 100644 --- a/sqlx-cli/src/lib.rs +++ b/sqlx-cli/src/lib.rs @@ -4,7 +4,7 @@ use std::time::Duration; use futures_util::TryFutureExt; -use sqlx::{AnyConnection, Connection}; +use sqlx::AnyConnection; use tokio::{select, signal}; use crate::opt::{Command, ConnectOpts, DatabaseCommand, MigrateCommand}; @@ -189,7 +189,28 @@ async fn do_run(opt: Opt) -> anyhow::Result<()> { /// Attempt to connect to the database server, retrying up to `ops.connect_timeout`. async fn connect(opts: &ConnectOpts) -> anyhow::Result { - retry_connect_errors(opts, AnyConnection::connect).await + retry_connect_errors(opts, move |url| { + // This only handles the default case. For good support of + // the new command line options, we need to work out some + // way to make the appropriate ConfigOpt available here. I + // suspect that that infrastructure would be useful for + // other things in the future, as well, but it also seems + // like an extensive and intrusive change. + // + // On the other hand, the compile-time checking macros + // can't be configured to use a different config file at + // all, so I believe this is okay for the time being. + let config = Some(std::path::PathBuf::from("sqlx.toml")).and_then(|p| { + if p.exists() { + Some(p) + } else { + None + } + }); + + async move { AnyConnection::connect_with_config(url, config.clone()).await } + }) + .await } /// Attempt an operation that may return errors like `ConnectionRefused`, diff --git a/sqlx-core/src/any/connection/mod.rs b/sqlx-core/src/any/connection/mod.rs index 8cf8fc510c..34c7ba5ea5 100644 --- a/sqlx-core/src/any/connection/mod.rs +++ b/sqlx-core/src/any/connection/mod.rs @@ -40,6 +40,24 @@ impl AnyConnection { }) } + /// UNSTABLE: for use with `sqlx-cli` + /// + /// Connect to the database, and instruct the nested driver to + /// read options from the sqlx.toml file as appropriate. + #[cfg(feature = "sqlx-toml")] + #[doc(hidden)] + pub fn connect_with_config( + url: &str, + path: Option, + ) -> BoxFuture<'static, Result> + where + Self: Sized, + { + let options: Result = url.parse(); + + Box::pin(async move { Self::connect_with(&options?.with_config_file(path)).await }) + } + pub(crate) fn connect_with_db( options: &AnyConnectOptions, ) -> BoxFuture<'_, crate::Result> diff --git a/sqlx-core/src/any/options.rs b/sqlx-core/src/any/options.rs index bb29d817c9..66a35eb9b3 100644 --- a/sqlx-core/src/any/options.rs +++ b/sqlx-core/src/any/options.rs @@ -19,6 +19,7 @@ use url::Url; pub struct AnyConnectOptions { pub database_url: Url, pub log_settings: LogSettings, + pub enable_config: Option, } impl FromStr for AnyConnectOptions { type Err = Error; @@ -29,6 +30,7 @@ impl FromStr for AnyConnectOptions { .parse::() .map_err(|e| Error::Configuration(e.into()))?, log_settings: LogSettings::default(), + enable_config: None, }) } } @@ -40,6 +42,7 @@ impl ConnectOptions for AnyConnectOptions { Ok(AnyConnectOptions { database_url: url.clone(), log_settings: LogSettings::default(), + enable_config: None, }) } @@ -63,3 +66,15 @@ impl ConnectOptions for AnyConnectOptions { self } } + +impl AnyConnectOptions { + /// UNSTABLE: for use with `sqlx-cli` + /// + /// Allow nested drivers to extract configuration information from + /// the sqlx.toml file. + #[doc(hidden)] + pub fn with_config_file(mut self, path: Option>) -> Self { + self.enable_config = path.map(|p| p.into()); + self + } +} diff --git a/sqlx-core/src/config/common.rs b/sqlx-core/src/config/common.rs index 2d5342d5b8..e1809d6d2b 100644 --- a/sqlx-core/src/config/common.rs +++ b/sqlx-core/src/config/common.rs @@ -40,6 +40,14 @@ pub struct Config { /// The query macros used in `foo` will use `FOO_DATABASE_URL`, /// and the ones used in `bar` will use `BAR_DATABASE_URL`. pub database_url_var: Option, + + /// Settings for specific database drivers. + /// + /// These settings apply when checking queries, or when applying + /// migrations via `sqlx-cli`. These settings *do not* apply when + /// applying migrations via the macro, as that uses the run-time + /// database connection configured by the application. + pub drivers: Drivers, } impl Config { @@ -47,3 +55,34 @@ impl Config { self.database_url_var.as_deref().unwrap_or("DATABASE_URL") } } + +/// Configuration for specific database drivers. +#[derive(Debug, Default)] +#[cfg_attr( + feature = "sqlx-toml", + derive(serde::Deserialize), + serde(default, rename_all = "kebab-case", deny_unknown_fields) +)] +pub struct Drivers { + /// Specify options for the SQLite driver. + pub sqlite: SQLite, +} + +/// Configuration for the SQLite database driver. +#[derive(Debug, Default)] +#[cfg_attr( + feature = "sqlx-toml", + derive(serde::Deserialize), + serde(default, rename_all = "kebab-case", deny_unknown_fields) +)] +pub struct SQLite { + /// Specify extensions to load. + /// + /// # Example: Load the "uuid" and "vsv" extensions + /// `sqlx.toml`: + /// ```toml + /// [common.drivers.sqlite] + /// load-extensions = ["uuid", "vsv"] + /// ``` + pub load_extensions: Vec, +} diff --git a/sqlx-core/src/config/reference.toml b/sqlx-core/src/config/reference.toml index 77833fb5a8..787c3456db 100644 --- a/sqlx-core/src/config/reference.toml +++ b/sqlx-core/src/config/reference.toml @@ -15,6 +15,12 @@ # If not specified, defaults to `DATABASE_URL` database-url-var = "FOO_DATABASE_URL" +[common.drivers.sqlite] +# Load extensions into SQLite when running macros or migrations +# +# Defaults to an empty list, which has no effect. +load-extensions = ["uuid", "vsv"] + ############################################################################################### # Configuration for the `query!()` family of macros. diff --git a/sqlx-core/src/config/tests.rs b/sqlx-core/src/config/tests.rs index 0b0b590919..3d0f4fc871 100644 --- a/sqlx-core/src/config/tests.rs +++ b/sqlx-core/src/config/tests.rs @@ -14,6 +14,7 @@ fn reference_parses_as_config() { fn assert_common_config(config: &config::common::Config) { assert_eq!(config.database_url_var.as_deref(), Some("FOO_DATABASE_URL")); + assert_eq!(config.drivers.sqlite.load_extensions[1].as_str(), "vsv"); } fn assert_macros_config(config: &config::macros::Config) { diff --git a/sqlx-core/src/error.rs b/sqlx-core/src/error.rs index 9ad5eff464..5db51eca59 100644 --- a/sqlx-core/src/error.rs +++ b/sqlx-core/src/error.rs @@ -30,6 +30,10 @@ pub struct UnexpectedNullError; #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum Error { + /// Error occurred while reading configuration file + #[error("error reading configuration file: {0}")] + ConfigFile(#[source] crate::config::ConfigError), + /// Error occurred while parsing a connection string. #[error("error with configuration: {0}")] Configuration(#[source] BoxDynError), diff --git a/sqlx-macros-core/Cargo.toml b/sqlx-macros-core/Cargo.toml index 02b773af07..c8eb5760a4 100644 --- a/sqlx-macros-core/Cargo.toml +++ b/sqlx-macros-core/Cargo.toml @@ -27,7 +27,7 @@ derive = [] macros = [] migrate = ["sqlx-core/migrate"] -sqlx-toml = ["sqlx-core/sqlx-toml"] +sqlx-toml = ["sqlx-core/sqlx-toml", "sqlx-sqlite?/sqlx-toml"] # database mysql = ["sqlx-mysql"] diff --git a/sqlx-sqlite/Cargo.toml b/sqlx-sqlite/Cargo.toml index db7fb63cb8..a84dccc6dc 100644 --- a/sqlx-sqlite/Cargo.toml +++ b/sqlx-sqlite/Cargo.toml @@ -27,6 +27,8 @@ preupdate-hook = ["libsqlite3-sys/preupdate_hook"] bundled = ["libsqlite3-sys/bundled"] unbundled = ["libsqlite3-sys/buildtime_bindgen"] +sqlx-toml = ["sqlx-core/sqlx-toml"] + # Note: currently unused, only to satisfy "unexpected `cfg` condition" lint bigdecimal = [] rust_decimal = [] diff --git a/sqlx-sqlite/src/any.rs b/sqlx-sqlite/src/any.rs index c72370d0ff..7b71e2cf4e 100644 --- a/sqlx-sqlite/src/any.rs +++ b/sqlx-sqlite/src/any.rs @@ -201,6 +201,23 @@ impl<'a> TryFrom<&'a AnyConnectOptions> for SqliteConnectOptions { fn try_from(opts: &'a AnyConnectOptions) -> Result { let mut opts_out = SqliteConnectOptions::from_url(&opts.database_url)?; opts_out.log_settings = opts.log_settings.clone(); + + if let Some(ref path) = opts.enable_config { + if path.exists() { + let config = match sqlx_core::config::Config::try_from_path(path.to_path_buf()) { + Ok(cfg) => cfg, + Err(sqlx_core::config::ConfigError::NotFound { path: _ }) => { + return Ok(opts_out) + } + Err(err) => return Err(Self::Error::ConfigFile(err)), + }; + + for extension in config.common.drivers.sqlite.load_extensions.iter() { + opts_out = opts_out.extension(extension.to_owned()); + } + } + } + Ok(opts_out) } } diff --git a/sqlx-sqlite/src/lib.rs b/sqlx-sqlite/src/lib.rs index e4a122b6bd..b4878a4e2a 100644 --- a/sqlx-sqlite/src/lib.rs +++ b/sqlx-sqlite/src/lib.rs @@ -127,7 +127,18 @@ pub static CREATE_DB_WAL: AtomicBool = AtomicBool::new(true); /// UNSTABLE: for use by `sqlite-macros-core` only. #[doc(hidden)] pub fn describe_blocking(query: &str, database_url: &str) -> Result, Error> { - let opts: SqliteConnectOptions = database_url.parse()?; + let mut opts: SqliteConnectOptions = database_url.parse()?; + + match sqlx_core::config::Config::try_from_crate_or_default() { + Ok(config) => { + for extension in config.common.drivers.sqlite.load_extensions.iter() { + opts = opts.extension(extension.to_owned()); + } + } + Err(sqlx_core::config::ConfigError::NotFound { path: _ }) => {} + Err(err) => return Err(Error::ConfigFile(err)), + } + let params = EstablishParams::from_options(&opts)?; let mut conn = params.establish()?; diff --git a/tests/docker.py b/tests/docker.py index b1b81b07fb..5e8c74fb1f 100644 --- a/tests/docker.py +++ b/tests/docker.py @@ -17,9 +17,10 @@ def start_database(driver, database, cwd): database = path.join(cwd, database) (base_path, ext) = path.splitext(database) new_database = f"{base_path}.test{ext}" - shutil.copy(database, new_database) + if path.exists(database): + shutil.copy(database, new_database) # short-circuit for sqlite - return f"sqlite://{path.join(cwd, new_database)}" + return f"sqlite://{path.join(cwd, new_database)}?mode=rwc" res = subprocess.run( ["docker-compose", "up", "-d", driver],