Add a "sqlite-unbundled" feature that dynamically links to system libsqlite3.so library (#3507)

* Add a "sqlite-unbundled" feature that dynamically links to system libsqlite3.so library

* update README abouot the newly-added `sqlite-unbundled` feature

* Update README.md to make it clear with bulleted list

Co-authored-by: Austin Bonander <austin.bonander@gmail.com>

* more cfg feature updates

Co-authored-by: Austin Bonander <austin.bonander@gmail.com>

* update documentation in sqlx-sqlx/src/lib.rs too

and also mention possible build time increasement.

* cargo fmt

* Add "sqlite-unbundled" feature to sqlx-cli

* Add sqlite-unbundled to gituhb actions tests

* cfg(feature = "sqlite") => cfg(any(feature = "sqlite", feature = "sqlite-unbundled"))

* fix

* CI: make sqlite-unbundled tests workaround required-features

by duplicating the relevant test section

* use an internal "_sqlite" feature to do the conditional compilation

---------

Co-authored-by: Austin Bonander <austin.bonander@gmail.com>

Author: lilydjwg

.github/workflows/sqlx.yml

@@ -117,6 +117,7 @@ jobs:
     strategy:
       matrix:
         runtime: [async-std, tokio]
+        linking: [sqlite, sqlite-unbundled]
     needs: check
     steps:
       - uses: actions/checkout@v4
@@ -125,7 +126,11 @@ jobs:
 
       - uses: Swatinem/rust-cache@v2
         with:
-          key: "${{ runner.os }}-sqlite-${{ matrix.runtime }}-${{ matrix.tls }}"
+          key: "${{ runner.os }}-${{ matrix.linking }}-${{ matrix.runtime }}-${{ matrix.tls }}"
+
+      - name: Install system sqlite library
+        if: ${{ matrix.linking == 'sqlite-unbundled' }}
+        run: sudo apt-get install -y libsqlite3-dev
 
       - run: echo "using ${DATABASE_URL}"
 
@@ -135,7 +140,7 @@ jobs:
       - run: >
           cargo test
           --no-default-features
-          --features any,macros,sqlite,_unstable-all-types,runtime-${{ matrix.runtime }}
+          --features any,macros,${{ matrix.linking }},_unstable-all-types,runtime-${{ matrix.runtime }}
           --
           --test-threads=1
         env:
@@ -151,8 +156,8 @@ jobs:
       - run: >
           cargo build
           --no-default-features
-          --test sqlite-macros
-          --features any,macros,sqlite,_unstable-all-types,runtime-${{ matrix.runtime }}
+          --test ${{ matrix.linking }}-macros
+          --features any,macros,${{ matrix.linking }},_unstable-all-types,runtime-${{ matrix.runtime }}
         env:
           SQLX_OFFLINE: true
           SQLX_OFFLINE_DIR: .sqlx
@@ -163,8 +168,8 @@ jobs:
       - run: >
           cargo test
           --no-default-features
-          --test sqlite-macros
-          --features any,macros,sqlite,_unstable-all-types,runtime-${{ matrix.runtime }}
+          --test ${{ matrix.linking }}-macros
+          --features any,macros,${{ matrix.linking }},_unstable-all-types,runtime-${{ matrix.runtime }}
         env:
           DATABASE_URL: sqlite://tests/sqlite/sqlite.db
           SQLX_OFFLINE: true

Cargo.toml

@@ -98,12 +98,14 @@ runtime-tokio-rustls = ["runtime-tokio", "tls-rustls-ring"]
 # for conditional compilation
 _rt-async-std = []
 _rt-tokio = []
+_sqlite = []
 
 # database
 any = ["sqlx-core/any", "sqlx-mysql?/any", "sqlx-postgres?/any", "sqlx-sqlite?/any"]
 postgres = ["sqlx-postgres", "sqlx-macros?/postgres"]
 mysql = ["sqlx-mysql", "sqlx-macros?/mysql"]
-sqlite = ["sqlx-sqlite", "sqlx-macros?/sqlite"]
+sqlite = ["_sqlite", "sqlx-sqlite/bundled", "sqlx-macros?/sqlite"]
+sqlite-unbundled = ["_sqlite", "sqlx-sqlite/unbundled", "sqlx-macros?/sqlite-unbundled"]
 
 # types
 json = ["sqlx-macros?/json", "sqlx-mysql?/json", "sqlx-postgres?/json", "sqlx-sqlite?/json"]
@@ -250,6 +252,11 @@ name = "sqlite-macros"
 path = "tests/sqlite/macros.rs"
 required-features = ["sqlite", "macros"]
 
+[[test]]
+name = "sqlite-unbundled-macros"
+path = "tests/sqlite/macros.rs"
+required-features = ["sqlite-unbundled", "macros"]
+
 [[test]]
 name = "sqlite-derives"
 path = "tests/sqlite/derives.rs"

README.md

@@ -183,7 +183,14 @@ be removed in the future.
 
 -   `mssql`: Add support for the MSSQL database server.
 
--   `sqlite`: Add support for the self-contained [SQLite](https://sqlite.org/) database engine.
+-   `sqlite`: Add support for the self-contained [SQLite](https://sqlite.org/) database engine with SQLite bundled and statically-linked.
+
+-   `sqlite-unbundled`: The same as above (`sqlite`), but link SQLite from the system instead of the bundled version.
+    * Allows updating SQLite independently of SQLx or using forked versions.
+    * You must have SQLite installed on the system or provide a path to the library at build time.
+       See [the `rusqlite` README](https://github.com/rusqlite/rusqlite?tab=readme-ov-file#notes-on-building-rusqlite-and-libsqlite3-sys) for details.
+    * May result in link errors if the SQLite version is too old. Version `3.20.0` or newer is recommended.
+    * Can increase build time due to the use of bindgen.
 
 -   `any`: Add support for the `Any` database driver, which can proxy to a database driver at runtime.
 

sqlx-cli/Cargo.toml

@@ -57,6 +57,7 @@ native-tls = ["sqlx/runtime-tokio-native-tls"]
 mysql = ["sqlx/mysql"]
 postgres = ["sqlx/postgres"]
 sqlite = ["sqlx/sqlite"]
+sqlite-unbundled = ["sqlx/sqlite-unbundled"]
 
 # workaround for musl + openssl issues
 openssl-vendored = ["openssl/vendored"]

sqlx-cli/README.md

@@ -19,6 +19,9 @@ $ cargo install sqlx-cli --features openssl-vendored
 
 # use Rustls rather than OpenSSL (be sure to add the features for the databases you intend to use!)
 $ cargo install sqlx-cli --no-default-features --features rustls
+
+# only for sqlite and use the system sqlite library
+$ cargo install sqlx-cli --no-default-features --features sqlite-unbundled
 ```
 
 ## Usage

sqlx-cli/src/database.rs

@@ -11,7 +11,7 @@ pub async fn create(connect_opts: &ConnectOpts) -> anyhow::Result<()> {
     let exists = crate::retry_connect_errors(connect_opts, Any::database_exists).await?;
 
     if !exists {
-        #[cfg(feature = "sqlite")]
+        #[cfg(feature = "_sqlite")]
         sqlx::sqlite::CREATE_DB_WAL.store(
             connect_opts.sqlite_create_db_wal,
             std::sync::atomic::Ordering::Release,

sqlx-cli/src/opt.rs

@@ -258,7 +258,7 @@ pub struct ConnectOpts {
     /// However, if your application sets a `journal_mode` on `SqliteConnectOptions` to something
     /// other than `Wal`, then it will have to take the database file out of WAL mode on connecting,
     /// which requires an exclusive lock and may return a `database is locked` (`SQLITE_BUSY`) error.
-    #[cfg(feature = "sqlite")]
+    #[cfg(feature = "_sqlite")]
     #[clap(long, action = clap::ArgAction::Set, default_value = "true")]
     pub sqlite_create_db_wal: bool,
 }

sqlx-core/src/any/kind.rs

@@ -16,7 +16,7 @@ pub enum AnyKind {
     #[cfg(feature = "mysql")]
     MySql,
 
-    #[cfg(feature = "sqlite")]
+    #[cfg(feature = "_sqlite")]
     Sqlite,
 
     #[cfg(feature = "mssql")]
@@ -48,12 +48,12 @@ impl FromStr for AnyKind {
                 Err(Error::Configuration("database URL has the scheme of a MySQL database but the `mysql` feature is not enabled".into()))
             }
 
-            #[cfg(feature = "sqlite")]
+            #[cfg(feature = "_sqlite")]
             _ if url.starts_with("sqlite:") => {
                 Ok(AnyKind::Sqlite)
             }
 
-            #[cfg(not(feature = "sqlite"))]
+            #[cfg(not(feature = "_sqlite"))]
             _ if url.starts_with("sqlite:") => {
                 Err(Error::Configuration("database URL has the scheme of a SQLite database but the `sqlite` feature is not enabled".into()))
             }

sqlx-macros-core/Cargo.toml

@@ -18,6 +18,8 @@ _tls-native-tls = ["sqlx-core/_tls-native-tls"]
 _tls-rustls-aws-lc-rs = ["sqlx-core/_tls-rustls-aws-lc-rs"]
 _tls-rustls-ring = ["sqlx-core/_tls-rustls-ring"]
 
+_sqlite = []
+
 # SQLx features
 derive = []
 macros = []
@@ -26,7 +28,8 @@ migrate = ["sqlx-core/migrate"]
 # database
 mysql = ["sqlx-mysql"]
 postgres = ["sqlx-postgres"]
-sqlite = ["sqlx-sqlite"]
+sqlite = ["_sqlite", "sqlx-sqlite/bundled"]
+sqlite-unbundled = ["_sqlite", "sqlx-sqlite/unbundled"]
 
 # type integrations
 json = ["sqlx-core/json", "sqlx-mysql?/json", "sqlx-postgres?/json", "sqlx-sqlite?/json"]

sqlx-macros-core/src/database/impls.rs

@@ -44,7 +44,7 @@ mod sqlx {
     #[cfg(feature = "postgres")]
     pub use sqlx_postgres as postgres;
 
-    #[cfg(feature = "sqlite")]
+    #[cfg(feature = "_sqlite")]
     pub use sqlx_sqlite as sqlite;
 }
 
@@ -61,7 +61,7 @@ impl_database_ext! {
     row: sqlx::postgres::PgRow,
 }
 
-#[cfg(feature = "sqlite")]
+#[cfg(feature = "_sqlite")]
 impl_database_ext! {
     sqlx::sqlite::Sqlite,
     row: sqlx::sqlite::SqliteRow,