sqldb: ensure schema definitions are fully SQLite compatible

Previously, we applied replacements to our schema definitions
to make them compatible with both SQLite and Postgres backends,
as the files were not fully compatible with either.

With this change, the only replacement required for SQLite has
been moved to the generator script. This adjustment ensures
compatibility by enabling auto-incrementing primary keys that
are treated as 64-bit integers by sqlc.

Author: bhandras

invoices/sql_migration.go

@@ -100,7 +100,7 @@ func createInvoiceHashIndex(ctx context.Context, db kvdb.Backend,
 
 			return tx.InsertKVInvoiceKeyAndAddIndex(ctx,
 				sqlc.InsertKVInvoiceKeyAndAddIndexParams{
-					ID:       int32(invoiceKey),
+					ID:       int64(invoiceKey),
 					AddIndex: int64(addIndexNo),
 				},
 			)
@@ -132,7 +132,7 @@ func createInvoiceHashIndex(ctx context.Context, db kvdb.Backend,
 
 			return tx.SetKVInvoicePaymentHash(ctx,
 				sqlc.SetKVInvoicePaymentHashParams{
-					ID:   int32(invoiceKey),
+					ID:   int64(invoiceKey),
 					Hash: k,
 				},
 			)

scripts/gen_sqlc_docker.sh

@@ -2,12 +2,40 @@
 
 set -e
 
+# restore_files is a function to restore original schema files.
+restore_files() {
+	echo "Restoring SQLite bigint patch..."
+	for file in sqldb/sqlc/migrations/*.up.sql.bak; do
+		mv "$file" "${file%.bak}"
+	done
+}
+
+
+# Set trap to call restore_files on script exit. This makes sure the old files
+# are always restored.
+trap restore_files EXIT
+
 # Directory of the script file, independent of where it's called from.
 DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
 # Use the user's cache directories
 GOCACHE=`go env GOCACHE`
 GOMODCACHE=`go env GOMODCACHE`
 
+# SQLite doesn't support "BIGINT PRIMARY KEY" for auto-incrementing primary
+# keys, only "INTEGER PRIMARY KEY". Internally it uses 64-bit integers for
+# numbers anyway, independent of the column type. So we can just use
+# "INTEGER PRIMARY KEY" and it will work the same under the hood, giving us
+# auto incrementing 64-bit integers.
+# _BUT_, sqlc will generate Go code with int32 if we use "INTEGER PRIMARY KEY",
+# even though we want int64. So before we run sqlc, we need to patch the
+# source schema SQL files to use "BIGINT PRIMARY KEY" instead of "INTEGER
+# PRIMARY KEY".
+echo "Applying SQLite bigint patch..."
+for file in sqldb/sqlc/migrations/*.up.sql; do
+	echo "Patching $file"
+	sed -i.bak -E 's/INTEGER PRIMARY KEY/BIGINT PRIMARY KEY/g' "$file"
+done
+
 echo "Generating sql models and queries in go..."
 
 docker run \

sqldb/postgres.go

@@ -29,8 +29,7 @@ var (
 	// has some differences.
 	postgresSchemaReplacements = map[string]string{
 		"BLOB":                "BYTEA",
-		"INTEGER PRIMARY KEY": "SERIAL PRIMARY KEY",
-		"BIGINT PRIMARY KEY":  "BIGSERIAL PRIMARY KEY",
+		"INTEGER PRIMARY KEY": "BIGSERIAL PRIMARY KEY",
 		"TIMESTAMP":           "TIMESTAMP WITHOUT TIME ZONE",
 	}
 

sqldb/sqlc/invoices.sql.go

@@ -11,7 +11,7 @@ INSERT INTO invoice_sequences(name, current_value) VALUES ('settle_index', 0);
 -- invoices table contains all the information shared by all the invoice types. 
 CREATE TABLE IF NOT EXISTS invoices (
     -- The id of the invoice. Translates to the AddIndex.
-    id BIGINT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
 
     -- The hash for this invoice. The invoice hash will always identify that 
     -- invoice.
@@ -102,16 +102,16 @@ CREATE INDEX IF NOT EXISTS invoice_feature_invoice_id_idx ON invoice_features(in
 CREATE TABLE IF NOT EXISTS invoice_htlcs (
     -- The id for this htlc. Used in foreign keys instead of the 
     -- htlc_id/chan_id combination.
-    id BIGINT PRIMARY KEY,
-    
+    id INTEGER PRIMARY KEY,
+
     -- Short chan id indicating the htlc's origin. uint64 stored as text.
     chan_id TEXT NOT NULL, 
 
     -- The uint64 htlc id. This field is a counter so it is safe to store it as 
     -- int64 in the database. The application layer must check that there is no 
     -- overflow when storing/loading this column.
     htlc_id BIGINT NOT NULL,
-    
+
     -- The htlc's amount in millisatoshis.
     amount_msat BIGINT NOT NULL,
 

sqldb/sqlc/migrations/000001_invoices.up.sql

@@ -29,7 +29,7 @@ VALUES
 -- AMP sub invoices. This table can be used to create a historical view of what
 -- happened to the node's invoices.
 CREATE TABLE IF NOT EXISTS invoice_events (
-    id BIGINT PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
 
     -- added_at is the timestamp when this event was added.
     added_at TIMESTAMP NOT NULL,

sqldb/sqlc/migrations/000003_invoice_events.up.sql

@@ -28,13 +28,10 @@ const (
 )
 
 var (
-	// sqliteSchemaReplacements is a map of schema strings that need to be
-	// replaced for sqlite. This is needed because sqlite doesn't directly
-	// support the BIGINT type for primary keys, so we need to replace it
-	// with INTEGER.
-	sqliteSchemaReplacements = map[string]string{
-		"BIGINT PRIMARY KEY": "INTEGER PRIMARY KEY",
-	}
+	// sqliteSchemaReplacements maps schema strings to their SQLite
+	// compatible replacements. Currently, no replacements are needed as our
+	// SQL schema definition files are designed for SQLite compatibility.
+	sqliteSchemaReplacements = map[string]string{}
 
 	// Make sure SqliteStore implements the MigrationExecutor interface.
 	_ MigrationExecutor = (*SqliteStore)(nil)