Merge pull request #8831 from bhandras/sql-invoice-migration

invoices: migrate KV invoices to native SQL for users of KV SQL backends

Author: guggero

config_builder.go

@@ -51,6 +51,7 @@ import (
 	"github.com/lightningnetwork/lnd/rpcperms"
 	"github.com/lightningnetwork/lnd/signal"
 	"github.com/lightningnetwork/lnd/sqldb"
+	"github.com/lightningnetwork/lnd/sqldb/sqlc"
 	"github.com/lightningnetwork/lnd/sweep"
 	"github.com/lightningnetwork/lnd/walletunlocker"
 	"github.com/lightningnetwork/lnd/watchtower"
@@ -60,6 +61,16 @@ import (
 	"gopkg.in/macaroon-bakery.v2/bakery"
 )
 
+const (
+	// invoiceMigrationBatchSize is the number of invoices that will be
+	// migrated in a single batch.
+	invoiceMigrationBatchSize = 1000
+
+	// invoiceMigration is the version of the migration that will be used to
+	// migrate invoices from the kvdb to the sql database.
+	invoiceMigration = 7
+)
+
 // GrpcRegistrar is an interface that must be satisfied by an external subserver
 // that wants to be able to register its own gRPC server onto lnd's main
 // grpc.Server instance.
@@ -932,10 +943,10 @@ type DatabaseInstances struct {
 	// the btcwallet's loader.
 	WalletDB btcwallet.LoaderOption
 
-	// NativeSQLStore is a pointer to a native SQL store that can be used
-	// for native SQL queries for tables that already support it. This may
-	// be nil if the use-native-sql flag was not set.
-	NativeSQLStore *sqldb.BaseDB
+	// NativeSQLStore holds a reference to the native SQL store that can
+	// be used for native SQL queries for tables that already support it.
+	// This may be nil if the use-native-sql flag was not set.
+	NativeSQLStore sqldb.DB
 }
 
 // DefaultDatabaseBuilder is a type that builds the default database backends
@@ -1038,7 +1049,7 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 	if err != nil {
 		cleanUp()
 
-		err := fmt.Errorf("unable to open graph DB: %w", err)
+		err = fmt.Errorf("unable to open graph DB: %w", err)
 		d.logger.Error(err)
 
 		return nil, nil, err
@@ -1072,51 +1083,69 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 	case err != nil:
 		cleanUp()
 
-		err := fmt.Errorf("unable to open graph DB: %w", err)
+		err = fmt.Errorf("unable to open graph DB: %w", err)
 		d.logger.Error(err)
 		return nil, nil, err
 	}
 
-	// Instantiate a native SQL invoice store if the flag is set.
+	// Instantiate a native SQL store if the flag is set.
 	if d.cfg.DB.UseNativeSQL {
-		// KV invoice db resides in the same database as the channel
-		// state DB. Let's query the database to see if we have any
-		// invoices there. If we do, we won't allow the user to start
-		// lnd with native SQL enabled, as we don't currently migrate
-		// the invoices to the new database schema.
-		invoiceSlice, err := dbs.ChanStateDB.QueryInvoices(
-			ctx, invoices.InvoiceQuery{
-				NumMaxInvoices: 1,
-			},
-		)
-		if err != nil {
-			cleanUp()
-			d.logger.Errorf("Unable to query KV invoice DB: %v",
-				err)
+		migrations := sqldb.GetMigrations()
+
+		// If the user has not explicitly disabled the SQL invoice
+		// migration, attach the custom migration function to invoice
+		// migration (version 7). Even if this custom migration is
+		// disabled, the regular native SQL store migrations will still
+		// run. If the database version is already above this custom
+		// migration's version (7), it will be skipped permanently,
+		// regardless of the flag.
+		if !d.cfg.DB.SkipSQLInvoiceMigration {
+			migrationFn := func(tx *sqlc.Queries) error {
+				return invoices.MigrateInvoicesToSQL(
+					ctx, dbs.ChanStateDB.Backend,
+					dbs.ChanStateDB, tx,
+					invoiceMigrationBatchSize,
+				)
+			}
 
-			return nil, nil, err
+			// Make sure we attach the custom migration function to
+			// the correct migration version.
+			for i := 0; i < len(migrations); i++ {
+				if migrations[i].Version != invoiceMigration {
+					continue
+				}
+
+				migrations[i].MigrationFn = migrationFn
+			}
 		}
 
-		if len(invoiceSlice.Invoices) > 0 {
+		// We need to apply all migrations to the native SQL store
+		// before we can use it.
+		err = dbs.NativeSQLStore.ApplyAllMigrations(ctx, migrations)
+		if err != nil {
 			cleanUp()
-			err := fmt.Errorf("found invoices in the KV invoice " +
-				"DB, migration to native SQL is not yet " +
-				"supported")
+			err = fmt.Errorf("faild to run migrations for the "+
+				"native SQL store: %w", err)
 			d.logger.Error(err)
 
 			return nil, nil, err
 		}
 
+		// With the DB ready and migrations applied, we can now create
+		// the base DB and transaction executor for the native SQL
+		// invoice store.
+		baseDB := dbs.NativeSQLStore.GetBaseDB()
 		executor := sqldb.NewTransactionExecutor(
-			dbs.NativeSQLStore,
-			func(tx *sql.Tx) invoices.SQLInvoiceQueries {
-				return dbs.NativeSQLStore.WithTx(tx)
+			baseDB, func(tx *sql.Tx) invoices.SQLInvoiceQueries {
+				return baseDB.WithTx(tx)
 			},
 		)
 
-		dbs.InvoiceDB = invoices.NewSQLStore(
+		sqlInvoiceDB := invoices.NewSQLStore(
 			executor, clock.NewDefaultClock(),
 		)
+
+		dbs.InvoiceDB = sqlInvoiceDB
 	} else {
 		dbs.InvoiceDB = dbs.ChanStateDB
 	}
@@ -1129,7 +1158,7 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 		if err != nil {
 			cleanUp()
 
-			err := fmt.Errorf("unable to open %s database: %w",
+			err = fmt.Errorf("unable to open %s database: %w",
 				lncfg.NSTowerClientDB, err)
 			d.logger.Error(err)
 			return nil, nil, err
@@ -1144,7 +1173,7 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 		if err != nil {
 			cleanUp()
 
-			err := fmt.Errorf("unable to open %s database: %w",
+			err = fmt.Errorf("unable to open %s database: %w",
 				lncfg.NSTowerServerDB, err)
 			d.logger.Error(err)
 			return nil, nil, err

docs/release-notes/release-notes-0.19.0.md

@@ -264,6 +264,11 @@ The underlying functionality between those two options remain the same.
   transactions can run at once, increasing efficiency. Includes several bugfixes
   to allow this to work properly.
 
+* [Migrate KV invoices to
+  SQL](https://github.com/lightningnetwork/lnd/pull/8831) as part of a larger
+  effort to support SQL databases natively in LND.
+
+
 ## Code Health
 
 * A code refactor that [moves all the graph related DB code out of the 
@@ -292,6 +297,7 @@ The underlying functionality between those two options remain the same.
 
 * Abdullahi Yunus
 * Alex Akselrod
+* Andras Banki-Horvath
 * Animesh Bilthare
 * Boris Nagaev
 * Carla Kirk-Cohen

go.mod

@@ -138,7 +138,7 @@ require (
 	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/opencontainers/runc v1.1.12 // indirect
 	github.com/ory/dockertest/v3 v3.10.0 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.26.0 // indirect
 	github.com/prometheus/procfs v0.6.0 // indirect
@@ -207,6 +207,10 @@ replace github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.2
 // allows us to specify that as an option.
 replace google.golang.org/protobuf => github.com/lightninglabs/protobuf-go-hex-display v1.30.0-hex-display
 
+// Temporary replace until https://github.com/lightningnetwork/lnd/pull/8831 is
+// merged.
+replace github.com/lightningnetwork/lnd/sqldb => ./sqldb
+
 // If you change this please also update docs/INSTALL.md and GO_VERSION in
 // Makefile (then run `make lint` to see where else it needs to be updated as
 // well).

go.sum

@@ -464,8 +464,6 @@ github.com/lightningnetwork/lnd/kvdb v1.4.12 h1:Y0WY5Tbjyjn6eCYh068qkWur5oFtioJl
 github.com/lightningnetwork/lnd/kvdb v1.4.12/go.mod h1:hx9buNcxsZpZwh8m1sjTQwy2SOeBoWWOZ3RnOQkMsxI=
 github.com/lightningnetwork/lnd/queue v1.1.1 h1:99ovBlpM9B0FRCGYJo6RSFDlt8/vOkQQZznVb18iNMI=
 github.com/lightningnetwork/lnd/queue v1.1.1/go.mod h1:7A6nC1Qrm32FHuhx/mi1cieAiBZo5O6l8IBIoQxvkz4=
-github.com/lightningnetwork/lnd/sqldb v1.0.6 h1:LJdDSVdN33bVBIefsaJlPW9PDAm6GrXlyFucmzSJ3Ts=
-github.com/lightningnetwork/lnd/sqldb v1.0.6/go.mod h1:OG09zL/PHPaBJefp4HsPz2YLUJ+zIQHbpgCtLnOx8I4=
 github.com/lightningnetwork/lnd/ticker v1.1.1 h1:J/b6N2hibFtC7JLV77ULQp++QLtCwT6ijJlbdiZFbSM=
 github.com/lightningnetwork/lnd/ticker v1.1.1/go.mod h1:waPTRAAcwtu7Ji3+3k+u/xH5GHovTsCoSVpho0KDvdA=
 github.com/lightningnetwork/lnd/tlv v1.3.0 h1:exS/KCPEgpOgviIttfiXAPaUqw2rHQrnUOpP7HPBPiY=

invoices/invoices.go

@@ -187,6 +187,11 @@ func (r InvoiceRef) Modifier() RefModifier {
 	return r.refModifier
 }
 
+// IsHashOnly returns true if the invoice ref only contains a payment hash.
+func (r InvoiceRef) IsHashOnly() bool {
+	return r.payHash != nil && r.payAddr == nil && r.setID == nil
+}
+
 // String returns a human-readable representation of an InvoiceRef.
 func (r InvoiceRef) String() string {
 	var ids []string