accounts: iterate over bbolt buckets in migration

Update the SQL migration to instead of using the public functions of the
BoltStore struct, we instead iterate over the buckets directly. This
ensures that the BoltStore struct Acccounts & LastIndexes functions of
the BoltStore implementation can be changed, without effecting the
SQL migration.

Author: ViktorTigerstrom

accounts/sql_migration.go

@@ -1,6 +1,7 @@
 package accounts
 
 import (
+	"bytes"
 	"context"
 	"database/sql"
 	"errors"
@@ -11,6 +12,7 @@ import (
 
 	"github.com/davecgh/go-spew/spew"
 	"github.com/lightninglabs/lightning-terminal/db/sqlc"
+	"github.com/lightningnetwork/lnd/kvdb"
 	"github.com/pmezard/go-difflib/difflib"
 )
 
@@ -24,7 +26,7 @@ var (
 // MigrateAccountStoreToSQL runs the migration of all accounts and indices from
 // the KV database to the SQL database. The migration is done in a single
 // transaction to ensure that all accounts are migrated or none at all.
-func MigrateAccountStoreToSQL(ctx context.Context, kvStore *BoltStore,
+func MigrateAccountStoreToSQL(ctx context.Context, kvStore kvdb.Backend,
 	tx SQLQueries) error {
 
 	log.Infof("Starting migration of the KV accounts store to SQL")
@@ -47,12 +49,12 @@ func MigrateAccountStoreToSQL(ctx context.Context, kvStore *BoltStore,
 // migrateAccountsToSQL runs the migration of all accounts from the KV database
 // to the SQL database. The migration is done in a single transaction to ensure
 // that all accounts are migrated or none at all.
-func migrateAccountsToSQL(ctx context.Context, kvStore *BoltStore,
+func migrateAccountsToSQL(ctx context.Context, kvStore kvdb.Backend,
 	tx SQLQueries) error {
 
 	log.Infof("Starting migration of accounts from KV to SQL")
 
-	kvAccounts, err := kvStore.Accounts(ctx)
+	kvAccounts, err := getBBoltAccounts(kvStore)
 	if err != nil {
 		return err
 	}
@@ -104,6 +106,51 @@ func migrateAccountsToSQL(ctx context.Context, kvStore *BoltStore,
 	return nil
 }
 
+// getBBoltAccounts is a helper function that fetches all accounts from the
+// Bbolt store, by iterating directly over the buckets, without needing to
+// use any public functions of the BoltStore struct.
+func getBBoltAccounts(db kvdb.Backend) ([]*OffChainBalanceAccount, error) {
+	var accounts []*OffChainBalanceAccount
+	err := db.View(func(tx kvdb.RTx) error {
+		// This function will be called in the ForEach and receive
+		// the key and value of each account in the DB. The key, which
+		// is also the ID is not used because it is also marshaled into
+		// the value.
+		readFn := func(k, v []byte) error {
+			// Skip the two special purpose keys.
+			if bytes.Equal(k, lastAddIndexKey) ||
+				bytes.Equal(k, lastSettleIndexKey) {
+
+				return nil
+			}
+
+			// There should be no sub-buckets.
+			if v == nil {
+				return fmt.Errorf("invalid bucket structure")
+			}
+
+			account, err := deserializeAccount(v)
+			if err != nil {
+				return err
+			}
+
+			accounts = append(accounts, account)
+			return nil
+		}
+
+		// We know the bucket should exist since it's created when
+		// the account storage is initialized.
+		return tx.ReadBucket(accountBucketName).ForEach(readFn)
+	}, func() {
+		accounts = nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return accounts, nil
+}
+
 // migrateSingleAccountToSQL runs the migration for a single account from the
 // KV database to the SQL database.
 func migrateSingleAccountToSQL(ctx context.Context,
@@ -163,12 +210,12 @@ func migrateSingleAccountToSQL(ctx context.Context,
 
 // migrateAccountsIndicesToSQL runs the migration for the account indices from
 // the KV database to the SQL database.
-func migrateAccountsIndicesToSQL(ctx context.Context, kvStore *BoltStore,
+func migrateAccountsIndicesToSQL(ctx context.Context, kvStore kvdb.Backend,
 	tx SQLQueries) error {
 
 	log.Infof("Starting migration of accounts indices from KV to SQL")
 
-	addIndex, settleIndex, err := kvStore.LastIndexes(ctx)
+	addIndex, settleIndex, err := getBBoltIndices(kvStore)
 	if errors.Is(err, ErrNoInvoiceIndexKnown) {
 		log.Infof("No indices found in KV store, skipping migration")
 		return nil
@@ -211,6 +258,40 @@ func migrateAccountsIndicesToSQL(ctx context.Context, kvStore *BoltStore,
 	return nil
 }
 
+// getBBoltIndices is a helper function that fetches the índices from the
+// Bbolt store, by iterating directly over the buckets, without needing to
+// use any public functions of the BoltStore struct.
+func getBBoltIndices(db kvdb.Backend) (uint64, uint64, error) {
+	var (
+		addValue, settleValue []byte
+	)
+	err := db.View(func(tx kvdb.RTx) error {
+		bucket := tx.ReadBucket(accountBucketName)
+		if bucket == nil {
+			return ErrAccountBucketNotFound
+		}
+
+		addValue = bucket.Get(lastAddIndexKey)
+		if len(addValue) == 0 {
+			return ErrNoInvoiceIndexKnown
+		}
+
+		settleValue = bucket.Get(lastSettleIndexKey)
+		if len(settleValue) == 0 {
+			return ErrNoInvoiceIndexKnown
+		}
+
+		return nil
+	}, func() {
+		addValue, settleValue = nil, nil
+	})
+	if err != nil {
+		return 0, 0, err
+	}
+
+	return byteOrder.Uint64(addValue), byteOrder.Uint64(settleValue), nil
+}
+
 // overrideAccountTimeZone overrides the time zone of the account to the local
 // time zone and chops off the nanosecond part for comparison. This is needed
 // because KV database stores times as-is which as an unwanted side effect would

accounts/sql_migration_test.go

@@ -344,7 +344,7 @@ func TestAccountStoreMigration(t *testing.T) {
 			err = txEx.ExecTx(ctx, &opts,
 				func(tx SQLQueries) error {
 					return MigrateAccountStoreToSQL(
-						ctx, kvStore, tx,
+						ctx, kvStore.db, tx,
 					)
 				},
 			)