Merge pull request #9945 from ziggie1984/optional-migration

Decayed log optional migration

Author: Roasbeef

channeldb/db.go

@@ -29,6 +29,7 @@ import (
 	"github.com/lightningnetwork/lnd/channeldb/migration31"
 	"github.com/lightningnetwork/lnd/channeldb/migration32"
 	"github.com/lightningnetwork/lnd/channeldb/migration33"
+	"github.com/lightningnetwork/lnd/channeldb/migration34"
 	"github.com/lightningnetwork/lnd/channeldb/migration_01_to_11"
 	"github.com/lightningnetwork/lnd/clock"
 	graphdb "github.com/lightningnetwork/lnd/graph/db"
@@ -74,12 +75,14 @@ type mandatoryVersion struct {
 // optional migrations.
 type MigrationConfig interface {
 	migration30.MigrateRevLogConfig
+	migration34.MigrationConfig
 }
 
 // MigrationConfigImpl is a super set of all the various migration configs and
 // an implementation of MigrationConfig.
 type MigrationConfigImpl struct {
 	migration30.MigrateRevLogConfigImpl
+	migration34.MigrationConfigImpl
 }
 
 // optionalMigration defines an optional migration function. When a migration
@@ -308,13 +311,23 @@ var (
 	// to determine its state.
 	optionalVersions = []optionalVersion{
 		{
-			name: "prune revocation log",
+			name: "prune_revocation_log",
 			migration: func(db kvdb.Backend,
 				cfg MigrationConfig) error {
 
 				return migration30.MigrateRevocationLog(db, cfg)
 			},
 		},
+		{
+			name: "gc_decayed_log",
+			migration: func(db kvdb.Backend,
+				cfg MigrationConfig) error {
+
+				return migration34.MigrateDecayedLog(
+					db, cfg,
+				)
+			},
+		},
 	}
 
 	// Big endian is the preferred byte order, due to cursor scans over
@@ -1731,10 +1744,8 @@ func (d *DB) syncVersions(versions []mandatoryVersion) error {
 	}, func() {})
 }
 
-// applyOptionalVersions takes a config to determine whether the optional
-// migrations will be applied.
-//
-// NOTE: only support the prune_revocation_log optional migration atm.
+// applyOptionalVersions applies the optional migrations to the database if
+// specified in the config.
 func (d *DB) applyOptionalVersions(cfg OptionalMiragtionConfig) error {
 	// TODO(yy): need to design the db to support dry run for optional
 	// migrations.
@@ -1751,50 +1762,71 @@ func (d *DB) applyOptionalVersions(cfg OptionalMiragtionConfig) error {
 				Versions: make(map[uint64]string),
 			}
 		} else {
-			return err
+			return fmt.Errorf("unable to fetch optional "+
+				"meta: %w", err)
 		}
 	}
 
-	log.Infof("Checking for optional update: prune_revocation_log=%v, "+
-		"db_version=%s", cfg.PruneRevocationLog, om)
-
-	// Exit early if the optional migration is not specified.
-	if !cfg.PruneRevocationLog {
-		return nil
-	}
-
-	// Exit early if the optional migration has already been applied.
-	if _, ok := om.Versions[0]; ok {
-		return nil
-	}
-
-	// Get the optional version.
-	version := optionalVersions[0]
-	log.Infof("Performing database optional migration: %s", version.name)
-
+	// migrationCfg is the parent configuration which implements the config
+	// interfaces of all the single optional migrations.
 	migrationCfg := &MigrationConfigImpl{
 		migration30.MigrateRevLogConfigImpl{
 			NoAmountData: d.noRevLogAmtData,
 		},
+		migration34.MigrationConfigImpl{
+			DecayedLog: cfg.DecayedLog,
+		},
 	}
 
-	// Migrate the data.
-	if err := version.migration(d, migrationCfg); err != nil {
-		log.Errorf("Unable to apply optional migration: %s, error: %v",
-			version.name, err)
-		return err
-	}
+	log.Infof("Applying %d optional migrations", len(optionalVersions))
 
-	// Update the optional meta. Notice that unlike the mandatory db
-	// migrations where we perform the migration and updating meta in a
-	// single db transaction, we use different transactions here. Even when
-	// the following update is failed, we should be fine here as we would
-	// re-run the optional migration again, which is a noop, during next
-	// startup.
-	om.Versions[0] = version.name
-	if err := d.putOptionalMeta(om); err != nil {
-		log.Errorf("Unable to update optional meta: %v", err)
-		return err
+	// Apply the optional migrations if requested.
+	for number, version := range optionalVersions {
+		log.Infof("Checking for optional update: name=%v", version.name)
+
+		// Exit early if the optional migration is not specified.
+		if !cfg.MigrationFlags[number] {
+			log.Debugf("Skipping optional migration: name=%s as "+
+				"it is not specified in the config",
+				version.name)
+
+			continue
+		}
+
+		// Exit early if the optional migration has already been
+		// applied.
+		if _, ok := om.Versions[uint64(number)]; ok {
+			log.Debugf("Skipping optional migration: name=%s as "+
+				"it has already been applied", version.name)
+
+			continue
+		}
+
+		log.Infof("Performing database optional migration: %s",
+			version.name)
+
+		// Call the migration function for the specific optional
+		// migration.
+		if err := version.migration(d, migrationCfg); err != nil {
+			log.Errorf("Unable to apply optional migration: %s, "+
+				"error: %v", version.name, err)
+			return err
+		}
+
+		// Update the optional meta. Notice that unlike the mandatory db
+		// migrations where we perform the migration and updating meta
+		// in a single db transaction, we use different transactions
+		// here. Even when the following update is failed, we should be
+		// fine here as we would re-run the optional migration again,
+		// which is a noop, during next startup.
+		om.Versions[uint64(number)] = version.name
+		if err := d.putOptionalMeta(om); err != nil {
+			log.Errorf("Unable to update optional meta: %v", err)
+			return err
+		}
+
+		log.Infof("Successfully applied optional migration: %s",
+			version.name)
 	}
 
 	return nil

channeldb/log.go

@@ -12,6 +12,7 @@ import (
 	"github.com/lightningnetwork/lnd/channeldb/migration31"
 	"github.com/lightningnetwork/lnd/channeldb/migration32"
 	"github.com/lightningnetwork/lnd/channeldb/migration33"
+	"github.com/lightningnetwork/lnd/channeldb/migration34"
 	"github.com/lightningnetwork/lnd/channeldb/migration_01_to_11"
 	"github.com/lightningnetwork/lnd/kvdb"
 )
@@ -46,5 +47,6 @@ func UseLogger(logger btclog.Logger) {
 	migration31.UseLogger(logger)
 	migration32.UseLogger(logger)
 	migration33.UseLogger(logger)
+	migration34.UseLogger(logger)
 	kvdb.UseLogger(logger)
 }

channeldb/meta.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"slices"
+	"strings"
 
 	"github.com/lightningnetwork/lnd/kvdb"
 	"github.com/lightningnetwork/lnd/tlv"
@@ -30,6 +32,10 @@ var (
 	// ErrMarkerNotPresent is the error that is returned if the queried
 	// marker is not present in the given database.
 	ErrMarkerNotPresent = errors.New("marker not present")
+
+	// ErrInvalidOptionalVersion is the error that is returned if the
+	// optional version persisted in the database is invalid.
+	ErrInvalidOptionalVersion = errors.New("invalid optional version")
 )
 
 // Meta structure holds the database meta information.
@@ -104,15 +110,28 @@ type OptionalMeta struct {
 	Versions map[uint64]string
 }
 
+// String returns a string representation of the optional meta.
 func (om *OptionalMeta) String() string {
-	s := ""
-	for index, name := range om.Versions {
-		s += fmt.Sprintf("%d: %s", index, name)
+	if len(om.Versions) == 0 {
+		return "empty"
+	}
+
+	// Create a slice of indices to sort
+	indices := make([]uint64, 0, len(om.Versions))
+	for index := range om.Versions {
+		indices = append(indices, index)
 	}
-	if s == "" {
-		s = "empty"
+
+	// Sort the indices in ascending order.
+	slices.Sort(indices)
+
+	// Create the string parts in sorted order.
+	parts := make([]string, len(indices))
+	for i, index := range indices {
+		parts[i] = fmt.Sprintf("%d: %s", index, om.Versions[index])
 	}
-	return s
+
+	return strings.Join(parts, ", ")
 }
 
 // fetchOptionalMeta reads the optional meta from the database.
@@ -146,7 +165,20 @@ func (d *DB) fetchOptionalMeta() (*OptionalMeta, error) {
 			if err != nil {
 				return err
 			}
-			om.Versions[version] = optionalVersions[i].name
+
+			// This check would not allow to downgrade LND software
+			// to a version with an optional migration when an
+			// optional migration not known to the current version
+			// has already been applied.
+			if version >= uint64(len(optionalVersions)) {
+				return fmt.Errorf("optional version read "+
+					"from db is %d, but only optional "+
+					"migrations up to %d are known: %w",
+					version, len(optionalVersions)-1,
+					ErrInvalidOptionalVersion)
+			}
+
+			om.Versions[version] = optionalVersions[version].name
 		}
 
 		return nil
@@ -174,8 +206,12 @@ func (d *DB) putOptionalMeta(om *OptionalMeta) error {
 			return err
 		}
 
-		// Write the version indexes.
+		// Write the version indexes of the single migrations.
 		for v := range om.Versions {
+			if v >= uint64(len(optionalVersions)) {
+				return ErrInvalidOptionalVersion
+			}
+
 			err := tlv.WriteVarInt(&b, v, &[8]byte{})
 			if err != nil {
 				return err

channeldb/meta_test.go

@@ -498,6 +498,7 @@ func TestOptionalMeta(t *testing.T) {
 	om = &OptionalMeta{
 		Versions: map[uint64]string{
 			0: optionalVersions[0].name,
+			1: optionalVersions[1].name,
 		},
 	}
 	err = db.putOptionalMeta(om)
@@ -506,43 +507,59 @@ func TestOptionalMeta(t *testing.T) {
 	om1, err := db.fetchOptionalMeta()
 	require.NoError(t, err, "error getting optional meta")
 	require.Equal(t, om, om1, "unexpected empty versions")
-	require.Equal(t, "0: prune revocation log", om.String())
+	require.Equal(
+		t, "0: prune_revocation_log, 1: gc_decayed_log",
+		om1.String(),
+	)
 }
 
 // TestApplyOptionalVersions checks that the optional migration is applied as
 // expected based on the config.
+//
+// NOTE: Cannot be run in parallel because we alter the optionalVersions
+// global variable which could be used by other tests.
 func TestApplyOptionalVersions(t *testing.T) {
-	t.Parallel()
-
 	db, err := MakeTestDB(t)
 	require.NoError(t, err)
 
-	// Overwrite the migration function so we can count how many times the
-	// migration has happened.
-	migrateCount := 0
-	optionalVersions[0].migration = func(_ kvdb.Backend,
-		_ MigrationConfig) error {
+	// migrateCount is the number of migrations that have been run. It
+	// counts the number of times a migration function is called.
+	var migrateCount int
 
-		migrateCount++
-		return nil
+	// Modify all migrations to track their execution.
+	for i := range optionalVersions {
+		optionalVersions[i].migration = func(_ kvdb.Backend,
+			_ MigrationConfig) error {
+
+			migrateCount++
+
+			return nil
+		}
 	}
 
-	// Test that when the flag is false, no migration happens.
-	cfg := OptionalMiragtionConfig{}
+	// All migrations are disabled by default.
+	cfg := NewOptionalMiragtionConfig()
+
+	// Run the optional migrations.
 	err = db.applyOptionalVersions(cfg)
 	require.NoError(t, err, "failed to apply optional migration")
 	require.Equal(t, 0, migrateCount, "expected no migration")
 
 	// Check the optional meta is not updated.
 	om, err := db.fetchOptionalMeta()
 	require.NoError(t, err, "error getting optional meta")
-	require.Empty(t, om.Versions, "expected empty versions")
 
-	// Test that when specified, the optional migration is applied.
-	cfg.PruneRevocationLog = true
+	// Enable all optional migrations.
+	for i := range cfg.MigrationFlags {
+		cfg.MigrationFlags[i] = true
+	}
+
 	err = db.applyOptionalVersions(cfg)
 	require.NoError(t, err, "failed to apply optional migration")
-	require.Equal(t, 1, migrateCount, "expected migration")
+	require.Equal(
+		t, len(optionalVersions), migrateCount,
+		"expected all migrations to be run",
+	)
 
 	// Fetch the updated optional meta.
 	om, err = db.fetchOptionalMeta()
@@ -552,16 +569,20 @@ func TestApplyOptionalVersions(t *testing.T) {
 	omExpected := &OptionalMeta{
 		Versions: map[uint64]string{
 			0: optionalVersions[0].name,
+			1: optionalVersions[1].name,
 		},
 	}
 	require.Equal(t, omExpected, om, "unexpected empty versions")
 
-	// Test that though specified, the optional migration is not run since
-	// it's already been applied.
-	cfg.PruneRevocationLog = true
+	// We make sure running the migrations again does not call the
+	// migrations again because the meta data should signal that they have
+	// already been run.
 	err = db.applyOptionalVersions(cfg)
 	require.NoError(t, err, "failed to apply optional migration")
-	require.Equal(t, 1, migrateCount, "expected no migration")
+	require.Equal(
+		t, len(optionalVersions), migrateCount,
+		"expected all migrations to be run",
+	)
 }
 
 // TestFetchMeta tests that the FetchMeta returns the latest DB version for a

channeldb/migration34/log.go

@@ -0,0 +1,14 @@
+package migration34
+
+import (
+	"github.com/btcsuite/btclog/v2"
+)
+
+// log is a logger that is initialized as disabled.  This means the package will
+// not perform any logging by default until a logger is set.
+var log = btclog.Disabled
+
+// UseLogger uses a specified Logger to output package logging info.
+func UseLogger(logger btclog.Logger) {
+	log = logger
+}