multi: Add decayedlog db migration code

This commit adds the migration code for the decayedlog db which
is optional and will default to true.

Author: ziggie1984

channeldb/db.go

@@ -29,6 +29,7 @@ import (
 	"github.com/lightningnetwork/lnd/channeldb/migration31"
 	"github.com/lightningnetwork/lnd/channeldb/migration32"
 	"github.com/lightningnetwork/lnd/channeldb/migration33"
+	"github.com/lightningnetwork/lnd/channeldb/migration34"
 	"github.com/lightningnetwork/lnd/channeldb/migration_01_to_11"
 	"github.com/lightningnetwork/lnd/clock"
 	graphdb "github.com/lightningnetwork/lnd/graph/db"
@@ -74,12 +75,14 @@ type mandatoryVersion struct {
 // optional migrations.
 type MigrationConfig interface {
 	migration30.MigrateRevLogConfig
+	migration34.MigrationConfig
 }
 
 // MigrationConfigImpl is a super set of all the various migration configs and
 // an implementation of MigrationConfig.
 type MigrationConfigImpl struct {
 	migration30.MigrateRevLogConfigImpl
+	migration34.MigrationConfigImpl
 }
 
 // optionalMigration defines an optional migration function. When a migration
@@ -315,6 +318,16 @@ var (
 				return migration30.MigrateRevocationLog(db, cfg)
 			},
 		},
+		{
+			name: "gc_decayed_log",
+			migration: func(db kvdb.Backend,
+				cfg MigrationConfig) error {
+
+				return migration34.MigrateDecayedLog(
+					db, cfg,
+				)
+			},
+		},
 	}
 
 	// Big endian is the preferred byte order, due to cursor scans over
@@ -1731,10 +1744,8 @@ func (d *DB) syncVersions(versions []mandatoryVersion) error {
 	}, func() {})
 }
 
-// applyOptionalVersions takes a config to determine whether the optional
-// migrations will be applied.
-//
-// NOTE: only support the prune_revocation_log optional migration atm.
+// applyOptionalVersions applies the optional migrations to the database if
+// specified in the config.
 func (d *DB) applyOptionalVersions(cfg OptionalMiragtionConfig) error {
 	// TODO(yy): need to design the db to support dry run for optional
 	// migrations.
@@ -1762,6 +1773,9 @@ func (d *DB) applyOptionalVersions(cfg OptionalMiragtionConfig) error {
 		migration30.MigrateRevLogConfigImpl{
 			NoAmountData: d.noRevLogAmtData,
 		},
+		migration34.MigrationConfigImpl{
+			DecayedLog: cfg.DecayedLog,
+		},
 	}
 
 	log.Infof("Applying %d optional migrations", len(optionalVersions))

channeldb/log.go

@@ -12,6 +12,7 @@ import (
 	"github.com/lightningnetwork/lnd/channeldb/migration31"
 	"github.com/lightningnetwork/lnd/channeldb/migration32"
 	"github.com/lightningnetwork/lnd/channeldb/migration33"
+	"github.com/lightningnetwork/lnd/channeldb/migration34"
 	"github.com/lightningnetwork/lnd/channeldb/migration_01_to_11"
 	"github.com/lightningnetwork/lnd/kvdb"
 )
@@ -46,5 +47,6 @@ func UseLogger(logger btclog.Logger) {
 	migration31.UseLogger(logger)
 	migration32.UseLogger(logger)
 	migration33.UseLogger(logger)
+	migration34.UseLogger(logger)
 	kvdb.UseLogger(logger)
 }

channeldb/meta_test.go

@@ -498,6 +498,7 @@ func TestOptionalMeta(t *testing.T) {
 	om = &OptionalMeta{
 		Versions: map[uint64]string{
 			0: optionalVersions[0].name,
+			1: optionalVersions[1].name,
 		},
 	}
 	err = db.putOptionalMeta(om)
@@ -506,7 +507,10 @@ func TestOptionalMeta(t *testing.T) {
 	om1, err := db.fetchOptionalMeta()
 	require.NoError(t, err, "error getting optional meta")
 	require.Equal(t, om, om1, "unexpected empty versions")
-	require.Equal(t, "0: prune_revocation_log", om.String())
+	require.Equal(
+		t, "0: prune_revocation_log, 1: gc_decayed_log",
+		om1.String(),
+	)
 }
 
 // TestApplyOptionalVersions checks that the optional migration is applied as
@@ -565,6 +569,7 @@ func TestApplyOptionalVersions(t *testing.T) {
 	omExpected := &OptionalMeta{
 		Versions: map[uint64]string{
 			0: optionalVersions[0].name,
+			1: optionalVersions[1].name,
 		},
 	}
 	require.Equal(t, omExpected, om, "unexpected empty versions")

channeldb/options.go

@@ -2,6 +2,7 @@ package channeldb
 
 import (
 	"github.com/lightningnetwork/lnd/clock"
+	"github.com/lightningnetwork/lnd/kvdb"
 )
 
 const (
@@ -29,6 +30,14 @@ type OptionalMiragtionConfig struct {
 	// migrations should be run. The index in the array corresponds to the
 	// migration number in optionalVersions.
 	MigrationFlags []bool
+
+	// DecayedLog is a reference to the decayed log database. The channeldb
+	// is inherently part of the optional migration flow so there is no need
+	// to specify it here. The DecayedLog is a separate database in case the
+	// kvdb backend is set to `bbolt`. And also for the kvdb SQL backend
+	// case it is a separate table therefore we need to reference it here
+	// as well to use the right query to access the decayed log.
+	DecayedLog kvdb.Backend
 }
 
 // NewOptionalMiragtionConfig creates a new OptionalMiragtionConfig with the
@@ -136,3 +145,21 @@ func OptionPruneRevocationLog(prune bool) OptionModifier {
 		o.OptionalMiragtionConfig.MigrationFlags[0] = prune
 	}
 }
+
+// OptionWithDecayedLogDB sets the decayed log database reference which might
+// be used for some migrations because generally we only touch the channeldb
+// databases in the migrations, this is a way to allow also access to the
+// decayed log database.
+func OptionWithDecayedLogDB(decayedLog kvdb.Backend) OptionModifier {
+	return func(o *Options) {
+		o.OptionalMiragtionConfig.DecayedLog = decayedLog
+	}
+}
+
+// OptionGcDecayedLog specifies whether the decayed log migration has to
+// take place.
+func OptionGcDecayedLog(noGc bool) OptionModifier {
+	return func(o *Options) {
+		o.OptionalMiragtionConfig.MigrationFlags[1] = !noGc
+	}
+}

config_builder.go

@@ -1073,6 +1073,8 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 		),
 		channeldb.OptionPruneRevocationLog(cfg.DB.PruneRevocation),
 		channeldb.OptionNoRevLogAmtData(cfg.DB.NoRevLogAmtData),
+		channeldb.OptionGcDecayedLog(cfg.DB.NoGcDecayedLog),
+		channeldb.OptionWithDecayedLogDB(dbs.DecayedLogDB),
 	}
 
 	// Otherwise, we'll open two instances, one for the state we only need

lncfg/db.go

@@ -95,6 +95,8 @@ type DB struct {
 	PruneRevocation bool `long:"prune-revocation" description:"Run the optional migration that prunes the revocation logs to save disk space."`
 
 	NoRevLogAmtData bool `long:"no-rev-log-amt-data" description:"If set, the to-local and to-remote output amounts of revoked commitment transactions will not be stored in the revocation log. Note that once this data is lost, a watchtower client will not be able to back up the revoked state."`
+
+	NoGcDecayedLog bool `long:"no-gc-decayed-log" description:"Do not run the optional migration that garbage collects the decayed log to save disk space."`
 }
 
 // DefaultDB creates and returns a new default DB config.

sample-lnd.conf

@@ -1476,6 +1476,11 @@
 ; channels prior to lnd@v0.15.0.
 ; db.prune-revocation=false
 
+; Specify whether the optional migration for garbage collecting the decayed
+; sphinx logs should be applied. By default, the decayed log will be garbage
+; collected.
+; db.no-gc-decayed-log=false
+
 ; If set to true, then the to-local and to-remote output amount data of revoked
 ; commitment transactions will not be stored in the revocation log. Note that
 ; this flag can only be set if --wtclient.active is not set. It is not