Merge branch 'mempool-space'

Author: Beerosagos

CHANGELOG.md

@@ -8,6 +8,7 @@
 - Support pasting different localized number formats, i.e. dot and comma separated amounts
 - Fix BitBoxApp crash on GrapheneOS and other phones without Google Play Services when scanning QR codes.
 - Add DMG installer for macOS
+- Use mempool.space as preferred fee estimation source for BTC
 
 ## 4.42.0
 - Preselect backup when there's only one backup available

backend/accounts/feetarget.go

@@ -16,6 +16,7 @@ package accounts
 
 import (
 	"github.com/BitBoxSwiss/bitbox-wallet-app/util/errp"
+	"github.com/btcsuite/btcd/btcutil"
 )
 
 // FeeTarget interface has priority codes.
@@ -39,6 +40,10 @@ func NewFeeTargetCode(code string) (FeeTargetCode, error) {
 	case string(FeeTargetCodeNormal):
 	case string(FeeTargetCodeHigh):
 	case string(FeeTargetCodeCustom):
+	case string(FeeTargetCodeMempoolFastest):
+	case string(FeeTargetCodeMempoolHalfHour):
+	case string(FeeTargetCodeMempoolHour):
+	case string(FeeTargetCodeMempoolEconomy):
 	default:
 		return "", errp.WithStack(errp.Newf("Unrecognized fee target code %s", code))
 	}
@@ -58,10 +63,51 @@ const (
 	// FeeTargetCodeHigh is the high priority fee target.
 	FeeTargetCodeHigh FeeTargetCode = "high"
 
+	// FeeTargetCodeMempoolFastest is the mempool highest priority fee target.
+	FeeTargetCodeMempoolFastest FeeTargetCode = "mFastest"
+
+	// FeeTargetCodeMempoolHalfHour is the mempool half hour fee target.
+	FeeTargetCodeMempoolHalfHour FeeTargetCode = "mHalfHour"
+
+	// FeeTargetCodeMempoolHour is the mempool hour fee target.
+	FeeTargetCodeMempoolHour FeeTargetCode = "mHour"
+
+	// FeeTargetCodeMempoolEconomy is the mempool economy fee target.
+	FeeTargetCodeMempoolEconomy FeeTargetCode = "mEconomy"
+
 	// FeeTargetCodeCustom means that the actual feerate is supplied separately instead of being
 	// estimated automatically.
 	FeeTargetCodeCustom FeeTargetCode = "custom"
 
+	// DefaultMempoolFeeTarget is the default fee target for mempool fees.
+	DefaultMempoolFeeTarget = FeeTargetCodeMempoolHalfHour
+
 	// DefaultFeeTarget is the default fee target.
 	DefaultFeeTarget = FeeTargetCodeNormal
 )
+
+// MempoolSpaceFees contains mempool.space recommended fees API response
+// (https://mempool.space/docs/api/rest#get-recommended-fees)
+type MempoolSpaceFees struct {
+	FastestFee  int64 `json:"fastestFee"`
+	HalfHourFee int64 `json:"halfHourFee"`
+	HourFee     int64 `json:"hourFee"`
+	EconomyFee  int64 `json:"economyFee"`
+	MinimumFee  int64 `json:"minimumFee"`
+}
+
+// GetFeeRate returns the btcutil.Amount of the fee for the given FeeTargetCode.
+func (fees MempoolSpaceFees) GetFeeRate(code FeeTargetCode) btcutil.Amount {
+	var feeRatePerByte int64
+	switch code {
+	case FeeTargetCodeMempoolFastest:
+		feeRatePerByte = fees.FastestFee
+	case FeeTargetCodeMempoolHalfHour:
+		feeRatePerByte = fees.HalfHourFee
+	case FeeTargetCodeMempoolHour:
+		feeRatePerByte = fees.HourFee
+	case FeeTargetCodeMempoolEconomy:
+		feeRatePerByte = fees.EconomyFee
+	}
+	return btcutil.Amount(feeRatePerByte * 1000)
+}

backend/accounts/types/event.go

@@ -31,7 +31,4 @@ const (
 
 	// EventHeadersSynced is fired when the headers finished syncing.
 	EventHeadersSynced Event = "headersSynced"
-
-	// EventFeeTargetsChanged is fired when the fee targets change.
-	EventFeeTargetsChanged Event = "feeTargetsChanged"
 )

backend/backend.go

@@ -240,6 +240,15 @@ func NewBackend(arguments *arguments.Arguments, environment Environment) (*Backe
 	}
 	log.Infof("backend config: %+v", config.AppConfig().Backend)
 	log.Infof("frontend config: %+v", config.AppConfig().Frontend)
+	backendProxy := socksproxy.NewSocksProxy(
+		config.AppConfig().Backend.Proxy.UseProxy,
+		config.AppConfig().Backend.Proxy.ProxyAddress,
+	)
+	hclient, err := backendProxy.GetHTTPClient()
+	if err != nil {
+		return nil, err
+	}
+
 	backend := &Backend{
 		arguments:   arguments,
 		environment: environment,
@@ -252,7 +261,7 @@ func NewBackend(arguments *arguments.Arguments, environment Environment) (*Backe
 		aopp:     AOPP{State: aoppStateInactive},
 
 		makeBtcAccount: func(config *accounts.AccountConfig, coin *btc.Coin, gapLimits *types.GapLimits, log *logrus.Entry) accounts.Interface {
-			return btc.NewAccount(config, coin, gapLimits, log)
+			return btc.NewAccount(config, coin, gapLimits, log, hclient)
 		},
 		makeEthAccount: func(config *accounts.AccountConfig, coin *eth.Coin, httpClient *http.Client, log *logrus.Entry) accounts.Interface {
 			return eth.NewAccount(config, coin, httpClient, log)
@@ -265,14 +274,7 @@ func NewBackend(arguments *arguments.Arguments, environment Environment) (*Backe
 		return nil, err
 	}
 	backend.notifier = notifier
-	backend.socksProxy = socksproxy.NewSocksProxy(
-		backend.config.AppConfig().Backend.Proxy.UseProxy,
-		backend.config.AppConfig().Backend.Proxy.ProxyAddress,
-	)
-	hclient, err := backend.socksProxy.GetHTTPClient()
-	if err != nil {
-		return nil, err
-	}
+	backend.socksProxy = backendProxy
 	backend.httpClient = hclient
 	backend.etherScanHTTPClient = ratelimit.FromTransport(hclient.Transport, etherscan.CallInterval)
 

backend/coins/btc/account.go

@@ -18,6 +18,7 @@ package btc
 import (
 	"encoding/base64"
 	"fmt"
+	"net/http"
 	"os"
 	"path"
 	"sort"
@@ -36,6 +37,7 @@ import (
 	"github.com/BitBoxSwiss/bitbox-wallet-app/backend/coins/coin"
 	"github.com/BitBoxSwiss/bitbox-wallet-app/backend/coins/ltc"
 	"github.com/BitBoxSwiss/bitbox-wallet-app/backend/signing"
+	"github.com/BitBoxSwiss/bitbox-wallet-app/backend/util"
 	"github.com/BitBoxSwiss/bitbox-wallet-app/util/errp"
 	"github.com/BitBoxSwiss/bitbox-wallet-app/util/locker"
 	"github.com/BitBoxSwiss/bitbox-wallet-app/util/observable"
@@ -56,6 +58,10 @@ const (
 	// maxGapLimit limits the maximum gap limit that can be used. It is an arbitrary number with the
 	// goal that the scanning will stop in a reasonable amount of time.
 	maxGapLimit = 2000
+
+	// mempoolSpaceMirror is Shift server that mirrors "https://mempool.space/api/v1/fees/recommended"
+	// rest call.
+	mempoolSpaceMirror = "https://fees1.shiftcrypto.io"
 )
 
 type subaccount struct {
@@ -101,8 +107,6 @@ type Account struct {
 	activeTxProposal     *maketx.TxProposal
 	activeTxProposalLock locker.Locker
 
-	feeTargets     []*FeeTarget
-	feeTargetsLock locker.Locker
 	// Access this only via getMinRelayFeeRate(). sat/kB.
 	minRelayFeeRate     *btcutil.Amount
 	minRelayFeeRateLock locker.Locker
@@ -116,6 +120,8 @@ type Account struct {
 	closed bool
 
 	log *logrus.Entry
+
+	httpClient *http.Client
 }
 
 // NewAccount creates a new account.
@@ -126,6 +132,7 @@ func NewAccount(
 	coin *Coin,
 	forceGapLimits *types.GapLimits,
 	log *logrus.Entry,
+	httpClient *http.Client,
 ) *Account {
 	log = log.WithField("group", "btc").
 		WithFields(logrus.Fields{"coin": coin.String(), "code": config.Config.Code, "name": config.Config.Name})
@@ -137,14 +144,8 @@ func NewAccount(
 		dbSubfolder:    "", // set in Initialize()
 		forceGapLimits: forceGapLimits,
 
-		// feeTargets must be sorted by ascending priority.
-		feeTargets: []*FeeTarget{
-			{blocks: 24, code: accounts.FeeTargetCodeEconomy},
-			{blocks: 12, code: accounts.FeeTargetCodeLow},
-			{blocks: 6, code: accounts.FeeTargetCodeNormal},
-			{blocks: 2, code: accounts.FeeTargetCodeHigh},
-		},
-		log: log,
+		log:        log,
+		httpClient: httpClient,
 	}
 	return account
 }
@@ -430,8 +431,6 @@ func (account *Account) onNewHeader(header *electrumTypes.Header) {
 		return
 	}
 	account.log.WithField("block-height", header.Height).Debug("Received new header")
-	// Fee estimates change with each block.
-	account.updateFeeTargets()
 }
 
 // FatalError returns true if the account had a fatal error.
@@ -476,26 +475,68 @@ func (account *Account) Notifier() accounts.Notifier {
 	return account.notifier
 }
 
-func (account *Account) updateFeeTargets() {
-	defer account.feeTargetsLock.Lock()()
+// feeTargets fetches the available fees. For mainnet BTC it uses mempool.space estimation.
+//
+// For the other coins or in case mempool.space is not available it fallbacks on Bitcoin Core.
+// The minimum relay fee is used as a last resource fallback in case also Bitcoin Core is
+// unavailable.
+func (account *Account) feeTargets() []*FeeTarget {
+	// for mainnet BTC we fetch mempool.space fees, as they should be more reliable.
+	var mempoolFees *accounts.MempoolSpaceFees
+	if account.coin.Code() == coin.CodeBTC {
+		mempoolFees = &accounts.MempoolSpaceFees{}
+		_, err := util.APIGet(account.httpClient, mempoolSpaceMirror, "", 1000, mempoolFees)
+		if err != nil {
+			mempoolFees = nil
+			account.log.WithError(err).Errorf("Fetching fees from %s failed", mempoolSpaceMirror)
+		}
+	}
+
+	// feeTargets must be sorted by ascending priority.
+	var feeTargets []*FeeTarget
+	if mempoolFees != nil {
+		feeTargets = []*FeeTarget{
+			{blocks: 12, code: accounts.FeeTargetCodeMempoolEconomy},
+			{blocks: 3, code: accounts.FeeTargetCodeMempoolHour},
+			{blocks: 2, code: accounts.FeeTargetCodeMempoolHalfHour},
+			{blocks: 1, code: accounts.FeeTargetCodeMempoolFastest},
+		}
+	} else {
+		feeTargets = []*FeeTarget{
+			{blocks: 24, code: accounts.FeeTargetCodeEconomy},
+			{blocks: 12, code: accounts.FeeTargetCodeLow},
+			{blocks: 6, code: accounts.FeeTargetCodeNormal},
+			{blocks: 2, code: accounts.FeeTargetCodeHigh},
+		}
+	}
+
 	var minRelayFeeRate *btcutil.Amount
 	minRelayFeeRateVal, err := account.getMinRelayFeeRate()
 	if err == nil {
 		minRelayFeeRate = &minRelayFeeRateVal
 	}
-	for _, feeTarget := range account.feeTargets {
-		feeRatePerKb, err := account.coin.Blockchain().EstimateFee(feeTarget.blocks)
-		if err != nil {
-			if account.coin.Code() != coin.CodeTLTC {
-				account.log.WithField("fee-target", feeTarget.blocks).
-					Warning("Fee could not be estimated. Taking the minimum relay fee instead")
-			}
-			if minRelayFeeRate == nil {
-				account.log.WithField("fee-target", feeTarget.blocks).
-					Warning("Minimum relay fee could not be determined")
-				continue
+
+	for _, feeTarget := range feeTargets {
+		var feeRatePerKb btcutil.Amount
+
+		if mempoolFees != nil {
+			feeRatePerKb = mempoolFees.GetFeeRate(feeTarget.code)
+		} else {
+			// If mempool.space fees are not available, we fallback on Bitcoin Core estimation.
+			// If even that one is not available, we just offer the min relay fee.
+			feeRatePerKb, err = account.coin.Blockchain().EstimateFee(feeTarget.blocks)
+			if err != nil {
+				if account.coin.Code() != coin.CodeTLTC {
+					account.log.WithField("fee-target", feeTarget.blocks).
+						Warning("Fee could not be estimated. Taking the minimum relay fee instead")
+				}
+				if minRelayFeeRate == nil {
+					account.log.WithField("fee-target", feeTarget.blocks).
+						Warning("Minimum relay fee could not be determined")
+					continue
+				}
+				feeRatePerKb = *minRelayFeeRate
 			}
-			feeRatePerKb = *minRelayFeeRate
 		}
 		// If the minrelayfee is available the estimated fee rate is smaller than the minrelayfee,
 		// we use the minrelayfee instead. If the minrelayfee is unknown, we leave the fee
@@ -506,42 +547,36 @@ func (account *Account) updateFeeTargets() {
 		feeTarget.feeRatePerKb = &feeRatePerKb
 		account.log.WithFields(logrus.Fields{"blocks": feeTarget.blocks,
 			"fee-rate-per-kb": feeRatePerKb}).Debug("Fee estimate per kb")
-		account.Config().OnEvent(accountsTypes.EventFeeTargetsChanged)
 	}
+
+	return feeTargets
 }
 
 // FeeTargets returns the fee targets and the default fee target.
 func (account *Account) FeeTargets() ([]accounts.FeeTarget, accounts.FeeTargetCode) {
-	defer account.feeTargetsLock.RLock()()
-	// Return only fee targets with a valid fee rate (drop if fee could not be estimated). Also
-	// remove all duplicate fee rates.
+	// Return only fee targets with a valid fee rate (drop if fee could not be estimated).
+	fetchedFeeTargets := account.feeTargets()
 	feeTargets := []accounts.FeeTarget{}
-	defaultAvailable := false
-outer:
-	for i := len(account.feeTargets) - 1; i >= 0; i-- {
-		feeTarget := account.feeTargets[i]
+	defaultFee := accounts.FeeTargetCodeCustom
+
+	for _, feeTarget := range fetchedFeeTargets {
 		if feeTarget.feeRatePerKb == nil {
 			continue
 		}
-		for j := i - 1; j >= 0; j-- {
-			checkFeeTarget := account.feeTargets[j]
-			if checkFeeTarget.feeRatePerKb != nil && *checkFeeTarget.feeRatePerKb == *feeTarget.feeRatePerKb {
-				continue outer
-			}
-		}
-		if feeTarget.code == accounts.DefaultFeeTarget {
-			defaultAvailable = true
+
+		switch feeTarget.code {
+		case accounts.DefaultFeeTarget:
+			fallthrough
+		case accounts.DefaultMempoolFeeTarget:
+			defaultFee = feeTarget.code
 		}
 		feeTargets = append(feeTargets, feeTarget)
 	}
 	// If the default fee level was dropped, use the cheapest.
 	// If no fee targets are available, use custom (the user can manually enter a fee rate).
-	defaultFee := accounts.DefaultFeeTarget
-	if !defaultAvailable {
+	if defaultFee == accounts.FeeTargetCodeCustom {
 		if len(feeTargets) != 0 {
 			defaultFee = feeTargets[0].Code()
-		} else {
-			defaultFee = accounts.FeeTargetCodeCustom
 		}
 	}
 	return feeTargets, defaultFee

backend/coins/btc/account_test.go

@@ -103,6 +103,7 @@ func mockAccount(t *testing.T, accountConfig *config.Account) *btc.Account {
 		},
 		coin, nil,
 		logging.Get().WithGroup("account_test"),
+		nil,
 	)
 }
 

backend/coins/btc/transaction.go

@@ -59,7 +59,7 @@ func (account *Account) getFeePerKb(args *accounts.TxProposalArgs) (btcutil.Amou
 		return feePerKb, nil
 	}
 	var feeTarget *FeeTarget
-	for _, target := range account.feeTargets {
+	for _, target := range account.feeTargets() {
 		if target.code == args.FeeTargetCode {
 			feeTarget = target
 			break

backend/util/util.go

@@ -15,10 +15,12 @@
 package util
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
+	"time"
 
 	"github.com/BitBoxSwiss/bitbox-wallet-app/util/errp"
 )
@@ -32,7 +34,10 @@ import (
 // - `result` object that should be used to unmarshal the response body
 // Returns the error code (if available) and possibly an error.
 func APIGet(httpClient *http.Client, endpoint string, apiKey string, maxSize int64, result interface{}) (int, error) {
-	req, err := http.NewRequest(http.MethodGet, endpoint, nil)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
 	if err != nil {
 		return 0, errp.WithStack(err)
 	}