Broadcast UserOperation batch to list of builders in searcher mode (#352)

Author: hazim-j

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/go-logr/zerologr v1.2.3
 	github.com/go-playground/validator/v10 v10.12.0
 	github.com/google/go-cmp v0.5.9
-	github.com/metachris/flashbotsrpc v0.5.0
+	github.com/metachris/flashbotsrpc v0.6.0
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/puzpuzpuz/xsync/v3 v3.0.1
 	github.com/rs/zerolog v1.29.0
@@ -101,3 +101,5 @@ require (
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
+
+replace github.com/metachris/flashbotsrpc => github.com/stackup-wallet/flashbotsrpc v0.6.1-rc1

go.sum

@@ -300,8 +300,6 @@ github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPn
 github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
-github.com/metachris/flashbotsrpc v0.5.0 h1:5OLpm6+6n4kXxeh3TZBeSj0PQWDxqUsOFwy7xertXQQ=
-github.com/metachris/flashbotsrpc v0.5.0/go.mod h1:UrS249kKA1PK27sf12M6tUxo/M4ayfFrBk7IMFY1TNw=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
@@ -369,6 +367,8 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU=
 github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA=
+github.com/stackup-wallet/flashbotsrpc v0.6.1-rc1 h1:kCB1WQZgD2edUiPZh+mghl1Ir/cuH5/4bI+a03Ki6Tc=
+github.com/stackup-wallet/flashbotsrpc v0.6.1-rc1/go.mod h1:UrS249kKA1PK27sf12M6tUxo/M4ayfFrBk7IMFY1TNw=
 github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=

internal/config/constants.go

@@ -5,6 +5,7 @@ import "math/big"
 var (
 	EthereumChainID        = big.NewInt(1)
 	GoerliChainID          = big.NewInt(5)
+	SepoliaChainID         = big.NewInt(11155111)
 	ArbitrumOneChainID     = big.NewInt(42161)
 	ArbitrumGoerliChainID  = big.NewInt(421613)
 	ArbitrumSepoliaChainID = big.NewInt(421614)

internal/config/values.go

@@ -26,7 +26,7 @@ type Values struct {
 	Beneficiary             string
 
 	// Searcher mode variables.
-	EthBuilderUrl     string
+	EthBuilderUrls    []string
 	BlocksInTheFuture int
 
 	// Observability variables.
@@ -88,7 +88,7 @@ func GetValues() *Values {
 	viper.SetDefault("erc4337_bundler_max_batch_gas_limit", 25000000)
 	viper.SetDefault("erc4337_bundler_max_op_ttl_seconds", 180)
 	viper.SetDefault("erc4337_bundler_max_ops_for_unstaked_sender", 4)
-	viper.SetDefault("erc4337_bundler_blocks_in_the_future", 25)
+	viper.SetDefault("erc4337_bundler_blocks_in_the_future", 6)
 	viper.SetDefault("erc4337_bundler_otel_insecure_mode", false)
 	viper.SetDefault("erc4337_bundler_debug_mode", false)
 	viper.SetDefault("erc4337_bundler_gin_mode", gin.ReleaseMode)
@@ -117,7 +117,7 @@ func GetValues() *Values {
 	_ = viper.BindEnv("erc4337_bundler_max_batch_gas_limit")
 	_ = viper.BindEnv("erc4337_bundler_max_op_ttl_seconds")
 	_ = viper.BindEnv("erc4337_bundler_max_ops_for_unstaked_sender")
-	_ = viper.BindEnv("erc4337_bundler_eth_builder_url")
+	_ = viper.BindEnv("erc4337_bundler_eth_builder_urls")
 	_ = viper.BindEnv("erc4337_bundler_blocks_in_the_future")
 	_ = viper.BindEnv("erc4337_bundler_otel_service_name")
 	_ = viper.BindEnv("erc4337_bundler_otel_collector_headers")
@@ -147,8 +147,8 @@ func GetValues() *Values {
 
 	switch viper.GetString("mode") {
 	case "searcher":
-		if variableNotSetOrIsNil("erc4337_bundler_eth_builder_url") {
-			panic("Fatal config error: erc4337_bundler_eth_builder_url not set")
+		if variableNotSetOrIsNil("erc4337_bundler_eth_builder_urls") {
+			panic("Fatal config error: erc4337_bundler_eth_builder_urls not set")
 		}
 	}
 
@@ -175,7 +175,7 @@ func GetValues() *Values {
 	maxBatchGasLimit := big.NewInt(int64(viper.GetInt("erc4337_bundler_max_batch_gas_limit")))
 	maxOpTTL := time.Second * viper.GetDuration("erc4337_bundler_max_op_ttl_seconds")
 	maxOpsForUnstakedSender := viper.GetInt("erc4337_bundler_max_ops_for_unstaked_sender")
-	ethBuilderUrl := viper.GetString("erc4337_bundler_eth_builder_url")
+	ethBuilderUrls := envArrayToStringSlice(viper.GetString("erc4337_bundler_eth_builder_urls"))
 	blocksInTheFuture := viper.GetInt("erc4337_bundler_blocks_in_the_future")
 	otelServiceName := viper.GetString("erc4337_bundler_otel_service_name")
 	otelCollectorHeader := envKeyValStringToMap(viper.GetString("erc4337_bundler_otel_collector_headers"))
@@ -196,7 +196,7 @@ func GetValues() *Values {
 		MaxBatchGasLimit:        maxBatchGasLimit,
 		MaxOpTTL:                maxOpTTL,
 		MaxOpsForUnstakedSender: maxOpsForUnstakedSender,
-		EthBuilderUrl:           ethBuilderUrl,
+		EthBuilderUrls:          ethBuilderUrls,
 		BlocksInTheFuture:       blocksInTheFuture,
 		OTELServiceName:         otelServiceName,
 		OTELCollectorHeaders:    otelCollectorHeader,

internal/start/searcher.go

@@ -60,7 +60,7 @@ func SearcherMode() {
 
 	eth := ethclient.NewClient(rpc)
 
-	fb := flashbotsrpc.NewFlashbotsRPC(conf.EthBuilderUrl)
+	fb := flashbotsrpc.NewBuilderBroadcastRPC(conf.EthBuilderUrls)
 
 	chain, err := eth.ChainID(context.Background())
 	if err != nil {
@@ -117,6 +117,7 @@ func SearcherMode() {
 
 	// TODO: Create separate go-routine for tracking transactions sent to the block builder.
 	builder := builder.New(eoa, eth, fb, beneficiary, conf.BlocksInTheFuture)
+
 	paymaster := paymaster.New(db)
 
 	// Init Client
@@ -191,6 +192,7 @@ func SearcherMode() {
 	}
 	r.POST("/", handlers...)
 	r.POST("/rpc", handlers...)
+
 	if err := r.Run(fmt.Sprintf(":%d", conf.Port)); err != nil {
 		log.Fatal(err)
 	}

internal/testutils/buildermock.go

@@ -0,0 +1,21 @@
+package testutils
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+
+	"github.com/metachris/flashbotsrpc"
+)
+
+func BadBuilderRpcMock() *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		res := &flashbotsrpc.RelayErrorResponse{
+			Error: "Mock upstream builder error",
+		}
+		w.WriteHeader(http.StatusOK)
+		if err := json.NewEncoder(w).Encode(res); err != nil {
+			panic(err)
+		}
+	}))
+}

internal/testutils/constants.go

@@ -5,7 +5,10 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/stackup-wallet/stackup-bundler/pkg/entrypoint"
+	"github.com/stackup-wallet/stackup-bundler/pkg/signer"
 )
 
 var (
@@ -44,4 +47,8 @@ var (
 		UnstakeDelaySec: uint32(0),
 		WithdrawTime:    big.NewInt(0),
 	}
+
+	pk, _       = crypto.GenerateKey()
+	DummyEOA, _ = signer.New(hexutil.Encode(crypto.FromECDSA(pk))[2:])
+	MockHash    = "0xdeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddead"
 )

internal/testutils/ethmock.go

@@ -1,51 +1,44 @@
 package testutils
 
 import (
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-)
+	"math/big"
+	"time"
 
-type mockReq struct {
-	JsonRpc string  `json:"jsonrpc"`
-	ID      float64 `json:"id"`
-	Method  string  `json:"method"`
-}
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+)
 
-type mockRes struct {
-	JsonRpc string  `json:"jsonrpc"`
-	ID      float64 `json:"id"`
-	Result  any     `json:"result"`
+func NewBlockMock() map[string]any {
+	return map[string]any{
+		"parentHash":       MockHash,
+		"sha3Uncles":       MockHash,
+		"stateRoot":        MockHash,
+		"transactionsRoot": MockHash,
+		"receiptsRoot":     MockHash,
+		"logsBloom":        "0xdeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddead",
+		"difficulty":       "0x0",
+		"number":           "0x1",
+		"gasLimit":         hexutil.EncodeBig(big.NewInt(30000000)),
+		"gasUsed":          hexutil.EncodeBig(big.NewInt(5000000)),
+		"timestamp":        hexutil.EncodeUint64(uint64(time.Now().Unix())),
+		"extraData":        "0x",
+	}
 }
 
-type MethodMocks map[string]any
-
-// EthMock returns a httptest.Server for mocking the return value of a JSON-RPC method call to an Ethereum node.
-func EthMock(mocks MethodMocks) *httptest.Server {
-	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		var req mockReq
-		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-			panic(err)
-		}
-
-		mock, ok := mocks[req.Method]
-		if !ok {
-			w.WriteHeader(http.StatusBadRequest)
-			if _, err := w.Write([]byte(fmt.Sprintf("method not in mocks: %s", req.Method))); err != nil {
-				panic(err)
-			}
-			return
-		}
-
-		res := &mockRes{
-			JsonRpc: req.JsonRpc,
-			ID:      req.ID,
-			Result:  mock,
-		}
-		w.WriteHeader(http.StatusOK)
-		if err := json.NewEncoder(w).Encode(res); err != nil {
-			panic(err)
-		}
-	}))
+func NewTransactionReceiptMock() map[string]any {
+	return map[string]any{
+		"blockHash":         MockHash,
+		"blockNumber":       "0x1",
+		"cumulativeGasUsed": "0x1",
+		"effectiveGasPrice": "0x1",
+		"from":              common.HexToAddress("0x").Hex(),
+		"gasUsed":           "0x1",
+		"logs":              []any{},
+		"logsBloom":         "0xdeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddead",
+		"status":            "0x1",
+		"to":                common.HexToAddress("0x").Hex(),
+		"transactionHash":   MockHash,
+		"transactionIndex":  "0x1",
+		"type":              "0x2",
+	}
 }

internal/testutils/rpcmock.go

@@ -0,0 +1,49 @@
+package testutils
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+)
+
+type mockReq struct {
+	JsonRpc string  `json:"jsonrpc"`
+	ID      float64 `json:"id"`
+	Method  string  `json:"method"`
+}
+
+type mockRes struct {
+	JsonRpc string  `json:"jsonrpc"`
+	ID      float64 `json:"id"`
+	Result  any     `json:"result"`
+}
+
+type MethodMocks map[string]any
+
+func RpcMock(mocks MethodMocks) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var req mockReq
+		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+			panic(err)
+		}
+		mock, ok := mocks[req.Method]
+		if !ok {
+			w.WriteHeader(http.StatusBadRequest)
+			if _, err := w.Write([]byte(fmt.Sprintf("method not in mocks: %s", req.Method))); err != nil {
+				panic(err)
+			}
+			return
+		}
+
+		res := &mockRes{
+			JsonRpc: req.JsonRpc,
+			ID:      req.ID,
+			Result:  mock,
+		}
+		w.WriteHeader(http.StatusOK)
+		if err := json.NewEncoder(w).Encode(res); err != nil {
+			panic(err)
+		}
+	}))
+}

pkg/entrypoint/transaction/handleops.go

@@ -1,7 +1,6 @@
 package transaction
 
 import (
-	bytesPkg "bytes"
 	"context"
 	"errors"
 	"math"
@@ -11,7 +10,6 @@ import (
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/stackup-wallet/stackup-bundler/pkg/entrypoint"
@@ -38,6 +36,7 @@ type Opts struct {
 	Tip         *big.Int
 	GasPrice    *big.Int
 	GasLimit    uint64
+	NoSend      bool
 	WaitTimeout time.Duration
 }
 
@@ -104,6 +103,7 @@ func HandleOps(opts *Opts) (txn *types.Transaction, err error) {
 		return nil, err
 	}
 	auth.GasLimit = opts.GasLimit
+	auth.NoSend = opts.NoSend
 
 	nonce, err := opts.Eth.NonceAt(context.Background(), opts.EOA.Address, nil)
 	if err != nil {
@@ -123,55 +123,11 @@ func HandleOps(opts *Opts) (txn *types.Transaction, err error) {
 	txn, err = ep.HandleOps(auth, toAbiType(opts.Batch), opts.Beneficiary)
 	if err != nil {
 		return nil, err
-	} else if opts.WaitTimeout == 0 {
+	} else if opts.WaitTimeout == 0 || opts.NoSend {
 		// Don't wait for transaction to be included. All userOps in the current batch will be dropped
 		// regardless of the transaction status.
 		return txn, nil
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), opts.WaitTimeout)
-	defer cancel()
-	if receipt, err := bind.WaitMined(ctx, opts.Eth, txn); err != nil {
-		return nil, err
-	} else if receipt.Status == types.ReceiptStatusFailed {
-		// Return an error here so that the current batch stays in the mempool. In the next bundler iteration,
-		// the offending userOps will be dropped during gas estimation.
-		return nil, errors.New("transaction: failed status")
-	}
-	return txn, nil
-}
-
-// CreateRawHandleOps returns a raw transaction string that calls handleOps() on the EntryPoint with a given
-// batch, gas limit, and tip.
-func CreateRawHandleOps(opts *Opts) (string, error) {
-	ep, err := entrypoint.NewEntrypoint(opts.EntryPoint, opts.Eth)
-	if err != nil {
-		return "", err
-	}
-
-	auth, err := bind.NewKeyedTransactorWithChainID(opts.EOA.PrivateKey, opts.ChainID)
-	if err != nil {
-		return "", err
-	}
-	auth.GasLimit = opts.GasLimit
-	auth.NoSend = true
-	if opts.BaseFee != nil {
-		tip, err := opts.Eth.SuggestGasTipCap(context.Background())
-		if err != nil {
-			return "", err
-		}
-
-		auth.GasTipCap = tip
-		auth.GasFeeCap = big.NewInt(0).Add(opts.BaseFee, tip)
-	}
-
-	tx, err := ep.HandleOps(auth, toAbiType(opts.Batch), opts.Beneficiary)
-	if err != nil {
-		return "", err
-	}
-
-	ts := types.Transactions{tx}
-	rawTxBytes := new(bytesPkg.Buffer)
-	ts.EncodeIndex(0, rawTxBytes)
-	return hexutil.Encode(rawTxBytes.Bytes()), nil
+	return Wait(txn, opts.Eth, opts.WaitTimeout)
 }