feat: Add builder-hub component and buildernet-recipe (#76)

* feat: Add builder-hub component and buildernet-recipe

* adapt to the newest changes from builder-hub

* More fixes

* Everything working

* Add orderflow proxy sender component

* Revert "Add orderflow proxy sender component"

This reverts commit e97ccc6.

---------

Co-authored-by: Ferran Borreguero <ferran.borreguero@gmail.com>

Author: MoeMahhouk

internal/artifacts.go

@@ -341,8 +341,12 @@ func getPrivKey(privStr string) (*ecdsa.PrivateKey, error) {
 	return priv, nil
 }
 
+func ConnectRaw(service, port, protocol string) string {
+	return fmt.Sprintf(`{{Service "%s" "%s" "%s"}}`, service, port, protocol)
+}
+
 func Connect(service, port string) string {
-	return fmt.Sprintf(`{{Service "%s" "%s"}}`, service, port)
+	return ConnectRaw(service, port, "http")
 }
 
 type output struct {

internal/catalog.go

@@ -17,6 +17,9 @@ func init() {
 	register(&MevBoostRelay{})
 	register(&RollupBoost{})
 	register(&OpReth{})
+	register(&BuilderHub{})
+	register(&BuilderHubPostgres{})
+	register(&BuilderHubMockProxy{})
 }
 
 func FindComponent(name string) Service {

internal/components.go

@@ -429,6 +429,70 @@ func (m *MevBoostRelay) Watchdog(out io.Writer, service *service, ctx context.Co
 	return watchGroup.wait()
 }
 
+type BuilderHubPostgres struct {
+}
+
+func (b *BuilderHubPostgres) Run(service *service, ctx *ExContext) {
+	service.
+		WithImage("docker.io/flashbots/builder-hub-db").
+		WithTag("latest").
+		WithPort("postgres", 5432).
+		WithEnv("POSTGRES_USER", "postgres").
+		WithEnv("POSTGRES_PASSWORD", "postgres").
+		WithEnv("POSTGRES_DB", "postgres").
+		WithReady(ReadyCheck{
+			Test:        []string{"CMD-SHELL", "pg_isready -U postgres -d postgres"},
+			Interval:    1 * time.Second,
+			Timeout:     30 * time.Second,
+			Retries:     3,
+			StartPeriod: 1 * time.Second,
+		})
+}
+
+func (b *BuilderHubPostgres) Name() string {
+	return "builder-hub-postgres"
+}
+
+type BuilderHub struct {
+	postgres string
+}
+
+func (b *BuilderHub) Run(service *service, ctx *ExContext) {
+	service.
+		WithImage("docker.io/flashbots/builder-hub").
+		WithTag("latest").
+		WithEntrypoint("/app/builder-hub").
+		WithEnv("POSTGRES_DSN", "postgres://postgres:postgres@"+ConnectRaw(b.postgres, "postgres", "")+"/postgres?sslmode=disable").
+		WithEnv("LISTEN_ADDR", "0.0.0.0:"+`{{Port "http" 8080}}`).
+		WithEnv("ADMIN_ADDR", "0.0.0.0:"+`{{Port "admin" 8081}}`).
+		WithEnv("INTERNAL_ADDR", "0.0.0.0:"+`{{Port "internal" 8082}}`).
+		WithEnv("METRICS_ADDR", "0.0.0.0:"+`{{Port "metrics" 8090}}`).
+		DependsOnHealthy(b.postgres)
+}
+
+func (b *BuilderHub) Name() string {
+	return "builder-hub"
+}
+
+type BuilderHubMockProxy struct {
+	TargetService string
+}
+
+func (b *BuilderHubMockProxy) Run(service *service, ctx *ExContext) {
+	service.
+		WithImage("docker.io/flashbots/builder-hub-mock-proxy").
+		WithTag("latest").
+		WithPort("http", 8888)
+
+	if b.TargetService != "" {
+		service.DependsOnHealthy(b.TargetService)
+	}
+}
+
+func (b *BuilderHubMockProxy) Name() string {
+	return "builder-hub-mock-proxy"
+}
+
 type OpReth struct {
 }
 

internal/local_runner.go

@@ -333,7 +333,7 @@ func (d *LocalRunner) getService(name string) *service {
 
 // applyTemplate resolves the templates from the manifest (Dir, Port, Connect) into
 // the actual values for this specific docker execution.
-func (d *LocalRunner) applyTemplate(s *service) ([]string, error) {
+func (d *LocalRunner) applyTemplate(s *service) ([]string, map[string]string, error) {
 	var input map[string]interface{}
 
 	// For {{.Dir}}:
@@ -350,7 +350,12 @@ func (d *LocalRunner) applyTemplate(s *service) ([]string, error) {
 	}
 
 	funcs := template.FuncMap{
-		"Service": func(name string, portLabel string) string {
+		"Service": func(name string, portLabel, protocol string) string {
+			protocolPrefix := ""
+			if protocol == "http" {
+				protocolPrefix = "http://"
+			}
+
 			// For {{Service "name" "portLabel"}}:
 			// - Service runs on host:
 			//   A: target is inside docker: access with localhost:hostPort
@@ -365,14 +370,14 @@ func (d *LocalRunner) applyTemplate(s *service) ([]string, error) {
 
 			if d.isHostService(s.Name) {
 				// A and B
-				return fmt.Sprintf("http://localhost:%d", port.HostPort)
+				return fmt.Sprintf("%slocalhost:%d", protocolPrefix, port.HostPort)
 			} else {
 				if d.isHostService(svc.Name) {
 					// D
-					return fmt.Sprintf("http://host.docker.internal:%d", port.HostPort)
+					return fmt.Sprintf("%shost.docker.internal:%d", protocolPrefix, port.HostPort)
 				}
 				// C
-				return fmt.Sprintf("http://%s:%d", svc.Name, port.Port)
+				return fmt.Sprintf("%s%s:%d", protocolPrefix, svc.Name, port.Port)
 			}
 		},
 		"Port": func(name string, defaultPort int) int {
@@ -386,28 +391,48 @@ func (d *LocalRunner) applyTemplate(s *service) ([]string, error) {
 		},
 	}
 
-	var argsResult []string
-	for _, arg := range s.args {
+	runTemplate := func(arg string) (string, error) {
 		tpl, err := template.New("").Funcs(funcs).Parse(arg)
 		if err != nil {
-			return nil, err
+			return "", err
 		}
 
 		var out strings.Builder
 		if err := tpl.Execute(&out, input); err != nil {
-			return nil, err
+			return "", err
+		}
+
+		return out.String(), nil
+	}
+
+	// apply the templates to the arguments
+	var argsResult []string
+	for _, arg := range s.args {
+		newArg, err := runTemplate(arg)
+		if err != nil {
+			return nil, nil, err
+		}
+		argsResult = append(argsResult, newArg)
+	}
+
+	// apply the templates to the environment variables
+	envs := map[string]string{}
+	for k, v := range s.env {
+		newV, err := runTemplate(v)
+		if err != nil {
+			return nil, nil, err
 		}
-		argsResult = append(argsResult, out.String())
+		envs[k] = newV
 	}
 
-	return argsResult, nil
+	return argsResult, envs, nil
 }
 
 func (d *LocalRunner) toDockerComposeService(s *service) (map[string]interface{}, error) {
 	// apply the template again on the arguments to figure out the connections
 	// at this point all of them are valid, we just have to resolve them again. We assume for now
 	// everyone is going to be on docker at the same network.
-	args, err := d.applyTemplate(s)
+	args, envs, err := d.applyTemplate(s)
 	if err != nil {
 		return nil, fmt.Errorf("failed to apply template, err: %w", err)
 	}
@@ -433,8 +458,8 @@ func (d *LocalRunner) toDockerComposeService(s *service) (map[string]interface{}
 		"labels": map[string]string{"playground": "true"},
 	}
 
-	if len(s.env) > 0 {
-		service["environment"] = s.env
+	if len(envs) > 0 {
+		service["environment"] = envs
 	}
 
 	if s.readyCheck != nil {
@@ -544,7 +569,8 @@ func (d *LocalRunner) generateDockerCompose() ([]byte, error) {
 
 // runOnHost runs the service on the host machine
 func (d *LocalRunner) runOnHost(ss *service) error {
-	args, err := d.applyTemplate(ss)
+	// TODO: Use env vars in host processes
+	args, _, err := d.applyTemplate(ss)
 	if err != nil {
 		return fmt.Errorf("failed to apply template, err: %w", err)
 	}

internal/manifest.go

@@ -332,6 +332,7 @@ func (s *service) WithEnv(key, value string) *service {
 	if s.env == nil {
 		s.env = make(map[string]string)
 	}
+	s.applyTemplate(value)
 	s.env[key] = value
 	return s
 }
@@ -378,17 +379,21 @@ func (s *service) WithPort(name string, portNumber int) *service {
 	return s
 }
 
+func (s *service) applyTemplate(arg string) {
+	var port []Port
+	var nodeRef []NodeRef
+	_, port, nodeRef = applyTemplate(arg)
+	for _, p := range port {
+		s.WithPort(p.Name, p.Port)
+	}
+	for _, n := range nodeRef {
+		s.nodeRefs = append(s.nodeRefs, &n)
+	}
+}
+
 func (s *service) WithArgs(args ...string) *service {
-	for i, arg := range args {
-		var port []Port
-		var nodeRef []NodeRef
-		args[i], port, nodeRef = applyTemplate(arg)
-		for _, p := range port {
-			s.WithPort(p.Name, p.Port)
-		}
-		for _, n := range nodeRef {
-			s.nodeRefs = append(s.nodeRefs, &n)
-		}
+	for _, arg := range args {
+		s.applyTemplate(arg)
 	}
 	s.args = append(s.args, args...)
 	return s
@@ -419,6 +424,8 @@ func (s *service) DependsOnRunning(name string) *service {
 }
 
 func applyTemplate(templateStr string) (string, []Port, []NodeRef) {
+	// TODO: Can we remove the return argument string?
+
 	// use template substitution to load constants
 	// pass-through the Dir template because it has to be resolved at the runtime
 	input := map[string]interface{}{
@@ -430,7 +437,7 @@ func applyTemplate(templateStr string) (string, []Port, []NodeRef) {
 	// ther can be multiple port and nodere because in the case of op-geth we pass a whole string as nested command args
 
 	funcs := template.FuncMap{
-		"Service": func(name string, portLabel string) string {
+		"Service": func(name string, portLabel, protocol string) string {
 			if name == "" {
 				panic("BUG: service name cannot be empty")
 			}

internal/recipe_buildernet.go

@@ -0,0 +1,90 @@
+package internal
+
+import (
+	"fmt"
+
+	flag "github.com/spf13/pflag"
+)
+
+var _ Recipe = &BuilderNetRecipe{}
+
+// BuilderNetRecipe is a recipe that extends the L1 recipe to include builder-hub
+type BuilderNetRecipe struct {
+	// Embed the L1Recipe to reuse its functionality
+	l1Recipe L1Recipe
+
+	// Add mock proxy for testing
+	includeMockProxy bool
+}
+
+func (b *BuilderNetRecipe) Name() string {
+	return "buildernet"
+}
+
+func (b *BuilderNetRecipe) Description() string {
+	return "Deploy a full L1 stack with mev-boost and builder-hub"
+}
+
+func (b *BuilderNetRecipe) Flags() *flag.FlagSet {
+	// Reuse the L1Recipe flags
+	flags := b.l1Recipe.Flags()
+
+	// Add a flag to enable/disable the mock proxy
+	flags.BoolVar(&b.includeMockProxy, "mock-proxy", false, "include a mock proxy for builder-hub with attestation headers")
+
+	return flags
+}
+
+func (b *BuilderNetRecipe) Artifacts() *ArtifactsBuilder {
+	// Reuse the L1Recipe artifacts builder
+	return b.l1Recipe.Artifacts()
+}
+
+func (b *BuilderNetRecipe) Apply(ctx *ExContext, artifacts *Artifacts) *Manifest {
+	// Start with the L1Recipe manifest
+	svcManager := b.l1Recipe.Apply(ctx, artifacts)
+
+	// Add builder-hub-postgres service (now includes migrations)
+	svcManager.AddService("builder-hub-postgres", &BuilderHubPostgres{})
+
+	// Add the builder-hub service
+	svcManager.AddService("builder-hub", &BuilderHub{
+		postgres: "builder-hub-postgres",
+	})
+
+	// Optionally add mock proxy for testing
+	if b.includeMockProxy {
+		svcManager.AddService("builder-hub-proxy", &BuilderHubMockProxy{
+			TargetService: "builder-hub",
+		})
+	}
+
+	return svcManager
+}
+
+func (b *BuilderNetRecipe) Output(manifest *Manifest) map[string]interface{} {
+	// Start with the L1Recipe output
+	output := b.l1Recipe.Output(manifest)
+
+	// Add builder-hub service info
+	builderHubService, ok := manifest.GetService("builder-hub")
+	if ok {
+		http := builderHubService.MustGetPort("http")
+		admin := builderHubService.MustGetPort("admin")
+		internal := builderHubService.MustGetPort("internal")
+
+		output["builder-hub-http"] = fmt.Sprintf("http://localhost:%d", http.HostPort)
+		output["builder-hub-admin"] = fmt.Sprintf("http://localhost:%d", admin.HostPort)
+		output["builder-hub-internal"] = fmt.Sprintf("http://localhost:%d", internal.HostPort)
+	}
+
+	if b.includeMockProxy {
+		proxyService, ok := manifest.GetService("builder-hub-proxy")
+		if ok {
+			http := proxyService.MustGetPort("http")
+			output["builder-hub-proxy"] = fmt.Sprintf("http://localhost:%d", http.HostPort)
+		}
+	}
+
+	return output
+}

main.go

@@ -116,6 +116,7 @@ var artifactsAllCmd = &cobra.Command{
 var recipes = []internal.Recipe{
 	&internal.L1Recipe{},
 	&internal.OpRecipe{},
+	&internal.BuilderNetRecipe{},
 }
 
 func main() {