feat: Add builder-hub component and buildernet-recipe

internal/artifacts.go

@@ -341,8 +341,12 @@ func getPrivKey(privStr string) (*ecdsa.PrivateKey, error) {
 	return priv, nil
 }
 
+func ConnectRaw(service, port, protocol string) string {
+	return fmt.Sprintf(`{{Service "%s" "%s" "%s"}}`, service, port, protocol)
+}
+
 func Connect(service, port string) string {
-	return fmt.Sprintf(`{{Service "%s" "%s"}}`, service, port)
+	return ConnectRaw(service, port, "http")
 }
 
 type output struct {

internal/catalog.go

@@ -17,6 +17,10 @@ func init() {
 	register(&MevBoostRelay{})
 	register(&RollupBoost{})
 	register(&OpReth{})
+	register(&BuilderHub{})
+	register(&BuilderHubPostgres{})
+	register(&BuilderHubMockProxy{})
+	register(&OrderflowProxySender{})
 }
 
 func FindComponent(name string) Service {

internal/components.go

@@ -429,6 +429,102 @@ func (m *MevBoostRelay) Watchdog(out io.Writer, service *service, ctx context.Co
 	return watchGroup.wait()
 }
 
+type BuilderHubPostgres struct {
+}
+
+func (b *BuilderHubPostgres) Run(service *service, ctx *ExContext) {
+	service.
+		WithImage("docker.io/flashbots/builder-hub-db").
+		WithTag("latest").
+		WithPort("postgres", 5432).
+		WithEnv("POSTGRES_USER", "postgres").
+		WithEnv("POSTGRES_PASSWORD", "postgres").
+		WithEnv("POSTGRES_DB", "postgres").
+		WithReady(ReadyCheck{
+			Test:        []string{"CMD-SHELL", "pg_isready -U postgres -d postgres"},
+			Interval:    1 * time.Second,
+			Timeout:     30 * time.Second,
+			Retries:     3,
+			StartPeriod: 1 * time.Second,
+		})
+}
+
+func (b *BuilderHubPostgres) Name() string {
+	return "builder-hub-postgres"
+}
+
+type BuilderHub struct {
+	postgres string
+}
+
+func (b *BuilderHub) Run(service *service, ctx *ExContext) {
+	service.
+		WithImage("docker.io/flashbots/builder-hub").
+		WithTag("latest").
+		WithEntrypoint("/app/builder-hub").
+		WithEnv("POSTGRES_DSN", "postgres://postgres:postgres@"+ConnectRaw(b.postgres, "postgres", "")+"/postgres?sslmode=disable").
+		WithEnv("LISTEN_ADDR", "0.0.0.0:"+`{{Port "http" 8080}}`).
+		WithEnv("ADMIN_ADDR", "0.0.0.0:"+`{{Port "admin" 8081}}`).
+		WithEnv("INTERNAL_ADDR", "0.0.0.0:"+`{{Port "internal" 8082}}`).
+		WithEnv("METRICS_ADDR", "0.0.0.0:"+`{{Port "metrics" 8090}}`).
+		DependsOnHealthy(b.postgres)
+}
+
+func (b *BuilderHub) Name() string {
+	return "builder-hub"
+}
+
+type BuilderHubMockProxy struct {
+	TargetService string
+}
+
+func (b *BuilderHubMockProxy) Run(service *service, ctx *ExContext) {
+	service.
+		WithImage("docker.io/flashbots/builder-hub-mock-proxy").
+		WithTag("latest").
+		WithPort("http", 8888)
+
+	if b.TargetService != "" {
+		service.DependsOnHealthy(b.TargetService)
+	}
+}
+
+func (b *BuilderHubMockProxy) Name() string {
+	return "builder-hub-mock-proxy"
+}
+
+type OrderflowProxySender struct {
+	ConfigHubEndpoint string
+	SignerKey         string
+}
+
+func (o *OrderflowProxySender) Run(service *service, ctx *ExContext) {
+	service.
+		WithImage("docker.io/flashbots/orderflow-proxy-sender").
+		WithTag("latest").
+		WithArgs(
+			"--listen-address", fmt.Sprintf("0.0.0.0:%s", `{{Port "http" 8080}}`),
+			"--builder-confighub-endpoint", Connect(o.ConfigHubEndpoint, "http"),
+			"--orderflow-signer-key", o.SignerKey,
+			"--connections-per-peer", "10",
+			"--metrics-addr", fmt.Sprintf("0.0.0.0:%s", `{{Port "metrics" 8090}}`),
+			"--log-json",
+		).
+		WithPort("http", 8080).
+		WithPort("metrics", 8090).
+		WithReady(ReadyCheck{
+			Test:        []string{"CMD-SHELL", "wget -q --spider http://localhost:8080 || curl -s http://localhost:8080 > /dev/null"},
+			Interval:    1 * time.Second,
+			Timeout:     5 * time.Second,
+			Retries:     3,
+			StartPeriod: 3 * time.Second,
+		})
+}
+
+func (o *OrderflowProxySender) Name() string {
+	return "orderflow-proxy-sender"
+}
+
 type OpReth struct {
 }
 

internal/local_runner.go

@@ -333,7 +333,7 @@ func (d *LocalRunner) getService(name string) *service {
 
 // applyTemplate resolves the templates from the manifest (Dir, Port, Connect) into
 // the actual values for this specific docker execution.
-func (d *LocalRunner) applyTemplate(s *service) ([]string, error) {
+func (d *LocalRunner) applyTemplate(s *service) ([]string, map[string]string, error) {
 	var input map[string]interface{}
 
 	// For {{.Dir}}:
@@ -350,7 +350,12 @@ func (d *LocalRunner) applyTemplate(s *service) ([]string, error) {
 	}
 
 	funcs := template.FuncMap{
-		"Service": func(name string, portLabel string) string {
+		"Service": func(name string, portLabel, protocol string) string {
+			protocolPrefix := ""
+			if protocol == "http" {
+				protocolPrefix = "http://"
+			}
+
 			// For {{Service "name" "portLabel"}}:
 			// - Service runs on host:
 			//   A: target is inside docker: access with localhost:hostPort
@@ -365,14 +370,14 @@ func (d *LocalRunner) applyTemplate(s *service) ([]string, error) {
 
 			if d.isHostService(s.Name) {
 				// A and B
-				return fmt.Sprintf("http://localhost:%d", port.HostPort)
+				return fmt.Sprintf("%slocalhost:%d", protocolPrefix, port.HostPort)
 			} else {
 				if d.isHostService(svc.Name) {
 					// D
-					return fmt.Sprintf("http://host.docker.internal:%d", port.HostPort)
+					return fmt.Sprintf("%shost.docker.internal:%d", protocolPrefix, port.HostPort)
 				}
 				// C
-				return fmt.Sprintf("http://%s:%d", svc.Name, port.Port)
+				return fmt.Sprintf("%s%s:%d", protocolPrefix, svc.Name, port.Port)
 			}
 		},
 		"Port": func(name string, defaultPort int) int {
@@ -386,28 +391,48 @@ func (d *LocalRunner) applyTemplate(s *service) ([]string, error) {
 		},
 	}
 
-	var argsResult []string
-	for _, arg := range s.args {
+	runTemplate := func(arg string) (string, error) {
 		tpl, err := template.New("").Funcs(funcs).Parse(arg)
 		if err != nil {
-			return nil, err
+			return "", err
 		}
 
 		var out strings.Builder
 		if err := tpl.Execute(&out, input); err != nil {
-			return nil, err
+			return "", err
+		}
+
+		return out.String(), nil
+	}
+
+	// apply the templates to the arguments
+	var argsResult []string
+	for _, arg := range s.args {
+		newArg, err := runTemplate(arg)
+		if err != nil {
+			return nil, nil, err
+		}
+		argsResult = append(argsResult, newArg)
+	}
+
+	// apply the templates to the environment variables
+	envs := map[string]string{}
+	for k, v := range s.env {
+		newV, err := runTemplate(v)
+		if err != nil {
+			return nil, nil, err
 		}
-		argsResult = append(argsResult, out.String())
+		envs[k] = newV
 	}
 
-	return argsResult, nil
+	return argsResult, envs, nil
 }
 
 func (d *LocalRunner) toDockerComposeService(s *service) (map[string]interface{}, error) {
 	// apply the template again on the arguments to figure out the connections
 	// at this point all of them are valid, we just have to resolve them again. We assume for now
 	// everyone is going to be on docker at the same network.
-	args, err := d.applyTemplate(s)
+	args, envs, err := d.applyTemplate(s)
 	if err != nil {
 		return nil, fmt.Errorf("failed to apply template, err: %w", err)
 	}
@@ -433,8 +458,8 @@ func (d *LocalRunner) toDockerComposeService(s *service) (map[string]interface{}
 		"labels": map[string]string{"playground": "true"},
 	}
 
-	if len(s.env) > 0 {
-		service["environment"] = s.env
+	if len(envs) > 0 {
+		service["environment"] = envs
 	}
 
 	if s.readyCheck != nil {
@@ -544,7 +569,8 @@ func (d *LocalRunner) generateDockerCompose() ([]byte, error) {
 
 // runOnHost runs the service on the host machine
 func (d *LocalRunner) runOnHost(ss *service) error {
-	args, err := d.applyTemplate(ss)
+	// TODO: Use env vars in host processes
+	args, _, err := d.applyTemplate(ss)
 	if err != nil {
 		return fmt.Errorf("failed to apply template, err: %w", err)
 	}

internal/manifest.go

@@ -332,6 +332,7 @@ func (s *service) WithEnv(key, value string) *service {
 	if s.env == nil {
 		s.env = make(map[string]string)
 	}
+	s.applyTemplate(value)
 	s.env[key] = value
 	return s
 }
@@ -378,17 +379,21 @@ func (s *service) WithPort(name string, portNumber int) *service {
 	return s
 }
 
+func (s *service) applyTemplate(arg string) {
+	var port []Port
+	var nodeRef []NodeRef
+	_, port, nodeRef = applyTemplate(arg)
+	for _, p := range port {
+		s.WithPort(p.Name, p.Port)
+	}
+	for _, n := range nodeRef {
+		s.nodeRefs = append(s.nodeRefs, &n)
+	}
+}
+
 func (s *service) WithArgs(args ...string) *service {
-	for i, arg := range args {
-		var port []Port
-		var nodeRef []NodeRef
-		args[i], port, nodeRef = applyTemplate(arg)
-		for _, p := range port {
-			s.WithPort(p.Name, p.Port)
-		}
-		for _, n := range nodeRef {
-			s.nodeRefs = append(s.nodeRefs, &n)
-		}
+	for _, arg := range args {
+		s.applyTemplate(arg)
 	}
 	s.args = append(s.args, args...)
 	return s
@@ -419,6 +424,8 @@ func (s *service) DependsOnRunning(name string) *service {
 }
 
 func applyTemplate(templateStr string) (string, []Port, []NodeRef) {
+	// TODO: Can we remove the return argument string?
+
 	// use template substitution to load constants
 	// pass-through the Dir template because it has to be resolved at the runtime
 	input := map[string]interface{}{
@@ -430,7 +437,7 @@ func applyTemplate(templateStr string) (string, []Port, []NodeRef) {
 	// ther can be multiple port and nodere because in the case of op-geth we pass a whole string as nested command args
 
 	funcs := template.FuncMap{
-		"Service": func(name string, portLabel string) string {
+		"Service": func(name string, portLabel, protocol string) string {
 			if name == "" {
 				panic("BUG: service name cannot be empty")
 			}