Use concrete artifacts on services (#121)

* Use concrete artifacts

* Change one thing

* Add fix

Author: ferranbt

internal/artifacts.go

@@ -433,6 +433,19 @@ func (o *output) Remove(path string) error {
 	return os.RemoveAll(filepath.Join(o.dst, path))
 }
 
+// CreateDir creates a new dir in the output folder and returns the
+// absolute file path
+func (o *output) CreateDir(path string) (string, error) {
+	absPath, err := filepath.Abs(filepath.Join(o.dst, path))
+	if err != nil {
+		return "", err
+	}
+	if err := os.MkdirAll(absPath, 0755); err != nil {
+		return "", fmt.Errorf("failed to create directory: %w", err)
+	}
+	return absPath, nil
+}
+
 func (o *output) CopyFile(src string, dst string) error {
 	// Open the source file
 	sourceFile, err := os.Open(src)

internal/components.go

@@ -22,11 +22,11 @@ func (r *RollupBoost) Run(service *service, ctx *ExContext) {
 		WithTag("0.4rc1").
 		WithArgs(
 			"--rpc-port", `{{Port "authrpc" 8551}}`,
-			"--l2-jwt-path", "{{.Dir}}/jwtsecret",
+			"--l2-jwt-path", "/data/jwtsecret",
 			"--l2-url", Connect(r.ELNode, "authrpc"),
-			"--builder-jwt-path", "{{.Dir}}/jwtsecret",
+			"--builder-jwt-path", "/data/jwtsecret",
 			"--builder-url", r.Builder,
-		)
+		).WithArtifact("/data/jwtsecret", "jwtsecret")
 }
 
 func (r *RollupBoost) Name() string {
@@ -82,12 +82,12 @@ func (o *OpNode) Run(service *service, ctx *ExContext) {
 			"--l1.epoch-poll-interval", "12s",
 			"--l1.http-poll-interval", "6s",
 			"--l2", Connect(o.L2Node, "authrpc"),
-			"--l2.jwt-secret", "{{.Dir}}/jwtsecret",
+			"--l2.jwt-secret", "/data/jwtsecret",
 			"--sequencer.enabled",
 			"--sequencer.l1-confs", "0",
 			"--verifier.l1-confs", "0",
 			"--p2p.sequencer.key", "8b3a350cf5c34c9194ca85829a2df0ec3153be0318b5e2d3348e872092edffba",
-			"--rollup.config", "{{.Dir}}/rollup.json",
+			"--rollup.config", "/data/rollup.json",
 			"--rpc.addr", "0.0.0.0",
 			"--rpc.port", `{{Port "http" 8549}}`,
 			"--p2p.listen.ip", "0.0.0.0",
@@ -100,8 +100,11 @@ func (o *OpNode) Run(service *service, ctx *ExContext) {
 			"--metrics.port", `{{Port "metrics" 7300}}`,
 			"--pprof.enabled",
 			"--rpc.enable-admin",
-			"--safedb.path", "{{.Dir}}/db",
-		)
+			"--safedb.path", "/data_db",
+		).
+		WithArtifact("/data/jwtsecret", "jwtsecret").
+		WithArtifact("/data/rollup.json", "rollup.json").
+		WithVolume("data", "/data_db")
 }
 
 func (o *OpNode) Name() string {
@@ -135,7 +138,7 @@ func logLevelToGethVerbosity(logLevel LogLevel) string {
 func (o *OpGeth) Run(service *service, ctx *ExContext) {
 	var nodeKeyFlag string
 	if o.UseDeterministicP2PKey {
-		nodeKeyFlag = "--nodekey {{.Dir}}/deterministic_p2p_key.txt "
+		nodeKeyFlag = "--nodekey /data/deterministic_p2p_key.txt "
 	}
 
 	service.
@@ -145,9 +148,9 @@ func (o *OpGeth) Run(service *service, ctx *ExContext) {
 		WithLabel("metrics_path", "/debug/metrics/prometheus").
 		WithArgs(
 			"-c",
-			"geth init --datadir {{.Dir}}/data_opgeth --state.scheme hash {{.Dir}}/l2-genesis.json && "+
+			"geth init --datadir /data_opgeth --state.scheme hash /data/l2-genesis.json && "+
 				"exec geth "+
-				"--datadir {{.Dir}}/data_opgeth "+
+				"--datadir /data_opgeth "+
 				"--verbosity "+logLevelToGethVerbosity(ctx.LogLevel)+" "+
 				"--http "+
 				"--http.corsdomain \"*\" "+
@@ -167,15 +170,19 @@ func (o *OpGeth) Run(service *service, ctx *ExContext) {
 				"--authrpc.addr 0.0.0.0 "+
 				"--authrpc.port "+`{{Port "authrpc" 8551}} `+
 				"--authrpc.vhosts \"*\" "+
-				"--authrpc.jwtsecret {{.Dir}}/jwtsecret "+
+				"--authrpc.jwtsecret /data/jwtsecret "+
 				"--gcmode archive "+
 				"--state.scheme hash "+
 				"--port "+`{{Port "rpc" 30303}} `+
 				nodeKeyFlag+
 				"--metrics "+
 				"--metrics.addr 0.0.0.0 "+
 				"--metrics.port "+`{{Port "metrics" 6061}}`,
-		)
+		).
+		WithVolume("data", "/data_opgeth").
+		WithArtifact("/data/l2-genesis.json", "l2-genesis.json").
+		WithArtifact("/data/jwtsecret", "jwtsecret").
+		WithArtifact("/data/deterministic_p2p_key.txt", "deterministic_p2p_key.txt")
 }
 
 func (o *OpGeth) Name() string {
@@ -253,10 +260,10 @@ func (r *RethEL) Run(svc *service, ctx *ExContext) {
 		WithEntrypoint("/usr/local/bin/reth").
 		WithArgs(
 			"node",
-			"--chain", "{{.Dir}}/genesis.json",
-			"--datadir", "{{.Dir}}/data_reth",
+			"--chain", "/data/genesis.json",
+			"--datadir", "/data_reth",
 			"--color", "never",
-			"--ipcpath", "{{.Dir}}/reth.ipc",
+			"--ipcpath", "/data_reth/reth.ipc",
 			"--addr", "127.0.0.1",
 			"--port", `{{Port "rpc" 30303}}`,
 			// "--disable-discovery",
@@ -267,12 +274,15 @@ func (r *RethEL) Run(svc *service, ctx *ExContext) {
 			"--http.port", `{{Port "http" 8545}}`,
 			"--authrpc.port", `{{Port "authrpc" 8551}}`,
 			"--authrpc.addr", "0.0.0.0",
-			"--authrpc.jwtsecret", "{{.Dir}}/jwtsecret",
+			"--authrpc.jwtsecret", "/data/jwtsecret",
 			"--metrics", `0.0.0.0:{{Port "metrics" 9090}}`,
 			// For reth version 1.2.0 the "legacy" engine was removed, so we now require these arguments:
 			"--engine.persistence-threshold", "0", "--engine.memory-block-buffer-target", "0",
 			logLevelToRethVerbosity(ctx.LogLevel),
-		)
+		).
+		WithArtifact("/data/genesis.json", "genesis.json").
+		WithArtifact("/data/jwtsecret", "jwtsecret").
+		WithVolume("data", "/data_reth")
 
 	if r.UseNativeReth {
 		// we need to use this otherwise the db cannot be binded
@@ -303,8 +313,8 @@ func (l *LighthouseBeaconNode) Run(svc *service, ctx *ExContext) {
 		WithEntrypoint("lighthouse").
 		WithArgs(
 			"bn",
-			"--datadir", "{{.Dir}}/data_beacon_node",
-			"--testnet-dir", "{{.Dir}}/testnet",
+			"--datadir", "/data_beacon",
+			"--testnet-dir", "/data/testnet-dir",
 			"--enable-private-discovery",
 			"--disable-peer-scoring",
 			"--staking",
@@ -321,11 +331,14 @@ func (l *LighthouseBeaconNode) Run(svc *service, ctx *ExContext) {
 			"--disable-packet-filter",
 			"--target-peers", "0",
 			"--execution-endpoint", Connect(l.ExecutionNode, "authrpc"),
-			"--execution-jwt", "{{.Dir}}/jwtsecret",
+			"--execution-jwt", "/data/jwtsecret",
 			"--always-prepare-payload",
 			"--prepare-payload-lookahead", "8000",
 			"--suggested-fee-recipient", "0x690B9A9E9aa1C9dB991C7721a92d351Db4FaC990",
 		).
+		WithArtifact("/data/testnet-dir", "testnet").
+		WithArtifact("/data/jwtsecret", "jwtsecret").
+		WithVolume("data", "/data_beacon").
 		WithReady(ReadyCheck{
 			QueryURL:    "http://localhost:3500/eth/v1/node/syncing",
 			Interval:    1 * time.Second,
@@ -359,14 +372,16 @@ func (l *LighthouseValidator) Run(service *service, ctx *ExContext) {
 		WithEntrypoint("lighthouse").
 		WithArgs(
 			"vc",
-			"--datadir", "{{.Dir}}/data_validator",
-			"--testnet-dir", "{{.Dir}}/testnet",
+			"--datadir", "/data/validator",
+			"--testnet-dir", "/data/testnet-dir",
 			"--init-slashing-protection",
 			"--beacon-nodes", Connect(l.BeaconNode, "http"),
 			"--suggested-fee-recipient", "0x690B9A9E9aa1C9dB991C7721a92d351Db4FaC990",
 			"--builder-proposals",
 			"--prefer-builder-proposals",
-		)
+		).
+		WithArtifact("/data/validator", "data_validator").
+		WithArtifact("/data/testnet-dir", "testnet")
 }
 
 func (l *LighthouseValidator) Name() string {
@@ -512,16 +527,19 @@ func (o *OpReth) Run(service *service, ctx *ExContext) {
 			"node",
 			"--authrpc.port", `{{Port "authrpc" 8551}}`,
 			"--authrpc.addr", "0.0.0.0",
-			"--authrpc.jwtsecret", "{{.Dir}}/jwtsecret",
+			"--authrpc.jwtsecret", "/data/jwtsecret",
 			"--http",
 			"--http.addr", "0.0.0.0",
 			"--http.port", `{{Port "http" 8545}}`,
-			"--chain", "{{.Dir}}/l2-genesis.json",
-			"--datadir", "{{.Dir}}/data_op_reth",
+			"--chain", "/data/l2-genesis.json",
+			"--datadir", "/data_op_reth",
 			"--disable-discovery",
 			"--color", "never",
 			"--metrics", `0.0.0.0:{{Port "metrics" 9090}}`,
-			"--port", `{{Port "rpc" 30303}}`)
+			"--port", `{{Port "rpc" 30303}}`).
+		WithArtifact("/data/jwtsecret", "jwtsecret").
+		WithArtifact("/data/l2-genesis.json", "l2-genesis.json").
+		WithVolume("data", "/data_op_reth")
 }
 
 func (o *OpReth) Name() string {

internal/local_runner.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"runtime"
 	"strings"
 	"sync"
@@ -562,14 +563,34 @@ func (d *LocalRunner) toDockerComposeService(s *service) (map[string]interface{}
 		labels[fmt.Sprintf("port.%s", port.Name)] = fmt.Sprintf("%d", port.Port)
 	}
 
+	// Use files mapped to figure out which files from the artifacts is using the service
+	volumes := map[string]string{
+		outputFolder: "/artifacts", // placeholder
+	}
+	for k, v := range s.filesMapped {
+		volumes[filepath.Join(outputFolder, v)] = k
+	}
+
+	// create the bind volumes
+	for localPath, volumeName := range s.volumesMapped {
+		volumeDirAbsPath, err := d.createVolume(s.Name, volumeName)
+		if err != nil {
+			return nil, err
+		}
+		volumes[volumeDirAbsPath] = localPath
+	}
+
+	volumesInLine := []string{}
+	for k, v := range volumes {
+		volumesInLine = append(volumesInLine, fmt.Sprintf("%s:%s", k, v))
+	}
+
 	// add the ports to the labels as well
 	service := map[string]interface{}{
 		"image":   imageName,
 		"command": args,
 		// Add volume mount for the output directory
-		"volumes": []string{
-			fmt.Sprintf("%s:/artifacts", outputFolder),
-		},
+		"volumes": volumesInLine,
 		// Add the ethereum network
 		"networks": []string{d.networkName},
 		"labels":   labels,
@@ -693,6 +714,15 @@ func (d *LocalRunner) generateDockerCompose() ([]byte, error) {
 	return yamlData, nil
 }
 
+func (d *LocalRunner) createVolume(service, volumeName string) (string, error) {
+	// create the volume in the output folder
+	volumeDirAbsPath, err := d.out.CreateDir(fmt.Sprintf("volume-%s-%s", service, volumeName))
+	if err != nil {
+		return "", fmt.Errorf("failed to create volume dir %s: %w", volumeName, err)
+	}
+	return volumeDirAbsPath, nil
+}
+
 // runOnHost runs the service on the host machine
 func (d *LocalRunner) runOnHost(ss *service) error {
 	// TODO: Use env vars in host processes
@@ -701,6 +731,34 @@ func (d *LocalRunner) runOnHost(ss *service) error {
 		return fmt.Errorf("failed to apply template, err: %w", err)
 	}
 
+	// Create the volumes for this service
+	volumesMapped := map[string]string{}
+	for pathInDocker, volumeName := range ss.volumesMapped {
+		volumeDirAbsPath, err := d.createVolume(ss.Name, volumeName)
+		if err != nil {
+			return err
+		}
+		volumesMapped[pathInDocker] = volumeDirAbsPath
+	}
+
+	// We have to replace the names of the files it is using as artifacts for the full names
+	// Just a string replacement should be enough
+	for i, arg := range args {
+		// If any of the args contains any of the files mapped, we need to replace it
+		for pathInDocker, artifactName := range ss.filesMapped {
+			if strings.Contains(arg, pathInDocker) {
+				args[i] = strings.ReplaceAll(arg, pathInDocker, filepath.Join(d.out.dst, artifactName))
+			}
+		}
+		// If any of the args contains any of the volumes mapped, we need to create
+		// the volume and replace it
+		for pathInDocker, volumeAbsPath := range volumesMapped {
+			if strings.Contains(arg, pathInDocker) {
+				args[i] = strings.ReplaceAll(arg, pathInDocker, volumeAbsPath)
+			}
+		}
+	}
+
 	execPath := d.overrides[ss.Name]
 	cmd := exec.Command(execPath, args...)
 

internal/manifest.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strings"
 	"text/template"
 	"time"
@@ -207,6 +208,21 @@ func (s *Manifest) Validate() error {
 			s.overrides[ss.Name] = bin
 		}
 	}
+
+	// validate that the mounts are correct
+	for _, ss := range s.services {
+		for _, fileNameRef := range ss.filesMapped {
+			fileLoc := filepath.Join(s.out.dst, fileNameRef)
+
+			if _, err := os.Stat(fileLoc); err != nil {
+				if os.IsNotExist(err) {
+					return fmt.Errorf("service %s includes an unknown file %s does not exist", ss.Name, fileLoc)
+				}
+				return fmt.Errorf("failed to stat file %s: %w", fileLoc, err)
+			}
+		}
+	}
+
 	return nil
 }
 
@@ -282,6 +298,9 @@ type service struct {
 	ports    []*Port
 	nodeRefs []*NodeRef
 
+	filesMapped   map[string]string
+	volumesMapped map[string]string
+
 	tag        string
 	image      string
 	entrypoint string
@@ -411,6 +430,22 @@ func (s *service) WithArgs(args ...string) *service {
 	return s
 }
 
+func (s *service) WithVolume(name string, localPath string) *service {
+	if s.volumesMapped == nil {
+		s.volumesMapped = make(map[string]string)
+	}
+	s.volumesMapped[localPath] = name
+	return s
+}
+
+func (s *service) WithArtifact(localPath string, artifactName string) *service {
+	if s.filesMapped == nil {
+		s.filesMapped = make(map[string]string)
+	}
+	s.filesMapped[localPath] = artifactName
+	return s
+}
+
 func (s *service) WithReady(check ReadyCheck) *service {
 	s.readyCheck = &check
 	return s