From 0867aedd820d9aa50eb9dbac9076ae87e0b4c477 Mon Sep 17 00:00:00 2001 From: Ferran Borreguero Date: Tue, 13 May 2025 10:57:44 +0100 Subject: [PATCH 1/2] Add read function for the manifest --- go.mod | 2 +- go.sum | 5 +++ main.go | 2 +- playground/local_runner.go | 20 ++++++------ playground/manifest.go | 65 ++++++++++++++++++++++++------------- playground/manifest_test.go | 31 ++++++++++++++++++ 6 files changed, 91 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index 58cbbf5..adc76be 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.9.1 github.com/spf13/pflag v1.0.6 + github.com/stretchr/testify v1.10.0 github.com/wealdtech/go-eth2-wallet-encryptor-keystorev4 v1.1.3 gopkg.in/yaml.v2 v2.4.0 ) @@ -119,7 +120,6 @@ require ( github.com/rubenv/sql-migrate v1.7.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect - github.com/stretchr/testify v1.10.0 // indirect github.com/supranational/blst v0.3.14 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e // indirect diff --git a/go.sum b/go.sum index b0cfe06..4793dca 100644 --- a/go.sum +++ b/go.sum @@ -305,18 +305,22 @@ github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1n github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE= +github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= @@ -563,6 +567,7 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index c09b752..8e2784e 100644 --- a/main.go +++ b/main.go @@ -277,7 +277,7 @@ func runIt(recipe playground.Recipe) error { if !interactive { // print services info fmt.Printf("\n========= Services started =========\n") - for _, ss := range svcManager.Services() { + for _, ss := range svcManager.Services { ports := ss.GetPorts() sort.Slice(ports, func(i, j int) bool { return ports[i].Name < ports[j].Name diff --git a/playground/local_runner.go b/playground/local_runner.go index 26f20e5..9d76e31 100644 --- a/playground/local_runner.go +++ b/playground/local_runner.go @@ -126,7 +126,7 @@ func NewLocalRunner(out *output, manifest *Manifest, overrides map[string]string // Create the concrete instances to run instances := []*instance{} - for _, service := range manifest.Services() { + for _, service := range manifest.Services { component := FindComponent(service.ComponentName) if component == nil { return nil, fmt.Errorf("component not found '%s'", service.ComponentName) @@ -192,7 +192,7 @@ func NewLocalRunner(out *output, manifest *Manifest, overrides map[string]string } tasks := map[string]*task{} - for _, svc := range manifest.services { + for _, svc := range manifest.Services { tasks[svc.Name] = &task{ status: taskStatusPending, logs: nil, @@ -241,8 +241,8 @@ func (d *LocalRunner) printStatus() { lineOffset := 0 // Get ordered service names from manifest - orderedServices := make([]string, 0, len(d.manifest.services)) - for _, svc := range d.manifest.services { + orderedServices := make([]string, 0, len(d.manifest.Services)) + for _, svc := range d.manifest.Services { orderedServices = append(orderedServices, svc.Name) } @@ -453,7 +453,7 @@ func (d *LocalRunner) reservePort(startPort int, protocol string) int { } func (d *LocalRunner) getService(name string) *Service { - for _, svc := range d.manifest.services { + for _, svc := range d.manifest.Services { if svc.Name == name { return svc } @@ -748,13 +748,13 @@ func (d *LocalRunner) generateDockerCompose() ([]byte, error) { // for each service, reserve a port on the host machine. We use this ports // both to have access to the services from localhost but also to do communication // between services running inside docker and the ones running on the host machine. - for _, svc := range d.manifest.services { + for _, svc := range d.manifest.Services { for _, port := range svc.Ports { port.HostPort = d.reservePort(port.Port, port.Protocol) } } - for _, svc := range d.manifest.services { + for _, svc := range d.manifest.Services { if d.isHostService(svc.Name) { // skip services that are going to be launched on host continue @@ -924,7 +924,7 @@ func CreatePrometheusServices(manifest *Manifest, out *output) error { }, }) - for _, c := range manifest.services { + for _, c := range manifest.Services { for _, port := range c.Ports { if port.Name == "metrics" { metricsPath := "/metrics" @@ -967,7 +967,7 @@ func CreatePrometheusServices(manifest *Manifest, out *output) error { WithPort("metrics", 9090, "tcp"). WithArtifact("/data/prometheus.yaml", "prometheus.yaml") srv.ComponentName = "null" // For now, later on we can create a Prometheus component - manifest.services = append(manifest.services, srv) + manifest.Services = append(manifest.Services, srv) return nil } @@ -1004,7 +1004,7 @@ func (d *LocalRunner) Run() error { // Second, start the services that are running on the host machine errCh := make(chan error) go func() { - for _, svc := range d.manifest.services { + for _, svc := range d.manifest.Services { if d.isHostService(svc.Name) { if err := d.runOnHost(svc); err != nil { errCh <- err diff --git a/playground/manifest.go b/playground/manifest.go index c7d0e12..175e0ea 100644 --- a/playground/manifest.go +++ b/playground/manifest.go @@ -1,6 +1,7 @@ package playground import ( + "encoding/json" "fmt" "os" "path/filepath" @@ -26,8 +27,8 @@ type Recipe interface { type Manifest struct { ctx *ExContext - // list of services - services []*Service + // list of Services + Services []*Service `json:"services"` // overrides is a map of service name to the path of the executable to run // on the host machine instead of a container. @@ -88,10 +89,6 @@ type ServiceReady interface { Ready(instance *instance) error } -func (s *Manifest) Services() []*Service { - return s.services -} - // ReleaseService is a service that can also be runned as an artifact in the host machine type ReleaseService interface { ReleaseArtifact() *release @@ -102,7 +99,7 @@ func (s *Manifest) AddService(name string, srv ServiceGen) { service.ComponentName = srv.Name() srv.Run(service, s.ctx) - s.services = append(s.services, service) + s.Services = append(s.Services, service) } func (s *Manifest) MustGetService(name string) *Service { @@ -114,7 +111,7 @@ func (s *Manifest) MustGetService(name string) *Service { } func (s *Manifest) GetService(name string) (*Service, bool) { - for _, ss := range s.services { + for _, ss := range s.Services { if ss.Name == name { return ss, true } @@ -126,7 +123,7 @@ func (s *Manifest) GetService(name string) (*Service, bool) { // - checks if all the port dependencies are met from the service description // - downloads any local release artifacts for the services that require host execution func (s *Manifest) Validate() error { - for _, ss := range s.services { + for _, ss := range s.Services { // validate node port references for _, nodeRef := range ss.NodeRefs { targetService, ok := s.GetService(nodeRef.Service) @@ -156,7 +153,7 @@ func (s *Manifest) Validate() error { } // validate that the mounts are correct - for _, ss := range s.services { + for _, ss := range s.Services { for _, fileNameRef := range ss.FilesMapped { fileLoc := filepath.Join(s.out.dst, fileNameRef) @@ -170,7 +167,7 @@ func (s *Manifest) Validate() error { } // validate that the mounts are correct - for _, ss := range s.services { + for _, ss := range s.Services { for _, fileNameRef := range ss.FilesMapped { fileLoc := filepath.Join(s.out.dst, fileNameRef) @@ -194,10 +191,10 @@ const ( // Port describes a port that a service exposes type Port struct { // Name is the name of the port - Name string + Name string `json:"name"` // Port is the port number - Port int + Port int `json:"port"` // Protocol (tcp or udp) Protocol string @@ -210,10 +207,10 @@ type Port struct { // NodeRef describes a reference from one service to another type NodeRef struct { - Service string - PortLabel string - Protocol string - User string + Service string `json:"service"` + PortLabel string `json:"port_label"` + Protocol string `json:"protocol"` + User string `json:"user"` } // serviceLogs is a service to access the logs of the running service @@ -501,12 +498,12 @@ func (s *Manifest) GenerateDotGraph() string { // Create a map of services for easy lookup servicesMap := make(map[string]*Service) - for _, ss := range s.services { + for _, ss := range s.Services { servicesMap[ss.Name] = ss } // Add nodes (services) with their ports as labels - for _, ss := range s.services { + for _, ss := range s.Services { var ports []string for _, p := range ss.Ports { ports = append(ports, fmt.Sprintf("%s:%d", p.Name, p.Port)) @@ -523,7 +520,7 @@ func (s *Manifest) GenerateDotGraph() string { b.WriteString("\n") // Add edges (connections between services) - for _, ss := range s.services { + for _, ss := range s.Services { sourceNode := strings.ReplaceAll(ss.Name, "-", "_") for _, ref := range ss.NodeRefs { targetNode := strings.ReplaceAll(ref.Service, "-", "_") @@ -536,7 +533,7 @@ func (s *Manifest) GenerateDotGraph() string { } // Add edges for dependws_on - for _, ss := range s.services { + for _, ss := range s.Services { for _, dep := range ss.DependsOn { sourceNode := strings.ReplaceAll(ss.Name, "-", "_") targetNode := strings.ReplaceAll(dep.Name, "-", "_") @@ -558,7 +555,31 @@ func saveDotGraph(svcManager *Manifest, out *output) error { func (m *Manifest) SaveJson() error { format := map[string]interface{}{ - "services": m.services, + "services": m.Services, } return m.out.WriteFile("manifest.json", format) } + +func ReadManifest(outputFolder string) (*Manifest, error) { + // read outputFolder/manifest.json file + manifestFile := filepath.Join(outputFolder, "manifest.json") + if _, err := os.Stat(manifestFile); os.IsNotExist(err) { + return nil, fmt.Errorf("manifest file %s does not exist", manifestFile) + } + manifest, err := os.ReadFile(manifestFile) + if err != nil { + return nil, fmt.Errorf("failed to read manifest file %s: %w", manifestFile, err) + } + + // parse the manifest file + var manifestData Manifest + if err := json.Unmarshal(manifest, &manifestData); err != nil { + return nil, fmt.Errorf("failed to parse manifest file %s: %w", manifestFile, err) + } + + // set the output folder + manifestData.out = &output{ + dst: outputFolder, + } + return &manifestData, nil +} diff --git a/playground/manifest_test.go b/playground/manifest_test.go index d3761e8..8b833bf 100644 --- a/playground/manifest_test.go +++ b/playground/manifest_test.go @@ -2,6 +2,8 @@ package playground import ( "testing" + + "github.com/stretchr/testify/assert" ) func TestNodeRefString(t *testing.T) { @@ -56,3 +58,32 @@ func TestNodeRefString(t *testing.T) { } } } + +func TestManifestWriteRead(t *testing.T) { + out := &output{ + dst: "./example", + } + + recipe := &L1Recipe{} + + builder := recipe.Artifacts() + builder.OutputDir(out.dst) + + artifacts, err := builder.Build() + assert.NoError(t, err) + + manifest := recipe.Apply(&ExContext{}, artifacts) + assert.NoError(t, manifest.SaveJson()) + + manifest2, err := ReadManifest(out.dst) + assert.NoError(t, err) + + for _, svc := range manifest.Services { + svc2 := manifest2.MustGetService(svc.Name) + assert.Equal(t, svc.Name, svc2.Name) + assert.Equal(t, svc.Args, svc2.Args) + assert.Equal(t, svc.Env, svc2.Env) + assert.Equal(t, svc.Labels, svc2.Labels) + assert.Equal(t, svc.VolumesMapped, svc2.VolumesMapped) + } +} From b6627cd671fe75eb4858cbd666313deb36455fbf Mon Sep 17 00:00:00 2001 From: Ferran Borreguero Date: Tue, 13 May 2025 10:58:58 +0100 Subject: [PATCH 2/2] Fix --- playground/manifest.go | 5 +---- playground/manifest_test.go | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/playground/manifest.go b/playground/manifest.go index 175e0ea..8429327 100644 --- a/playground/manifest.go +++ b/playground/manifest.go @@ -554,10 +554,7 @@ func saveDotGraph(svcManager *Manifest, out *output) error { } func (m *Manifest) SaveJson() error { - format := map[string]interface{}{ - "services": m.Services, - } - return m.out.WriteFile("manifest.json", format) + return m.out.WriteFile("manifest.json", m) } func ReadManifest(outputFolder string) (*Manifest, error) { diff --git a/playground/manifest_test.go b/playground/manifest_test.go index 8b833bf..8431679 100644 --- a/playground/manifest_test.go +++ b/playground/manifest_test.go @@ -60,9 +60,7 @@ func TestNodeRefString(t *testing.T) { } func TestManifestWriteRead(t *testing.T) { - out := &output{ - dst: "./example", - } + out := newTestOutput(t) recipe := &L1Recipe{}