feat: implement reproducible build and get cached image

README.md

@@ -343,4 +343,6 @@ On MacOS or Windows systems, we recommend either using a VM or the provided `.de
 | `--coder-agent-url` | `CODER_AGENT_URL` |  | URL of the Coder deployment. If CODER_AGENT_TOKEN is also set, logs from envbuilder will be forwarded here and will be visible in the workspace build logs. |
 | `--coder-agent-token` | `CODER_AGENT_TOKEN` |  | Authentication token for a Coder agent. If this is set, then CODER_AGENT_URL must also be set. |
 | `--coder-agent-subsystem` | `CODER_AGENT_SUBSYSTEM` |  | Coder agent subsystems to report when forwarding logs. The envbuilder subsystem is always included. |
+| `--push-image` | `ENVBUILDER_PUSH_IMAGE` |  | Push the built image to a remote registry.This option forces a reproducible build. |
+| `--get-cached-image` | `ENVBUILDER_GET_CACHED_IMAGE` |  | Print the digest of the cached image, if available.Exits with an error if not found. |
 <!--- END docsgen --->

envbuilder.go

@@ -99,6 +99,9 @@ func Run(ctx context.Context, options Options) error {
 	if options.InitCommand == "" {
 		options.InitCommand = "/bin/sh"
 	}
+	if options.CacheRepo == "" && options.PushImage {
+		return fmt.Errorf("--cache-repo must be set when using --push-image!")
+	}
 	// Default to the shell!
 	initArgs := []string{"-c", options.InitScript}
 	if options.InitArgs != "" {
@@ -119,11 +122,11 @@ func Run(ctx context.Context, options Options) error {
 		options.WorkspaceFolder = f
 	}
 
-	stageNumber := 1
+	stageNumber := 0
 	startStage := func(format string, args ...any) func(format string, args ...any) {
 		now := time.Now()
-		stageNum := stageNumber
 		stageNumber++
+		stageNum := stageNumber
 		options.Logger(notcodersdk.LogLevelInfo, "#%d: %s", stageNum, fmt.Sprintf(format, args...))
 
 		return func(format string, args ...any) {
@@ -338,7 +341,7 @@ func Run(ctx context.Context, options Options) error {
 
 	HijackLogrus(func(entry *logrus.Entry) {
 		for _, line := range strings.Split(entry.Message, "\r") {
-			options.Logger(notcodersdk.LogLevelInfo, "#2: %s", color.HiBlackString(line))
+			options.Logger(notcodersdk.LogLevelInfo, "#%d: %s", stageNumber, color.HiBlackString(line))
 		}
 	})
 
@@ -474,20 +477,24 @@ func Run(ctx context.Context, options Options) error {
 			cacheTTL = time.Hour * 24 * time.Duration(options.CacheTTLDays)
 		}
 
-		endStage := startStage("🏗️ Building image...")
 		// At this point we have all the context, we can now build!
 		registryMirror := []string{}
 		if val, ok := os.LookupEnv("KANIKO_REGISTRY_MIRROR"); ok {
 			registryMirror = strings.Split(val, ";")
 		}
-		image, err := executor.DoBuild(&config.KanikoOptions{
+		var destinations []string
+		if options.CacheRepo != "" {
+			destinations = append(destinations, options.CacheRepo)
+		}
+		opts := &config.KanikoOptions{
 			// Boilerplate!
 			CustomPlatform:    platforms.Format(platforms.Normalize(platforms.DefaultSpec())),
 			SnapshotMode:      "redo",
 			RunV2:             true,
 			RunStdout:         stdoutWriter,
 			RunStderr:         stderrWriter,
-			Destinations:      []string{"local"},
+			Destinations:      destinations,
+			NoPush:            !options.PushImage || len(destinations) == 0,
 			CacheRunLayers:    true,
 			CacheCopyLayers:   true,
 			CompressedCaching: true,
@@ -518,11 +525,40 @@ func Run(ctx context.Context, options Options) error {
 				RegistryMirrors: registryMirror,
 			},
 			SrcContext: buildParams.BuildContext,
-		})
+
+			// For cached image utilization, produce reproducible builds.
+			Reproducible: options.PushImage,
+		}
+
+		if options.GetCachedImage {
+			endStage := startStage("🏗️ Checking for cached image...")
+			image, err := executor.DoCacheProbe(opts)
+			if err != nil {
+				return nil, xerrors.Errorf("unable to get cached image: %w", err)
+			}
+			digest, err := image.Digest()
+			if err != nil {
+				return nil, xerrors.Errorf("get cached image digest: %w", err)
+			}
+			endStage("🏗️ Found cached image!")
+			_, _ = fmt.Fprintf(os.Stdout, "%s@%s\n", options.CacheRepo, digest.String())
+			os.Exit(0)
+		}
+
+		endStage := startStage("🏗️ Building image...")
+		image, err := executor.DoBuild(opts)
 		if err != nil {
-			return nil, err
+			return nil, xerrors.Errorf("do build: %w", err)
 		}
 		endStage("🏗️ Built image!")
+		if options.PushImage {
+			endStage = startStage("🏗️ Pushing image...")
+			if err := executor.DoPush(image, opts); err != nil {
+				return nil, xerrors.Errorf("do push: %w", err)
+			}
+			endStage("🏗️ Pushed image!")
+		}
+
 		return image, err
 	}
 

go.mod

@@ -6,7 +6,7 @@ toolchain go1.22.3
 
 // There are a few options we need added to Kaniko!
 // See: https://github.com/GoogleContainerTools/kaniko/compare/main...coder:kaniko:main
-replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240524082248-9d0d55902c34
+replace github.com/GoogleContainerTools/kaniko => github.com/coder/kaniko v0.0.0-20240610145227-0a73fcd89e3f
 
 require (
 	cdr.dev/slog v1.6.2-0.20240126064726-20367d4aede6

go.sum

@@ -121,8 +121,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
-github.com/coder/kaniko v0.0.0-20240524082248-9d0d55902c34 h1:Wm7sMNc1aTN5l0NerYHb3LZdQJVQp4QrW4v83N21sfc=
-github.com/coder/kaniko v0.0.0-20240524082248-9d0d55902c34/go.mod h1:YMK7BlxerzLlMwihGxNWUaFoN9LXCij4P+w/8/fNlcM=
+github.com/coder/kaniko v0.0.0-20240610145227-0a73fcd89e3f h1:o5p2u5sF1KVMgH+pcoMNfStpfXe76O3Enwg0h1KlzIU=
+github.com/coder/kaniko v0.0.0-20240610145227-0a73fcd89e3f/go.mod h1:YMK7BlxerzLlMwihGxNWUaFoN9LXCij4P+w/8/fNlcM=
 github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0 h1:3A0ES21Ke+FxEM8CXx9n47SZOKOpgSE1bbJzlE4qPVs=
 github.com/coder/pretty v0.0.0-20230908205945-e89ba86370e0/go.mod h1:5UuS2Ts+nTToAMeOjNlnHFkPahrtDkmpydBen/3wgZc=
 github.com/coder/retry v1.5.1 h1:iWu8YnD8YqHs3XwqrqsjoBTAVqT9ml6z9ViJ2wlMiqc=

integration/integration_test.go

@@ -34,6 +34,8 @@ import (
 	"github.com/go-git/go-billy/v5/memfs"
 	"github.com/google/go-containerregistry/pkg/authn"
 	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/registry"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/remote/transport"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -984,6 +986,186 @@ COPY %s .`, testImageAlpine, inclFile)
 	}
 }
 
+func TestPushImage(t *testing.T) {
+	t.Parallel()
+
+	t.Run("CacheWithoutPush", func(t *testing.T) {
+		t.Parallel()
+
+		srv := createGitServer(t, gitServerOptions{
+			files: map[string]string{
+				".devcontainer/Dockerfile": fmt.Sprintf("FROM %s\nRUN date --utc > /root/date.txt", testImageAlpine),
+				".devcontainer/devcontainer.json": `{
+				"name": "Test",
+				"build": {
+					"dockerfile": "Dockerfile"
+				},
+			}`,
+			},
+		})
+
+		// Given: an empty registry
+		testReg := setupInMemoryRegistry(t)
+		testRepo := testReg + "/test"
+		ref, err := name.ParseReference(testRepo + ":latest")
+		require.NoError(t, err)
+		_, err = remote.Image(ref)
+		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
+
+		// When: we run envbuilder with GET_CACHED_IMAGE
+		_, err = runEnvbuilder(t, options{env: []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", testRepo),
+			envbuilderEnv("GET_CACHED_IMAGE", "1"),
+		}})
+		require.ErrorContains(t, err, "not supported in fake build")
+		// Then: it should fail to build the image and nothing should be pushed
+		_, err = remote.Image(ref)
+		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
+
+		// When: we run envbuilder with PUSH_IMAGE set
+		_, err = runEnvbuilder(t, options{env: []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", testRepo),
+		}})
+		require.NoError(t, err)
+
+		// Then: the image tag should not be present, only the layers
+		_, err = remote.Image(ref)
+		require.ErrorContains(t, err, "MANIFEST_UNKNOWN", "expected image to not be present before build + push")
+
+		// Then: re-running envbuilder with GET_CACHED_IMAGE should succeed
+		_, err = runEnvbuilder(t, options{env: []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", testRepo),
+			envbuilderEnv("GET_CACHED_IMAGE", "1"),
+		}})
+		require.NoError(t, err)
+	})
+
+	t.Run("CacheAndPush", func(t *testing.T) {
+		t.Parallel()
+
+		srv := createGitServer(t, gitServerOptions{
+			files: map[string]string{
+				".devcontainer/Dockerfile": fmt.Sprintf("FROM %s\nRUN date --utc > /root/date.txt", testImageAlpine),
+				".devcontainer/devcontainer.json": `{
+				"name": "Test",
+				"build": {
+					"dockerfile": "Dockerfile"
+				},
+			}`,
+			},
+		})
+
+		// Given: an empty registry
+		testReg := setupInMemoryRegistry(t)
+		testRepo := testReg + "/test"
+		ref, err := name.ParseReference(testRepo + ":latest")
+		require.NoError(t, err)
+		_, err = remote.Image(ref)
+		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
+
+		// When: we run envbuilder with GET_CACHED_IMAGE
+		_, err = runEnvbuilder(t, options{env: []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", testRepo),
+			envbuilderEnv("GET_CACHED_IMAGE", "1"),
+		}})
+		require.ErrorContains(t, err, "not supported in fake build")
+		// Then: it should fail to build the image and nothing should be pushed
+		_, err = remote.Image(ref)
+		require.ErrorContains(t, err, "NAME_UNKNOWN", "expected image to not be present before build + push")
+
+		// When: we run envbuilder with PUSH_IMAGE set
+		_, err = runEnvbuilder(t, options{env: []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", testRepo),
+			envbuilderEnv("PUSH_IMAGE", "1"),
+		}})
+		require.NoError(t, err)
+
+		// Then: the image should be pushed
+		_, err = remote.Image(ref)
+		require.NoError(t, err, "expected image to be present after build + push")
+
+		// Then: re-running envbuilder with GET_CACHED_IMAGE should succeed
+		_, err = runEnvbuilder(t, options{env: []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", testRepo),
+			envbuilderEnv("GET_CACHED_IMAGE", "1"),
+		}})
+		require.NoError(t, err)
+	})
+
+	t.Run("PushImageRequiresCache", func(t *testing.T) {
+		t.Parallel()
+
+		srv := createGitServer(t, gitServerOptions{
+			files: map[string]string{
+				".devcontainer/Dockerfile": fmt.Sprintf("FROM %s\nRUN date --utc > /root/date.txt", testImageAlpine),
+				".devcontainer/devcontainer.json": `{
+				"name": "Test",
+				"build": {
+					"dockerfile": "Dockerfile"
+				},
+			}`,
+			},
+		})
+
+		// When: we run envbuilder with PUSH_IMAGE set but no cache repo set
+		_, err := runEnvbuilder(t, options{env: []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("PUSH_IMAGE", "1"),
+		}})
+
+		// Then: Envbuilder should fail explicitly, as it does not make sense to
+		// specify PUSH_IMAGE
+		require.ErrorContains(t, err, "--cache-repo must be set when using --push-image")
+	})
+
+	t.Run("PushErr", func(t *testing.T) {
+		t.Parallel()
+
+		srv := createGitServer(t, gitServerOptions{
+			files: map[string]string{
+				".devcontainer/Dockerfile": fmt.Sprintf("FROM %s\nRUN date --utc > /root/date.txt", testImageAlpine),
+				".devcontainer/devcontainer.json": `{
+				"name": "Test",
+				"build": {
+					"dockerfile": "Dockerfile"
+				},
+			}`,
+			},
+		})
+
+		// Given: registry is not set up (in this case, not a registry)
+		notRegSrv := httptest.NewServer(http.NotFoundHandler())
+		notRegURL := strings.TrimPrefix(notRegSrv.URL, "http://") + "/test"
+
+		// When: we run envbuilder with PUSH_IMAGE set
+		_, err := runEnvbuilder(t, options{env: []string{
+			envbuilderEnv("GIT_URL", srv.URL),
+			envbuilderEnv("CACHE_REPO", notRegURL),
+			envbuilderEnv("PUSH_IMAGE", "1"),
+		}})
+
+		// Then: envbuilder should fail with a descriptive error
+		require.ErrorContains(t, err, "failed to push to destination")
+	})
+}
+
+func setupInMemoryRegistry(t *testing.T) string {
+	t.Helper()
+	tempDir := t.TempDir()
+	testReg := registry.New(registry.WithBlobHandler(registry.NewDiskBlobHandler(tempDir)))
+	regSrv := httptest.NewServer(testReg)
+	t.Cleanup(func() { regSrv.Close() })
+	regSrvURL, err := url.Parse(regSrv.URL)
+	require.NoError(t, err)
+	return fmt.Sprintf("localhost:%s", regSrvURL.Port())
+}
+
 // TestMain runs before all tests to build the envbuilder image.
 func TestMain(m *testing.M) {
 	checkTestRegistry()
@@ -1187,6 +1369,34 @@ func streamContainerLogs(t *testing.T, cli *client.Client, containerID string) (
 	return logChan, errChan
 }
 
+func requireContainerLog(t *testing.T, ctrID, needle string) {
+	t.Helper()
+	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		cli.Close()
+	})
+	ctx := context.Background()
+	rawLogs, err := cli.ContainerLogs(ctx, ctrID, container.LogsOptions{
+		ShowStdout: true,
+		ShowStderr: true,
+		Follow:     true,
+		Timestamps: false,
+	})
+	require.NoError(t, err)
+	defer rawLogs.Close()
+	scanner := bufio.NewScanner(rawLogs)
+	var found bool
+	for scanner.Scan() {
+		haystack := scanner.Text()
+		if strings.Contains(haystack, needle) {
+			found = true
+			break
+		}
+	}
+	require.True(t, found, "did not find expected log %q", needle)
+}
+
 func envbuilderEnv(env string, value string) string {
 	return fmt.Sprintf("%s=%s", envbuilder.WithEnvPrefix(env), value)
 }

options.go

@@ -138,6 +138,13 @@ type Options struct {
 	// CoderAgentSubsystem is the Coder agent subsystems to report when forwarding
 	// logs. The envbuilder subsystem is always included.
 	CoderAgentSubsystem []string
+
+	// PushImage is a flag to determine if the image should be pushed to the
+	// container registry. This option implies reproducible builds.
+	PushImage bool
+	// GetCachedImage is a flag to determine if the cached image is available,
+	// and if it is, to return it.
+	GetCachedImage bool
 }
 
 const envPrefix = "ENVBUILDER_"
@@ -395,6 +402,20 @@ func (o *Options) CLI() serpent.OptionSet {
 			Description: "Coder agent subsystems to report when forwarding logs. " +
 				"The envbuilder subsystem is always included.",
 		},
+		{
+			Flag:  "push-image",
+			Env:   WithEnvPrefix("PUSH_IMAGE"),
+			Value: serpent.BoolOf(&o.PushImage),
+			Description: "Push the built image to a remote registry." +
+				"This option forces a reproducible build.",
+		},
+		{
+			Flag:  "get-cached-image",
+			Env:   WithEnvPrefix("GET_CACHED_IMAGE"),
+			Value: serpent.BoolOf(&o.GetCachedImage),
+			Description: "Print the digest of the cached image, if available." +
+				"Exits with an error if not found.",
+		},
 	}
 
 	// Add options without the prefix for backward compatibility. These options