Skip to content

feat: add support for copying certs to dockerd's registry directory #101

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Sep 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cli/clitest/cli.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func New(t *testing.T, cmd string, args ...string) (context.Context, *cobra.Comm

var (
execer = NewFakeExecer()
fs = NewMemFS()
fs = xunixfake.NewMemFS()
mnt = &mount.FakeMounter{}
client = NewFakeDockerClient()
iface = GetNetLink(t)
Expand Down
8 changes: 0 additions & 8 deletions cli/clitest/fake.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,13 @@ import (
"strings"

dockertypes "github.com/docker/docker/api/types"
"github.com/spf13/afero"
testingexec "k8s.io/utils/exec/testing"

"github.com/coder/envbox/dockerutil"
"github.com/coder/envbox/dockerutil/dockerfake"
"github.com/coder/envbox/xunix/xunixfake"
)

func NewMemFS() *xunixfake.MemFS {
return &xunixfake.MemFS{
MemMapFs: &afero.MemMapFs{},
Owner: map[string]xunixfake.FileOwner{},
}
}

func NewFakeExecer() *xunixfake.FakeExec {
return &xunixfake.FakeExec{
Commands: map[string]*xunixfake.FakeCmd{},
Expand Down
37 changes: 29 additions & 8 deletions cli/docker.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,20 +157,20 @@ func dockerCmd() *cobra.Command {
RunE: func(cmd *cobra.Command, args []string) (err error) {
var (
ctx = cmd.Context()
log = slog.Make(slogjson.Sink(io.Discard))
blog buildlog.Logger = buildlog.NopLogger{}
log = slog.Make(slogjson.Sink(cmd.ErrOrStderr()), slogkubeterminate.Make()).Leveled(slog.LevelDebug)
blog buildlog.Logger = buildlog.JSONLogger{Encoder: json.NewEncoder(os.Stderr)}
)

if flags.noStartupLogs {
log = slog.Make(slogjson.Sink(io.Discard))
blog = buildlog.NopLogger{}
}

httpClient, err := xhttp.Client(log, flags.extraCertsPath)
if err != nil {
return xerrors.Errorf("http client: %w", err)
}

if !flags.noStartupLogs {
log = slog.Make(slogjson.Sink(cmd.ErrOrStderr()), slogkubeterminate.Make()).Leveled(slog.LevelDebug)
blog = buildlog.JSONLogger{Encoder: json.NewEncoder(os.Stderr)}
}

if !flags.noStartupLogs && flags.agentToken != "" && flags.coderURL != "" {
coderURL, err := url.Parse(flags.coderURL)
if err != nil {
Expand All @@ -197,7 +197,6 @@ func dockerCmd() *cobra.Command {
}
}(&err)

blog.Info("Waiting for dockerd to startup...")
sysboxArgs := []string{}
if flags.disableIDMappedMount {
sysboxArgs = append(sysboxArgs, "--disable-idmapped-mount")
Expand Down Expand Up @@ -231,6 +230,7 @@ func dockerCmd() *cobra.Command {

log.Debug(ctx, "starting dockerd", slog.F("args", args))

blog.Info("Waiting for sysbox processes to startup...")
dockerd := background.New(ctx, log, "dockerd", dargs...)
err = dockerd.Start()
if err != nil {
Expand Down Expand Up @@ -289,11 +289,32 @@ func dockerCmd() *cobra.Command {
// We wait for the daemon after spawning the goroutine in case
// startup causes the daemon to encounter encounter a 'no space left
// on device' error.
blog.Info("Waiting for dockerd to startup...")
err = dockerutil.WaitForDaemon(ctx, client)
if err != nil {
return xerrors.Errorf("wait for dockerd: %w", err)
}

if flags.extraCertsPath != "" {
// Parse the registry from the inner image
registry, err := name.ParseReference(flags.innerImage)
if err != nil {
return xerrors.Errorf("invalid image: %w", err)
}
registryName := registry.Context().RegistryStr()

// Write certificates for the registry
err = dockerutil.WriteCertsForRegistry(ctx, registryName, flags.extraCertsPath)
if err != nil {
return xerrors.Errorf("write certs for registry: %w", err)
}

blog.Infof("Successfully copied certificates from %q to %q", flags.extraCertsPath, filepath.Join("/etc/docker/certs.d", registryName))
log.Debug(ctx, "wrote certificates for registry", slog.F("registry", registryName),
slog.F("extra_certs_path", flags.extraCertsPath),
)
}

err = runDockerCVM(ctx, log, client, blog, flags)
if err != nil {
// It's possible we failed because we ran out of disk while
Expand Down
13 changes: 13 additions & 0 deletions dockerutil/image.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"encoding/json"
"fmt"
"io"
"strings"
"time"

dockertypes "github.com/docker/docker/api/types"
Expand Down Expand Up @@ -65,6 +66,14 @@ func PullImage(ctx context.Context, config *PullImageConfig) error {
}

err = pullImageFn()
// We should bail early if we're going to fail due to a
// certificate error. We can't xerrors.As here since this is
// returned from the daemon so the client is reporting
// essentially an unwrapped error.
if isTLSVerificationErr(err) {
return err
}

if err == nil {
return nil
}
Expand Down Expand Up @@ -253,3 +262,7 @@ func DefaultLogImagePullFn(log buildlog.Logger) func(ImagePullEvent) error {
return nil
}
}

func isTLSVerificationErr(err error) bool {
return err != nil && strings.Contains(err.Error(), "tls: failed to verify certificate: x509: certificate signed by unknown authority")
}
92 changes: 92 additions & 0 deletions dockerutil/registry.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
package dockerutil

import (
"context"
"io"
"path/filepath"

"github.com/spf13/afero"
"golang.org/x/xerrors"

"github.com/coder/envbox/xunix"
)

// WriteCertsForRegistry writes the certificates found in the provided directory
// to the correct subdirectory that the Docker daemon uses when pulling images
// from the specified private registry.
func WriteCertsForRegistry(ctx context.Context, registryName, certsDir string) error {
fs := xunix.GetFS(ctx)

// Docker certs directory.
registryCertsDir := filepath.Join("/etc/docker/certs.d", registryName)

// If the directory already exists it means someone
// has either wrapped the image or has mounted in certs
// manually. We should assume the user knows what they're
// doing and avoid mucking with their solution.
if _, err := fs.Stat(registryCertsDir); err == nil {
return nil
}

// Ensure the registry certs directory exists.
err := fs.MkdirAll(registryCertsDir, 0o755)
if err != nil {
return xerrors.Errorf("create registry certs directory: %w", err)
}

// Check if certsDir is a file.
fileInfo, err := fs.Stat(certsDir)
if err != nil {
return xerrors.Errorf("stat certs directory/file: %w", err)
}

if !fileInfo.IsDir() {
// If it's a file, copy it directly
err = copyCertFile(fs, certsDir, filepath.Join(registryCertsDir, "ca.crt"))
if err != nil {
return xerrors.Errorf("copy cert file: %w", err)
}
return nil
}

// If it's a directory, copy all cert files in the root of the directory
entries, err := afero.ReadDir(fs, certsDir)
if err != nil {
return xerrors.Errorf("read certs directory: %w", err)
}

for _, entry := range entries {
if entry.IsDir() {
continue
}
srcPath := filepath.Join(certsDir, entry.Name())
dstPath := filepath.Join(registryCertsDir, entry.Name())
err = copyCertFile(fs, srcPath, dstPath)
if err != nil {
return xerrors.Errorf("copy cert file %s: %w", entry.Name(), err)
}
}

return nil
}

func copyCertFile(fs xunix.FS, src, dst string) error {
srcFile, err := fs.Open(src)
if err != nil {
return xerrors.Errorf("open source file: %w", err)
}
defer srcFile.Close()

dstFile, err := fs.Create(dst)
if err != nil {
return xerrors.Errorf("create destination file: %w", err)
}
defer dstFile.Close()

_, err = io.Copy(dstFile, srcFile)
if err != nil {
return xerrors.Errorf("copy file contents: %w", err)
}

return nil
}
100 changes: 100 additions & 0 deletions dockerutil/registry_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
package dockerutil_test

import (
"context"
"os"
"path/filepath"
"testing"

"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/coder/envbox/dockerutil"
"github.com/coder/envbox/xunix"
"github.com/coder/envbox/xunix/xunixfake"
)

func TestWriteCertsForRegistry(t *testing.T) {
t.Parallel()

t.Run("SingleCertFile", func(t *testing.T) {
t.Parallel()
// Test setup
fs := xunixfake.NewMemFS()
ctx := xunix.WithFS(context.Background(), fs)

// Create a test certificate file
certContent := []byte("test certificate content")
err := afero.WriteFile(fs, "/certs/ca.crt", certContent, 0o644)
require.NoError(t, err)

// Run the function
err = dockerutil.WriteCertsForRegistry(ctx, "test.registry.com", "/certs/ca.crt")
require.NoError(t, err)

// Check the result
copiedContent, err := afero.ReadFile(fs, "/etc/docker/certs.d/test.registry.com/ca.crt")
require.NoError(t, err)
assert.Equal(t, certContent, copiedContent)
})

t.Run("MultipleCertFiles", func(t *testing.T) {
t.Parallel()
// Test setup
fs := xunixfake.NewMemFS()
ctx := xunix.WithFS(context.Background(), fs)

// Create test certificate files
certFiles := []string{"ca.crt", "client.cert", "client.key"}
for _, file := range certFiles {
err := afero.WriteFile(fs, filepath.Join("/certs", file), []byte("content of "+file), 0o644)
require.NoError(t, err)
}

// Run the function
err := dockerutil.WriteCertsForRegistry(ctx, "test.registry.com", "/certs")
require.NoError(t, err)

// Check the results
for _, file := range certFiles {
copiedContent, err := afero.ReadFile(fs, filepath.Join("/etc/docker/certs.d/test.registry.com", file))
require.NoError(t, err)
assert.Equal(t, []byte("content of "+file), copiedContent)
}
})
t.Run("ExistingRegistryCertsDir", func(t *testing.T) {
t.Parallel()
// Test setup
fs := xunixfake.NewMemFS()
ctx := xunix.WithFS(context.Background(), fs)

// Create an existing registry certs directory
registryCertsDir := "/etc/docker/certs.d/test.registry.com"
err := fs.MkdirAll(registryCertsDir, 0o755)
require.NoError(t, err)

// Create a file in the existing directory
existingContent := []byte("existing certificate content")
err = afero.WriteFile(fs, filepath.Join(registryCertsDir, "existing.crt"), existingContent, 0o644)
require.NoError(t, err)

// Create a test certificate file in the source directory
certContent := []byte("new certificate content")
err = afero.WriteFile(fs, "/certs/ca.crt", certContent, 0o644)
require.NoError(t, err)

// Run the function
err = dockerutil.WriteCertsForRegistry(ctx, "test.registry.com", "/certs")
require.NoError(t, err)

// Check that the existing file was not modified
existingFileContent, err := afero.ReadFile(fs, filepath.Join(registryCertsDir, "existing.crt"))
require.NoError(t, err)
assert.Equal(t, existingContent, existingFileContent)

// Check that the new file was not copied
_, err = fs.Stat(filepath.Join(registryCertsDir, "ca.crt"))
assert.True(t, os.IsNotExist(err), "New certificate file should not have been copied")
})
}
Loading
Loading