feat: add tls to internal registry (#630)

* feat: add tls to internal registry

this pr adds tls configuration to the registry add-on. certs are valid
for 1 year and there is no rotation (this will be implemented later).

containerd configuration to access the registry through http was removed.

* chore: bump kots

* use ECO v0.32.0 and remove ttl override

* bump ECO go mod to 0.32.0 too

* Revert "bump ECO go mod to 0.32.0 too"

This reverts commit cc94f26.

* eco 0.32.1

* eco 0.32.2

* label registry-tls secret for DR

---------

Co-authored-by: Craig O'Donnell <craig@replicated.com>

Author: ricardomaraschini

Makefile

@@ -4,12 +4,12 @@ ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_URL = oci://registry.replicated.com/library
 ADMIN_CONSOLE_CHART_NAME = admin-console
-ADMIN_CONSOLE_CHART_VERSION = 1.109.4-build.1
+ADMIN_CONSOLE_CHART_VERSION = 1.109.5
 ADMIN_CONSOLE_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = 0.31.1
+EMBEDDED_OPERATOR_CHART_VERSION = 0.32.2
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
 OPENEBS_CHART_URL = https://openebs.github.io/openebs
@@ -31,6 +31,7 @@ PREVIOUS_K0S_VERSION ?= v1.28.9+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.92.1
 KOTS_VERSION = v$(shell echo $(ADMIN_CONSOLE_CHART_VERSION) | sed 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/')
+KOTS_BINARY_URL_OVERRIDE =
 LD_FLAGS = -X github.com/replicatedhq/embedded-cluster/pkg/defaults.K0sVersion=$(K0S_VERSION) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/defaults.Version=$(VERSION) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/defaults.K0sBinaryURL=$(K0S_BINARY_SOURCE_OVERRIDE) \
@@ -102,7 +103,11 @@ pkg/goods/bins/local-artifact-mirror: Makefile
 pkg/goods/internal/bins/kubectl-kots: Makefile
 	mkdir -p pkg/goods/internal/bins
 	mkdir -p output/tmp/kots
-	curl -L -o output/tmp/kots/kots.tar.gz https://github.com/replicatedhq/kots/releases/download/$(KOTS_VERSION)/kots_linux_amd64.tar.gz
+	if [ "$(KOTS_BINARY_URL_OVERRIDE)" != "" ]; then \
+	    curl -L -o output/tmp/kots/kots.tar.gz "$(KOTS_BINARY_URL_OVERRIDE)" ; \
+	else \
+	    curl -L -o output/tmp/kots/kots.tar.gz https://github.com/replicatedhq/kots/releases/download/$(KOTS_VERSION)/kots_linux_amd64.tar.gz ; \
+	fi
 	tar -xzf output/tmp/kots/kots.tar.gz -C output/tmp/kots
 	mv output/tmp/kots/kots pkg/goods/internal/bins/kubectl-kots
 	touch pkg/goods/internal/bins/kubectl-kots

cmd/local-artifact-mirror/artifact.go

@@ -2,11 +2,14 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"os"
 
 	"github.com/sirupsen/logrus"
+	"go.uber.org/multierr"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
@@ -97,14 +100,32 @@ func pullArtifact(ctx context.Context, from string) (string, error) {
 	}
 	defer fs.Close()
 
-	// setup the pull options. XXX we are using a registry over http.
-	repo.Client = &auth.Client{Credential: store.Get}
-	repo.PlainHTTP = true
+	transp, ok := http.DefaultTransport.(*http.Transport)
+	if !ok {
+		return "", fmt.Errorf("unable to get default transport")
+	}
+
+	transp = transp.Clone()
+	transp.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+	repo.Client = &auth.Client{
+		Client:     &http.Client{Transport: transp},
+		Credential: store.Get,
+	}
 
 	tag := imgref.Reference
+	_, tlserr := oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions)
+	if tlserr == nil {
+		return tmpdir, nil
+	}
+
+	// if we fail to fetch the artifact using https we gonna try once more using plain
+	// http as some versions of the registry were deployed without tls.
+	repo.PlainHTTP = true
+	logrus.Infof("unable to fetch artifact using tls, retrying with http")
 	if _, err := oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions); err != nil {
 		os.RemoveAll(tmpdir)
-		return "", fmt.Errorf("unable to copy: %w", err)
+		err = multierr.Combine(tlserr, err)
+		return "", fmt.Errorf("unable to fetch artifacts with or without tls: %w", err)
 	}
 	return tmpdir, nil
 }

go.mod

@@ -132,7 +132,7 @@ require (
 	github.com/xrash/smetrics v0.0.0-20240312152122-5f08fbb34913 // indirect
 	github.com/zitadel/oidc/v2 v2.7.0 // indirect
 	go.mongodb.org/mongo-driver v1.11.3 // indirect
-	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/net v0.25.0 // indirect
 	golang.org/x/oauth2 v0.18.0 // indirect

pkg/addons/registry/registry.go

@@ -3,23 +3,27 @@ package registry
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
 	"github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 	"golang.org/x/crypto/bcrypt"
 	"gopkg.in/yaml.v2"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/replicatedhq/embedded-cluster/pkg/airgap"
+	"github.com/replicatedhq/embedded-cluster/pkg/certs"
 	"github.com/replicatedhq/embedded-cluster/pkg/helpers"
 	"github.com/replicatedhq/embedded-cluster/pkg/kubeutils"
 	"github.com/replicatedhq/embedded-cluster/pkg/spinner"
 )
 
 const (
-	releaseName = "docker-registry"
+	releaseName   = "docker-registry"
+	tlsSecretName = "registry-tls"
 )
 
 // Overwritten by -ldflags in Makefile
@@ -134,6 +138,10 @@ func (o *Registry) GenerateHelmConfig(onlyDefaults bool) ([]v1beta1.Chart, []v1b
 		"clusterIP": registryServiceIP.String(),
 	}
 
+	if !onlyDefaults {
+		helmValues["tlsSecretName"] = tlsSecretName
+	}
+
 	valuesStringData, err := yaml.Marshal(helmValues)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to marshal helm values: %w", err)
@@ -147,6 +155,34 @@ func (o *Registry) GetAdditionalImages() []string {
 	return nil
 }
 
+func (o *Registry) generateRegistryTLS(ctx context.Context, cli client.Client) (string, string, error) {
+	nsn := types.NamespacedName{Name: "registry", Namespace: o.namespace}
+	var svc corev1.Service
+	if err := cli.Get(ctx, nsn, &svc); err != nil {
+		return "", "", fmt.Errorf("unable to get registry service: %w", err)
+	}
+
+	opts := []certs.Option{
+		certs.WithCommonName("registry"),
+		certs.WithDuration(365 * 24 * time.Hour),
+		certs.WithIPAddress(registryAddress),
+	}
+
+	for _, name := range []string{
+		"registry",
+		fmt.Sprintf("registry.%s.svc", o.namespace),
+		fmt.Sprintf("registry.%s.svc.cluster.local", o.namespace),
+	} {
+		opts = append(opts, certs.WithDNSName(name))
+	}
+
+	builder, err := certs.NewBuilder(opts...)
+	if err != nil {
+		return "", "", fmt.Errorf("failed to create cert builder: %w", err)
+	}
+	return builder.Generate()
+}
+
 // Outro is executed after the cluster deployment.
 func (o *Registry) Outro(ctx context.Context, cli client.Client) error {
 	if !o.isAirgap {
@@ -157,13 +193,13 @@ func (o *Registry) Outro(ctx context.Context, cli client.Client) error {
 	loading.Infof("Waiting for Registry to be ready")
 
 	if err := kubeutils.WaitForNamespace(ctx, cli, o.namespace); err != nil {
-		loading.Close()
+		loading.CloseWithError()
 		return err
 	}
 
 	hashPassword, err := bcrypt.GenerateFromPassword([]byte(registryPassword), bcrypt.DefaultCost)
 	if err != nil {
-		loading.Close()
+		loading.CloseWithError()
 		return fmt.Errorf("unable to hash registry password: %w", err)
 	}
 
@@ -186,27 +222,51 @@ func (o *Registry) Outro(ctx context.Context, cli client.Client) error {
 	}
 	err = cli.Create(ctx, &htpasswd)
 	if err != nil {
-		loading.Close()
+		loading.CloseWithError()
 		return fmt.Errorf("unable to create registry-auth secret: %w", err)
 	}
 
-	if err := kubeutils.WaitForDeployment(ctx, cli, o.namespace, "registry"); err != nil {
-		loading.Close()
-		return err
-	}
 	if err := kubeutils.WaitForService(ctx, cli, o.namespace, "registry"); err != nil {
-		loading.Close()
+		loading.CloseWithError()
 		return err
 	}
 
 	if err := InitRegistryClusterIP(ctx, cli, o.namespace); err != nil {
-		loading.Close()
+		loading.CloseWithError()
 		return fmt.Errorf("failed to determine registry cluster IP: %w", err)
 	}
 
+	tlsCert, tlsKey, err := o.generateRegistryTLS(ctx, cli)
+	if err != nil {
+		loading.CloseWithError()
+		return fmt.Errorf("unable to generate registry tls: %w", err)
+	}
+
+	tlsSecret := &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      tlsSecretName,
+			Namespace: o.namespace,
+			Labels: map[string]string{
+				"app": "docker-registry", // this is the backup/restore label for the registry component
+			},
+		},
+		StringData: map[string]string{"tls.crt": tlsCert, "tls.key": tlsKey},
+		Type:       "Opaque",
+	}
+	if err := cli.Create(ctx, tlsSecret); err != nil {
+		loading.CloseWithError()
+		return fmt.Errorf("unable to create %s secret: %w", tlsSecretName, err)
+	}
+
+	if err := kubeutils.WaitForDeployment(ctx, cli, o.namespace, "registry"); err != nil {
+		loading.CloseWithError()
+		return err
+	}
+
 	if err := airgap.AddInsecureRegistry(fmt.Sprintf("%s:5000", registryAddress)); err != nil {
-		loading.Close()
-		return fmt.Errorf("failed to add insecure registry: %w", err)
+		loading.CloseWithError()
+		return fmt.Errorf("unable to add containerd registry config: %w", err)
 	}
 
 	loading.Closef("Registry is ready!")

pkg/airgap/containerd.go

@@ -10,9 +10,6 @@ import (
 
 const registryConfigTemplate = `
 [plugins."io.containerd.grpc.v1.cri".registry]
-  [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
-    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."%s"]
-      endpoint = ["http://%s"]
   [plugins."io.containerd.grpc.v1.cri".registry.configs]
     [plugins."io.containerd.grpc.v1.cri".registry.configs."%s".tls]
       insecure_skip_verify = true
@@ -22,7 +19,7 @@ const registryConfigTemplate = `
 // are allowed to be accessed over HTTP.
 func AddInsecureRegistry(registry string) error {
 	parentDir := defaults.PathToK0sContainerdConfig()
-	contents := fmt.Sprintf(registryConfigTemplate, registry, registry, registry)
+	contents := fmt.Sprintf(registryConfigTemplate, registry)
 
 	if err := os.MkdirAll(parentDir, 0755); err != nil {
 		return fmt.Errorf("failed to ensure containerd directory exists: %w", err)

pkg/certs/certs.go

@@ -0,0 +1,129 @@
+package certs
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"math/big"
+	"net"
+	"time"
+)
+
+// NewBuilder returns a new certificate builder.
+func NewBuilder(opts ...Option) (*Builder, error) {
+	builder := &Builder{
+		organizations: []string{"replicated"},
+		expiration:    time.Now().Add(time.Hour * 24 * 365),
+		commonName:    "localhost",
+		dnsNames:      []string{"localhost"},
+		ipAddresses:   []net.IP{net.ParseIP("127.0.0.1")},
+	}
+	for _, opt := range opts {
+		if err := opt(builder); err != nil {
+			return nil, fmt.Errorf("unable to apply option: %w", err)
+		}
+	}
+	return builder, nil
+}
+
+// Builder is a helper to generate self signed certificates.
+type Builder struct {
+	organizations []string
+	expiration    time.Time
+	commonName    string
+	dnsNames      []string
+	ipAddresses   []net.IP
+	signBy        *SignCA
+}
+
+// SignCA holds a path to a key and a certificate we use to sign the new certificate.
+type SignCA struct {
+	crt []byte
+	key []byte
+}
+
+// parse reads and parses the CA certificate and key from disk.
+func (s *SignCA) parse() (*x509.Certificate, *rsa.PrivateKey, error) {
+	caCertBlock, _ := pem.Decode(s.crt)
+	if caCertBlock == nil {
+		return nil, nil, fmt.Errorf("unable to decode certificate PEM")
+	}
+	caCert, err := x509.ParseCertificate(caCertBlock.Bytes)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	caKeyBlock, _ := pem.Decode(s.key)
+	if caKeyBlock == nil {
+		return nil, nil, fmt.Errorf("unable to decode key PEM")
+	}
+	caKey, err := x509.ParsePKCS1PrivateKey(caKeyBlock.Bytes)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return caCert, caKey, nil
+}
+
+// generatePair creates a new key/crt pair.
+func (b *Builder) Generate() (string, string, error) {
+	pkey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return "", "", fmt.Errorf("unable to generate private key: %w", err)
+	}
+
+	kusage := x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature
+	tpl := x509.Certificate{
+		SerialNumber:          big.NewInt(time.Now().UnixNano()),
+		NotBefore:             time.Now(),
+		NotAfter:              b.expiration,
+		KeyUsage:              kusage,
+		BasicConstraintsValid: true,
+		IPAddresses:           b.ipAddresses,
+		DNSNames:              b.dnsNames,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageAny},
+		Subject: pkix.Name{
+			Organization: b.organizations,
+			CommonName:   b.commonName,
+		},
+	}
+
+	cacert := &tpl
+	cakey := pkey
+	if b.signBy != nil {
+		cacert, cakey, err = b.signBy.parse()
+		if err != nil {
+			return "", "", fmt.Errorf("unable to parse signer ca: %w", err)
+		}
+	}
+
+	cert, err := x509.CreateCertificate(rand.Reader, &tpl, cacert, &pkey.PublicKey, cakey)
+	if err != nil {
+		return "", "", fmt.Errorf("unable to create certificate: %w", err)
+	}
+
+	crtbuf := bytes.NewBuffer(nil)
+	if err := pem.Encode(
+		crtbuf, &pem.Block{Type: "CERTIFICATE", Bytes: cert},
+	); err != nil {
+		return "", "", fmt.Errorf("unable to encode certificate: %w", err)
+	}
+
+	pbytes, err := x509.MarshalPKCS8PrivateKey(pkey)
+	if err != nil {
+		return "", "", fmt.Errorf("unable to marshal private key: %w", err)
+	}
+
+	keybuf := bytes.NewBuffer(nil)
+	if err := pem.Encode(
+		keybuf, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: pbytes},
+	); err != nil {
+		return "", "", fmt.Errorf("unable to encode private key: %w", err)
+	}
+
+	return crtbuf.String(), keybuf.String(), nil
+}