feat: generate a self-signed cert for registry addon (#1127)

**What problem does this PR solve?**:
This PR adds the functionality to set a self-signed certificate for the
registry addon.
A new `Certificate` deployed by the Helm chart generates a 10 year CA
Secret `registry-addon-root-ca` in the namespace where this controller
is running.

The `ca.crt` data is copied into a Cluster specific Secret (with an
OwnerRef set) in a `BeforeClusterCreate` hook and is then used by the
mirror handler.

This CA is then used to sign a new 2 year TLS certificate for every
cluster that is copied to the remote cluster
`registry-system/registry-tls` and used by the registry Pods. A new cert
is generated and copied on every invocation of the handler, on
`BeforeClusterUpgrade` and `AfterControlPlaneInitialized` lifecycle
hooks, so that we can punt on auto renewal for now.

**Which issue(s) this PR fixes**:
Fixes #

**How Has This Been Tested?**:
<!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
-->
1. Tested manually by create a Docker cluster and verifying I can push
an image into the registry, observe it get synced across the replicas,
and then get pulled with Containerd.

```
$ kubectl get secret -n registry-system
NAME                                                TYPE                 DATA   AGE
cncf-distribution-registry-docker-registry-secret   Opaque               3      5h16m
registry-tls                                        kubernetes.io/tls    3      5h16m
sh.helm.release.v1.cncf-distribution-registry.v1    helm.sh/release.v1   1      5h16m


$ kubectl port-forward --address=127.0.0.1 --namespace registry-system pod/cncf-distribution-registry-docker-registry-0 5000:5000
$ crane copy nginx:stable 127.0.0.1:5000/library/nginx:dkoshkin --insecure
$ kubectl run nginx-working --image=docker.io/library/nginx:dkoshkin
$ kubectl get pods
NAME                                                              READY   STATUS              RESTARTS   AGE
cluster-autoscaler-0196db54-35b0-73fd-ad5b-14f998751820-7bzwnm9   0/1     ContainerCreating   0          5h15m
nginx-working                                                     1/1     Running             0          53m
```

2. New integration tests.
3. Existing e2e tests already wait for the STS Pods to be Ready which
won't happen unless the TLS Secret is there and is valid.

**Special notes for your reviewer**:
<!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
-->
I've considered other approaches here:
1. Using cert-manager to generate a unique CA per cluster. This makes it
more difficult for clients to trust the registries when pushing images
to multiple clusters.
2. Use cert-manager to also generate the TLS certificate, it got pretty
complicated to trigger a renewal and needing to wait for it to reconcile
the cert Secret. Using cert-manager was also not providing much value as
we want better control when the certs get rotated -
118b39e.
3. Use the CAPI generated CA cert to sign a new cert, but decided
against using an externally managed CA for a separate usecase.

Author: dkoshkin

charts/cluster-api-runtime-extensions-nutanix/addons/registry/cncf-distribution/values-template.yaml

@@ -1,12 +1,13 @@
-replicaCount: 2
+replicaCount: {{ .Replicas }}
 persistence:
   enabled: true
   size: 50Gi
 service:
   type: ClusterIP
   clusterIP: {{ .ServiceIP }}
-  port: 80
+  port: 443
 statefulSet:
   enabled: true
   syncer:
     interval: 2m
+tlsSecretName: {{ .TLSSecretName }}

charts/cluster-api-runtime-extensions-nutanix/templates/certificates.yaml

@@ -32,3 +32,20 @@ spec:
     kind: {{ .Values.certificates.issuer.kind }}
     name: {{ template "chart.issuerName" . }}
   secretName: {{ template "chart.name" . }}-admission-tls
+---
+# CA used to sign certificates for the clusters' registry addons
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: registry-addon-root-ca
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "chart.labels" . | nindent 4 }}
+spec:
+  isCA: true
+  commonName: registry-addon
+  secretName: registry-addon-root-ca
+  issuerRef:
+    kind: {{ .Values.certificates.issuer.kind }}
+    name: {{ template "chart.issuerName" . }}
+  duration: 87600h  # 10 years

charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml

@@ -25,7 +25,7 @@ data:
     RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes.github.io/autoscaler{{ end }}'
   cncf-distribution-registry: |
     ChartName: docker-registry
-    ChartVersion: 2.3.1
+    ChartVersion: 2.3.2
     RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://mesosphere.github.io/charts/staging/{{ end }}'
   cosi-controller: |
     ChartName: cosi

docs/content/addons/registry-certificate.png

@@ -31,6 +31,24 @@ spec:
             registry: {}
 ```
 
+## Registry Certificate
+
+1. A root CA Certificate is deployed in the provider's namespace.
+2. cert-manager generates a 10-year self-signed root Certificate
+   and creates a Secret `registry-addon-root-ca` in the provider's namespace.
+3. BCC handler copies `ca.crt` from the `registry-addon-root-ca` Secret
+   to a new cluster Secret `<cluster-name>-registry-addon-ca`.
+   A client pushing to the registry can use either the root CA Secret or the cluster Secret to trust the registry.
+4. The cluster CA Secret contents (`ca.crt`) is written out as files on the Nodes
+   and used by Containerd to trust the registry addon.
+5. During the initial cluster creation, the ACPI handler uses the root CA to create a new 2-year server certificate
+   for the registry and creates a Secret `registry-tls` on the remote cluster.
+6. During cluster upgrades, the BCU handler renews the server certificate
+   and updates the Secret `registry-tls` on the remote cluster with the new certificate.
+   It is expected that clusters will be upgraded at least once every 2 years to avoid certificate expiration.
+
+![registry-certificate.png](registry-certificate.png)
+
 [Distribution]: https://github.com/distribution/distribution
 [Cluster API Add-on Provider for Helm]: https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm
 [Regsync]: https://regclient.org/usage/regsync/

docs/content/addons/registry.md

@@ -35,7 +35,7 @@ repositories:
     repoURL: https://mesosphere.github.io/charts/staging/
     charts:
       docker-registry:
-      - 2.3.1
+      - 2.3.2
   local-path-provisioner:
     repoURL: https://charts.containeroo.ch
     charts:

hack/addons/helm-chart-bundler/repos.yaml

@@ -18,7 +18,7 @@ helmCharts:
 - name: docker-registry
   repo: https://mesosphere.github.io/charts/staging/
   releaseName: cncf-distribution-registry
-  version: 2.3.1
+  version: 2.3.2
   valuesFile: helm-values.yaml
   includeCRDs: true
   skipTests: true

hack/addons/kustomize/cncf-distribution-registry/kustomization.yaml.tmpl

@@ -24,6 +24,11 @@ import (
 const (
 	DefaultHelmReleaseName      = "cncf-distribution-registry"
 	DefaultHelmReleaseNamespace = "registry-system"
+
+	stsName                = "cncf-distribution-registry-docker-registry"
+	stsHeadlessServiceName = "cncf-distribution-registry-docker-registry-headless"
+	stsReplicas            = 2
+	tlsSecretName          = "registry-tls"
 )
 
 type Config struct {
@@ -59,14 +64,64 @@ func New(
 	}
 }
 
+// Setup ensures any pre-requisites for the CNCF Distribution registry addon are met.
+// It is expected to be called before the cluster is created.
+// Specifically, it ensures that the CA secret for the registry is created in the cluster's namespace.
+func (n *CNCFDistribution) Setup(
+	ctx context.Context,
+	_ v1alpha1.RegistryAddon,
+	cluster *clusterv1.Cluster,
+	log logr.Logger,
+) error {
+	log.Info("Setting up CA for CNCF Distribution registry")
+	err := utils.EnsureCASecretForCluster(
+		ctx,
+		n.client,
+		cluster,
+	)
+	if err != nil {
+		return fmt.Errorf("failed to ensure CA secret for CNCF Distribution registry addon: %w", err)
+	}
+	return nil
+}
+
+// Apply applies the CNCF Distribution registry addon to the cluster.
 func (n *CNCFDistribution) Apply(
 	ctx context.Context,
 	_ v1alpha1.RegistryAddon,
 	cluster *clusterv1.Cluster,
 	log logr.Logger,
 ) error {
-	log.Info("Applying CNCF Distribution registry installation")
+	// Copy the TLS secret to the remote cluster.
+	serviceIP, err := utils.ServiceIPForCluster(cluster)
+	if err != nil {
+		return fmt.Errorf("error getting service IP for the CNCF distribution registry: %w", err)
+	}
+	opts := &utils.EnsureCertificateOpts{
+		RemoteSecretKey: ctrlclient.ObjectKey{
+			Name:      tlsSecretName,
+			Namespace: DefaultHelmReleaseNamespace,
+		},
+		Spec: utils.CertificateSpec{
+			CommonName:  stsName,
+			DNSNames:    certificateDNSNames(),
+			IPAddresses: certificateIPAddresses(serviceIP),
+		},
+	}
+	err = utils.EnsureRegistryServerCertificateSecretOnRemoteCluster(
+		ctx,
+		n.client,
+		cluster,
+		opts,
+	)
+	if err != nil {
+		return fmt.Errorf(
+			"failed to copy certificate secret for CNCF Distribution registry addon to remote cluster: %w",
+			err,
+		)
+	}
 
+	log.Info("Applying CNCF Distribution registry installation")
 	helmChartInfo, err := n.helmChartInfoGetter.For(ctx, log, config.CNCFDistributionRegistry)
 	if err != nil {
 		return fmt.Errorf("failed to get CNCF Distribution registry helm chart: %w", err)
@@ -101,11 +156,15 @@ func templateValues(cluster *clusterv1.Cluster, text string) (string, error) {
 	}
 
 	type input struct {
-		ServiceIP string
+		ServiceIP     string
+		Replicas      int32
+		TLSSecretName string
 	}
 
 	templateInput := input{
-		ServiceIP: serviceIP,
+		Replicas:      stsReplicas,
+		ServiceIP:     serviceIP,
+		TLSSecretName: tlsSecretName,
 	}
 
 	var b bytes.Buffer
@@ -119,3 +178,34 @@ func templateValues(cluster *clusterv1.Cluster, text string) (string, error) {
 
 	return b.String(), nil
 }
+
+func certificateDNSNames() []string {
+	names := []string{
+		stsName,
+		fmt.Sprintf("%s.%s", stsName, DefaultHelmReleaseNamespace),
+		fmt.Sprintf("%s.%s.svc", stsName, DefaultHelmReleaseNamespace),
+		fmt.Sprintf("%s.%s.svc.cluster.local", stsName, DefaultHelmReleaseNamespace),
+	}
+	for i := 0; i < stsReplicas; i++ {
+		names = append(names,
+			[]string{
+				fmt.Sprintf("%s-%d", stsName, i),
+				fmt.Sprintf("%s-%d.%s.%s", stsName, i, stsHeadlessServiceName, DefaultHelmReleaseNamespace),
+				fmt.Sprintf("%s-%d.%s.%s.svc", stsName, i, stsHeadlessServiceName, DefaultHelmReleaseNamespace),
+				fmt.Sprintf(
+					"%s-%d.%s.%s.svc.cluster.local",
+					stsName, i, stsHeadlessServiceName, DefaultHelmReleaseNamespace,
+				),
+			}...,
+		)
+	}
+
+	return names
+}
+
+func certificateIPAddresses(serviceIP string) []string {
+	return []string{
+		serviceIP,
+		"127.0.0.1",
+	}
+}

pkg/handlers/generic/lifecycle/registry/cncfdistribution/handler.go

@@ -0,0 +1,29 @@
+// Copyright 2025 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package cncfdistribution
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_certificateDNSNames(t *testing.T) {
+	//nolint:lll // Keep long lines for readability.
+	expected := []string{
+		"cncf-distribution-registry-docker-registry",
+		"cncf-distribution-registry-docker-registry.registry-system",
+		"cncf-distribution-registry-docker-registry.registry-system.svc",
+		"cncf-distribution-registry-docker-registry.registry-system.svc.cluster.local",
+		"cncf-distribution-registry-docker-registry-0",
+		"cncf-distribution-registry-docker-registry-0.cncf-distribution-registry-docker-registry-headless.registry-system",
+		"cncf-distribution-registry-docker-registry-0.cncf-distribution-registry-docker-registry-headless.registry-system.svc",
+		"cncf-distribution-registry-docker-registry-0.cncf-distribution-registry-docker-registry-headless.registry-system.svc.cluster.local",
+		"cncf-distribution-registry-docker-registry-1",
+		"cncf-distribution-registry-docker-registry-1.cncf-distribution-registry-docker-registry-headless.registry-system",
+		"cncf-distribution-registry-docker-registry-1.cncf-distribution-registry-docker-registry-headless.registry-system.svc",
+		"cncf-distribution-registry-docker-registry-1.cncf-distribution-registry-docker-registry-headless.registry-system.svc.cluster.local",
+	}
+	assert.Equal(t, expected, certificateDNSNames())
+}

pkg/handlers/generic/lifecycle/registry/cncfdistribution/handler_test.go

@@ -20,6 +20,12 @@ import (
 )
 
 type RegistryProvider interface {
+	Setup(
+		ctx context.Context,
+		registryVar v1alpha1.RegistryAddon,
+		cluster *clusterv1.Cluster,
+		log logr.Logger,
+	) error
 	Apply(
 		ctx context.Context,
 		registryVar v1alpha1.RegistryAddon,
@@ -37,6 +43,7 @@ type RegistryHandler struct {
 
 var (
 	_ commonhandlers.Named                   = &RegistryHandler{}
+	_ lifecycle.BeforeClusterCreate          = &RegistryHandler{}
 	_ lifecycle.AfterControlPlaneInitialized = &RegistryHandler{}
 	_ lifecycle.BeforeClusterUpgrade         = &RegistryHandler{}
 )
@@ -57,6 +64,17 @@ func (r *RegistryHandler) Name() string {
 	return "RegistryHandler"
 }
 
+func (r *RegistryHandler) BeforeClusterCreate(
+	ctx context.Context,
+	req *runtimehooksv1.BeforeClusterCreateRequest,
+	resp *runtimehooksv1.BeforeClusterCreateResponse,
+) {
+	commonResponse := &runtimehooksv1.CommonResponse{}
+	r.setup(ctx, &req.Cluster, commonResponse)
+	resp.Status = commonResponse.GetStatus()
+	resp.Message = commonResponse.GetMessage()
+}
+
 func (r *RegistryHandler) AfterControlPlaneInitialized(
 	ctx context.Context,
 	req *runtimehooksv1.AfterControlPlaneInitializedRequest,
@@ -79,6 +97,91 @@ func (r *RegistryHandler) BeforeClusterUpgrade(
 	resp.Message = commonResponse.GetMessage()
 }
 
+func (r *RegistryHandler) setup(
+	ctx context.Context,
+	cluster *clusterv1.Cluster,
+	resp *runtimehooksv1.CommonResponse,
+) {
+	clusterKey := ctrlclient.ObjectKeyFromObject(cluster)
+
+	log := ctrl.LoggerFrom(ctx).WithValues(
+		"cluster",
+		clusterKey,
+	)
+
+	varMap := variables.ClusterVariablesToVariablesMap(cluster.Spec.Topology.Variables)
+	registryVar, err := variables.Get[v1alpha1.RegistryAddon](
+		varMap,
+		r.variableName,
+		r.variablePath...)
+	if err != nil {
+		if variables.IsNotFoundError(err) {
+			log.V(5).
+				Info(
+					"Skipping RegistryAddon, field is not specified",
+					"error",
+					err,
+				)
+			return
+		}
+		log.Error(
+			err,
+			"failed to read RegistryAddon provider from cluster definition",
+		)
+		resp.SetStatus(runtimehooksv1.ResponseStatusFailure)
+		resp.SetMessage(
+			fmt.Sprintf("failed to read RegistryAddon provider from cluster definition: %v",
+				err,
+			),
+		)
+		return
+	}
+
+	handler, ok := r.ProviderHandler[registryVar.Provider]
+	if !ok {
+		err = fmt.Errorf("unknown RegistryAddon Provider")
+		log.Error(err, "provider", registryVar.Provider)
+		resp.SetStatus(runtimehooksv1.ResponseStatusFailure)
+		resp.SetMessage(
+			fmt.Sprintf("%s %s", err, registryVar.Provider),
+		)
+		return
+	}
+
+	log.Info(fmt.Sprintf("Setting up RegistryAddon provider prerequisites %s", registryVar.Provider))
+	err = handler.Setup(
+		ctx,
+		registryVar,
+		cluster,
+		log,
+	)
+	if err != nil {
+		log.Error(
+			err,
+			fmt.Sprintf(
+				"failed to set up RegistryAddon provider prerequisites %s",
+				registryVar.Provider,
+			),
+		)
+		resp.SetStatus(runtimehooksv1.ResponseStatusFailure)
+		resp.SetMessage(
+			fmt.Sprintf(
+				"failed to set up RegistryAddon provider prerequisites: %v",
+				err,
+			),
+		)
+		return
+	}
+
+	resp.SetStatus(runtimehooksv1.ResponseStatusSuccess)
+	resp.SetMessage(
+		fmt.Sprintf(
+			"Set up RegistryAddon provider prerequisites %s",
+			registryVar.Provider,
+		),
+	)
+}
+
 func (r *RegistryHandler) apply(
 	ctx context.Context,
 	cluster *clusterv1.Cluster,