feat: deploy registry syncer for workload clusters (#1189)

**What problem does this PR solve?**:
This adds a new feature gate `SynchronizeWorkloadClusterRegistry` that
will deploy another
[HCP](mesosphere/charts#1591) against the
management cluster that will setup a Job and a Deployment to sync images
from the management cluster to the workload cluster being depoyed.
HCPs use a local clusterRef so we will always to create the HCP in the
management cluster's namespace but deploy the Pods into the workload
cluster's namespace.

Stacked on
#1175

**Which issue(s) this PR fixes**:
Fixes #

**How Has This Been Tested?**:
<!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
-->
Added unit and integration tests.

Tested manually:
<details>
<summary>1. Create a management cluster and make it
self-managed</summary>

```bash
# Create the bootstrap cluster
make dev.run-on-kind CAREN_FEATURE_GATES="SynchronizeWorkloadClusterRegistry=true"
eval $(make kind.kubeconfig)

# Create the management cluster
export CLUSTER_NAME=dk-agf-syncer      
export CLUSTER_FILE=examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml
export KUBERNETES_VERSION=v1.33.1
clusterctl generate cluster ${CLUSTER_NAME} \
  --from ${CLUSTER_FILE} \
  --kubernetes-version ${KUBERNETES_VERSION} \
  --worker-machine-count 1 | \
  kubectl apply --server-side -f -

# Get the management cluster kubeconfig
kubectl wait clusters/${CLUSTER_NAME} \
  --for=condition=ControlPlaneInitialized \
  --timeout=1m
clusterctl get kubeconfig ${CLUSTER_NAME} > ${CLUSTER_NAME}.conf
kubectl config set-cluster ${CLUSTER_NAME} \
  --kubeconfig ${CLUSTER_NAME}.conf \
  --server=https://$(docker container port ${CLUSTER_NAME}-lb 6443/tcp)
kubectl --kubeconfig ${CLUSTER_NAME}.conf wait nodes \
  --all \
  --for=condition=Ready \
  --timeout=5m

# Deploy CAREN on the management cluster
make dev.run-on-kind \
  CAREN_FEATURE_GATES="SynchronizeWorkloadClusterRegistry=true" \
  SKIP_BUILD=true KIND_KUBECONFIG=$CLUSTER_NAME.conf \
  KIND_CLUSTER_NAME=$CLUSTER_NAME

# Move the management cluster
clusterctl move --to-kubeconfig=$CLUSTER_NAME.conf

# Delete the bootstrap cluster
kind delete cluster -n cluster-api-runtime-extensions-nutanix-dev
```

</details>

<details>
<summary>2. Push a test image into the management cluster's
registry</summary>

```bash
export KUBECONFIG=$CLUSTER_NAME.conf

kubectl port-forward --address=127.0.0.1 --namespace registry-system service/cncf-distribution-registry-docker-registry-headless 5000:5000 &
crane copy alpine:latest 127.0.0.1:5000/library/alpine:agf-test --insecure
```

</details>

<details>
<summary>3. Create a workload cluster in a new namespace</summary>

```bash
export KUBECONFIG=$CLUSTER_NAME.conf

# Create the workload cluster
kubectl create ns wrkld
kubectl label ns wrkld caren.nutanix.com/namespace-sync=true
export CLUSTER_NAME=dk-agf-syncer-wrkld
clusterctl generate cluster ${CLUSTER_NAME} \
  -n wrkld \
  --from ${CLUSTER_FILE} \
  --kubernetes-version ${KUBERNETES_VERSION} \
  --worker-machine-count 1 | \
  kubectl apply --server-side -f -
```

</details>

<details>
<summary>4. Deploy a test workload in the workload cluster</summary>

```bash
clusterctl get kubeconfig ${CLUSTER_NAME} -n wrkld > ${CLUSTER_NAME}.conf
kubectl config set-cluster ${CLUSTER_NAME} \
  --kubeconfig ${CLUSTER_NAME}.conf \
  --server=https://$(docker container port ${CLUSTER_NAME}-lb 6443/tcp)
kubectl run alpine \
  --kubeconfig ${CLUSTER_NAME}.conf \
  --image=alpine:agf-test \
  --restart=Never \
  -it \
  --command -- sh -c "echo hello"
```

</details>

<details>
<summary>5. Clean everything up</summary>

```bash
kind delete clusters -A
```

</details>


**Special notes for your reviewer**:
<!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
-->

Author: dkoshkin

.pre-commit-config.yaml

@@ -153,7 +153,7 @@ repos:
     name: License headers - YAML and Makefiles
     stages: [pre-commit]
     files: (^Makefile|\.(ya?ml|mk))$
-    exclude: ^(internal/test|pkg/handlers/.+/embedded|examples|charts/cluster-api-runtime-extensions-nutanix/(defaultclusterclasses|addons))/.+\.ya?ml|docs/static/helm/index\.yaml|charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml|hack/examples/files/kube-vip.yaml$
+    exclude: ^(internal/test|pkg/handlers/.+/embedded|pkg/handlers/.+/testdata|examples|charts/cluster-api-runtime-extensions-nutanix/(defaultclusterclasses|addons))/.+\.ya?ml|docs/static/helm/index\.yaml|charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml|hack/examples/files/kube-vip.yaml|$
     args:
       - --license-filepath
       - hack/license-header.txt

api/v1alpha1/constants.go

@@ -41,8 +41,15 @@ const (
 
 	ClusterUUIDAnnotationKey = APIGroup + "/cluster-uuid"
 
+	// SkipAutoEnablingWorkloadClusterRegistry is the key of the annotation on the Cluster
+	// used to skip enabling the registry addon on workload cluster.
 	SkipAutoEnablingWorkloadClusterRegistry = APIGroup + "/skip-auto-enabling-workload-cluster-registry"
 
+	// SkipSynchronizingWorkloadClusterRegistry is the key of the annotation on the Cluster
+	// used to skip deploying the components that will sync OCI artifacts from the registry
+	// running on the management cluster to registry running on the workload cluster.
+	SkipSynchronizingWorkloadClusterRegistry = APIGroup + "/skip-synchronizing-workload-cluster-registry"
+
 	// PreflightChecksSkipAnnotationKey is the key of the annotation on the Cluster used to skip preflight checks.
 	PreflightChecksSkipAnnotationKey = "preflight.cluster.caren.nutanix.com/skip"
 

api/variables/getters.go

@@ -0,0 +1,29 @@
+// Copyright 2025 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package variables
+
+import (
+	"fmt"
+
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+
+	carenv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
+)
+
+// RegistryAddon retrieves the RegistryAddon from the cluster's topology variables.
+// Returns nil if the addon is not defined.
+func RegistryAddon(cluster *clusterv1.Cluster) (*carenv1.RegistryAddon, error) {
+	spec, err := UnmarshalClusterConfigVariable(cluster.Spec.Topology.Variables)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal cluster variable: %w", err)
+	}
+	if spec == nil {
+		return nil, nil
+	}
+	if spec.Addons == nil {
+		return nil, nil
+	}
+
+	return spec.Addons.Registry, nil
+}

charts/cluster-api-runtime-extensions-nutanix/README.md

@@ -89,6 +89,8 @@ A Helm chart for cluster-api-runtime-extensions-nutanix
 | hooks.nfd.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-nfd-helm-values-template"` |  |
 | hooks.registry.cncfDistribution.defaultValueTemplateConfigMap.create | bool | `true` |  |
 | hooks.registry.cncfDistribution.defaultValueTemplateConfigMap.name | string | `"default-cncf-distribution-registry-helm-values-template"` |  |
+| hooks.registrySyncer.defaultValueTemplateConfigMap.create | bool | `true` |  |
+| hooks.registrySyncer.defaultValueTemplateConfigMap.name | string | `"default-registry-syncer-helm-values-template"` |  |
 | hooks.serviceLoadBalancer.metalLB.defaultValueTemplateConfigMap.create | bool | `true` |  |
 | hooks.serviceLoadBalancer.metalLB.defaultValueTemplateConfigMap.name | string | `"default-metallb-helm-values-template"` |  |
 | hooks.virtualIP.kubeVip.defaultTemplateConfigMap.create | bool | `true` |  |

charts/cluster-api-runtime-extensions-nutanix/addons/registry-syncer/values-template.yaml

@@ -0,0 +1,86 @@
+initContainers:
+  # The regsync container does not fail when it cannot connect to the destination registry.
+  # In the case when it runs as a Job, it will prematurely exit.
+  # This init container will wait for the destination registry to be ready.
+  - name: wait-for-registry
+    image: ghcr.io/d2iq-labs/kubectl-betterwait:{{ .KubernetesVersion }}
+    args:
+      - --for=condition=Ready
+      - --timeout=-1s # a negative number here means wait forever
+      - --interval=5s # poll every 5 seconds to the resources to be created
+      - --namespace={{ .DestinationRegistryHeadlessServiceNamespace }}
+      - --kubeconfig=/kubeconfig/admin.conf
+      # Ideally we would wait for the Service to be ready, but Kubernetes does not have a condition for that.
+      - pod/{{ .DestinationRegistryAnyPodName }}
+    volumeMounts:
+      - mountPath: /kubeconfig
+        name: kubeconfig
+        readOnly: true
+  - name: port-forward-registry
+    image: ghcr.io/d2iq-labs/kubectl-betterwait:{{ .KubernetesVersion }}
+    command:
+      - /bin/kubectl
+    args:
+      - port-forward
+      - --address=127.0.0.1
+      - --namespace={{ .DestinationRegistryHeadlessServiceNamespace }}
+      - --kubeconfig=/kubeconfig/admin.conf
+      # This will port-forward to a single Pod in the Service.
+      - service/{{ .DestinationRegistryHeadlessServiceName }}
+      - 5000:{{ .DestinationRegistryHeadlessServicePort }}
+    resources:
+      requests:
+        cpu: 25m
+        memory: 32Mi
+      limits:
+        cpu: 100m
+        memory: 50Mi
+    volumeMounts:
+      - mountPath: /kubeconfig
+        name: kubeconfig
+        readOnly: true
+    # Kubernetes will treat this as a Sidecar container
+    # https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/
+    restartPolicy: Always
+
+extraVolumes:
+  - name: kubeconfig
+    secret:
+      items:
+        - key: value
+          path: admin.conf
+      secretName: {{ .CusterName }}-kubeconfig
+  - name: ca-cert
+    secret:
+      secretName: {{ .RegistryCASecretName }}
+
+extraVolumeMounts:
+  # Assume both the source and the target registries have the same CA.
+  # Source registry running in the cluster.
+  - mountPath: /etc/docker/certs.d/{{ .SourceRegistryAddress }}/
+    name: ca-cert
+    readOnly: true
+  # Destination registry running in the remote cluster being port-forwarded.
+  - mountPath: /etc/docker/certs.d/127.0.0.1:5000/
+    name: ca-cert
+    readOnly: true
+
+deployment:
+  config:
+    creds:
+      - registry: {{ .SourceRegistryAddress }}
+        reqPerSec: 1
+    sync:
+      - source: {{ .SourceRegistryAddress }}
+        target: 127.0.0.1:5000
+        type: registry
+        interval: 1m
+
+job:
+  enabled: true
+  config:
+    sync:
+      - source: {{ .SourceRegistryAddress }}
+        target: 127.0.0.1:5000
+        type: registry
+        interval: 1m

charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml

@@ -51,6 +51,10 @@ data:
     ChartName: nutanix-csi-storage
     ChartVersion: 3.3.4
     RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://nutanix.github.io/helm-releases/{{ end }}'
+  registry-syncer: |
+    ChartName: registry-syncer
+    ChartVersion: 0.1.0
+    RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://mesosphere.github.io/charts/staging/{{ end }}'
   snapshot-controller: |
     ChartName: snapshot-controller
     ChartVersion: 4.0.2

charts/cluster-api-runtime-extensions-nutanix/templates/registry-syncer/helm-addon-installation.yaml

@@ -0,0 +1,12 @@
+# Copyright 2025 Nutanix. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+{{- if .Values.hooks.registrySyncer.defaultValueTemplateConfigMap.name }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: '{{ .Values.hooks.registrySyncer.defaultValueTemplateConfigMap.name }}'
+data:
+  values.yaml: |-
+    {{- .Files.Get "addons/registry-syncer/values-template.yaml" | nindent 4 }}
+{{- end -}}

charts/cluster-api-runtime-extensions-nutanix/values.schema.json

@@ -536,6 +536,22 @@
                     },
                     "type": "object"
                 },
+                "registrySyncer": {
+                    "properties": {
+                        "defaultValueTemplateConfigMap": {
+                            "properties": {
+                                "create": {
+                                    "type": "boolean"
+                                },
+                                "name": {
+                                    "type": "string"
+                                }
+                            },
+                            "type": "object"
+                        }
+                    },
+                    "type": "object"
+                },
                 "serviceLoadBalancer": {
                     "properties": {
                         "metalLB": {

charts/cluster-api-runtime-extensions-nutanix/values.yaml

@@ -117,6 +117,10 @@ hooks:
       defaultValueTemplateConfigMap:
         create: true
         name: default-cncf-distribution-registry-helm-values-template
+  registrySyncer:
+    defaultValueTemplateConfigMap:
+      create: true
+      name: default-registry-syncer-helm-values-template
 
 helmAddonsConfigMap: default-helm-addons-config
 

docs/content/addons/registry.md

@@ -47,6 +47,20 @@ annotations:
    caren.nutanix.com/skip-auto-enabling-workload-cluster-registry: "true"
 ```
 
+All images pushed to the management cluster's registry can be automatically synced to the workload cluster's registry.
+To enable this behavior, set the following feature gate on the controller:
+
+```text
+  --feature-gates=SynchronizeWorkloadClusterRegistry=true
+```
+
+It is also possible to disable this behavior by setting the following annotation on the Cluster resource:
+
+```yaml
+annotations:
+   caren.nutanix.com/skip-synchronizing-workload-cluster-registry: "true"
+```
+
 ## Registry Certificate
 
 1. A root CA Certificate is deployed in the provider's namespace.