Skip to content

feat: use registryMirror addon as Containerd mirror #1117

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
May 16, 2025
Merged

feat: use registryMirror addon as Containerd mirror #1117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e934240
feat: use registryMirror addon as Containerd mirror
dkoshkin May 2, 2025
d92fa8b
fixup! fix: remove unused function args
dkoshkin May 7, 2025
e8b1a8d
fixup! after rebase
dkoshkin May 8, 2025
0eb39bf
fixup! tests: add more unit tests
dkoshkin May 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 47 additions & 4 deletions pkg/handlers/generic/mutation/mirrors/inject.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
Expand All @@ -22,6 +23,7 @@ import (
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches"
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors"
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables"
registryutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/registry/utils"
handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils"
)

Expand Down Expand Up @@ -62,8 +64,8 @@ func (h *globalMirrorPatchHandler) Mutate(
obj *unstructured.Unstructured,
vars map[string]apiextensionsv1.JSON,
holderRef runtimehooksv1.HolderReference,
clusterKey ctrlclient.ObjectKey,
_ mutation.ClusterGetter,
_ ctrlclient.ObjectKey,
clusterGetter mutation.ClusterGetter,
) error {
log := ctrl.LoggerFrom(ctx).WithValues(
"holderRef", holderRef,
Expand All @@ -82,14 +84,24 @@ func (h *globalMirrorPatchHandler) Mutate(
v1alpha1.ImageRegistriesVariableName,
)

_, registryAddonErr := variables.Get[v1alpha1.RegistryAddon](
vars,
v1alpha1.ClusterConfigVariableName,
[]string{"addons", v1alpha1.RegistryAddonVariableName}...)

switch {
case variables.IsNotFoundError(imageRegistriesErr) && variables.IsNotFoundError(globalMirrorErr):
log.V(5).Info("Image Registry Credentials and Global Registry Mirror variable not defined")
case variables.IsNotFoundError(imageRegistriesErr) &&
variables.IsNotFoundError(globalMirrorErr) &&
variables.IsNotFoundError(registryAddonErr):
log.V(5).
Info("Image Registry Credentials and Global Registry Mirror and Registry Addon variable not defined")
return nil
case imageRegistriesErr != nil && !variables.IsNotFoundError(imageRegistriesErr):
return imageRegistriesErr
case globalMirrorErr != nil && !variables.IsNotFoundError(globalMirrorErr):
return globalMirrorErr
case registryAddonErr != nil && !variables.IsNotFoundError(registryAddonErr):
return registryAddonErr
}

var registriesWithOptionalCA []containerdConfig //nolint:prealloc // We don't know the size of the slice yet.
Expand Down Expand Up @@ -121,6 +133,22 @@ func (h *globalMirrorPatchHandler) Mutate(
registryWithOptionalCredentials,
)
}
if registryAddonErr == nil {
cluster, err := clusterGetter(ctx)
if err != nil {
log.Error(
err,
"failed to get cluster from Global Mirror mutation handler",
)
return err
}

registryConfig, err := containerdConfigFromRegistryAddon(cluster)
if err != nil {
return err
}
registriesWithOptionalCA = append(registriesWithOptionalCA, registryConfig)
}

needConfiguration := needContainerdConfiguration(
registriesWithOptionalCA,
Expand Down Expand Up @@ -234,6 +262,21 @@ func containerdConfigFromImageRegistry(
return configWithOptionalCACert, nil
}

func containerdConfigFromRegistryAddon(cluster *clusterv1.Cluster) (containerdConfig, error) {
serviceIP, err := registryutils.ServiceIPForCluster(cluster)
if err != nil {
return containerdConfig{}, fmt.Errorf("error getting service IP for the registry addon: %w", err)
}

config := containerdConfig{
// FIXME: Generate a self-signed CA.
URL: fmt.Sprintf("http://%s", serviceIP),
Mirror: true,
}

return config, nil
}

func generateFiles(
registriesWithOptionalCA []containerdConfig,
) ([]bootstrapv1.File, error) {
Expand Down
162 changes: 159 additions & 3 deletions pkg/handlers/generic/mutation/mirrors/inject_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,20 @@
package mirrors

import (
"fmt"
"testing"

. "github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/storage/names"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"

"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
Expand All @@ -32,6 +38,10 @@ func TestMirrorsPatch(t *testing.T) {
}

var _ = Describe("Generate Global mirror patches", func() {
clientScheme := runtime.NewScheme()
utilruntime.Must(clientgoscheme.AddToScheme(clientScheme))
utilruntime.Must(clusterv1.AddToScheme(clientScheme))

patchGenerator := func() mutation.GeneratePatches {
// Always initialize the testEnv variable in the closure.
// This will allow ginkgo to initialize testEnv variable during test execution time.
Expand All @@ -40,7 +50,7 @@ var _ = Describe("Generate Global mirror patches", func() {
// that are written by the tests.
// Test cases writes credentials secret that the mutator handler reads.
// Using direct client will enable reading it immediately.
client, err := testEnv.GetK8sClient()
client, err := testEnv.GetK8sClientWithScheme(clientScheme)
gomega.Expect(err).To(gomega.BeNil())
return mutation.NewMetaGeneratePatchesHandler("", client, NewPatch(client)).(mutation.GeneratePatches)
}
Expand Down Expand Up @@ -330,11 +340,69 @@ var _ = Describe("Generate Global mirror patches", func() {
},
},
},
{
Name: "files added in KubeadmControlPlaneTemplate for registry addon",
Vars: []runtimehooksv1.Variable{
capitest.VariableWithValue(
v1alpha1.ClusterConfigVariableName,
v1alpha1.RegistryAddon{},
[]string{"addons", v1alpha1.RegistryAddonVariableName}...,
),
},
RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
{
Operation: "add",
Path: "/spec/template/spec/kubeadmConfigSpec/files",
ValueMatcher: gomega.HaveExactElements(
gomega.HaveKeyWithValue(
"path", "/etc/containerd/certs.d/_default/hosts.toml",
),
gomega.HaveKeyWithValue(
"path", "/etc/caren/containerd/patches/registry-config.toml",
),
),
},
},
},
{
Name: "files added in KubeadmConfigTemplate for registry addon",
Vars: []runtimehooksv1.Variable{
capitest.VariableWithValue(
v1alpha1.ClusterConfigVariableName,
v1alpha1.RegistryAddon{},
[]string{"addons", v1alpha1.RegistryAddonVariableName}...,
),
capitest.VariableWithValue(
"builtin",
map[string]any{
"machineDeployment": map[string]any{
"class": names.SimpleNameGenerator.GenerateName("worker-"),
},
},
),
},
RequestItem: request.NewKubeadmConfigTemplateRequestItem(""),
ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
{
Operation: "add",
Path: "/spec/template/spec/files",
ValueMatcher: gomega.HaveExactElements(
gomega.HaveKeyWithValue(
"path", "/etc/containerd/certs.d/_default/hosts.toml",
),
gomega.HaveKeyWithValue(
"path", "/etc/caren/containerd/patches/registry-config.toml",
),
),
},
},
},
}

// Create credentials secret before each test
BeforeEach(func(ctx SpecContext) {
client, err := helpers.TestEnv.GetK8sClient()
client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme)
gomega.Expect(err).To(gomega.BeNil())
gomega.Expect(client.Create(
ctx,
Expand All @@ -344,11 +412,28 @@ var _ = Describe("Generate Global mirror patches", func() {
ctx,
newMirrorSecretWithoutCA(validMirrorNoCASecretName, request.Namespace),
)).To(gomega.BeNil())

gomega.Expect(client.Create(
ctx,
&clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: request.ClusterName,
Namespace: request.Namespace,
},
Spec: clusterv1.ClusterSpec{
ClusterNetwork: &clusterv1.ClusterNetwork{
Services: &clusterv1.NetworkRanges{
CIDRBlocks: []string{"192.168.0.1/16"},
},
},
},
},
)).To(gomega.BeNil())
})

// Delete credentials secret after each test
AfterEach(func(ctx SpecContext) {
client, err := helpers.TestEnv.GetK8sClient()
client, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme)
gomega.Expect(err).To(gomega.BeNil())
gomega.Expect(client.Delete(
ctx,
Expand All @@ -358,6 +443,16 @@ var _ = Describe("Generate Global mirror patches", func() {
ctx,
newMirrorSecretWithoutCA(validMirrorNoCASecretName, request.Namespace),
)).To(gomega.BeNil())

gomega.Expect(client.Delete(
ctx,
&clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: request.ClusterName,
Namespace: request.Namespace,
},
},
)).To(gomega.BeNil())
})

// create test node for each case
Expand Down Expand Up @@ -406,6 +501,67 @@ func newMirrorSecretWithoutCA(name, namespace string) *corev1.Secret {
}
}

func Test_containerdConfigFromRegistryAddon(t *testing.T) {
t.Parallel()
tests := []struct {
name string
cluster *clusterv1.Cluster
want containerdConfig
wantErr error
}{
{
name: "valid input",
cluster: &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: request.ClusterName,
Namespace: request.Namespace,
},
Spec: clusterv1.ClusterSpec{
ClusterNetwork: &clusterv1.ClusterNetwork{
Services: &clusterv1.NetworkRanges{
CIDRBlocks: []string{"192.168.0.1/16"},
},
},
},
},
want: containerdConfig{
URL: "http://192.168.0.20",
Mirror: true,
},
},
{
name: "missing Services CIDR",
cluster: &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: request.ClusterName,
Namespace: request.Namespace,
},
Spec: clusterv1.ClusterSpec{
ClusterNetwork: &clusterv1.ClusterNetwork{},
},
},
wantErr: fmt.Errorf(
"error getting service IP for the registry addon: " +
"error getting a service IP for a cluster: " +
"unexpected empty service Subnets",
),
},
}
for idx := range tests {
tt := tests[idx]
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got, err := containerdConfigFromRegistryAddon(tt.cluster)
if tt.wantErr != nil {
require.EqualError(t, err, tt.wantErr.Error())
} else {
require.NoError(t, err)
}
assert.Equal(t, tt.want, got)
})
}
}

func Test_needContainerdConfiguration(t *testing.T) {
t.Parallel()
tests := []struct {
Expand Down
Loading