Implement RemoteNodeSet controllers (#181)

Author: kobzonega

.github/workflows/run-tests.yml

@@ -164,8 +164,8 @@ jobs:
           kind load docker-image k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.0
 
           # TODO would be cool to parse YDB image from manifests to avoid duplicating information
-          docker pull cr.yandex/crptqonuodf51kdj7a7d/ydb:22.4.44
-          kind load docker-image cr.yandex/crptqonuodf51kdj7a7d/ydb:22.4.44
+          docker pull cr.yandex/crptqonuodf51kdj7a7d/ydb:23.3.17
+          kind load docker-image cr.yandex/crptqonuodf51kdj7a7d/ydb:23.3.17
       - name: run-tests
         run: |
           go test -v -timeout 1800s -p 1 ./... -args -ginkgo.v

Makefile

@@ -70,16 +70,21 @@ kind-init:
 	kind create cluster --config e2e/kind-cluster-config.yaml --name kind-ydb-operator; \
 	docker pull k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.0; \
 	kind load docker-image k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.0 --name kind-ydb-operator; \
-	docker pull cr.yandex/crptqonuodf51kdj7a7d/ydb:22.4.44; \
-	kind load docker-image cr.yandex/crptqonuodf51kdj7a7d/ydb:22.4.44 --name kind-ydb-operator
+	docker pull cr.yandex/crptqonuodf51kdj7a7d/ydb:23.3.17; \
+	kind load docker-image cr.yandex/crptqonuodf51kdj7a7d/ydb:23.3.17 --name kind-ydb-operator
 
 kind-load:
 	docker tag cr.yandex/yc/ydb-operator:latest kind/ydb-operator:current
 	kind load docker-image kind/ydb-operator:current --name kind-ydb-operator
 
+unit-test: manifests generate fmt vet envtest ## Run unit tests
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test -v -timeout 1800s -p 1 ./internal/controllers/... -ginkgo.vv -coverprofile cover.out
+
+e2e-test: docker-build kind-init kind-load ## Run e2e tests
+	go test -v -timeout 1800s -p 1 ./e2e/... -args -ginkgo.vv
+
 .PHONY: test
-test: manifests generate fmt vet docker-build kind-init kind-load envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test -timeout 1800s -p 1 ./... -ginkgo.v -coverprofile cover.out
+test: unit-test test ## Run all tests
 
 .PHONY: clean
 clean:

api/v1alpha1/databasenodeset_types.go

@@ -65,3 +65,17 @@ type DatabaseNodeSetList struct {
 func init() {
 	SchemeBuilder.Register(&DatabaseNodeSet{}, &DatabaseNodeSetList{})
 }
+
+func RecastDatabaseNodeSet(databaseNodeSet *DatabaseNodeSet) *Database {
+	return &Database{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      databaseNodeSet.Spec.DatabaseRef.Name,
+			Namespace: databaseNodeSet.Spec.DatabaseRef.Namespace,
+			Labels:    databaseNodeSet.Labels,
+		},
+		Spec: DatabaseSpec{
+			DatabaseClusterSpec: databaseNodeSet.Spec.DatabaseClusterSpec,
+			DatabaseNodeSpec:    databaseNodeSet.Spec.DatabaseNodeSpec,
+		},
+	}
+}

api/v1alpha1/storagenodeset_types.go

@@ -65,3 +65,17 @@ type StorageNodeSetList struct {
 func init() {
 	SchemeBuilder.Register(&StorageNodeSet{}, &StorageNodeSetList{})
 }
+
+func RecastStorageNodeSet(storageNodeSet *StorageNodeSet) *Storage {
+	return &Storage{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      storageNodeSet.Spec.StorageRef.Name,
+			Namespace: storageNodeSet.Spec.StorageRef.Namespace,
+			Labels:    storageNodeSet.Labels,
+		},
+		Spec: StorageSpec{
+			StorageClusterSpec: storageNodeSet.Spec.StorageClusterSpec,
+			StorageNodeSpec:    storageNodeSet.Spec.StorageNodeSpec,
+		},
+	}
+}

cmd/ydb-kubernetes-operator/main.go

@@ -9,14 +9,19 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/client-go/tools/clientcmd"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/cluster"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	ydbv1alpha1 "github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1"
 	"github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/database"
 	"github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/databasenodeset"
 	"github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/monitoring"
+	"github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/remotedatabasenodeset"
+	"github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/remotestoragenodeset"
 	"github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/storage"
 	"github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/storagenodeset"
 )
@@ -39,13 +44,17 @@ func main() {
 	var disableWebhooks bool
 	var enableServiceMonitors bool
 	var probeAddr string
+	var mgmtClusterKubeconfig string
+	var mgmtClusterName string
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
 	flag.BoolVar(&disableWebhooks, "disable-webhooks", false, "Disable webhooks registration on start.")
 	flag.BoolVar(&enableServiceMonitors, "with-service-monitors", false, "Enables service monitoring")
+	flag.StringVar(&mgmtClusterKubeconfig, "mgmt-cluster-kubeconfig", "/mgmt-cluster/kubeconfig", "Path to kubeconfig for mgmt remote k8s cluster. Only required if using Remote objects")
+	flag.StringVar(&mgmtClusterName, "mgmt-cluster-name", "", "The name of mgmt remote cluster to sync k8s resources. Only required if using Remote objects")
 	opts := zap.Options{
 		Development: true,
 	}
@@ -143,6 +152,36 @@ func main() {
 			os.Exit(1)
 		}
 	}
+
+	if mgmtClusterName != "" && mgmtClusterKubeconfig != "" {
+		remoteCluster, err := createRemoteCluster(mgmtClusterName, mgmtClusterKubeconfig)
+		if err != nil {
+			setupLog.Error(err, "unable to create mgmt cluster client")
+			os.Exit(1)
+		}
+
+		if err = mgr.Add(remoteCluster); err != nil {
+			setupLog.Error(err, "unable to add mgmt cluster client to controller manager")
+			os.Exit(1)
+		}
+
+		if err = (&remotestoragenodeset.Reconciler{
+			Client: mgr.GetClient(),
+			Scheme: mgr.GetScheme(),
+		}).SetupWithManager(mgr, &remoteCluster); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "RemoteStorageNodeSet")
+			os.Exit(1)
+		}
+
+		if err = (&remotedatabasenodeset.Reconciler{
+			Client: mgr.GetClient(),
+			Scheme: mgr.GetScheme(),
+		}).SetupWithManager(mgr, &remoteCluster); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "RemoteDatabaseNodeSet")
+			os.Exit(1)
+		}
+	}
+
 	//+kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
@@ -160,3 +199,33 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func createRemoteCluster(mgmtClusterName, mgmtClusterKubeconfig string) (cluster.Cluster, error) {
+	remoteConfig, err := clientcmd.BuildConfigFromFlags("", mgmtClusterKubeconfig)
+	if err != nil {
+		setupLog.Error(err, "unable to read mgmt cluster kubeconfig")
+		return nil, err
+	}
+
+	storageSelector, err := remotestoragenodeset.BuildRemoteSelector(mgmtClusterName)
+	if err != nil {
+		setupLog.Error(err, "unable to create label selector", "selector", "RemoteStorageNodeSet")
+		return nil, err
+	}
+
+	databaseSelector, err := remotedatabasenodeset.BuildRemoteSelector(mgmtClusterName)
+	if err != nil {
+		setupLog.Error(err, "unable to create label selector", "selector", "RemoteDatabaseNodeSet")
+		return nil, err
+	}
+
+	return cluster.New(remoteConfig, func(o *cluster.Options) {
+		o.Scheme = scheme
+		o.NewCache = cache.BuilderWithOptions(cache.Options{
+			SelectorsByObject: cache.SelectorsByObject{
+				&ydbv1alpha1.RemoteStorageNodeSet{}:  {Label: storageSelector},
+				&ydbv1alpha1.RemoteDatabaseNodeSet{}: {Label: databaseSelector},
+			},
+		})
+	})
+}

deploy/ydb-operator/templates/deployment.yaml

@@ -30,6 +30,10 @@ spec:
             {{- if .Values.metrics.enabled }}
             - --with-service-monitors=true
             {{- end }}
+            {{- if .Values.mgmtCluster.enabled }}
+            - --mgmt-cluster-name={{- .Values.mgmtCluster.name }}
+            - --mgmt-cluster-kubeconfig=/mgmt-cluster/kubeconfig
+            {{- end }}
           command:
             - /manager
           image: {{ default "cr.yandex/yc/ydb-kubernetes-operator" .Values.image.repository }}:
@@ -60,17 +64,24 @@ spec:
             {{- toYaml .Values.resources | nindent 12  }}
           securityContext:
             allowPrivilegeEscalation: false
+          {{- if or .Values.webhook.enabled .Values.mgmtCluster.enabled }}
           {{- if .Values.webhook.enabled }}
           volumeMounts:
             - mountPath: /tmp/k8s-webhook-server/serving-certs
               name: webhook-tls
           {{- end }}
+          {{- if .Values.mgmtCluster.enabled }}
+            - mountPath: /mgmt-cluster
+              name: mgmt-cluster-kubeconfig
+          {{- end }}
+          {{- end }}
       securityContext:
         runAsNonRoot: true
       serviceAccountName: {{ include "ydb.fullname" . }}
       terminationGracePeriodSeconds: 10
-      {{- if .Values.webhook.enabled }}
+      {{- if or .Values.webhook.enabled .Values.mgmtCluster.enabled }}
       volumes:
+        {{- if .Values.webhook.enabled }}
         - name: webhook-tls
           secret:
             secretName: {{ include "ydb.fullname" . }}-webhook
@@ -83,6 +94,12 @@ spec:
               - key: key
                 path: tls.key
             {{- end }}
+        {{- end }}
+        {{- if .Values.mgmtCluster.enabled }}
+        - name: mgmt-cluster-kubeconfig
+          secret:
+            secretName: {{ .Values.mgmtCluster.kubeconfig }}
+        {{- end }}
       {{- end }}
       {{- if .Values.imagePullSecrets }}
       {{- with .Values.imagePullSecrets }}

deploy/ydb-operator/values.yaml

@@ -47,6 +47,14 @@ metrics:
   ##
   enabled: false
 
+mgmtCluster:
+  ## Watch resources from mgmtCluster
+  ##
+  enabled: false
+  name: ""
+  ## Define existing kubeconfig Secret name in current namespace
+  kubeconfig: "remote-kubeconfig"
+
 webhook:
   enabled: true
 

docs/tests.md

@@ -88,14 +88,14 @@ kind create cluster \
 kubectl config use-context kind-local-kind
 
 # Within tests, the following two images are used:
-# cr.yandex/crptqonuodf51kdj7a7d/ydb:22.4.44
+# cr.yandex/crptqonuodf51kdj7a7d/ydb:23.3.17
 # kind/ydb-operator:current
 
 # You have to download the ydb image and build the operator image yourself. Then, explicitly
 # upload them into the kind cluster. Refer to `./github/e2e.yaml` github workflow which essentially
 # does the same thing.
 kind --name local-kind load docker-image kind/ydb-operator:current
-kind --name local-kind load docker-image ydb:22.4.44
+kind --name local-kind load docker-image ydb:23.3.17
 
 # Run all tests with disabled concurrency, because there is only one cluster to run tests against
 go test -p 1 -v ./...

e2e/tests/test-objects/objects.go

@@ -11,7 +11,7 @@ import (
 )
 
 const (
-	YdbImage      = "cr.yandex/crptqonuodf51kdj7a7d/ydb:22.4.44"
+	YdbImage      = "cr.yandex/crptqonuodf51kdj7a7d/ydb:23.3.17"
 	YdbNamespace  = "ydb"
 	StorageName   = "storage"
 	DatabaseName  = "database"

internal/annotations/annotations.go

@@ -2,6 +2,14 @@ package annotations
 
 import "strings"
 
+const (
+	PrimaryResourceStorageAnnotation  = "ydb.tech/primary-resource-storage"
+	PrimaryResourceDatabaseAnnotation = "ydb.tech/primary-resource-database"
+	RemoteResourceVersionAnnotation   = "ydb.tech/remote-resource-version"
+	RemoteFinalizerKey                = "ydb.tech/remote-finalizer"
+	LastAppliedAnnotation             = "ydb.tech/last-applied"
+)
+
 func GetYdbTechAnnotations(annotations map[string]string) map[string]string {
 	result := make(map[string]string)
 	for key, value := range annotations {