diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 35fc5a64d..030698269 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -140,11 +140,16 @@ jobs: - name: E2E Remote Tests shell: bash - run: make e2e-test + run: | + make undeploy + make uninstall-crds + make e2e-test - name: Helm Chart Tests shell: bash run: | + make undeploy + make uninstall-crds make e2e-helm-test - name: Upload Manifests diff --git a/.github/workflows/istio-tests.yaml b/.github/workflows/istio-tests.yaml index 5cb83eec3..599927a4a 100644 --- a/.github/workflows/istio-tests.yaml +++ b/.github/workflows/istio-tests.yaml @@ -105,7 +105,6 @@ jobs: sudo echo nameserver 8.8.8.8 > /run/systemd/resolve/stub-resolv.conf - name: Start KinD Cluster -# Start a KinD K8s cluster with single worker node shell: bash run: | echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin @@ -119,7 +118,6 @@ jobs: make all - name: Load Images to KinD -# Load the images just built to the KinD cluster shell: bash run: | make kind-load @@ -130,8 +128,9 @@ jobs: - name: Istio Tests shell: bash run: | - make deploy + make reset-namespace ISTIO_VERSION=${{ matrix.istioVersion }} make install-istio + make deploy make e2e-client-test make e2e-test make undeploy diff --git a/Makefile b/Makefile index 63efdc828..20c784c86 100644 --- a/Makefile +++ b/Makefile @@ -732,13 +732,25 @@ $(BUILD_TARGETS)/java: $(JAVA_FILES) .PHONY: helm-chart helm-chart: $(BUILD_PROPS) $(BUILD_HELM)/coherence-operator-$(VERSION).tgz ## Build the Coherence Operator Helm chart +CRD_TEMPLATE := $(BUILD_HELM)/coherence-operator/templates/crd.yaml $(BUILD_HELM)/coherence-operator-$(VERSION).tgz: $(BUILD_PROPS) $(HELM_FILES) $(BUILD_TARGETS)/generate $(BUILD_TARGETS)/manifests $(TOOLS_BIN)/kustomize # Copy the Helm chart from the source location to the distribution folder - -mkdir -p $(BUILD_HELM) + -mkdir -p $(BUILD_HELM)/temp cp -R ./helm-charts/coherence-operator $(BUILD_HELM) + $(KUSTOMIZE) build $(BUILD_DEPLOY)/overlays/helm -o $(BUILD_HELM)/temp + rm $(CRD_TEMPLATE) || true + echo "{{- if (eq .Values.installCrd true) }}" > $(CRD_TEMPLATE) + cat $(BUILD_HELM)/temp/apiextensions.k8s.io_v1_customresourcedefinition_coherence.coherence.oracle.com.yaml >> $(CRD_TEMPLATE) + printf "\n{{- if (eq .Values.allowCoherenceJobs true) }}\n" >> $(CRD_TEMPLATE) + echo "---" >> $(CRD_TEMPLATE) + cat $(BUILD_HELM)/temp/apiextensions.k8s.io_v1_customresourcedefinition_coherencejob.coherence.oracle.com.yaml >> $(CRD_TEMPLATE) + echo "" >> $(CRD_TEMPLATE) + echo "{{- end }}" >> $(CRD_TEMPLATE) + echo "{{- end }}" >> $(CRD_TEMPLATE) $(call replaceprop,$(BUILD_HELM)/coherence-operator/Chart.yaml $(BUILD_HELM)/coherence-operator/values.yaml $(BUILD_HELM)/coherence-operator/templates/deployment.yaml $(BUILD_HELM)/coherence-operator/templates/rbac.yaml) helm lint $(BUILD_HELM)/coherence-operator helm package $(BUILD_HELM)/coherence-operator --destination $(BUILD_HELM) + rm -rf $(BUILD_HELM)/temp # --------------------------------------------------------------------------- # Do a search and replace of properties in selected files in the Helm charts. diff --git a/api/v1/coherence_types.go b/api/v1/coherence_types.go index 8f0215635..3876788c0 100644 --- a/api/v1/coherence_types.go +++ b/api/v1/coherence_types.go @@ -880,6 +880,11 @@ type CoherenceUtilsSpec struct { // More info: https://kubernetes.io/docs/concepts/containers/images#updating-images // +optional ImagePullPolicy *corev1.PullPolicy `json:"imagePullPolicy,omitempty"` + // Image is used to set the utils image used in Coherence Pods. + // + // Deprecated: This field is deprecated and no longer used, any value set will be ignored. + // +optional + Image *string `json:"image,omitempty"` } // EnsureImage ensures that the image value is set. diff --git a/api/v1/coherenceresource_utils.go b/api/v1/coherenceresource_utils.go deleted file mode 100644 index fb73cd19a..000000000 --- a/api/v1/coherenceresource_utils.go +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright (c) 2020, 2025, Oracle and/or its affiliates. - * Licensed under the Universal Permissive License v 1.0 as shown at - * http://oss.oracle.com/licenses/upl. - */ - -package v1 - -import ( - "context" - "fmt" - "github.com/ghodss/yaml" - "github.com/go-logr/logr" - "github.com/oracle/coherence-operator/pkg/data" - "github.com/oracle/coherence-operator/pkg/operator" - "github.com/pkg/errors" - "golang.org/x/mod/semver" - "io" - crdv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/version" - "sigs.k8s.io/controller-runtime/pkg/client" - logf "sigs.k8s.io/controller-runtime/pkg/log" -) - -// Utility and helper functions for the Coherence API - -// EnsureCRDs ensures that the Operator configuration secret exists in the namespace. -// CRDs will be created depending on the server version of k8s. -func EnsureCRDs(ctx context.Context, v *version.Version, scheme *runtime.Scheme, cl client.Client) error { - logger := logf.Log.WithName("operator") - logger.Info(fmt.Sprintf("Ensuring operator CRDs are present (K8s version %v)", v)) - return EnsureV1CRDs(ctx, logger, scheme, cl) -} - -// EnsureV1CRDs ensures that the Operator configuration secret exists in the namespace. -func EnsureV1CRDs(ctx context.Context, logger logr.Logger, scheme *runtime.Scheme, cl client.Client) error { - err := ensureV1CRDs(ctx, logger, scheme, cl, "apiextensions.k8s.io_v1_customresourcedefinition_coherence.coherence.oracle.com.yaml") - if err != nil { - return err - } - if operator.ShouldInstallJobCRD() { - return ensureV1CRDs(ctx, logger, scheme, cl, "apiextensions.k8s.io_v1_customresourcedefinition_coherencejob.coherence.oracle.com.yaml") - } - return nil -} - -// ensureV1CRDs ensures that the specified V1 CRDs are loaded using the specified embedded CRD files -func ensureV1CRDs(ctx context.Context, logger logr.Logger, scheme *runtime.Scheme, cl client.Client, fileNames ...string) error { - if err := crdv1.AddToScheme(scheme); err != nil { - return err - } - for _, fileName := range fileNames { - if err := ensureV1CRD(ctx, logger, cl, fileName); err != nil { - return err - } - } - return nil -} - -// ensureV1CRD ensures that the specified V1 CRD is loaded using the specified embedded CRD file -func ensureV1CRD(ctx context.Context, logger logr.Logger, cl client.Client, fileName string) error { - f, err := data.Assets.Open("assets/" + fileName) - if err != nil { - return errors.Wrap(err, "opening embedded CRD asset "+fileName) - } - //goland:noinspection GoUnhandledErrorResult - defer f.Close() - - yml, err := io.ReadAll(f) - if err != nil { - return errors.Wrap(err, "reading embedded CRD asset "+fileName) - } - - u := unstructured.Unstructured{} - err = yaml.Unmarshal(yml, &u) - if err != nil { - return err - } - - oldCRD := crdv1.CustomResourceDefinition{} - newCRD := crdv1.CustomResourceDefinition{} - err = yaml.Unmarshal(yml, &newCRD) - if err != nil { - return err - } - - logger.Info("Loading operator CRD yaml from '" + fileName + "'") - - // Get the existing CRD - err = cl.Get(ctx, client.ObjectKey{Name: newCRD.Name}, &oldCRD) - switch { - case err == nil: - // CRD exists so update it - logger.Info("Updating operator CRD '" + newCRD.Name + "'") - newCRD.ResourceVersion = oldCRD.ResourceVersion - err = cl.Update(ctx, &newCRD) - if err != nil { - return errors.Wrapf(err, "updating Coherence CRD %s", newCRD.Name) - } - case apierrors.IsNotFound(err): - // CRD does not exist so create it - logger.Info("Creating operator CRD '" + newCRD.Name + "'") - err = cl.Create(ctx, &newCRD) - if err != nil { - return errors.Wrapf(err, "creating Coherence CRD %s", newCRD.Name) - } - default: - // An error occurred - logger.Error(err, "checking for existing Coherence CRD "+newCRD.Name) - return errors.Wrapf(err, "checking for existing Coherence CRD %s", newCRD.Name) - } - - return nil -} - -// EnsureVersionForAllCRDs ensures that the required CRD version is present -func EnsureVersionForAllCRDs(ctx context.Context, scheme *runtime.Scheme, cl client.Client, required string) error { - if required == "" { - required = operator.GetVersion() - } - if err := ensureVersionForCRD(ctx, scheme, cl, "coherence.coherence.oracle.com", required); err != nil { - return err - } - if err := ensureVersionForCRD(ctx, scheme, cl, "coherencejob.coherence.oracle.com", required); err != nil { - return err - } - return nil -} - -// ensureCRDVersion ensures that the required CRD version is present -func ensureVersionForCRD(ctx context.Context, scheme *runtime.Scheme, cl client.Client, name, required string) error { - label := "app.kubernetes.io/version" - if err := crdv1.AddToScheme(scheme); err != nil { - return err - } - crd := crdv1.CustomResourceDefinition{} - if err := cl.Get(ctx, client.ObjectKey{Name: name}, &crd); err != nil { - return err - } - v, found := crd.GetLabels()[label] - if !found { - return errors.New("cannot verify CRD version, label " + name + " not found in CRD" + name) - } - if v[0] != 'v' { - v = "v" + v - } - if required[0] != 'v' { - required = "v" + required - } - if semver.Compare(v, required) < 0 { - return errors.New("crd " + name + " is version " + v + " but must be a minimum of " + required) - } - return nil -} diff --git a/config/components/helm/kustomization.yaml b/config/components/helm/kustomization.yaml new file mode 100644 index 000000000..1f470defe --- /dev/null +++ b/config/components/helm/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +labels: + - pairs: + control-plane: coherence + app.kubernetes.io/name: coherence-operator + app.kubernetes.io/version: "3.5.0" + app.kubernetes.io/part-of: coherence-operator diff --git a/config/components/no-coherencejob/no-jobs-patch.yaml b/config/components/no-coherencejob/no-jobs-patch.yaml index 29b507ea5..e51c50ee0 100644 --- a/config/components/no-coherencejob/no-jobs-patch.yaml +++ b/config/components/no-coherencejob/no-jobs-patch.yaml @@ -1,4 +1,4 @@ - op: add path: /spec/template/spec/containers/0/args/- value: - - --install-job-crd=false + - --enable-jobs=false diff --git a/config/components/restricted/single-namespace-patch.yaml b/config/components/restricted/single-namespace-patch.yaml index 3a95dab28..25ae37ed2 100644 --- a/config/components/restricted/single-namespace-patch.yaml +++ b/config/components/restricted/single-namespace-patch.yaml @@ -12,9 +12,6 @@ - op: add path: /spec/template/spec/containers/0/args/- value: --enable-webhook=false -- op: add - path: /spec/template/spec/containers/0/args/- - value: --install-crd=false - op: add path: /spec/template/spec/containers/0/args/- value: --node-lookup-enabled=false diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index c53d1c5ab..a421dc941 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -61,7 +61,6 @@ spec: args: - operator - --enable-leader-election - - --install-crd=false envFrom: - configMapRef: name: env-vars diff --git a/config/overlays/helm/kustomization.yaml b/config/overlays/helm/kustomization.yaml new file mode 100644 index 000000000..ffa31fbc9 --- /dev/null +++ b/config/overlays/helm/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../crd-small + +components: + - ../../components/helm diff --git a/config/rbac/cluster_role.yaml b/config/rbac/cluster_role.yaml index 3eafd1858..18e002f19 100644 --- a/config/rbac/cluster_role.yaml +++ b/config/rbac/cluster_role.yaml @@ -18,10 +18,7 @@ rules: resources: - customresourcedefinitions verbs: - - create - - delete - get - - update - apiGroups: - admissionregistration.k8s.io resources: diff --git a/controllers/coherence_controller.go b/controllers/coherence_controller.go index 2e002ac71..873748179 100644 --- a/controllers/coherence_controller.go +++ b/controllers/coherence_controller.go @@ -218,7 +218,7 @@ func (in *CoherenceReconciler) Reconcile(ctx context.Context, request ctrl.Reque } // ensure that the state store exists - storage, err := utils.NewStorage(request.NamespacedName, in.GetManager()) + storage, err := utils.NewStorage(request.NamespacedName, in.GetManager(), in.GetPatcher()) if err != nil { err = errors.Wrap(err, "obtaining desired state store") in.GetEventRecorder().Event(deployment, coreV1.EventTypeWarning, reconciler.EventReasonFailed, @@ -235,7 +235,7 @@ func (in *CoherenceReconciler) Reconcile(ctx context.Context, request ctrl.Reque if hash == storeHash && deployment.IsBeforeOrSameVersion("3.4.3") { deployment.UpdateStatusVersion(operator.GetVersion()) - if err = storage.ResetHash(deployment); err != nil { + if err = storage.ResetHash(ctx, deployment); err != nil { return result, errors.Wrap(err, "error updating storage status hash") } hashNew := deployment.GetGenerationString() @@ -261,7 +261,7 @@ func (in *CoherenceReconciler) Reconcile(ctx context.Context, request ctrl.Reque desiredResources.SetHashLabelAndAnnotations(hash) // update the store to have the desired state as the latest state. - if err = storage.Store(desiredResources, deployment); err != nil { + if err = storage.Store(ctx, desiredResources, deployment); err != nil { err = errors.Wrap(err, "storing latest state in state store") return reconcile.Result{}, err } @@ -323,7 +323,7 @@ func (in *CoherenceReconciler) SetupWithManager(mgr ctrl.Manager, cs clients.Cli in.reconcilers = reconcilers in.SetCommonReconciler(controllerName, mgr, cs) - in.SetPatchType(types.MergePatchType) + in.GetPatcher().SetPatchType(types.MergePatchType) template := &coh.Coherence{} diff --git a/controllers/coherencejob_controller.go b/controllers/coherencejob_controller.go index fbd7be360..1a872e75b 100644 --- a/controllers/coherencejob_controller.go +++ b/controllers/coherencejob_controller.go @@ -147,7 +147,7 @@ func (in *CoherenceJobReconciler) ReconcileDeployment(ctx context.Context, reque } // ensure that the state store exists - storage, err := utils.NewStorage(request.NamespacedName, in.GetManager()) + storage, err := utils.NewStorage(request.NamespacedName, in.GetManager(), in.GetPatcher()) if err != nil { err = errors.Wrap(err, "obtaining desired state store") return in.HandleErrAndRequeue(ctx, err, nil, fmt.Sprintf(reconcileFailedMessage, request.Name, request.Namespace, err), in.Log) @@ -162,7 +162,7 @@ func (in *CoherenceJobReconciler) ReconcileDeployment(ctx context.Context, reque if hash == storeHash && deployment.IsBeforeOrSameVersion("3.4.3") { deployment.UpdateStatusVersion(operator.GetVersion()) - if err = storage.ResetHash(deployment); err != nil { + if err = storage.ResetHash(ctx, deployment); err != nil { return result, errors.Wrap(err, "error updating storage status hash") } hashNew := deployment.GetGenerationString() @@ -193,7 +193,7 @@ func (in *CoherenceJobReconciler) ReconcileDeployment(ctx context.Context, reque desiredResources.SetHashLabelAndAnnotations(hash) // update the store to have the desired state as the latest state. - if err = storage.Store(desiredResources, deployment); err != nil { + if err = storage.Store(ctx, desiredResources, deployment); err != nil { err = errors.Wrap(err, "storing latest state in state store") return reconcile.Result{}, err } @@ -252,7 +252,7 @@ func (in *CoherenceJobReconciler) SetupWithManager(mgr ctrl.Manager, cs clients. in.reconcilers = reconcilers in.SetCommonReconciler(jobControllerName, mgr, cs) - in.SetPatchType(types.MergePatchType) + in.GetPatcher().SetPatchType(types.MergePatchType) template := &coh.CoherenceJob{} diff --git a/controllers/job/job_controller.go b/controllers/job/job_controller.go index 7ffbec07f..5733b5ad6 100644 --- a/controllers/job/job_controller.go +++ b/controllers/job/job_controller.go @@ -13,6 +13,7 @@ import ( coh "github.com/oracle/coherence-operator/api/v1" "github.com/oracle/coherence-operator/controllers/reconciler" "github.com/oracle/coherence-operator/pkg/clients" + "github.com/oracle/coherence-operator/pkg/patching" "github.com/oracle/coherence-operator/pkg/probe" "github.com/oracle/coherence-operator/pkg/utils" "github.com/pkg/errors" @@ -72,7 +73,7 @@ func (in *ReconcileJob) Reconcile(ctx context.Context, request reconcile.Request // Make sure that the request is unlocked when this method exits defer in.Unlock(request) - storage, err := utils.NewStorage(request.NamespacedName, in.GetManager()) + storage, err := utils.NewStorage(request.NamespacedName, in.GetManager(), in.GetPatcher()) if err != nil { return reconcile.Result{}, err } @@ -305,7 +306,7 @@ func (in *ReconcileJob) patchJob(ctx context.Context, deployment coh.CoherenceRe // fix the CreationTimestamp so that it is not in the patch desired.SetCreationTimestamp(current.GetCreationTimestamp()) // create the patch to see whether there is anything to update - patch, data, err := in.CreateThreeWayPatch(current.GetName(), original, desired, current, reconciler.PatchIgnore) + patch, data, err := in.CreateThreeWayPatch(current.GetName(), original, desired, current, patching.PatchIgnore) if err != nil { return reconcile.Result{}, errors.Wrapf(err, "failed to create patch for Job/%s", current.GetName()) } diff --git a/controllers/reconciler/base_controller.go b/controllers/reconciler/base_controller.go index 7259ee6f4..fce3673ec 100644 --- a/controllers/reconciler/base_controller.go +++ b/controllers/reconciler/base_controller.go @@ -8,12 +8,12 @@ package reconciler import ( "context" - "encoding/json" "fmt" "github.com/go-logr/logr" coh "github.com/oracle/coherence-operator/api/v1" "github.com/oracle/coherence-operator/pkg/clients" "github.com/oracle/coherence-operator/pkg/operator" + "github.com/oracle/coherence-operator/pkg/patching" "github.com/oracle/coherence-operator/pkg/utils" "github.com/pkg/errors" "golang.org/x/mod/semver" @@ -23,10 +23,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/strategicpatch" - "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -39,9 +36,6 @@ import ( //goland:noinspection GoUnusedConst const ( - // PatchIgnore - If the patch json is this we can skip the patch. - PatchIgnore = "{\"metadata\":{\"creationTimestamp\":null},\"status\":{\"replicas\":0}}" - // EventReasonCreated is the reason description for a created event. EventReasonCreated string = "Created" // EventReasonUpdated is the reason description for an updated event. @@ -73,7 +67,7 @@ type BaseReconciler interface { GetEventRecorder() record.EventRecorder GetLog() logr.Logger GetReconciler() reconcile.Reconciler - SetPatchType(patchType types.PatchType) + GetPatcher() patching.ResourcePatcher } // CommonReconciler is a base controller structure. @@ -84,7 +78,7 @@ type CommonReconciler struct { locks map[types.NamespacedName]bool mutex *sync.Mutex logger logr.Logger - patchType types.PatchType + patcher patching.ResourcePatcher } func (in *CommonReconciler) GetControllerName() string { return in.name } @@ -92,22 +86,24 @@ func (in *CommonReconciler) GetManager() manager.Manager { return in.mgr } func (in *CommonReconciler) GetClient() client.Client { return in.mgr.GetClient() } func (in *CommonReconciler) GetClientSet() clients.ClientSet { return in.clientSet } func (in *CommonReconciler) GetMutex() *sync.Mutex { return in.mutex } -func (in *CommonReconciler) GetPatchType() types.PatchType { return in.patchType } -func (in *CommonReconciler) SetPatchType(pt types.PatchType) { in.patchType = pt } func (in *CommonReconciler) GetEventRecorder() record.EventRecorder { return in.mgr.GetEventRecorderFor(in.name) } func (in *CommonReconciler) GetLog() logr.Logger { return in.logger } +func (in *CommonReconciler) GetPatcher() patching.ResourcePatcher { + return in.patcher +} func (in *CommonReconciler) SetCommonReconciler(name string, mgr manager.Manager, cs clients.ClientSet) { + logger := logf.Log.WithName(name) in.name = name in.mgr = mgr in.clientSet = cs in.mutex = commonMutex - in.logger = logf.Log.WithName(name) - in.patchType = types.StrategicMergePatchType + in.logger = logger + in.patcher = patching.NewResourcePatcher(mgr, logger, types.StrategicMergePatchType) } // Lock attempts to lock the requested resource. @@ -317,6 +313,8 @@ func (in *CommonReconciler) IsVersionAnnotationEqualOrBefore(m metav1.Object, ve } // CanCreate determines whether any specified start quorum has been met. +// +//goland:noinspection GoDfaConstantCondition func (in *CommonReconciler) CanCreate(ctx context.Context, deployment coh.CoherenceResource) (bool, string) { spec := deployment.GetSpec() if len(spec.StartQuorum) == 0 { @@ -370,7 +368,7 @@ func (in *CommonReconciler) CanCreate(ctx context.Context, deployment coh.Cohere // TwoWayPatch performs a two-way merge patch on the resource. func (in *CommonReconciler) TwoWayPatch(ctx context.Context, name string, current, desired client.Object) (bool, error) { - patch, err := in.CreateTwoWayPatch(name, desired, current, PatchIgnore) + patch, err := in.CreateTwoWayPatch(name, desired, current, patching.PatchIgnore) if err != nil { kind := current.GetObjectKind().GroupVersionKind().Kind return false, errors.Wrapf(err, "failed to create patch for %s/%s", kind, name) @@ -392,148 +390,37 @@ func (in *CommonReconciler) TwoWayPatch(ctx context.Context, name string, curren // CreateTwoWayPatch creates a two-way patch between the original state, the current state and the desired state of a k8s resource. func (in *CommonReconciler) CreateTwoWayPatch(name string, desired, current runtime.Object, ignore ...string) (client.Patch, error) { - return in.CreateTwoWayPatchOfType(in.patchType, name, desired, current, ignore...) + return in.patcher.CreateTwoWayPatch(name, desired, current, ignore...) } // CreateTwoWayPatchOfType creates a two-way patch between the current state and the desired state of a k8s resource. func (in *CommonReconciler) CreateTwoWayPatchOfType(patchType types.PatchType, name string, desired, current runtime.Object, ignore ...string) (client.Patch, error) { - currentData, err := json.Marshal(current) - if err != nil { - return nil, errors.Wrap(err, "serializing current configuration") - } - desiredData, err := json.Marshal(desired) - if err != nil { - return nil, errors.Wrap(err, "serializing desired configuration") - } - - // Get a versioned object - versionedObject := in.asVersioned(desired) - - patchMeta, err := strategicpatch.NewPatchMetaFromStruct(versionedObject) - if err != nil { - return nil, errors.Wrap(err, "unable to create patch metadata from object") - } - - data, err := strategicpatch.CreateTwoWayMergePatchUsingLookupPatchMeta(currentData, desiredData, patchMeta) - if err != nil { - return nil, errors.Wrap(err, "creating three-way patch") - } - - // check whether the patch counts as no-patch - ignore = append(ignore, "{}") - for _, s := range ignore { - if string(data) == s { - // empty patch - return nil, err - } - } - - // log the patch - kind := current.GetObjectKind().GroupVersionKind().Kind - - in.GetLog().V(2).Info(fmt.Sprintf("Created patch for %s/%s\n%s", kind, name, string(data))) - - return client.RawPatch(patchType, data), nil + return in.patcher.CreateTwoWayPatchOfType(patchType, name, desired, current, ignore...) } // ThreeWayPatch performs a three-way merge patch on the resource returning true if a patch was required otherwise false. func (in *CommonReconciler) ThreeWayPatch(ctx context.Context, name string, current, original, desired client.Object) (bool, error) { - return in.ThreeWayPatchWithCallback(ctx, name, current, original, desired, nil) + return in.patcher.ThreeWayPatch(ctx, name, current, original, desired) } // ThreeWayPatchWithCallback performs a three-way merge patch on the resource returning true if a patch was required otherwise false. func (in *CommonReconciler) ThreeWayPatchWithCallback(ctx context.Context, name string, current, original, desired client.Object, callback func()) (bool, error) { - kind := current.GetObjectKind().GroupVersionKind().Kind - // fix the CreationTimestamp so that it is not in the patch - desired.(metav1.Object).SetCreationTimestamp(current.(metav1.Object).GetCreationTimestamp()) - // create the patch - patch, data, err := in.CreateThreeWayPatch(name, original, desired, current, PatchIgnore) - if err != nil { - return false, errors.Wrapf(err, "failed to create patch for %s/%s", kind, name) - } - - if patch == nil { - // nothing to patch so just return - return false, nil - } - - return in.ApplyThreeWayPatchWithCallback(ctx, name, current, patch, data, callback) + return in.patcher.ThreeWayPatchWithCallback(ctx, name, current, original, desired, callback) } // ApplyThreeWayPatchWithCallback performs a three-way merge patch on the resource returning true if a patch was required otherwise false. func (in *CommonReconciler) ApplyThreeWayPatchWithCallback(ctx context.Context, name string, current client.Object, patch client.Patch, data []byte, callback func()) (bool, error) { - kind := current.GetObjectKind().GroupVersionKind().Kind - - // execute any callback - if callback != nil { - callback() - } - - in.GetLog().WithValues().Info(fmt.Sprintf("Patching %s/%s", kind, name), "Patch", string(data)) - err := in.GetManager().GetClient().Patch(ctx, current, patch) - if err != nil { - return false, errors.Wrapf(err, "failed to patch %s/%s with %s", kind, name, string(data)) - } - - return true, nil + return in.patcher.ApplyThreeWayPatchWithCallback(ctx, name, current, patch, data, callback) } // CreateThreeWayPatch creates a three-way patch between the original state, the current state and the desired state of a k8s resource. func (in *CommonReconciler) CreateThreeWayPatch(name string, original, desired, current runtime.Object, ignore ...string) (client.Patch, []byte, error) { - data, err := in.CreateThreeWayPatchData(original, desired, current) - if err != nil { - return nil, data, errors.Wrap(err, "creating three-way patch") - } - - // check whether the patch counts as no-patch - ignore = append(ignore, "{}") - for _, s := range ignore { - if string(data) == s { - // empty patch - return nil, data, err - } - } - - // log the patch - kind := current.GetObjectKind().GroupVersionKind().Kind - in.GetLog().Info(fmt.Sprintf("Created patch for %s/%s", kind, name), "Patch", string(data)) - - return client.RawPatch(in.patchType, data), data, nil + return in.patcher.CreateThreeWayPatch(name, original, desired, current, ignore...) } // CreateThreeWayPatchData creates a three-way patch between the original state, the current state and the desired state of a k8s resource. func (in *CommonReconciler) CreateThreeWayPatchData(original, desired, current runtime.Object) ([]byte, error) { - originalData, err := json.Marshal(original) - if err != nil { - return nil, errors.Wrap(err, "serializing original configuration") - } - currentData, err := json.Marshal(current) - if err != nil { - return nil, errors.Wrap(err, "serializing current configuration") - } - desiredData, err := json.Marshal(desired) - if err != nil { - return nil, errors.Wrap(err, "serializing desired configuration") - } - - // Get a versioned object - versionedObject := in.asVersioned(desired) - - patchMeta, err := strategicpatch.NewPatchMetaFromStruct(versionedObject) - if err != nil { - return nil, errors.Wrap(err, "unable to create patch metadata from object") - } - - return strategicpatch.CreateThreeWayMergePatch(originalData, desiredData, currentData, patchMeta, true) -} - -// asVersioned converts the given object into a runtime.Template with the correct group and version set. -func (in *CommonReconciler) asVersioned(obj runtime.Object) runtime.Object { - var gv = runtime.GroupVersioner(schema.GroupVersions(scheme.Scheme.PrioritizedVersionsAllGroups())) - if obj, err := runtime.ObjectConvertor(scheme.Scheme).ConvertToVersion(obj, gv); err == nil { - return obj - } - return obj + return in.patcher.CreateThreeWayPatchData(original, desired, current) } // HandleErrAndRequeue is the common error handler @@ -832,7 +719,7 @@ func (in *ReconcileSecondaryResource) ReconcileSingleResource(ctx context.Contex } if storage == nil && owner != nil { - if storage, err = utils.NewStorage(owner.GetNamespacedName(), in.GetManager()); err != nil { + if storage, err = utils.NewStorage(owner.GetNamespacedName(), in.GetManager(), in.GetPatcher()); err != nil { return err } } diff --git a/controllers/servicemonitor/servicemonitor_controller.go b/controllers/servicemonitor/servicemonitor_controller.go index c6a9d95b5..47445472a 100644 --- a/controllers/servicemonitor/servicemonitor_controller.go +++ b/controllers/servicemonitor/servicemonitor_controller.go @@ -74,7 +74,7 @@ func (in *ReconcileServiceMonitor) Reconcile(ctx context.Context, request reconc // Make sure that the request is unlocked when this method exits defer in.Unlock(request) - storage, err := utils.NewStorage(request.NamespacedName, in.GetManager()) + storage, err := utils.NewStorage(request.NamespacedName, in.GetManager(), in.GetPatcher()) if err != nil { return reconcile.Result{}, err } @@ -222,7 +222,7 @@ func (in *ReconcileServiceMonitor) UpdateServiceMonitor(ctx context.Context, nam } logger.Info("Patching ServiceMonitor") - _, err = in.monClient.ServiceMonitors(namespace).Patch(ctx, name, in.GetPatchType(), data, metav1.PatchOptions{}) + _, err = in.monClient.ServiceMonitors(namespace).Patch(ctx, name, in.GetPatcher().GetPatchType(), data, metav1.PatchOptions{}) if err != nil { // Patch or update failed - resort to an update with retry as sometimes custom resource (like ServiceMonitor) cannot be patched count := 1 diff --git a/controllers/statefulset/statefulset_controller.go b/controllers/statefulset/statefulset_controller.go index deafc8f74..77a397653 100644 --- a/controllers/statefulset/statefulset_controller.go +++ b/controllers/statefulset/statefulset_controller.go @@ -15,6 +15,7 @@ import ( "github.com/oracle/coherence-operator/pkg/clients" "github.com/oracle/coherence-operator/pkg/events" "github.com/oracle/coherence-operator/pkg/operator" + "github.com/oracle/coherence-operator/pkg/patching" "github.com/oracle/coherence-operator/pkg/probe" "github.com/oracle/coherence-operator/pkg/utils" "github.com/pkg/errors" @@ -95,7 +96,7 @@ func (in *ReconcileStatefulSet) Reconcile(ctx context.Context, request reconcile // Make sure that the request is unlocked when this method exits defer in.Unlock(request) - storage, err := utils.NewStorage(request.NamespacedName, in.GetManager()) + storage, err := utils.NewStorage(request.NamespacedName, in.GetManager(), in.GetPatcher()) if err != nil { return reconcile.Result{}, err } @@ -514,7 +515,7 @@ func (in *ReconcileStatefulSet) maybePatchStatefulSet(ctx context.Context, deplo // fix the CreationTimestamp so that it is not in the patch desired.SetCreationTimestamp(current.GetCreationTimestamp()) // create the patch to see whether there is anything to update - patch, data, err := in.CreateThreeWayPatch(current.GetName(), original, desired, current, reconciler.PatchIgnore) + patch, data, err := in.CreateThreeWayPatch(current.GetName(), original, desired, current, patching.PatchIgnore) if err != nil { return reconcile.Result{}, errors.Wrapf(err, "failed to create patch for StatefulSet/%s", current.GetName()) } diff --git a/docs/about/04_coherence_spec.adoc b/docs/about/04_coherence_spec.adoc index 43b8f5fb2..d98a19200 100644 --- a/docs/about/04_coherence_spec.adoc +++ b/docs/about/04_coherence_spec.adoc @@ -487,6 +487,9 @@ CoherenceUtilsSpec defines the settings for the Coherence Operator utilities ima |=== | Field | Description | Type | Required m| imagePullPolicy | Image pull policy. One of Always, Never, IfNotPresent. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images m| *https://pkg.go.dev/k8s.io/api/core/v1#PullPolicy | false +m| image | Image is used to set the utils image used in Coherence Pods. + + + +Deprecated: This field is deprecated and no longer used, any value set will be ignored. m| *string | false |=== <> diff --git a/docs/installation/01_installation.adoc b/docs/installation/01_installation.adoc index b55b8c5a9..c438898f6 100644 --- a/docs/installation/01_installation.adoc +++ b/docs/installation/01_installation.adoc @@ -729,7 +729,7 @@ deploymentAnnotations: By default, the Operator will install both CRDs, `Coherence` and `CoherenceJob`. If support for `CoherenceJob` is not required then it can be excluded from being installed setting the -Operator command line parameter `--install-job-crd` to `false`. +Operator command line parameter `--enable-jobs` to `false`. When installing with Helm, the `allowCoherenceJobs` value can be set to `false` to disable support for `CoherenceJob` resources (the default value is `true`). diff --git a/helm-charts/coherence-operator/templates/_helpers.tpl b/helm-charts/coherence-operator/templates/_helpers.tpl index 4aebbc4fc..09519820c 100644 --- a/helm-charts/coherence-operator/templates/_helpers.tpl +++ b/helm-charts/coherence-operator/templates/_helpers.tpl @@ -34,16 +34,3 @@ Create chart name and version as used by the chart label. {{- define "coherence-operator.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} - -{{/* -Create the release labels. -These are a common set of labels applied to all of the resources -generated from this chart. -*/}} -{{- define "coherence-operator.release_labels" }} -heritage: {{ .Release.Service | quote }} -release: {{ .Release.Name | quote }} -chart: {{ template "coherence-operator.chart" . }} -app: {{ template "coherence-operator.name" . }} -component: coherence-operator -{{- end }} diff --git a/helm-charts/coherence-operator/templates/deployment.yaml b/helm-charts/coherence-operator/templates/deployment.yaml index db17caebd..661798c42 100644 --- a/helm-charts/coherence-operator/templates/deployment.yaml +++ b/helm-charts/coherence-operator/templates/deployment.yaml @@ -5,8 +5,15 @@ kind: Secret metadata: name: {{ default "coherence-webhook-server-cert" .Values.webhookCertSecret }} namespace: {{ .Release.Namespace }} -{{- if (.Values.globalLabels) }} labels: + control-plane: coherence + app.kubernetes.io/name: coherence-operator + app.kubernetes.io/instance: coherence-operator-manager + app.kubernetes.io/version: "${VERSION}" + app.kubernetes.io/component: webhook-cert + app.kubernetes.io/part-of: coherence-operator + app.kubernetes.io/managed-by: helm +{{- if (.Values.globalLabels) }} {{ toYaml .Values.globalLabels | indent 4 }} {{- end }} {{- if (.Values.globalAnnotations) }} @@ -158,14 +165,13 @@ spec: {{- end }} {{- if (eq .Values.clusterRoles false) }} - --enable-webhook=false - - --install-crd=false {{- else }} {{- if (eq .Values.webhooks false) }} - --enable-webhook=false {{- end }} {{- end }} {{- if (eq .Values.allowCoherenceJobs false) }} - - --install-job-crd=false + - --enable-jobs=false {{- end }} {{- if (.Values.globalLabels) }} {{- range $k, $v := .Values.globalLabels }} @@ -202,10 +208,14 @@ spec: {{- else }} value: {{ printf "%s/%s:%s" .Values.defaultCoherenceImage.registry .Values.defaultCoherenceImage.name .Values.defaultCoherenceImage.tag | quote }} {{- end }} +{{- if .Values.rackLabel }} - name: RACK_LABEL value: {{ .Values.rackLabel | quote }} +{{- end }} +{{- if .Values.siteLabel }} - name: SITE_LABEL value: {{ .Values.siteLabel | quote }} +{{- end }} - name: OPERATOR_IMAGE {{- if kindIs "string" .Values.image }} value: {{ .Values.image | quote }} diff --git a/helm-charts/coherence-operator/templates/rbac.yaml b/helm-charts/coherence-operator/templates/rbac.yaml index 720f1e28f..35926437b 100644 --- a/helm-charts/coherence-operator/templates/rbac.yaml +++ b/helm-charts/coherence-operator/templates/rbac.yaml @@ -47,10 +47,7 @@ rules: resources: - customresourcedefinitions verbs: - - create - - delete - get - - update - apiGroups: - admissionregistration.k8s.io resources: diff --git a/helm-charts/coherence-operator/values.yaml b/helm-charts/coherence-operator/values.yaml index e59484d13..a88079028 100644 --- a/helm-charts/coherence-operator/values.yaml +++ b/helm-charts/coherence-operator/values.yaml @@ -212,6 +212,10 @@ nodeRoles: false webhooks: true # If set to false, the Operator will not support the CoherenceJob resource type. -# The CoherenceJob CRD will not be installed by the Operator and the Operator will -# not listen for any CoherenceJob resource events. +# The CoherenceJob CRD will not be installed and the Operator will not listen +# for any CoherenceJob resource events. allowCoherenceJobs: true + +# If set to false, the Helm chart will not install the CRDs. +# The CRDs must be manually installed before the Operator can be installed. +installCrd: true diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 3b7e3e586..aa8646fb0 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -54,6 +54,7 @@ const ( FlagCoherenceImage = "coherence-image" FlagCRD = "install-crd" FlagJobCRD = "install-job-crd" + FlagEnableCoherenceJobs = "enable-jobs" FlagDevMode = "coherence-dev-mode" FlagDryRun = "dry-run" FlagEnableWebhook = "enable-webhook" @@ -177,12 +178,17 @@ func SetupFlags(cmd *cobra.Command, v *viper.Viper) { cmd.Flags().Bool( FlagCRD, true, - "Enables automatic installation/update of all Coherence CRDs", + "This flag is deprecated and no longer has any function", ) cmd.Flags().Bool( FlagJobCRD, true, - "Enables automatic installation/update of CoherenceJob CRD", + "Enables CoherenceJob support, this flag is deprecated use --"+FlagEnableCoherenceJobs, + ) + cmd.Flags().Bool( + FlagEnableCoherenceJobs, + true, + "Enables CoherenceJob support", ) cmd.Flags().Bool( FlagEnableHttp2, @@ -347,12 +353,9 @@ func GetRackLabel() []string { return GetViper().GetStringSlice(FlagRackLabel) } -func ShouldInstallCRDs() bool { - return GetViper().GetBool(FlagCRD) && !IsDryRun() -} - -func ShouldInstallJobCRD() bool { - return GetViper().GetBool(FlagJobCRD) +func ShouldSupportCoherenceJob() bool { + v := GetViper() + return v.GetBool(FlagEnableCoherenceJobs) || v.GetBool(FlagJobCRD) } func ShouldEnableWebhooks() bool { diff --git a/pkg/operator/operator_test.go b/pkg/operator/operator_test.go deleted file mode 100644 index 2e6504c82..000000000 --- a/pkg/operator/operator_test.go +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright (c) 2020, 2025, Oracle and/or its affiliates. - * Licensed under the Universal Permissive License v 1.0 as shown at - * http://oss.oracle.com/licenses/upl. - */ - -package operator_test - -import ( - "context" - "github.com/go-logr/logr" - . "github.com/onsi/gomega" - v1 "github.com/oracle/coherence-operator/api/v1" - "github.com/oracle/coherence-operator/pkg/fakes" - "github.com/oracle/coherence-operator/pkg/operator" - "github.com/spf13/viper" - crdv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "testing" -) - -func TestShouldCreateV1CRDs(t *testing.T) { - var err error - - g := NewGomegaWithT(t) - mgr, err := fakes.NewFakeManager() - g.Expect(err).NotTo(HaveOccurred()) - - err = crdv1.AddToScheme(mgr.Scheme) - g.Expect(err).NotTo(HaveOccurred()) - - ctx := context.TODO() - log := logr.New(fakes.TestLogSink{T: t}) - - viper.GetViper().Set(operator.FlagJobCRD, true) - err = v1.EnsureV1CRDs(ctx, log, mgr.Scheme, mgr.Client) - g.Expect(err).NotTo(HaveOccurred()) - - crdList := crdv1.CustomResourceDefinitionList{} - err = mgr.Client.List(ctx, &crdList) - g.Expect(err).NotTo(HaveOccurred()) - - g.Expect(len(crdList.Items)).To(Equal(2)) - - expected := make(map[string]bool) - expected["coherence.coherence.oracle.com"] = false - expected["coherencejob.coherence.oracle.com"] = false - - for _, crd := range crdList.Items { - expected[crd.Name] = true - } - - for crd, found := range expected { - if !found { - t.Error("Failed to create CRD " + crd) - } - } -} - -func TestShouldNotCreateJobCRDWhenFlagIsFalse(t *testing.T) { - var err error - - g := NewGomegaWithT(t) - mgr, err := fakes.NewFakeManager() - g.Expect(err).NotTo(HaveOccurred()) - - err = crdv1.AddToScheme(mgr.Scheme) - g.Expect(err).NotTo(HaveOccurred()) - - ctx := context.TODO() - log := logr.New(fakes.TestLogSink{T: t}) - - viper.GetViper().Set(operator.FlagJobCRD, false) - err = v1.EnsureV1CRDs(ctx, log, mgr.Scheme, mgr.Client) - g.Expect(err).NotTo(HaveOccurred()) - - crdList := crdv1.CustomResourceDefinitionList{} - err = mgr.Client.List(ctx, &crdList) - g.Expect(err).NotTo(HaveOccurred()) - - g.Expect(len(crdList.Items)).To(Equal(1)) - - expected := make(map[string]bool) - expected["coherence.coherence.oracle.com"] = false - - for _, crd := range crdList.Items { - expected[crd.Name] = true - } - - for crd, found := range expected { - if !found { - t.Error("Failed to create CRD " + crd) - } - } -} - -func TestShouldUpdateV1CRDs(t *testing.T) { - var err error - - g := NewGomegaWithT(t) - mgr, err := fakes.NewFakeManager() - g.Expect(err).NotTo(HaveOccurred()) - err = crdv1.AddToScheme(mgr.Scheme) - g.Expect(err).NotTo(HaveOccurred()) - - viper.GetViper().Set(operator.FlagJobCRD, true) - - oldCRDs := make(map[string]*crdv1.CustomResourceDefinition) - oldCRDs["coherence.coherence.oracle.com"] = nil - oldCRDs["coherencejob.coherence.oracle.com"] = nil - - for name := range oldCRDs { - crd := crdv1.CustomResourceDefinition{} - crd.SetName(name) - crd.SetResourceVersion("1") - oldCRDs[name] = &crd - _ = mgr.GetClient().Create(context.TODO(), &crd) - } - - ctx := context.TODO() - log := logr.New(fakes.TestLogSink{T: t}) - - err = v1.EnsureV1CRDs(ctx, log, mgr.Scheme, mgr.Client) - g.Expect(err).NotTo(HaveOccurred()) - - crdList := crdv1.CustomResourceDefinitionList{} - err = mgr.Client.List(ctx, &crdList) - g.Expect(err).NotTo(HaveOccurred()) - - g.Expect(len(crdList.Items)).To(Equal(2)) - - for _, crd := range crdList.Items { - oldCRD := oldCRDs[crd.Name] - g.Expect(crd).NotTo(Equal(oldCRD)) - g.Expect(crd.GetResourceVersion()).To(Equal(oldCRD.GetResourceVersion())) - } -} - -func TestShouldNotUpdateJobCRDWhenFlagIsFalse(t *testing.T) { - var err error - - g := NewGomegaWithT(t) - mgr, err := fakes.NewFakeManager() - g.Expect(err).NotTo(HaveOccurred()) - err = crdv1.AddToScheme(mgr.Scheme) - g.Expect(err).NotTo(HaveOccurred()) - - viper.GetViper().Set(operator.FlagJobCRD, false) - - oldCRDs := make(map[string]*crdv1.CustomResourceDefinition) - oldCRDs["coherence.coherence.oracle.com"] = nil - - for name := range oldCRDs { - crd := crdv1.CustomResourceDefinition{} - crd.SetName(name) - crd.SetResourceVersion("1") - oldCRDs[name] = &crd - _ = mgr.GetClient().Create(context.TODO(), &crd) - } - - ctx := context.TODO() - log := logr.New(fakes.TestLogSink{T: t}) - - err = v1.EnsureV1CRDs(ctx, log, mgr.Scheme, mgr.Client) - g.Expect(err).NotTo(HaveOccurred()) - - crdList := crdv1.CustomResourceDefinitionList{} - err = mgr.Client.List(ctx, &crdList) - g.Expect(err).NotTo(HaveOccurred()) - - g.Expect(len(crdList.Items)).To(Equal(1)) - - for _, crd := range crdList.Items { - oldCRD := oldCRDs[crd.Name] - g.Expect(crd).NotTo(Equal(oldCRD)) - g.Expect(crd.GetResourceVersion()).To(Equal(oldCRD.GetResourceVersion())) - } -} diff --git a/pkg/patching/patcher.go b/pkg/patching/patcher.go new file mode 100644 index 000000000..b824f1515 --- /dev/null +++ b/pkg/patching/patcher.go @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2020, 2025, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ + +package patching + +import ( + "context" + "encoding/json" + "fmt" + "github.com/go-logr/logr" + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/strategicpatch" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +const ( + // PatchIgnore - If the patching json is this we can skip the patching. + PatchIgnore = "{\"metadata\":{\"creationTimestamp\":null},\"status\":{\"replicas\":0}}" +) + +type ResourcePatcher interface { + // TwoWayPatch performs a two-way merge patching on the resource. + TwoWayPatch(context.Context, string, client.Object, client.Object) (bool, error) + // CreateTwoWayPatch creates a two-way patching between the original state, the current state and the desired state of a k8s resource. + CreateTwoWayPatch(string, runtime.Object, runtime.Object, ...string) (client.Patch, error) + // CreateTwoWayPatchOfType creates a two-way patching between the current state and the desired state of a k8s resource. + CreateTwoWayPatchOfType(types.PatchType, string, runtime.Object, runtime.Object, ...string) (client.Patch, error) + // ThreeWayPatch performs a three-way merge patching on the resource returning true if a patching was required otherwise false. + ThreeWayPatch(context.Context, string, client.Object, client.Object, client.Object) (bool, error) + // ThreeWayPatchWithCallback performs a three-way merge patching on the resource returning true if a patching was required otherwise false. + ThreeWayPatchWithCallback(context.Context, string, client.Object, client.Object, client.Object, func()) (bool, error) + // ApplyThreeWayPatchWithCallback performs a three-way merge patching on the resource returning true if a patching was required otherwise false. + ApplyThreeWayPatchWithCallback(context.Context, string, client.Object, client.Patch, []byte, func()) (bool, error) + // CreateThreeWayPatch creates a three-way patching between the original state, the current state and the desired state of a k8s resource. + CreateThreeWayPatch(string, runtime.Object, runtime.Object, runtime.Object, ...string) (client.Patch, []byte, error) + // CreateThreeWayPatchData creates a three-way patching between the original state, the current state and the desired state of a k8s resource. + CreateThreeWayPatchData(original, desired, current runtime.Object) ([]byte, error) + // GetPatchType returns the patching type this patcher uses + GetPatchType() types.PatchType + // SetPatchType sets the patching type this patcher uses + SetPatchType(pt types.PatchType) +} + +// NewResourcePatcher creates a new ResourcePatcher +func NewResourcePatcher(mgr manager.Manager, logger logr.Logger, patchType types.PatchType) ResourcePatcher { + return &patcher{ + mgr: mgr, + logger: logger, + patchType: patchType, + } +} + +// compile time check to verify `patcher` implements `ResourcePatcher` +var _ ResourcePatcher = &patcher{} + +type patcher struct { + mgr manager.Manager + logger logr.Logger + patchType types.PatchType +} + +func (in *patcher) GetPatchType() types.PatchType { return in.patchType } + +func (in *patcher) SetPatchType(pt types.PatchType) { in.patchType = pt } + +// TwoWayPatch performs a two-way merge patching on the resource. +func (in *patcher) TwoWayPatch(ctx context.Context, name string, current, desired client.Object) (bool, error) { + patch, err := in.CreateTwoWayPatch(name, desired, current, PatchIgnore) + if err != nil { + kind := current.GetObjectKind().GroupVersionKind().Kind + return false, errors.Wrapf(err, "failed to create patching for %s/%s", kind, name) + } + + if patch == nil { + // nothing to patching so just return + return false, nil + } + + err = in.mgr.GetClient().Patch(ctx, current, patch) + if err != nil { + kind := current.GetObjectKind().GroupVersionKind().Kind + return false, errors.Wrapf(err, "cannot patching %s/%s", kind, name) + } + + return true, nil +} + +// CreateTwoWayPatch creates a two-way patching between the original state, the current state and the desired state of a k8s resource. +func (in *patcher) CreateTwoWayPatch(name string, desired, current runtime.Object, ignore ...string) (client.Patch, error) { + return in.CreateTwoWayPatchOfType(in.patchType, name, desired, current, ignore...) +} + +// CreateTwoWayPatchOfType creates a two-way patching between the current state and the desired state of a k8s resource. +func (in *patcher) CreateTwoWayPatchOfType(patchType types.PatchType, name string, desired, current runtime.Object, ignore ...string) (client.Patch, error) { + currentData, err := json.Marshal(current) + if err != nil { + return nil, errors.Wrap(err, "serializing current configuration") + } + desiredData, err := json.Marshal(desired) + if err != nil { + return nil, errors.Wrap(err, "serializing desired configuration") + } + + // Get a versioned object + versionedObject := in.asVersioned(desired) + + patchMeta, err := strategicpatch.NewPatchMetaFromStruct(versionedObject) + if err != nil { + return nil, errors.Wrap(err, "unable to create patching metadata from object") + } + + data, err := strategicpatch.CreateTwoWayMergePatchUsingLookupPatchMeta(currentData, desiredData, patchMeta) + if err != nil { + return nil, errors.Wrap(err, "creating three-way patching") + } + + // check whether the patching counts as no-patching + ignore = append(ignore, "{}") + for _, s := range ignore { + if string(data) == s { + // empty patching + return nil, err + } + } + + // log the patching + kind := current.GetObjectKind().GroupVersionKind().Kind + + in.logger.V(2).Info(fmt.Sprintf("Created patching for %s/%s\n%s", kind, name, string(data))) + + return client.RawPatch(patchType, data), nil +} + +// ThreeWayPatch performs a three-way merge patching on the resource returning true if a patching was required otherwise false. +func (in *patcher) ThreeWayPatch(ctx context.Context, name string, current, original, desired client.Object) (bool, error) { + return in.ThreeWayPatchWithCallback(ctx, name, current, original, desired, nil) +} + +// ThreeWayPatchWithCallback performs a three-way merge patching on the resource returning true if a patching was required otherwise false. +func (in *patcher) ThreeWayPatchWithCallback(ctx context.Context, name string, current, original, desired client.Object, callback func()) (bool, error) { + kind := current.GetObjectKind().GroupVersionKind().Kind + // fix the CreationTimestamp so that it is not in the patching + desired.(metav1.Object).SetCreationTimestamp(current.(metav1.Object).GetCreationTimestamp()) + // create the patching + patch, data, err := in.CreateThreeWayPatch(name, original, desired, current, PatchIgnore) + if err != nil { + return false, errors.Wrapf(err, "failed to create patching for %s/%s", kind, name) + } + + if patch == nil { + // nothing to patching so just return + return false, nil + } + + return in.ApplyThreeWayPatchWithCallback(ctx, name, current, patch, data, callback) +} + +// ApplyThreeWayPatchWithCallback performs a three-way merge patching on the resource returning true if a patching was required otherwise false. +func (in *patcher) ApplyThreeWayPatchWithCallback(ctx context.Context, name string, current client.Object, patch client.Patch, data []byte, callback func()) (bool, error) { + kind := current.GetObjectKind().GroupVersionKind().Kind + + // execute any callback + if callback != nil { + callback() + } + + in.logger.WithValues().Info(fmt.Sprintf("Patching %s/%s", kind, name), "Patch", string(data)) + err := in.mgr.GetClient().Patch(ctx, current, patch) + if err != nil { + return false, errors.Wrapf(err, "failed to patching %s/%s with %s", kind, name, string(data)) + } + + return true, nil +} + +// CreateThreeWayPatch creates a three-way patching between the original state, the current state and the desired state of a k8s resource. +func (in *patcher) CreateThreeWayPatch(name string, original, desired, current runtime.Object, ignore ...string) (client.Patch, []byte, error) { + data, err := in.CreateThreeWayPatchData(original, desired, current) + if err != nil { + return nil, data, errors.Wrap(err, "creating three-way patching") + } + + // check whether the patching counts as no-patching + ignore = append(ignore, "{}") + for _, s := range ignore { + if string(data) == s { + // empty patching + return nil, data, err + } + } + + // log the patching + kind := current.GetObjectKind().GroupVersionKind().Kind + in.logger.Info(fmt.Sprintf("Created patching for %s/%s", kind, name), "Patch", string(data)) + + return client.RawPatch(in.patchType, data), data, nil +} + +// CreateThreeWayPatchData creates a three-way patching between the original state, the current state and the desired state of a k8s resource. +func (in *patcher) CreateThreeWayPatchData(original, desired, current runtime.Object) ([]byte, error) { + originalData, err := json.Marshal(original) + if err != nil { + return nil, errors.Wrap(err, "serializing original configuration") + } + currentData, err := json.Marshal(current) + if err != nil { + return nil, errors.Wrap(err, "serializing current configuration") + } + desiredData, err := json.Marshal(desired) + if err != nil { + return nil, errors.Wrap(err, "serializing desired configuration") + } + + // Get a versioned object + versionedObject := in.asVersioned(desired) + + patchMeta, err := strategicpatch.NewPatchMetaFromStruct(versionedObject) + if err != nil { + return nil, errors.Wrap(err, "unable to create patching metadata from object") + } + + return strategicpatch.CreateThreeWayMergePatch(originalData, desiredData, currentData, patchMeta, true) +} + +// asVersioned converts the given object into a runtime.Template with the correct group and version set. +func (in *patcher) asVersioned(obj runtime.Object) runtime.Object { + var gv = runtime.GroupVersioner(schema.GroupVersions(scheme.Scheme.PrioritizedVersionsAllGroups())) + if obj, err := runtime.ObjectConvertor(scheme.Scheme).ConvertToVersion(obj, gv); err == nil { + return obj + } + return obj +} diff --git a/pkg/runner/cmd_operator.go b/pkg/runner/cmd_operator.go index 7b78e02f9..d39a483fc 100644 --- a/pkg/runner/cmd_operator.go +++ b/pkg/runner/cmd_operator.go @@ -7,7 +7,6 @@ package runner import ( - "context" "crypto/tls" "fmt" coh "github.com/oracle/coherence-operator/api/v1" @@ -21,7 +20,6 @@ import ( "github.com/spf13/viper" apiruntime "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/apimachinery/pkg/util/version" clientgoscheme "k8s.io/client-go/kubernetes/scheme" rest2 "k8s.io/client-go/rest" "k8s.io/utils/ptr" @@ -29,7 +27,6 @@ import ( "os" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -108,13 +105,6 @@ func execute() error { return errors.Wrap(err, "unable to create client set") } - // create the client here as we use it to install CRDs then inject it into the Manager - setupLog.Info("Creating Kubernetes client", "Host", cfg.Host) - cl, err := client.New(cfg, client.Options{Scheme: scheme}) - if err != nil { - return errors.Wrap(err, "unable to create client") - } - dryRun := operator.IsDryRun() secureMetrics := vpr.GetBool(operator.FlagSecureMetrics) @@ -191,14 +181,6 @@ func execute() error { return errors.Wrap(err, "unable to create controller manager") } - v, err := operator.DetectKubernetesVersion(cs) - if err != nil { - return errors.Wrap(err, "unable to detect Kubernetes version") - } - - ctx := context.TODO() - initialiseOperator(ctx, v, cl) - // Set up the Coherence reconciler if err = (&controllers.CoherenceReconciler{ Client: mgr.GetClient(), @@ -210,7 +192,7 @@ func execute() error { } // Set up the CoherenceJob reconciler - if operator.ShouldInstallJobCRD() { + if operator.ShouldSupportCoherenceJob() { if err = (&controllers.CoherenceJobReconciler{ Client: mgr.GetClient(), ClientSet: cs, @@ -280,16 +262,3 @@ func execute() error { return nil } - -func initialiseOperator(ctx context.Context, v *version.Version, cl client.Client) { - opLog := ctrl.Log.WithName("operator") - - // Ensure that the CRDs exist - if operator.ShouldInstallCRDs() { - err := coh.EnsureCRDs(ctx, v, scheme, cl) - if err != nil { - opLog.Error(err, "") - os.Exit(1) - } - } -} diff --git a/pkg/utils/storage.go b/pkg/utils/storage.go index 5ac3b9715..08d945833 100644 --- a/pkg/utils/storage.go +++ b/pkg/utils/storage.go @@ -11,6 +11,7 @@ import ( "encoding/json" "fmt" coh "github.com/oracle/coherence-operator/api/v1" + "github.com/oracle/coherence-operator/pkg/patching" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -37,24 +38,24 @@ type Storage interface { // GetPrevious obtains the deployment resources for the version prior to the specified version GetPrevious() coh.Resources // Store will store the deployment resources, this will create a new version in the store - Store(coh.Resources, coh.CoherenceResource) error + Store(context.Context, coh.Resources, coh.CoherenceResource) error // Destroy will destroy the store Destroy() // GetHash will return the hash label of the owning resource GetHash() (string, bool) // ResetHash resets the hash to match the Coherence resource - ResetHash(owner coh.CoherenceResource) error + ResetHash(context.Context, coh.CoherenceResource) error // IsJob returns true if the Coherence deployment is a Job IsJob(reconcile.Request) bool } // NewStorage creates a new storage for the given key. -func NewStorage(key client.ObjectKey, mgr manager.Manager) (Storage, error) { - return newStorage(key, mgr) +func NewStorage(key client.ObjectKey, mgr manager.Manager, patcher patching.ResourcePatcher) (Storage, error) { + return newStorage(key, mgr, patcher) } -func newStorage(key client.ObjectKey, mgr manager.Manager) (Storage, error) { - store := &secretStore{manager: mgr, key: key} +func newStorage(key client.ObjectKey, mgr manager.Manager, patcher patching.ResourcePatcher) (Storage, error) { + store := &secretStore{manager: mgr, key: key, patcher: patcher} err := store.loadVersions() return store, err } @@ -65,6 +66,7 @@ type secretStore struct { latest coh.Resources previous coh.Resources hash *string + patcher patching.ResourcePatcher } func (in *secretStore) IsJob(request reconcile.Request) bool { @@ -123,8 +125,8 @@ func (in *secretStore) GetPrevious() coh.Resources { return in.previous } -func (in *secretStore) ResetHash(owner coh.CoherenceResource) error { - secret, exists, err := in.getSecret() +func (in *secretStore) ResetHash(ctx context.Context, owner coh.CoherenceResource) error { + secret, _, err := in.getSecret() if err != nil { // an error occurred other than NotFound return err @@ -136,27 +138,27 @@ func (in *secretStore) ResetHash(owner coh.CoherenceResource) error { hash := owner.GetGenerationString() labels[coh.LabelCoherenceHash] = hash in.hash = &hash - return in.save(owner, secret, exists) + return in.save(ctx, owner, secret) } -func (in *secretStore) Store(r coh.Resources, owner coh.CoherenceResource) error { - secret, exists, err := in.getSecret() +func (in *secretStore) Store(ctx context.Context, res coh.Resources, owner coh.CoherenceResource) error { + secret, _, err := in.getSecret() if err != nil { // an error occurred other than NotFound return err } - r.Version = in.latest.Version + 1 + res.Version = in.latest.Version + 1 if secret.Data == nil { secret.Data = make(map[string][]byte) } - r.EnsureGVK(in.manager.GetScheme()) + res.EnsureGVK(in.manager.GetScheme()) hash := owner.GetGenerationString() oldLatest := secret.Data[storeKeyLatest] - newLatest, err := json.Marshal(r) + newLatest, err := json.Marshal(res) if err != nil { return err } @@ -188,31 +190,36 @@ func (in *secretStore) Store(r coh.Resources, owner coh.CoherenceResource) error secret.Data[storeKeyLatest] = newLatest secret.Data[storeKeyPrevious] = oldLatest - err = in.save(owner, secret, exists) + err = in.save(ctx, owner, secret) if err == nil { // everything was updated successfully so update the storage state in.previous = in.latest - in.latest = r + in.latest = res in.hash = &hash } return err } -func (in *secretStore) save(owner coh.CoherenceResource, secret *corev1.Secret, exists bool) error { +func (in *secretStore) save(ctx context.Context, owner coh.CoherenceResource, desired *corev1.Secret) error { var err error + current, exists, err := in.getSecret() + if err != nil { + return err + } + if !exists { // the resource does not exist so set the deployment as the controller/owner and create it - err = controllerutil.SetControllerReference(owner, secret, in.manager.GetScheme()) + err = controllerutil.SetControllerReference(owner, desired, in.manager.GetScheme()) if err != nil { - err = errors.Wrap(err, fmt.Sprintf("setting resource owner/controller in state store %s/%s", secret.Namespace, secret.Name)) + err = errors.Wrap(err, fmt.Sprintf("setting resource owner/controller in state store %s/%s", desired.Namespace, desired.Name)) } else { - err = in.manager.GetClient().Create(context.TODO(), secret) + err = in.manager.GetClient().Create(context.TODO(), desired) } } else { // the store secret exists so update it - err = in.manager.GetClient().Update(context.TODO(), secret) + _, err = in.patcher.TwoWayPatch(ctx, desired.Name, current, desired) } return err } diff --git a/test/e2e/compatibility/compatibility_helpers.go b/test/e2e/compatibility/compatibility_helpers.go deleted file mode 100644 index e226ad4e1..000000000 --- a/test/e2e/compatibility/compatibility_helpers.go +++ /dev/null @@ -1,249 +0,0 @@ -/* - * Copyright (c) 2020, 2022, Oracle and/or its affiliates. - * Licensed under the Universal Permissive License v 1.0 as shown at - * http://oss.oracle.com/licenses/upl. - */ - -package compatibility - -import ( - "encoding/json" - "fmt" - "github.com/ghodss/yaml" - "io" - k8serr "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - "strings" -) - -/* -func findContainer(name string, d *appsv1.Deployment) *corev1.Container { - for _, c := range d.Spec.Template.Spec.Containers { - if c.Name == name { - return &c - } - } - return nil -} - -func findEnvVar(name string, c *corev1.Container) *corev1.EnvVar { - for _, e := range c.Env { - if e.Name == name { - return &e - } - } - return nil -} - -func helmInstall(args ...string) (*HelmInstallResult, error) { - chart, err := helper.FindOperatorHelmChartDir() - if err != nil { - return nil, err - } - - ns := helper.GetTestNamespace() - - args = append([]string{"install", "--dry-run", "-o", "json"}, args...) - args = append(args, "--namespace", ns, "operator", chart) - - cmd := exec.Command("helm", args...) - b, err := cmd.CombinedOutput() - if err != nil { - return nil, err - } - - m := make(map[string]interface{}) - if err = json.Unmarshal(b, &m); err != nil { - return nil, err - } - - manifest := m["manifest"] - - return parseHelmManifest(fmt.Sprintf("%v", manifest)) -} - -func parseHelmManifest(manifest string) (*HelmInstallResult, error) { - resources := make(map[schema.GroupVersionResource]map[string]runtime.Object) - s := scheme.Scheme - decoder := scheme.Codecs.UniversalDecoder() - - parts := strings.Split(manifest, "\n---\n") - list := make([]runtime.Object, len(parts)) - ownerRefs := make([]metav1.OwnerReference, 0) - - index := 0 - for _, part := range parts { - trimmed := strings.TrimSpace(part) - if trimmed != "" { - u := unstructured.Unstructured{} - err := yaml.Unmarshal([]byte(trimmed), &u) - if err != nil { - return nil, err - } - gvr, _ := meta.UnsafeGuessKindToResource(u.GroupVersionKind()) - - // remove owner references - u.SetOwnerReferences(ownerRefs) - data, err := yaml.Marshal(u.Object) - if err != nil { - return nil, err - } - - o, err := s.New(u.GroupVersionKind()) - if err != nil { - return nil, err - } - _, _, err = decoder.Decode(data, nil, o) - if err != nil { - return nil, err - } - - m, ok := resources[gvr] - if !ok { - m = make(map[string]runtime.Object) - } - list[index] = o - index++ - m[u.GetName()] = o - resources[gvr] = m - } - } - - ordered := list[0:index] - return &HelmInstallResult{resources: resources, ordered: ordered, decoder: decoder}, nil -} -*/ - -type HelmInstallResult struct { - resources map[schema.GroupVersionResource]map[string]runtime.Object - ordered []runtime.Object - decoder runtime.Decoder -} - -type HelmInstallResultFilter func(runtime.Object) bool - -func (h *HelmInstallResult) ToString(filter HelmInstallResultFilter, w io.Writer) error { - var sep = []byte("\n---\n") - - for _, res := range h.ordered { - if filter == nil || filter(res) { - _, err := w.Write(sep) - if err != nil { - return err - } - - d, err := yaml.Marshal(res) - if err != nil { - return err - } - _, err = w.Write(d) - if err != nil { - return err - } - } - } - - return nil -} - -func (h *HelmInstallResult) Size() int { - if h == nil { - return 0 - } - return len(h.ordered) -} -func (h *HelmInstallResult) Get(name string, o runtime.Object) error { - if h == nil { - return fmt.Errorf("resource '%s' not found", name) - } - - gvr, err := h.getGVRFromObject(o, scheme.Scheme) - if err != nil { - return err - } - - if h.resources == nil { - return k8serr.NewNotFound(gvr.GroupResource(), name) - } - - m, ok := h.resources[gvr] - if !ok { - return k8serr.NewNotFound(gvr.GroupResource(), name) - } - - item, ok := m[name] - if !ok { - return k8serr.NewNotFound(gvr.GroupResource(), name) - } - - j, err := json.Marshal(item) - if err != nil { - return err - } - - _, _, err = h.decoder.Decode(j, nil, o) - if err != nil { - return err - } - - return nil -} - -func (h *HelmInstallResult) List(list runtime.Object) error { - if h == nil || h.resources == nil { - return nil - } - - gvk, err := getGVKFromList(list, scheme.Scheme) - if err != nil { - return err - } - - gvr, _ := meta.UnsafeGuessKindToResource(gvk) - m, ok := h.resources[gvr] - if ok { - items := make([]runtime.Object, len(m)) - i := 0 - for _, o := range m { - items[i] = o.DeepCopyObject() - i++ - } - - if err := meta.SetList(list, items); err != nil { - return err - } - } - - return nil -} - -func (h *HelmInstallResult) getGVRFromObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersionResource, error) { - gvk, err := apiutil.GVKForObject(obj, scheme) - if err != nil { - return schema.GroupVersionResource{}, err - } - gvr, _ := meta.UnsafeGuessKindToResource(gvk) - return gvr, nil -} - -func getGVKFromList(list runtime.Object, scheme *runtime.Scheme) (schema.GroupVersionKind, error) { - gvk, err := apiutil.GVKForObject(list, scheme) - if err != nil { - return schema.GroupVersionKind{}, err - } - - if gvk.Kind == "List" { - return schema.GroupVersionKind{}, fmt.Errorf("cannot derive GVK for generic List type %T (kind %q)", list, gvk) - } - - if !strings.HasSuffix(gvk.Kind, "List") { - return schema.GroupVersionKind{}, fmt.Errorf("non-list type %T (kind %q) passed as output", list, gvk) - } - // we need the non-list GVK, so chop off the "List" from the end of the kind - gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] - return gvk, nil -} diff --git a/test/e2e/compatibility/compatibility_test.go b/test/e2e/compatibility/compatibility_test.go index 3984553a5..066552cec 100644 --- a/test/e2e/compatibility/compatibility_test.go +++ b/test/e2e/compatibility/compatibility_test.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2023, Oracle and/or its affiliates. + * Copyright (c) 2020, 2025, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ @@ -11,7 +11,9 @@ import ( "fmt" . "github.com/onsi/gomega" cohv1 "github.com/oracle/coherence-operator/api/v1" + "github.com/oracle/coherence-operator/test/e2e/helm" "github.com/oracle/coherence-operator/test/e2e/helper" + "golang.org/x/mod/semver" appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/types" "os" @@ -57,6 +59,11 @@ func TestCompatibility(t *testing.T) { dir := fmt.Sprintf("%s-%s-before", t.Name(), version) helper.DumpState(testContext, ns, dir) + if semver.Compare("v"+version, "v3.5.0") < 0 { + // upgrading from a pre-3.5.0 version so we need to patch the CRDs + helm.PatchAllCRDs(t, g, name, ns) + } + // Upgrade to this version UpgradeToCurrentVersion(t, g, ns, name) diff --git a/test/e2e/helm/helm_helpers.go b/test/e2e/helm/helm_helpers.go index 103309e52..6346eb76b 100644 --- a/test/e2e/helm/helm_helpers.go +++ b/test/e2e/helm/helm_helpers.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2021, Oracle and/or its affiliates. + * Copyright (c) 2020, 2025, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ @@ -10,6 +10,7 @@ import ( "encoding/json" "fmt" "github.com/ghodss/yaml" + . "github.com/onsi/gomega" "github.com/oracle/coherence-operator/test/e2e/helper" "io" appsv1 "k8s.io/api/apps/v1" @@ -21,9 +22,11 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/scheme" + "os" "os/exec" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "strings" + "testing" ) func findContainer(name string, d *appsv1.Deployment) *corev1.Container { @@ -251,3 +254,23 @@ func getGVKFromList(list runtime.Object, scheme *runtime.Scheme) (schema.GroupVe gvk.Kind = gvk.Kind[:len(gvk.Kind)-4] return gvk, nil } + +func PatchAllCRDs(t *testing.T, g *GomegaWithT, name, namespace string) { + patchCRD(t, g, "coherence.coherence.oracle.com", name, namespace) + patchCRD(t, g, "coherencejob.coherence.oracle.com", name, namespace) +} + +func patchCRD(t *testing.T, g *GomegaWithT, crd, name, namespace string) { + applyPatch(t, g, crd, "{\"metadata\": {\"labels\": {\"app.kubernetes.io/managed-by\": \"Helm\"}}}") + applyPatch(t, g, crd, fmt.Sprintf("{\"metadata\": {\"annotations\": {\"meta.helm.sh/release-name\": \"%s\"}}}", name)) + applyPatch(t, g, crd, fmt.Sprintf("{\"metadata\": {\"annotations\": {\"meta.helm.sh/release-namespace\": \"%s\"}}}", namespace)) +} + +func applyPatch(t *testing.T, g *GomegaWithT, name, patch string) { + cmd := exec.Command("kubectl", "patch", "customresourcedefinition", name, "--patch", patch) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + t.Logf("Helm upgrade to current Operator version - patching CRDs\n") + err := cmd.Run() + g.Expect(err).NotTo(HaveOccurred()) +} diff --git a/test/e2e/helm/helm_test.go b/test/e2e/helm/helm_test.go index dd5248494..5853c1da5 100644 --- a/test/e2e/helm/helm_test.go +++ b/test/e2e/helm/helm_test.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2024, Oracle and/or its affiliates. + * Copyright (c) 2020, 2025, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ @@ -160,7 +160,7 @@ func TestDisableJobCRD(t *testing.T) { g.Expect(c).NotTo(BeNil()) g.Expect(c.Args).NotTo(BeNil()) - g.Expect(c.Args).Should(ContainElements("operator", "--enable-leader-election", "--install-job-crd=false")) + g.Expect(c.Args).Should(ContainElements("operator", "--enable-leader-election", "--enable-jobs=false")) } func TestSetOnlySameNamespace(t *testing.T) { diff --git a/test/e2e/helper/test_context.go b/test/e2e/helper/test_context.go index 8e0098c08..0aca7ffa0 100644 --- a/test/e2e/helper/test_context.go +++ b/test/e2e/helper/test_context.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2024, Oracle and/or its affiliates. + * Copyright (c) 2020, 2025, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ @@ -14,6 +14,7 @@ import ( "github.com/oracle/coherence-operator/controllers" "github.com/oracle/coherence-operator/pkg/clients" "github.com/oracle/coherence-operator/pkg/operator" + "github.com/oracle/coherence-operator/pkg/patching" oprest "github.com/oracle/coherence-operator/pkg/rest" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -21,6 +22,7 @@ import ( "golang.org/x/net/context" corev1 "k8s.io/api/core/v1" v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "net/http" @@ -53,6 +55,7 @@ type TestContext struct { namespaces []string RestServer oprest.Server RestEndpoints map[string]func(w http.ResponseWriter, r *http.Request) + Patcher patching.ResourcePatcher } func (in *TestContext) Logf(format string, a ...interface{}) { @@ -215,11 +218,6 @@ func NewContext(startController bool, watchNamespaces ...string) (TestContext, e return TestContext{}, err } - cl, err := client.New(k8sCfg, client.Options{Scheme: scheme.Scheme}) - if err != nil { - return TestContext{}, err - } - options := ctrl.Options{ Scheme: scheme.Scheme, } @@ -256,22 +254,11 @@ func NewContext(startController bool, watchNamespaces ...string) (TestContext, e return TestContext{}, err } - v, err := operator.DetectKubernetesVersion(cs) - if err != nil { - return TestContext{}, err - } - ctx, cancel := context.WithCancel(context.Background()) var stop chan struct{} if startController { - // Ensure CRDs exist - err = coh.EnsureCRDs(ctx, v, scheme.Scheme, cl) - if err != nil { - return TestContext{}, err - } - // Create the Coherence controller err = (&controllers.CoherenceReconciler{ Client: k8sManager.GetClient(), @@ -293,6 +280,7 @@ func NewContext(startController bool, watchNamespaces ...string) (TestContext, e ep := make(map[string]func(w http.ResponseWriter, r *http.Request)) + p := patching.NewResourcePatcher(k8sManager, ctrl.Log, types.StrategicMergePatchType) return TestContext{ Config: k8sCfg, Client: k8sClient, @@ -304,5 +292,6 @@ func NewContext(startController bool, watchNamespaces ...string) (TestContext, e stop: stop, Cancel: cancel, RestEndpoints: ep, + Patcher: p, }, nil } diff --git a/test/e2e/local/clustering_test.go b/test/e2e/local/clustering_test.go index 1c8217c72..385295a23 100644 --- a/test/e2e/local/clustering_test.go +++ b/test/e2e/local/clustering_test.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2024, Oracle and/or its affiliates. + * Copyright (c) 2020, 2025, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ @@ -180,7 +180,7 @@ func TestGlobalLabels(t *testing.T) { data, ok := deployments["global-label"] g.Expect(ok).To(BeTrue(), "did not find expected 'global-label' deployment") - storage, err := utils.NewStorage(data.GetNamespacedName(), testContext.Manager) + storage, err := utils.NewStorage(data.GetNamespacedName(), testContext.Manager, testContext.Patcher) g.Expect(err).NotTo(HaveOccurred()) latest := storage.GetLatest() for _, res := range latest.Items { @@ -205,7 +205,7 @@ func TestGlobalAnnotations(t *testing.T) { data, ok := deployments["global-annotation"] g.Expect(ok).To(BeTrue(), "did not find expected 'global-annotation' deployment") - storage, err := utils.NewStorage(data.GetNamespacedName(), testContext.Manager) + storage, err := utils.NewStorage(data.GetNamespacedName(), testContext.Manager, testContext.Patcher) g.Expect(err).NotTo(HaveOccurred()) latest := storage.GetLatest() for _, res := range latest.Items { diff --git a/test/e2e/local/doc.go b/test/e2e/local/doc.go index 06840e888..abc903b01 100644 --- a/test/e2e/local/doc.go +++ b/test/e2e/local/doc.go @@ -1,10 +1,10 @@ /* - * Copyright (c) 2019, 2020 Oracle and/or its affiliates. + * Copyright (c) 2020, 2025, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ -// The local package contains end-to-end Operator tests that do not require +// Package local contains end-to-end Operator tests that do not require // the Operator to be deployed into the K8s cluster. // These tests use the operator-sdk end-to-end framework and must be run // using the "operator-sdk test local" command with the "--up-local" parameter. diff --git a/test/e2e/remote/suspend_test.go b/test/e2e/remote/suspend_test.go index c6edba76d..83d362526 100644 --- a/test/e2e/remote/suspend_test.go +++ b/test/e2e/remote/suspend_test.go @@ -23,7 +23,6 @@ import ( "k8s.io/utils/ptr" "net/http" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "testing" "time" ) @@ -295,8 +294,14 @@ func addTestFinalizer(o client.Object) error { if err := testContext.Client.Get(ctx, k, o); err != nil { return err } - controllerutil.AddFinalizer(o, testFinalizer) - return testContext.Client.Update(ctx, o) + s := `{"metadata":{"finalizers":[` + for _, f := range o.GetFinalizers() { + s += fmt.Sprintf(`"%s",`, f) + } + s += fmt.Sprintf(`"%s"]}}`, testFinalizer) + + patch := client.RawPatch(types.MergePatchType, []byte(s)) + return testContext.Client.Patch(ctx, o, patch) } func removeAllFinalizers(o client.Object) error {