diff --git a/Makefile b/Makefile index 0e29ad50b..4cbb3ae30 100644 --- a/Makefile +++ b/Makefile @@ -1650,6 +1650,11 @@ kind: ## Run a default KinD cluster kind create cluster --name $(KIND_CLUSTER) --wait 10m --config $(SCRIPTS_DIR)/kind-config.yaml --image $(KIND_IMAGE) $(SCRIPTS_DIR)/kind-label-node.sh +.PHONY: kind-dual +kind-dual: ## Run a KinD cluster configured for a dual stack IPv4 and IPv6 network + kind create cluster --name $(KIND_CLUSTER) --wait 10m --config $(SCRIPTS_DIR)/kind-config-dual.yaml --image $(KIND_IMAGE) + $(SCRIPTS_DIR)/kind-label-node.sh + # ---------------------------------------------------------------------------------------------------------------------- # Start a Kind cluster # ---------------------------------------------------------------------------------------------------------------------- diff --git a/api/v1/coherence_types.go b/api/v1/coherence_types.go index 8516274aa..873c90360 100644 --- a/api/v1/coherence_types.go +++ b/api/v1/coherence_types.go @@ -562,7 +562,7 @@ func (in *CoherenceSpec) GetManagementPort() int32 { } } -// GetPersistenceSpec returns the Coherence persistence spcification. +// GetPersistenceSpec returns the Coherence persistence specification. func (in *CoherenceSpec) GetPersistenceSpec() *PersistenceSpec { if in == nil { return nil @@ -570,6 +570,14 @@ func (in *CoherenceSpec) GetPersistenceSpec() *PersistenceSpec { return in.Persistence } +// GetWkaIPFamily returns the IP Family of the headless Service used for Coherence WKA. +func (in *CoherenceSpec) GetWkaIPFamily() corev1.IPFamily { + if in == nil || in.WKA == nil || in.WKA.IPFamily == nil { + return corev1.IPFamilyUnknown + } + return *in.WKA.IPFamily +} + // ----- CoherenceWKASpec struct -------------------------------------------- // CoherenceWKASpec configures Coherence well-known-addressing to use an @@ -599,6 +607,11 @@ type CoherenceWKASpec struct { // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ // +optional Annotations map[string]string `json:"annotations,omitempty" protobuf:"bytes,12,rep,name=annotations"` + + // IPFamily is the IP family to use for the WKA service (and also the StatefulSet headless service). + // Valid values are "IPv4" or "IPv6". + // +optional + IPFamily *corev1.IPFamily `json:"ipFamily,omitempty"` } // ----- CoherenceTracingSpec struct ---------------------------------------- diff --git a/api/v1/coherencejobresource_types.go b/api/v1/coherencejobresource_types.go index 5aa2c1d9b..2da6a1c62 100644 --- a/api/v1/coherencejobresource_types.go +++ b/api/v1/coherencejobresource_types.go @@ -334,6 +334,19 @@ func (in *CoherenceJob) IsBeforeVersion(version string) bool { return true } +// GetWkaIPFamily returns the IP Family of the headless Service used for Coherence WKA. +func (in *CoherenceJob) GetWkaIPFamily() corev1.IPFamily { + if in == nil { + return corev1.IPFamilyUnknown + } + return in.Spec.GetWkaIPFamily() +} + +// GetHeadlessServiceIPFamily always returns an empty array as this is not applicable to Jobs. +func (in *CoherenceJob) GetHeadlessServiceIPFamily() []corev1.IPFamily { + return nil +} + // ----- CoherenceJobList type ---------------------------------------------- // CoherenceJobResourceSpec defines the specification of a CoherenceJob resource. @@ -487,6 +500,14 @@ func (in *CoherenceJobResourceSpec) GetReplicas() int32 { return *in.CoherenceResourceSpec.Replicas } +// GetWkaIPFamily returns the IP Family of the headless Service used for Coherence WKA. +func (in *CoherenceJobResourceSpec) GetWkaIPFamily() corev1.IPFamily { + if in == nil || in.Coherence == nil { + return corev1.IPFamilyUnknown + } + return in.Coherence.GetWkaIPFamily() +} + // UpdateJob updates a JobSpec from the fields in this spec func (in *CoherenceJobResourceSpec) UpdateJob(spec *batchv1.JobSpec) { if in == nil { diff --git a/api/v1/coherenceresource.go b/api/v1/coherenceresource.go index 936cc7142..1476bfe88 100644 --- a/api/v1/coherenceresource.go +++ b/api/v1/coherenceresource.go @@ -21,8 +21,12 @@ type CoherenceResource interface { GetCoherenceClusterName() string // GetWkaServiceName returns the name of the headless Service used for Coherence WKA. GetWkaServiceName() string + // GetWkaIPFamily returns the IP Family of the headless Service used for Coherence WKA. + GetWkaIPFamily() corev1.IPFamily // GetHeadlessServiceName returns the name of the headless Service used for the StatefulSet. GetHeadlessServiceName() string + // GetHeadlessServiceIPFamily always returns an empty array as this is not applicable to Jobs. + GetHeadlessServiceIPFamily() []corev1.IPFamily // GetReplicas returns the number of replicas required for a deployment. // The Replicas field is a pointer and may be nil so this method will // return either the actual Replicas value or the default (DefaultReplicas const) diff --git a/api/v1/coherenceresource_types.go b/api/v1/coherenceresource_types.go index bca656445..314eae1ca 100644 --- a/api/v1/coherenceresource_types.go +++ b/api/v1/coherenceresource_types.go @@ -150,6 +150,11 @@ func (in *Coherence) GetWkaServiceName() string { return in.Name + WKAServiceNameSuffix } +// GetWkaIPFamily returns the IP Family of the headless Service used for Coherence WKA. +func (in *Coherence) GetWkaIPFamily() corev1.IPFamily { + return in.Spec.GetWkaIPFamily() +} + // GetHeadlessServiceName returns the name of the headless Service used for the StatefulSet. func (in *Coherence) GetHeadlessServiceName() string { if in == nil { @@ -158,6 +163,14 @@ func (in *Coherence) GetHeadlessServiceName() string { return in.Name + HeadlessServiceNameSuffix } +// GetHeadlessServiceIPFamily returns the IP Family of the headless Service used for the StatefulSet. +func (in *Coherence) GetHeadlessServiceIPFamily() []corev1.IPFamily { + if in == nil { + return nil + } + return in.Spec.HeadlessServiceIpFamilies +} + // GetReplicas returns the number of replicas required for a deployment. // The Replicas field is a pointer and may be nil so this method will // return either the actual Replicas value or the default (DefaultReplicas const) @@ -527,6 +540,10 @@ type CoherenceStatefulSetResourceSpec struct { // one of the node labels used to set the Coherence site or rack value. // +optional RollingUpdateLabel *string `json:"rollingUpdateLabel,omitempty"` + // HeadlessServiceIpFamilies is the optional array of IP families that can be configured for + // the headless service used for the StatefulSet. + // +optional + HeadlessServiceIpFamilies []corev1.IPFamily `json:"headlessServiceIpFamilies,omitempty"` } // RollingUpdateStrategyType is a string enumeration type that enumerates diff --git a/api/v1/coherenceresourcespec_types.go b/api/v1/coherenceresourcespec_types.go index bbbb24d6b..f302eaee9 100644 --- a/api/v1/coherenceresourcespec_types.go +++ b/api/v1/coherenceresourcespec_types.go @@ -331,6 +331,14 @@ func (in *CoherenceResourceSpec) SetReplicas(replicas int32) { } } +// GetWkaIPFamily returns the IP Family of the headless Service used for Coherence WKA. +func (in *CoherenceResourceSpec) GetWkaIPFamily() corev1.IPFamily { + if in == nil || in.Coherence == nil { + return corev1.IPFamilyUnknown + } + return in.Coherence.GetWkaIPFamily() +} + // GetRestartPolicy returns the name of the application image to use func (in *CoherenceResourceSpec) GetRestartPolicy() *corev1.RestartPolicy { if in == nil { @@ -553,6 +561,12 @@ func (in *CoherenceResourceSpec) CreateWKAService(deployment CoherenceResource) }, } + ip := deployment.GetWkaIPFamily() + if ip != corev1.IPFamilyUnknown { + svc.Spec.IPFamilyPolicy = ptr.To(corev1.IPFamilyPolicySingleStack) + svc.Spec.IPFamilies = []corev1.IPFamily{ip} + } + return Resource{ Kind: ResourceTypeService, Name: svc.GetName(), @@ -592,6 +606,15 @@ func (in *CoherenceResourceSpec) CreateHeadlessService(deployment CoherenceResou }, } + ipFamilies := deployment.GetHeadlessServiceIPFamily() + if len(ipFamilies) == 1 { + svc.Spec.IPFamilyPolicy = ptr.To(corev1.IPFamilyPolicySingleStack) + svc.Spec.IPFamilies = ipFamilies + } else if len(ipFamilies) > 1 { + svc.Spec.IPFamilyPolicy = ptr.To(corev1.IPFamilyPolicyPreferDualStack) + svc.Spec.IPFamilies = ipFamilies + } + return Resource{ Kind: ResourceTypeService, Name: svc.GetName(), diff --git a/api/v1/create_job_wka_services_test.go b/api/v1/create_job_wka_services_test.go index 4c937f052..f813392b1 100644 --- a/api/v1/create_job_wka_services_test.go +++ b/api/v1/create_job_wka_services_test.go @@ -12,10 +12,11 @@ import ( coh "github.com/oracle/coherence-operator/api/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" "testing" ) -func TestCreateWKAServiceForMinimalJonDeployment(t *testing.T) { +func TestCreateWKAServiceForMinimalJsonDeployment(t *testing.T) { // Create the test deployment deployment := &coh.CoherenceJob{ ObjectMeta: metav1.ObjectMeta{ @@ -315,6 +316,60 @@ func TestCreateWKAServiceForJobWithAdditionalAnnotations(t *testing.T) { assertWKAServiceForJob(t, deployment, expected) } +func TestCreateJobWKAServiceWithIPFamily(t *testing.T) { + // Create the test deployment + deployment := &coh.CoherenceJob{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + Name: "test", + }, + Spec: coh.CoherenceJobResourceSpec{ + CoherenceResourceSpec: coh.CoherenceResourceSpec{ + Coherence: &coh.CoherenceSpec{ + WKA: &coh.CoherenceWKASpec{ + IPFamily: ptr.To(corev1.IPv4Protocol), + }, + }, + }, + Cluster: "test-cluster", + }, + } + + // create the expected WKA service + labels := deployment.CreateCommonLabels() + labels[coh.LabelCoherenceCluster] = "test-cluster" + labels[coh.LabelComponent] = coh.LabelComponentWKA + + // The selector for the service (match all Pods with the same cluster label) + selector := make(map[string]string) + selector[coh.LabelCoherenceCluster] = "test-cluster" + selector[coh.LabelComponent] = coh.LabelComponentCoherencePod + selector[coh.LabelCoherenceWKAMember] = "true" + + expected := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + Name: "test-wka", + Labels: labels, + Annotations: map[string]string{ + "service.alpha.kubernetes.io/tolerate-unready-endpoints": "true", + }, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: corev1.ClusterIPNone, + // Pods must be part of the WKA service even if not ready + PublishNotReadyAddresses: true, + Ports: getDefaultServicePorts(), + Selector: selector, + IPFamilyPolicy: ptr.To(corev1.IPFamilyPolicySingleStack), + IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol}, + }, + } + + // assert that the Services are as expected + assertWKAServiceForJob(t, deployment, expected) +} + func assertWKAServiceForJob(t *testing.T, deployment *coh.CoherenceJob, expected *corev1.Service) { g := NewGomegaWithT(t) diff --git a/api/v1/create_statefulset_headless_services_test.go b/api/v1/create_statefulset_headless_services_test.go new file mode 100644 index 000000000..b18edd700 --- /dev/null +++ b/api/v1/create_statefulset_headless_services_test.go @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ + +package v1_test + +import ( + "github.com/go-test/deep" + . "github.com/onsi/gomega" + coh "github.com/oracle/coherence-operator/api/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "testing" +) + +func TestCreateHeadlessServiceForMinimalDeployment(t *testing.T) { + // Create the test deployment + deployment := &coh.Coherence{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + Name: "test", + }, + } + + // create the expected WKA service + labels := deployment.CreateCommonLabels() + labels[coh.LabelCoherenceCluster] = "test" + labels[coh.LabelComponent] = coh.LabelComponentCoherenceHeadless + + // The selector for the service (match all Pods with the same cluster label) + selector := make(map[string]string) + selector[coh.LabelCoherenceDeployment] = "test" + selector[coh.LabelCoherenceCluster] = "test" + selector[coh.LabelCoherenceRole] = "test" + selector[coh.LabelComponent] = coh.LabelComponentCoherencePod + + expected := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + Name: "test-sts", + Labels: labels, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: corev1.ClusterIPNone, + PublishNotReadyAddresses: true, + Ports: getDefaultServicePorts(), + Selector: selector, + }, + } + + // assert that the Services are as expected + assertHeadlessService(t, deployment, expected) +} + +func TestCreateHeadlessServiceWithSingleIPFamily(t *testing.T) { + // Create the test deployment + deployment := &coh.Coherence{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + Name: "test", + }, + Spec: coh.CoherenceStatefulSetResourceSpec{ + HeadlessServiceIpFamilies: []corev1.IPFamily{ + corev1.IPv6Protocol, + }, + }, + } + + // create the expected WKA service + labels := deployment.CreateCommonLabels() + labels[coh.LabelCoherenceCluster] = "test" + labels[coh.LabelComponent] = coh.LabelComponentCoherenceHeadless + + // The selector for the service (match all Pods with the same cluster label) + selector := make(map[string]string) + selector[coh.LabelCoherenceDeployment] = "test" + selector[coh.LabelCoherenceCluster] = "test" + selector[coh.LabelCoherenceRole] = "test" + selector[coh.LabelComponent] = coh.LabelComponentCoherencePod + + expected := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + Name: "test-sts", + Labels: labels, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: corev1.ClusterIPNone, + PublishNotReadyAddresses: true, + Ports: getDefaultServicePorts(), + Selector: selector, + IPFamilyPolicy: ptr.To(corev1.IPFamilyPolicySingleStack), + IPFamilies: []corev1.IPFamily{corev1.IPv6Protocol}, + }, + } + + // assert that the Services are as expected + assertHeadlessService(t, deployment, expected) +} + +func TestCreateHeadlessServiceWithDualStackIPFamily(t *testing.T) { + // Create the test deployment + deployment := &coh.Coherence{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + Name: "test", + }, + Spec: coh.CoherenceStatefulSetResourceSpec{ + HeadlessServiceIpFamilies: []corev1.IPFamily{ + corev1.IPv4Protocol, + corev1.IPv6Protocol, + }, + }, + } + + // create the expected WKA service + labels := deployment.CreateCommonLabels() + labels[coh.LabelCoherenceCluster] = "test" + labels[coh.LabelComponent] = coh.LabelComponentCoherenceHeadless + + // The selector for the service (match all Pods with the same cluster label) + selector := make(map[string]string) + selector[coh.LabelCoherenceDeployment] = "test" + selector[coh.LabelCoherenceCluster] = "test" + selector[coh.LabelCoherenceRole] = "test" + selector[coh.LabelComponent] = coh.LabelComponentCoherencePod + + expected := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + Name: "test-sts", + Labels: labels, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: corev1.ClusterIPNone, + PublishNotReadyAddresses: true, + Ports: getDefaultServicePorts(), + Selector: selector, + IPFamilyPolicy: ptr.To(corev1.IPFamilyPolicyPreferDualStack), + IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol, corev1.IPv6Protocol}, + }, + } + + // assert that the Services are as expected + assertHeadlessService(t, deployment, expected) +} + +func assertHeadlessService(t *testing.T, deployment *coh.Coherence, expected *corev1.Service) { + g := NewGomegaWithT(t) + + resActual := deployment.Spec.CreateHeadlessService(deployment) + resExpected := coh.Resource{ + Kind: coh.ResourceTypeService, + Name: expected.GetName(), + Spec: expected, + } + + diffs := deep.Equal(resActual, resExpected) + g.Expect(diffs).To(BeNil()) +} diff --git a/api/v1/create_wka_services_test.go b/api/v1/create_wka_services_test.go index 3e7dd306c..4c3ac6eb7 100644 --- a/api/v1/create_wka_services_test.go +++ b/api/v1/create_wka_services_test.go @@ -317,6 +317,59 @@ func TestCreateWKAServiceForDeploymentWithAdditionalAnnotations(t *testing.T) { assertWKAService(t, deployment, expected) } +func TestCreateWKAServiceWithIPFamily(t *testing.T) { + // Create the test deployment + deployment := &coh.Coherence{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + Name: "test", + }, + Spec: coh.CoherenceStatefulSetResourceSpec{ + CoherenceResourceSpec: coh.CoherenceResourceSpec{ + Coherence: &coh.CoherenceSpec{ + WKA: &coh.CoherenceWKASpec{ + IPFamily: ptr.To(corev1.IPv4Protocol), + }, + }, + }, + }, + } + + // create the expected WKA service + labels := deployment.CreateCommonLabels() + labels[coh.LabelCoherenceCluster] = "test" + labels[coh.LabelComponent] = coh.LabelComponentWKA + + // The selector for the service (match all Pods with the same cluster label) + selector := make(map[string]string) + selector[coh.LabelCoherenceCluster] = "test" + selector[coh.LabelComponent] = coh.LabelComponentCoherencePod + selector[coh.LabelCoherenceWKAMember] = "true" + + expected := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test-ns", + Name: "test-wka", + Labels: labels, + Annotations: map[string]string{ + "service.alpha.kubernetes.io/tolerate-unready-endpoints": "true", + }, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: corev1.ClusterIPNone, + // Pods must be part of the WKA service even if not ready + PublishNotReadyAddresses: true, + Ports: getDefaultServicePorts(), + Selector: selector, + IPFamilyPolicy: ptr.To(corev1.IPFamilyPolicySingleStack), + IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol}, + }, + } + + // assert that the Services are as expected + assertWKAService(t, deployment, expected) +} + func assertWKAService(t *testing.T, deployment *coh.Coherence, expected *corev1.Service) { g := NewGomegaWithT(t) diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index ddf8ab6ac..ec177e7aa 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -917,6 +917,11 @@ func (in *CoherenceStatefulSetResourceSpec) DeepCopyInto(out *CoherenceStatefulS *out = new(string) **out = **in } + if in.HeadlessServiceIpFamilies != nil { + in, out := &in.HeadlessServiceIpFamilies, &out.HeadlessServiceIpFamilies + *out = make([]corev1.IPFamily, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoherenceStatefulSetResourceSpec. @@ -971,6 +976,11 @@ func (in *CoherenceWKASpec) DeepCopyInto(out *CoherenceWKASpec) { (*out)[key] = val } } + if in.IPFamily != nil { + in, out := &in.IPFamily, &out.IPFamily + *out = new(corev1.IPFamily) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoherenceWKASpec. diff --git a/docs/about/01_overview.adoc b/docs/about/01_overview.adoc index 0c0c65bac..637d688db 100644 --- a/docs/about/01_overview.adoc +++ b/docs/about/01_overview.adoc @@ -90,7 +90,10 @@ Configuring Coherence behaviour. -- Configure the behaviour of the JVM. -- +==== +[PILLARS] +==== [CARD] .Expose Ports & Services [icon=control_camera,link=docs/ports/010_overview.adoc] @@ -98,6 +101,13 @@ Configure the behaviour of the JVM. Configure services to expose ports provided by the application. -- +[CARD] +.Networking +[icon=share,link=docs/networking/010_overview.adoc] +-- +Configure networking settings. +-- + ==== [PILLARS] diff --git a/docs/about/02_introduction.adoc b/docs/about/02_introduction.adoc index cba510958..573423169 100644 --- a/docs/about/02_introduction.adoc +++ b/docs/about/02_introduction.adoc @@ -70,6 +70,9 @@ The Coherence CRD is designed to make the more commonly used configuration param to configure. The Coherence CRD is simple to use, in fact none of its fields are mandatory, so an application can be deployed with nothing more than a name, and a container image. +=== Dual-Stack Kubernetes Clusters +The Operator supports running Coherence on dual-stack IPv4 and IPv6 Kubernetes clusters. + === Consistency By using the Operator to manage Coherence clusters all clusters are configured and managed the same way making it easier for DevOps to manage multiple clusters and applications. diff --git a/docs/about/04_coherence_spec.adoc b/docs/about/04_coherence_spec.adoc index ef5c84942..1ae189a16 100644 --- a/docs/about/04_coherence_spec.adoc +++ b/docs/about/04_coherence_spec.adoc @@ -277,6 +277,7 @@ m| namespace | The optional namespace of the existing Coherence deployment to us m| addresses | A list of addresses to be used for WKA. If this field is set, the WKA property for the Coherence cluster will be set using this value and the other WKA fields will be ignored. m| []string | false m| labels | Labels is a map of optional additional labels to apply to the WKA Service. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ m| map[string]string | false m| annotations | Annotations is a map of optional additional labels to apply to the WKA Service. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ m| map[string]string | false +m| ipFamily | IPFamily is the IP family to use for the WKA service (and also the StatefulSet headless service). Valid values are "IPv4" or "IPv6". m| *https://pkg.go.dev/k8s.io/api/core/v1#IPFamily | false |=== <> @@ -928,6 +929,7 @@ m| initResources | InitResources is the optional resource requests and limits fo The Coherence operator does not apply any default resources. m| *https://{k8s-doc-link}/#resourcerequirements-v1-core[corev1.ResourceRequirements] | false m| rollingUpdateStrategy | The rolling upgrade strategy to use. If present, the value must be one of "UpgradeByPod", "UpgradeByNode" of "OnDelete". If not set, the default is "UpgradeByPod" UpgradeByPod will perform a rolling upgrade one Pod at a time. UpgradeByNode will update all Pods on a Node at the same time. OnDelete will not automatically apply any updates, Pods must be manually deleted for updates to be applied to the restarted Pod. m| *RollingUpdateStrategyType | false m| rollingUpdateLabel | The name of the Node label to use to group Pods during a rolling upgrade. This field ony applies if RollingUpdateStrategy is set to NodeLabel. If RollingUpdateStrategy is set to NodeLabel and this field is omitted then the rolling upgrade will be by Node. It is the users responsibility to ensure that Nodes actually have the label used for this field. The label should be one of the node labels used to set the Coherence site or rack value. m| *string | false +m| headlessServiceIpFamilies | HeadlessServiceIpFamilies is the optional array of IP families that can be configured for the headless service used for the StatefulSet. m| []https://pkg.go.dev/k8s.io/api/core/v1#IPFamily | false |=== <
> diff --git a/docs/networking/010_overview.adoc b/docs/networking/010_overview.adoc new file mode 100644 index 000000000..b22abe254 --- /dev/null +++ b/docs/networking/010_overview.adoc @@ -0,0 +1,24 @@ +/////////////////////////////////////////////////////////////////////////////// + + Copyright (c) 2024, Oracle and/or its affiliates. + Licensed under the Universal Permissive License v 1.0 as shown at + http://oss.oracle.com/licenses/upl. + +/////////////////////////////////////////////////////////////////////////////// + += Overview + +== Overview + +This section describes how Coherence and the Operator work with different Kubernetes networking configurations. + +[PILLARS] +==== +[CARD] +.Dual Stack Clusters +[link=docs/networking/020_dual_stack.adoc] +-- +Running on a dual-stack Kubernetes cluster +-- +==== + diff --git a/docs/networking/020_dual_stack.adoc b/docs/networking/020_dual_stack.adoc new file mode 100644 index 000000000..7899585fb --- /dev/null +++ b/docs/networking/020_dual_stack.adoc @@ -0,0 +1,202 @@ +/////////////////////////////////////////////////////////////////////////////// + + Copyright (c) 2024, Oracle and/or its affiliates. + Licensed under the Universal Permissive License v 1.0 as shown at + http://oss.oracle.com/licenses/upl. + +/////////////////////////////////////////////////////////////////////////////// + += Dual Stack Networking + +== Dual Stack Networking + +This section describes using Coherence and the Operator with a dual stack Kubernetes cluster, +where Pods and Services can have both IPv4 and IPv4 interfaces. + +[NOTE] +==== +This section only really applies to making Coherence bind to the correct local IP address for inter-cluster communication. +Normally for other Coherence endpoints, such as Extend, gRPC, management, metrics, etc. Coherence will bind to all +local addresses ubless specifically configured otherwise. +This means that in and environment such as dual-stack Kubernetes where a Pod has both an IPv4 and IPv6 +address, those Coherence endpoints will be reachable using either the IPv4 or IPv6 address of the Pod. +==== + +Normally, using Coherence on a dual-stack server can cause issues due to the way that Coherence decides which local IP +address to use for inter-cluster communication. Similar problems can occur on any server that multiple IP addresses. +When Coherence is configured to use well known addressing for cluster discovery, a Coherence JVM will choose a local +address that is either in the WKA list, or is on an interface that can route to the WKA addresses. +In a dual stack environment the problem comes when an interface has both IPv4 and IPv6 addresses and Coherence is +inconsistent about which one to choose. + +There are a few simple ways to fix this: + +* Set the JVM system property `java.net.preferIPv4Stack=true` or `java.net.preferIPv6Addresses=true` to set the Coherence +JVM to use the desired stack. If application code requires both stacks to be available though, this is not a good option. + +* Configure the WKA list to be only IPv4 addresses or IPv6 addresses. Coherence will then choose a matching local address. + +* Set the `coherence.localhost` system property (or `COHERENCE_LOCALHOST` environment variable) to the IP address +that Coherence should bind to. In a dual stack environment choose either the IPv4 address or IPv6 address and make sure +that the corresponding addresses are used in the WKA list. + +=== Dual Stack Kubernetes Clusters + +In a dual-stack Kubernetes cluster, Pods will have both an IPv4 and IPv6 address. +These can be seen by looking at the status section of a Pod spec: + +[source,yaml] +---- + podIP: 10.244.3.3 + podIPs: + - ip: 10.244.3.3 + - ip: fd00:10:244:3::3 +---- + +The status section will have a `podIP` field, which is the Pods primary address. +There is also an array of the dual-stack addresses in the `podIPs` field. +The first address in `podIPs` is always the same as `podIP` and is usually the IPv4 address. + +A Service in a dual-stack cluster can have a single IP family or multiple IP families configured in its spec. +The Operator will work out of the box if the default IP families configuration for Services is single stack, either IPv4 or IPv6. +When the WKA Service is created it will only be populated with one type of address, and Coherence will bind to the correct type. + +In Kubernetes clusters where the WKA service has multiple IP families by default, there are a few options to fix this: + +* Set the JVM system property `java.net.preferIPv4Stack=true` or `java.net.preferIPv6Addresses=true` to set the Coherence +JVM to use the desired stack. If application code requires both stacks to be available though, this is not a good option. + +[source,yaml] +---- +apiVersion: coherence.oracle.com/v1 +kind: Coherence +metadata: + name: storage +spec: + replicas: 3 + jvm: + args: + - "java.net.preferIPv4Stack=true" +---- + +* The `COHERENCE_LOCALHOST` environment variable can be configured to be the Pods IP address. +Typically, this will be the IPv4 address. + +[source,yaml] +---- +apiVersion: coherence.oracle.com/v1 +kind: Coherence +metadata: + name: storage +spec: + replicas: 3 + env: + - name: COHERENCE_LOCALHOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP +---- + +* Since Operator 3.4.1 it is possible to configure the IP family for the WKA Service. The `spec.coherence.wka.ipFamily` +field can be set to either "IPv4" or "IPv6". This will cause Coherence to bind to the relevant IP address type. + +For example, the yaml below will cause Coherence to bind to the IPv6 address. + +[source,yaml] +---- +apiVersion: coherence.oracle.com/v1 +kind: Coherence +metadata: + name: storage +spec: + replicas: 3 + coherence: + wka: + ipFamily: IPv6 +---- + +Since Operator 3.4.1 it is also possible to configure the IP families used by the headless service created for the StatefulSet +if this is required. + +The yaml below will configure WKA to use only IPv6, the headless Service created for the StatefulSet will be +a dual-stack, IPv4 and IPv6 service. + +[source,yaml] +---- +apiVersion: coherence.oracle.com/v1 +kind: Coherence +metadata: + name: storage +spec: + replicas: 3 + headlessServiceIpFamilies: + - IPv4 + - IPv6 + coherence: + wka: + ipFamily: IPv6 +---- + +The yaml below will configure both WKA and the headless Service created for the StatefulSet to use a single stack IPv6. + +[source,yaml] +---- +apiVersion: coherence.oracle.com/v1 +kind: Coherence +metadata: + name: storage +spec: + replicas: 3 + headlessServiceIpFamilies: + - IPv6 + coherence: + wka: + ipFamily: IPv6 +---- + +=== Dual Stack Kubernetes Clusters Without Using the Operator + +If not using the Coherence Operator to manage clusters the same techniques described above can be used to +manually configure Coherence to work correctly. + +The simplest option is to ensure that the headless service used for well known addressing is configured to be single stack. +For example, the yaml below configures the service `storage-sts` to be a single stack IPv6 service. + +[source,yaml] +---- +apiVersion: v1 +kind: Service +metadata: + name: storage-sts +spec: + clusterIP: None + clusterIPs: + - None + ipFamilies: + - IPv6 + ipFamilyPolicy: SingleStack +---- + +If for some reason it is not possible to ise a dedicated single stack service for WKA, then the `COHERENCE_LOCALHOST` +environment variable can be set in the Pod to be the Pod IP address. + +[source,yaml] +---- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: storage +spec: + template: + spec: + containers: + - name: coherence + env: + - name: COHERENCE_LOCALHOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP +---- + diff --git a/docs/sitegen.yaml b/docs/sitegen.yaml index 9d3ad3afe..4f731b45f 100644 --- a/docs/sitegen.yaml +++ b/docs/sitegen.yaml @@ -94,6 +94,14 @@ backend: items: - includes: - "docs/ports/*.adoc" + - title: "Networking" + pathprefix: "/networking" + glyph: + type: "icon" + value: "share" + items: + - includes: + - "docs/networking/*.adoc" - title: "Scaling Up & Down" pathprefix: "/scaling" glyph: diff --git a/docs/troubleshooting/01_trouble-shooting.adoc b/docs/troubleshooting/01_trouble-shooting.adoc index b2a9b9de0..3595ea8d9 100644 --- a/docs/troubleshooting/01_trouble-shooting.adoc +++ b/docs/troubleshooting/01_trouble-shooting.adoc @@ -23,6 +23,8 @@ This page will be updated and maintained over time to include common issues we s * <<#ready,Why are the Coherence Pods not reaching ready>> +* <<#messed,I messed up a Coherence deployment and now cannot apply a fixed yaml>> + * <<#stuck-pending,My Coherence cluster is stuck with some running Pods and some pending Pods, I want to scale down>> * <<#stuck-delete,My Coherence cluster is stuck with all pending/crashing Pods, I cannot delete the deployment>> @@ -115,6 +117,31 @@ When running in clusters with the Operator using custom main classes it is advis from within your `main` method. This can be done using the new Coherence bootstrap API available from CE release 20.12 or by calling `com.tangosol.net.DefaultCacheServer.startServerDaemon().waitForServiceStart();` +[#messed] +=== I messed up a Coherence deployment and now cannot apply a fixed yaml +If you deploy a Coherence resource or perform a rolling upgrade of a Coherence resource, and there is an error +somewhere that causes the Pods to fail to become ready, the Operator will refuse to allow any updates to be +applied to that resource. This means it is impossible to re-apply the old working yaml and roll back an update. +By default, the Operator will not apply an update to a Coherence cluster if any of the Pods is not ready or the +cluster's Status HA value is endangered. This is to stop updates happening when Coherence is recovering from the loss +of a Pod or still starting up. + +This can be very annoying as the cluster is broken and needs to be fixed. +The way to force the Operator to apply the update is to set the `haBeforeUpdate` field in the yaml to `false`. +This causes the Operator to skip the status checks for the cluster and apply the update. + +[source,yaml] +---- +apiVersion: coherence.oracle.com/v1 +kind: Coherence +metadata: + name: storage +spec: + haBeforeUpdate: false +---- + + + [#stuck-pending] === My Coherence cluster is stuck with some running Pods and some pending Pods, I want to scale down diff --git a/hack/kind-config-dual.yaml b/hack/kind-config-dual.yaml new file mode 100644 index 000000000..062a2c8e5 --- /dev/null +++ b/hack/kind-config-dual.yaml @@ -0,0 +1,9 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + ipFamily: dual +nodes: + - role: control-plane + - role: worker + - role: worker + - role: worker