From ee0ba2b8bb4279049df19c05f9cde3fc4f71d486 Mon Sep 17 00:00:00 2001 From: Pavel Basov Date: Sat, 12 Apr 2025 20:06:14 +0200 Subject: [PATCH 1/2] Allow API Loadbalancer Health Monitor configuration --- api/v1beta1/types.go | 29 +++++ ...re.cluster.x-k8s.io_openstackclusters.yaml | 30 +++++ ...er.x-k8s.io_openstackclustertemplates.yaml | 30 +++++ pkg/clients/loadbalancer.go | 10 ++ pkg/clients/mock/loadbalancer.go | 15 +++ .../services/loadbalancer/loadbalancer.go | 107 ++++++++++++++++-- pkg/webhooks/openstackcluster_webhook.go | 6 + 7 files changed, 216 insertions(+), 11 deletions(-) diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 99bc29d906..848ce0903e 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -879,6 +879,35 @@ type APIServerLoadBalancer struct { // Flavor is the flavor name that will be used to create the APIServerLoadBalancer Spec. //+optional Flavor optional.String `json:"flavor,omitempty"` + + // Monitor contains configuration for the load balancer health monitor. + //+optional + Monitor *APIServerLoadBalancerMonitor `json:"monitor,omitempty"` +} + +// APIServerLoadBalancerMonitor contains configuration for the load balancer health monitor. +type APIServerLoadBalancerMonitor struct { + // Delay is the time in seconds between sending probes to members. Default is 10. + //+optional + //+kubebuilder:validation:Minimum=1 + Delay *int `json:"delay,omitempty"` + + // Timeout is the maximum time in seconds for a monitor to wait for a connection to be established before it times out. Default is 5. + //+optional + //+kubebuilder:validation:Minimum=1 + Timeout *int `json:"timeout,omitempty"` + + // MaxRetries is the number of successful checks before changing the operating status of the member to ONLINE. Default is 5. + //+optional + //+kubebuilder:validation:Minimum=1 + //+kubebuilder:validation:Maximum=10 + MaxRetries *int `json:"maxRetries,omitempty"` + + // MaxRetriesDown is the number of allowed check failures before changing the operating status of the member to ERROR. Default is 3. + //+optional + //+kubebuilder:validation:Minimum=1 + //+kubebuilder:validation:Maximum=10 + MaxRetriesDown *int `json:"maxRetriesDown,omitempty"` } func (s *APIServerLoadBalancer) IsZero() bool { diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml index bdddd3d140..df3c62b9cd 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml @@ -124,6 +124,36 @@ spec: description: Flavor is the flavor name that will be used to create the APIServerLoadBalancer Spec. type: string + monitor: + description: Monitor contains configuration for the load balancer + health monitor. + properties: + delay: + description: Delay is the time in seconds between sending + probes to members. Default is 10. + minimum: 1 + type: integer + maxRetries: + description: MaxRetries is the number of successful checks + before changing the operating status of the member to ONLINE. + Default is 5. + maximum: 10 + minimum: 1 + type: integer + maxRetriesDown: + description: MaxRetriesDown is the number of allowed check + failures before changing the operating status of the member + to ERROR. Default is 3. + maximum: 10 + minimum: 1 + type: integer + timeout: + description: Timeout is the maximum time in seconds for a + monitor to wait for a connection to be established before + it times out. Default is 5. + minimum: 1 + type: integer + type: object network: description: Network defines which network should the load balancer be allocated on. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclustertemplates.yaml index e9c85270b0..3c019f0d9e 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclustertemplates.yaml @@ -108,6 +108,36 @@ spec: description: Flavor is the flavor name that will be used to create the APIServerLoadBalancer Spec. type: string + monitor: + description: Monitor contains configuration for the load + balancer health monitor. + properties: + delay: + description: Delay is the time in seconds between + sending probes to members. Default is 10. + minimum: 1 + type: integer + maxRetries: + description: MaxRetries is the number of successful + checks before changing the operating status of the + member to ONLINE. Default is 5. + maximum: 10 + minimum: 1 + type: integer + maxRetriesDown: + description: MaxRetriesDown is the number of allowed + check failures before changing the operating status + of the member to ERROR. Default is 3. + maximum: 10 + minimum: 1 + type: integer + timeout: + description: Timeout is the maximum time in seconds + for a monitor to wait for a connection to be established + before it times out. Default is 5. + minimum: 1 + type: integer + type: object network: description: Network defines which network should the load balancer be allocated on. diff --git a/pkg/clients/loadbalancer.go b/pkg/clients/loadbalancer.go index 2fd019ea3d..1c35823967 100644 --- a/pkg/clients/loadbalancer.go +++ b/pkg/clients/loadbalancer.go @@ -54,6 +54,7 @@ type LbClient interface { DeletePoolMember(poolID string, lbMemberID string) error CreateMonitor(opts monitors.CreateOptsBuilder) (*monitors.Monitor, error) ListMonitors(opts monitors.ListOptsBuilder) ([]monitors.Monitor, error) + UpdateMonitor(id string, opts monitors.UpdateOptsBuilder) (*monitors.Monitor, error) DeleteMonitor(id string) error ListLoadBalancerProviders() ([]providers.Provider, error) ListOctaviaVersions() ([]apiversions.APIVersion, error) @@ -239,6 +240,15 @@ func (l lbClient) ListMonitors(opts monitors.ListOptsBuilder) ([]monitors.Monito return monitors.ExtractMonitors(allPages) } +func (l lbClient) UpdateMonitor(id string, opts monitors.UpdateOptsBuilder) (*monitors.Monitor, error) { + mc := metrics.NewMetricPrometheusContext("loadbalancer_healthmonitor", "update") + monitor, err := monitors.Update(context.TODO(), l.serviceClient, id, opts).Extract() + if mc.ObserveRequest(err) != nil { + return nil, err + } + return monitor, nil +} + func (l lbClient) DeleteMonitor(id string) error { mc := metrics.NewMetricPrometheusContext("loadbalancer_healthmonitor", "delete") err := monitors.Delete(context.TODO(), l.serviceClient, id).ExtractErr() diff --git a/pkg/clients/mock/loadbalancer.go b/pkg/clients/mock/loadbalancer.go index 92b15886c1..075d92ad85 100644 --- a/pkg/clients/mock/loadbalancer.go +++ b/pkg/clients/mock/loadbalancer.go @@ -164,6 +164,21 @@ func (mr *MockLbClientMockRecorder) DeleteLoadBalancer(id, opts any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLoadBalancer", reflect.TypeOf((*MockLbClient)(nil).DeleteLoadBalancer), id, opts) } +// UpdateMonitor mocks base method. +func (m *MockLbClient) UpdateMonitor(id string, opts monitors.UpdateOptsBuilder) (*monitors.Monitor, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateMonitor", id, opts) + ret0, _ := ret[0].(*monitors.Monitor) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateMonitor indicates an expected call of UpdateMonitor. +func (mr *MockLbClientMockRecorder) UpdateMonitor(id, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateMonitor", reflect.TypeOf((*MockLbClient)(nil).UpdateMonitor), id, opts) +} + // DeleteMonitor mocks base method. func (m *MockLbClient) DeleteMonitor(id string) error { m.ctrl.T.Helper() diff --git a/pkg/cloud/services/loadbalancer/loadbalancer.go b/pkg/cloud/services/loadbalancer/loadbalancer.go index 62028145d2..4575c5fc47 100644 --- a/pkg/cloud/services/loadbalancer/loadbalancer.go +++ b/pkg/cloud/services/loadbalancer/loadbalancer.go @@ -406,8 +406,7 @@ func (s *Service) reconcileAPILoadBalancerListener(lb *loadbalancers.LoadBalance if err != nil { return err } - - if err := s.getOrCreateMonitor(openStackCluster, lbPortObjectsName, pool.ID, lb.ID); err != nil { + if err := s.getOrUpdateMonitor(openStackCluster, lbPortObjectsName, pool.ID, lb.ID); err != nil { return err } @@ -532,13 +531,83 @@ func (s *Service) getOrCreatePool(openStackCluster *infrav1.OpenStackCluster, po return pool, nil } -func (s *Service) getOrCreateMonitor(openStackCluster *infrav1.OpenStackCluster, monitorName, poolID, lbID string) error { +func (s *Service) getOrUpdateMonitor(openStackCluster *infrav1.OpenStackCluster, monitorName, poolID, lbID string) error { monitor, err := s.checkIfMonitorExists(monitorName) if err != nil { return err } + monitorConfig := openStackCluster.Spec.APIServerLoadBalancer.Monitor + + // Default values for monitor + const ( + defaultDelay = 10 + defaultTimeout = 5 + defaultMaxRetries = 5 + defaultMaxRetriesDown = 3 + ) + if monitor != nil { + needsUpdate := false + monitorUpdateOpts := monitors.UpdateOpts{} + + if (monitorConfig == nil || monitorConfig.Delay == nil) && monitor.Delay != defaultDelay { + s.scope.Logger().Info("Monitor delay needs update to default", "current", monitor.Delay, "default", defaultDelay) + monitorUpdateOpts.Delay = defaultDelay + needsUpdate = true + } else if monitorConfig != nil && monitorConfig.Delay != nil && monitor.Delay != *monitorConfig.Delay { + s.scope.Logger().Info("Monitor delay needs update", "current", monitor.Delay, "desired", *monitorConfig.Delay) + monitorUpdateOpts.Delay = *monitorConfig.Delay + needsUpdate = true + } + + if (monitorConfig == nil || monitorConfig.Timeout == nil) && monitor.Timeout != defaultTimeout { + s.scope.Logger().Info("Monitor timeout needs update to default", "current", monitor.Timeout, "default", defaultTimeout) + monitorUpdateOpts.Timeout = defaultTimeout + needsUpdate = true + } else if monitorConfig != nil && monitorConfig.Timeout != nil && monitor.Timeout != *monitorConfig.Timeout { + s.scope.Logger().Info("Monitor timeout needs update", "current", monitor.Timeout, "desired", *monitorConfig.Timeout) + monitorUpdateOpts.Timeout = *monitorConfig.Timeout + needsUpdate = true + } + + if (monitorConfig == nil || monitorConfig.MaxRetries == nil) && monitor.MaxRetries != defaultMaxRetries { + s.scope.Logger().Info("Monitor maxRetries needs update to default", "current", monitor.MaxRetries, "default", defaultMaxRetries) + monitorUpdateOpts.MaxRetries = defaultMaxRetries + needsUpdate = true + } else if monitorConfig != nil && monitorConfig.MaxRetries != nil && monitor.MaxRetries != *monitorConfig.MaxRetries { + s.scope.Logger().Info("Monitor maxRetries needs update", "current", monitor.MaxRetries, "desired", *monitorConfig.MaxRetries) + monitorUpdateOpts.MaxRetries = *monitorConfig.MaxRetries + needsUpdate = true + } + + if (monitorConfig == nil || monitorConfig.MaxRetriesDown == nil) && monitor.MaxRetriesDown != defaultMaxRetriesDown { + s.scope.Logger().Info("Monitor maxRetriesDown needs update to default", "current", monitor.MaxRetriesDown, "default", defaultMaxRetriesDown) + monitorUpdateOpts.MaxRetriesDown = defaultMaxRetriesDown + needsUpdate = true + } else if monitorConfig != nil && monitorConfig.MaxRetriesDown != nil && monitor.MaxRetriesDown != *monitorConfig.MaxRetriesDown { + s.scope.Logger().Info("Monitor maxRetriesDown needs update", "current", monitor.MaxRetriesDown, "desired", *monitorConfig.MaxRetriesDown) + monitorUpdateOpts.MaxRetriesDown = *monitorConfig.MaxRetriesDown + needsUpdate = true + } + + if needsUpdate { + s.scope.Logger().Info("Updating load balancer monitor", "loadBalancerID", lbID, "name", monitorName, "monitorID", monitor.ID) + + updatedMonitor, err := s.loadbalancerClient.UpdateMonitor(monitor.ID, monitorUpdateOpts) + if err != nil { + record.Warnf(openStackCluster, "FailedUpdateMonitor", "Failed to update monitor %s with id %s: %v", monitorName, monitor.ID, err) + return err + } + + if _, err = s.waitForLoadBalancerActive(lbID); err != nil { + record.Warnf(openStackCluster, "FailedUpdateMonitor", "Failed to update monitor %s with id %s: wait for load balancer active %s: %v", monitorName, monitor.ID, lbID, err) + return err + } + + record.Eventf(openStackCluster, "SuccessfulUpdateMonitor", "Updated monitor %s with id %s", monitorName, updatedMonitor.ID) + } + return nil } @@ -548,13 +617,29 @@ func (s *Service) getOrCreateMonitor(openStackCluster *infrav1.OpenStackCluster, Name: monitorName, PoolID: poolID, Type: "TCP", - Delay: 10, - MaxRetries: 5, - MaxRetriesDown: 3, - Timeout: 5, + Delay: defaultDelay, + Timeout: defaultTimeout, + MaxRetries: defaultMaxRetries, + MaxRetriesDown: defaultMaxRetriesDown, + } + + if monitorConfig != nil { + if monitorConfig.Delay != nil { + monitorCreateOpts.Delay = *monitorConfig.Delay + } + if monitorConfig.MaxRetries != nil { + monitorCreateOpts.MaxRetries = *monitorConfig.MaxRetries + } + if monitorConfig.MaxRetriesDown != nil { + monitorCreateOpts.MaxRetriesDown = *monitorConfig.MaxRetriesDown + } + if monitorConfig.Timeout != nil { + monitorCreateOpts.Timeout = *monitorConfig.Timeout + } } - monitor, err = s.loadbalancerClient.CreateMonitor(monitorCreateOpts) - // Skip creating monitor if it is not supported by Octavia provider + + newMonitor, err := s.loadbalancerClient.CreateMonitor(monitorCreateOpts) + if capoerrors.IsNotImplementedError(err) { record.Warnf(openStackCluster, "SkippedCreateMonitor", "Health Monitor is not created as it's not implemented with the current Octavia provider.") return nil @@ -566,11 +651,11 @@ func (s *Service) getOrCreateMonitor(openStackCluster *infrav1.OpenStackCluster, } if _, err = s.waitForLoadBalancerActive(lbID); err != nil { - record.Warnf(openStackCluster, "FailedCreateMonitor", "Failed to create monitor %s with id %s: wait for load balancer active %s: %v", monitorName, monitor.ID, lbID, err) + record.Warnf(openStackCluster, "FailedCreateMonitor", "Failed to create monitor %s with id %s: wait for load balancer active %s: %v", monitorName, newMonitor.ID, lbID, err) return err } - record.Eventf(openStackCluster, "SuccessfulCreateMonitor", "Created monitor %s with id %s", monitorName, monitor.ID) + record.Eventf(openStackCluster, "SuccessfulCreateMonitor", "Created monitor %s with id %s", monitorName, newMonitor.ID) return nil } diff --git a/pkg/webhooks/openstackcluster_webhook.go b/pkg/webhooks/openstackcluster_webhook.go index 19a2571b2a..61cc444ea7 100644 --- a/pkg/webhooks/openstackcluster_webhook.go +++ b/pkg/webhooks/openstackcluster_webhook.go @@ -199,6 +199,12 @@ func (*openStackClusterWebhook) ValidateUpdate(_ context.Context, oldObjRaw, new newObj.Spec.APIServerLoadBalancer.AllowedCIDRs = []string{} } + // Allow changes on APIServerLB monitors + if newObj.Spec.APIServerLoadBalancer != nil && oldObj.Spec.APIServerLoadBalancer != nil { + oldObj.Spec.APIServerLoadBalancer.Monitor = &infrav1.APIServerLoadBalancerMonitor{} + newObj.Spec.APIServerLoadBalancer.Monitor = &infrav1.APIServerLoadBalancerMonitor{} + } + // Allow changes to the availability zones. oldObj.Spec.ControlPlaneAvailabilityZones = []string{} newObj.Spec.ControlPlaneAvailabilityZones = []string{} From 2a68b8e2ebc9ab6412df40684c7eee2aa58bc8f3 Mon Sep 17 00:00:00 2001 From: s3rj1k Date: Tue, 29 Apr 2025 20:59:47 +0200 Subject: [PATCH 2/2] Code refactor and regenerate autogenerated files --- api/v1beta1/types.go | 26 +- api/v1beta1/zz_generated.deepcopy.go | 20 ++ cmd/models-schema/zz_generated.openapi.go | 50 ++- ...re.cluster.x-k8s.io_openstackclusters.yaml | 13 +- ...er.x-k8s.io_openstackclustertemplates.yaml | 14 +- docs/book/src/api/v1beta1/api.md | 81 +++++ pkg/clients/mock/loadbalancer.go | 30 +- .../services/loadbalancer/loadbalancer.go | 116 +++---- .../loadbalancer/loadbalancer_test.go | 311 +++++++++++++++++- .../api/v1beta1/apiserverloadbalancer.go | 25 +- .../v1beta1/apiserverloadbalancermonitor.go | 66 ++++ .../applyconfiguration/internal/internal.go | 18 + pkg/generated/applyconfiguration/utils.go | 2 + 13 files changed, 648 insertions(+), 124 deletions(-) create mode 100644 pkg/generated/applyconfiguration/api/v1beta1/apiserverloadbalancermonitor.go diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 848ce0903e..cc6cc0b3ce 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -887,27 +887,31 @@ type APIServerLoadBalancer struct { // APIServerLoadBalancerMonitor contains configuration for the load balancer health monitor. type APIServerLoadBalancerMonitor struct { - // Delay is the time in seconds between sending probes to members. Default is 10. + // Delay is the time in seconds between sending probes to members. //+optional - //+kubebuilder:validation:Minimum=1 - Delay *int `json:"delay,omitempty"` + //+kubebuilder:validation:Minimum=0 + //+kubebuilder:default:10 + Delay int `json:"delay,omitempty"` - // Timeout is the maximum time in seconds for a monitor to wait for a connection to be established before it times out. Default is 5. + // Timeout is the maximum time in seconds for a monitor to wait for a connection to be established before it times out. //+optional - //+kubebuilder:validation:Minimum=1 - Timeout *int `json:"timeout,omitempty"` + //+kubebuilder:validation:Minimum=0 + //+kubebuilder:default:5 + Timeout int `json:"timeout,omitempty"` - // MaxRetries is the number of successful checks before changing the operating status of the member to ONLINE. Default is 5. + // MaxRetries is the number of successful checks before changing the operating status of the member to ONLINE. //+optional - //+kubebuilder:validation:Minimum=1 + //+kubebuilder:validation:Minimum=0 //+kubebuilder:validation:Maximum=10 - MaxRetries *int `json:"maxRetries,omitempty"` + //+kubebuilder:default:5 + MaxRetries int `json:"maxRetries,omitempty"` - // MaxRetriesDown is the number of allowed check failures before changing the operating status of the member to ERROR. Default is 3. + // MaxRetriesDown is the number of allowed check failures before changing the operating status of the member to ERROR. //+optional //+kubebuilder:validation:Minimum=1 //+kubebuilder:validation:Maximum=10 - MaxRetriesDown *int `json:"maxRetriesDown,omitempty"` + //+kubebuilder:default:3 + MaxRetriesDown int `json:"maxRetriesDown,omitempty"` } func (s *APIServerLoadBalancer) IsZero() bool { diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index d10b48a05a..096419fd4e 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -72,6 +72,11 @@ func (in *APIServerLoadBalancer) DeepCopyInto(out *APIServerLoadBalancer) { *out = new(string) **out = **in } + if in.Monitor != nil { + in, out := &in.Monitor, &out.Monitor + *out = new(APIServerLoadBalancerMonitor) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIServerLoadBalancer. @@ -84,6 +89,21 @@ func (in *APIServerLoadBalancer) DeepCopy() *APIServerLoadBalancer { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *APIServerLoadBalancerMonitor) DeepCopyInto(out *APIServerLoadBalancerMonitor) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIServerLoadBalancerMonitor. +func (in *APIServerLoadBalancerMonitor) DeepCopy() *APIServerLoadBalancerMonitor { + if in == nil { + return nil + } + out := new(APIServerLoadBalancerMonitor) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AdditionalBlockDevice) DeepCopyInto(out *AdditionalBlockDevice) { *out = *in diff --git a/cmd/models-schema/zz_generated.openapi.go b/cmd/models-schema/zz_generated.openapi.go index 84da178f8d..87e44c528d 100644 --- a/cmd/models-schema/zz_generated.openapi.go +++ b/cmd/models-schema/zz_generated.openapi.go @@ -324,6 +324,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha1.ResolvedServerSpec": schema_sigsk8sio_cluster_api_provider_openstack_api_v1alpha1_ResolvedServerSpec(ref), "sigs.k8s.io/cluster-api-provider-openstack/api/v1alpha1.ServerResources": schema_sigsk8sio_cluster_api_provider_openstack_api_v1alpha1_ServerResources(ref), "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1.APIServerLoadBalancer": schema_sigsk8sio_cluster_api_provider_openstack_api_v1beta1_APIServerLoadBalancer(ref), + "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1.APIServerLoadBalancerMonitor": schema_sigsk8sio_cluster_api_provider_openstack_api_v1beta1_APIServerLoadBalancerMonitor(ref), "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1.AdditionalBlockDevice": schema_sigsk8sio_cluster_api_provider_openstack_api_v1beta1_AdditionalBlockDevice(ref), "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1.AddressPair": schema_sigsk8sio_cluster_api_provider_openstack_api_v1beta1_AddressPair(ref), "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1.AllocationPool": schema_sigsk8sio_cluster_api_provider_openstack_api_v1beta1_AllocationPool(ref), @@ -17301,12 +17302,59 @@ func schema_sigsk8sio_cluster_api_provider_openstack_api_v1beta1_APIServerLoadBa Format: "", }, }, + "monitor": { + SchemaProps: spec.SchemaProps{ + Description: "Monitor contains configuration for the load balancer health monitor.", + Ref: ref("sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1.APIServerLoadBalancerMonitor"), + }, + }, }, Required: []string{"enabled"}, }, }, Dependencies: []string{ - "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1.NetworkParam", "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1.SubnetParam"}, + "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1.APIServerLoadBalancerMonitor", "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1.NetworkParam", "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1.SubnetParam"}, + } +} + +func schema_sigsk8sio_cluster_api_provider_openstack_api_v1beta1_APIServerLoadBalancerMonitor(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "APIServerLoadBalancerMonitor contains configuration for the load balancer health monitor.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "delay": { + SchemaProps: spec.SchemaProps{ + Description: "Delay is the time in seconds between sending probes to members.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "timeout": { + SchemaProps: spec.SchemaProps{ + Description: "Timeout is the maximum time in seconds for a monitor to wait for a connection to be established before it times out.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "maxRetries": { + SchemaProps: spec.SchemaProps{ + Description: "MaxRetries is the number of successful checks before changing the operating status of the member to ONLINE.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "maxRetriesDown": { + SchemaProps: spec.SchemaProps{ + Description: "MaxRetriesDown is the number of allowed check failures before changing the operating status of the member to ERROR.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + }, + }, + }, } } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml index df3c62b9cd..ae4cc97c99 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml @@ -130,28 +130,27 @@ spec: properties: delay: description: Delay is the time in seconds between sending - probes to members. Default is 10. - minimum: 1 + probes to members. + minimum: 0 type: integer maxRetries: description: MaxRetries is the number of successful checks before changing the operating status of the member to ONLINE. - Default is 5. maximum: 10 - minimum: 1 + minimum: 0 type: integer maxRetriesDown: description: MaxRetriesDown is the number of allowed check failures before changing the operating status of the member - to ERROR. Default is 3. + to ERROR. maximum: 10 minimum: 1 type: integer timeout: description: Timeout is the maximum time in seconds for a monitor to wait for a connection to be established before - it times out. Default is 5. - minimum: 1 + it times out. + minimum: 0 type: integer type: object network: diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclustertemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclustertemplates.yaml index 3c019f0d9e..6913c78c84 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclustertemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclustertemplates.yaml @@ -114,28 +114,28 @@ spec: properties: delay: description: Delay is the time in seconds between - sending probes to members. Default is 10. - minimum: 1 + sending probes to members. + minimum: 0 type: integer maxRetries: description: MaxRetries is the number of successful checks before changing the operating status of the - member to ONLINE. Default is 5. + member to ONLINE. maximum: 10 - minimum: 1 + minimum: 0 type: integer maxRetriesDown: description: MaxRetriesDown is the number of allowed check failures before changing the operating status - of the member to ERROR. Default is 3. + of the member to ERROR. maximum: 10 minimum: 1 type: integer timeout: description: Timeout is the maximum time in seconds for a monitor to wait for a connection to be established - before it times out. Default is 5. - minimum: 1 + before it times out. + minimum: 0 type: integer type: object network: diff --git a/docs/book/src/api/v1beta1/api.md b/docs/book/src/api/v1beta1/api.md index 00e763491e..db0acfdd7b 100644 --- a/docs/book/src/api/v1beta1/api.md +++ b/docs/book/src/api/v1beta1/api.md @@ -985,6 +985,87 @@ string

Flavor is the flavor name that will be used to create the APIServerLoadBalancer Spec.

+ + +monitor
+ + +APIServerLoadBalancerMonitor + + + + +(Optional) +

Monitor contains configuration for the load balancer health monitor.

+ + + + +

APIServerLoadBalancerMonitor +

+

+(Appears on: +APIServerLoadBalancer) +

+

+

APIServerLoadBalancerMonitor contains configuration for the load balancer health monitor.

+

+ + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+delay
+ +int + +		 +(Optional) +

Delay is the time in seconds between sending probes to members.

+
+timeout
+ +int + +		 +(Optional) +

Timeout is the maximum time in seconds for a monitor to wait for a connection to be established before it times out.

+
+maxRetries
+ +int + +		 +(Optional) +

MaxRetries is the number of successful checks before changing the operating status of the member to ONLINE.

+
+maxRetriesDown
+ +int + +		 +(Optional) +

MaxRetriesDown is the number of allowed check failures before changing the operating status of the member to ERROR.

+

AdditionalBlockDevice diff --git a/pkg/clients/mock/loadbalancer.go b/pkg/clients/mock/loadbalancer.go index 075d92ad85..59024521f8 100644 --- a/pkg/clients/mock/loadbalancer.go +++ b/pkg/clients/mock/loadbalancer.go @@ -164,21 +164,6 @@ func (mr *MockLbClientMockRecorder) DeleteLoadBalancer(id, opts any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLoadBalancer", reflect.TypeOf((*MockLbClient)(nil).DeleteLoadBalancer), id, opts) } -// UpdateMonitor mocks base method. -func (m *MockLbClient) UpdateMonitor(id string, opts monitors.UpdateOptsBuilder) (*monitors.Monitor, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateMonitor", id, opts) - ret0, _ := ret[0].(*monitors.Monitor) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// UpdateMonitor indicates an expected call of UpdateMonitor. -func (mr *MockLbClientMockRecorder) UpdateMonitor(id, opts any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateMonitor", reflect.TypeOf((*MockLbClient)(nil).UpdateMonitor), id, opts) -} - // DeleteMonitor mocks base method. func (m *MockLbClient) DeleteMonitor(id string) error { m.ctrl.T.Helper() @@ -400,3 +385,18 @@ func (mr *MockLbClientMockRecorder) UpdateListener(id, opts any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateListener", reflect.TypeOf((*MockLbClient)(nil).UpdateListener), id, opts) } + +// UpdateMonitor mocks base method. +func (m *MockLbClient) UpdateMonitor(id string, opts monitors.UpdateOptsBuilder) (*monitors.Monitor, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateMonitor", id, opts) + ret0, _ := ret[0].(*monitors.Monitor) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateMonitor indicates an expected call of UpdateMonitor. +func (mr *MockLbClientMockRecorder) UpdateMonitor(id, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateMonitor", reflect.TypeOf((*MockLbClient)(nil).UpdateMonitor), id, opts) +} diff --git a/pkg/cloud/services/loadbalancer/loadbalancer.go b/pkg/cloud/services/loadbalancer/loadbalancer.go index 4575c5fc47..efcd79a161 100644 --- a/pkg/cloud/services/loadbalancer/loadbalancer.go +++ b/pkg/cloud/services/loadbalancer/loadbalancer.go @@ -17,6 +17,7 @@ limitations under the License. package loadbalancer import ( + "cmp" "context" "errors" "fmt" @@ -53,6 +54,14 @@ const ( loadBalancerProvisioningStatusPendingDelete = "PENDING_DELETE" ) +// Default values for Monitor, sync with `kubebuilder:default` annotations on APIServerLoadBalancerMonitor object. +const ( + defaultMonitorDelay = 10 + defaultMonitorTimeout = 5 + defaultMonitorMaxRetries = 5 + defaultMonitorMaxRetriesDown = 3 +) + // We wrap the LookupHost function in a variable to allow overriding it in unit tests. // //nolint:gocritic @@ -406,7 +415,7 @@ func (s *Service) reconcileAPILoadBalancerListener(lb *loadbalancers.LoadBalance if err != nil { return err } - if err := s.getOrUpdateMonitor(openStackCluster, lbPortObjectsName, pool.ID, lb.ID); err != nil { + if err := s.ensureMonitor(openStackCluster, lbPortObjectsName, pool.ID, lb.ID); err != nil { return err } @@ -531,63 +540,48 @@ func (s *Service) getOrCreatePool(openStackCluster *infrav1.OpenStackCluster, po return pool, nil } -func (s *Service) getOrUpdateMonitor(openStackCluster *infrav1.OpenStackCluster, monitorName, poolID, lbID string) error { +func (s *Service) ensureMonitor(openStackCluster *infrav1.OpenStackCluster, monitorName, poolID, lbID string) error { + var cfg infrav1.APIServerLoadBalancerMonitor + + if openStackCluster.Spec.APIServerLoadBalancer.Monitor != nil { + cfg = *openStackCluster.Spec.APIServerLoadBalancer.Monitor + } + + cfg.Delay = cmp.Or(cfg.Delay, defaultMonitorDelay) + cfg.Timeout = cmp.Or(cfg.Timeout, defaultMonitorTimeout) + cfg.MaxRetries = cmp.Or(cfg.MaxRetries, defaultMonitorMaxRetries) + cfg.MaxRetriesDown = cmp.Or(cfg.MaxRetriesDown, defaultMonitorMaxRetriesDown) + monitor, err := s.checkIfMonitorExists(monitorName) if err != nil { return err } - monitorConfig := openStackCluster.Spec.APIServerLoadBalancer.Monitor - - // Default values for monitor - const ( - defaultDelay = 10 - defaultTimeout = 5 - defaultMaxRetries = 5 - defaultMaxRetriesDown = 3 - ) - if monitor != nil { needsUpdate := false monitorUpdateOpts := monitors.UpdateOpts{} - if (monitorConfig == nil || monitorConfig.Delay == nil) && monitor.Delay != defaultDelay { - s.scope.Logger().Info("Monitor delay needs update to default", "current", monitor.Delay, "default", defaultDelay) - monitorUpdateOpts.Delay = defaultDelay - needsUpdate = true - } else if monitorConfig != nil && monitorConfig.Delay != nil && monitor.Delay != *monitorConfig.Delay { - s.scope.Logger().Info("Monitor delay needs update", "current", monitor.Delay, "desired", *monitorConfig.Delay) - monitorUpdateOpts.Delay = *monitorConfig.Delay + if monitor.Delay != cfg.Delay { + s.scope.Logger().Info("Monitor delay needs update", "current", monitor.Delay, "desired", cfg.Delay) + monitorUpdateOpts.Delay = cfg.Delay needsUpdate = true } - if (monitorConfig == nil || monitorConfig.Timeout == nil) && monitor.Timeout != defaultTimeout { - s.scope.Logger().Info("Monitor timeout needs update to default", "current", monitor.Timeout, "default", defaultTimeout) - monitorUpdateOpts.Timeout = defaultTimeout - needsUpdate = true - } else if monitorConfig != nil && monitorConfig.Timeout != nil && monitor.Timeout != *monitorConfig.Timeout { - s.scope.Logger().Info("Monitor timeout needs update", "current", monitor.Timeout, "desired", *monitorConfig.Timeout) - monitorUpdateOpts.Timeout = *monitorConfig.Timeout + if monitor.Timeout != cfg.Timeout { + s.scope.Logger().Info("Monitor timeout needs update", "current", monitor.Timeout, "desired", cfg.Timeout) + monitorUpdateOpts.Timeout = cfg.Timeout needsUpdate = true } - if (monitorConfig == nil || monitorConfig.MaxRetries == nil) && monitor.MaxRetries != defaultMaxRetries { - s.scope.Logger().Info("Monitor maxRetries needs update to default", "current", monitor.MaxRetries, "default", defaultMaxRetries) - monitorUpdateOpts.MaxRetries = defaultMaxRetries - needsUpdate = true - } else if monitorConfig != nil && monitorConfig.MaxRetries != nil && monitor.MaxRetries != *monitorConfig.MaxRetries { - s.scope.Logger().Info("Monitor maxRetries needs update", "current", monitor.MaxRetries, "desired", *monitorConfig.MaxRetries) - monitorUpdateOpts.MaxRetries = *monitorConfig.MaxRetries + if monitor.MaxRetries != cfg.MaxRetries { + s.scope.Logger().Info("Monitor maxRetries needs update", "current", monitor.MaxRetries, "desired", cfg.MaxRetries) + monitorUpdateOpts.MaxRetries = cfg.MaxRetries needsUpdate = true } - if (monitorConfig == nil || monitorConfig.MaxRetriesDown == nil) && monitor.MaxRetriesDown != defaultMaxRetriesDown { - s.scope.Logger().Info("Monitor maxRetriesDown needs update to default", "current", monitor.MaxRetriesDown, "default", defaultMaxRetriesDown) - monitorUpdateOpts.MaxRetriesDown = defaultMaxRetriesDown - needsUpdate = true - } else if monitorConfig != nil && monitorConfig.MaxRetriesDown != nil && monitor.MaxRetriesDown != *monitorConfig.MaxRetriesDown { - s.scope.Logger().Info("Monitor maxRetriesDown needs update", "current", monitor.MaxRetriesDown, "desired", *monitorConfig.MaxRetriesDown) - monitorUpdateOpts.MaxRetriesDown = *monitorConfig.MaxRetriesDown + if monitor.MaxRetriesDown != cfg.MaxRetriesDown { + s.scope.Logger().Info("Monitor maxRetriesDown needs update", "current", monitor.MaxRetriesDown, "desired", cfg.MaxRetriesDown) + monitorUpdateOpts.MaxRetriesDown = cfg.MaxRetriesDown needsUpdate = true } @@ -613,49 +607,31 @@ func (s *Service) getOrUpdateMonitor(openStackCluster *infrav1.OpenStackCluster, s.scope.Logger().Info("Creating load balancer monitor for pool", "loadBalancerID", lbID, "name", monitorName, "poolID", poolID) - monitorCreateOpts := monitors.CreateOpts{ + monitor, err = s.loadbalancerClient.CreateMonitor(monitors.CreateOpts{ Name: monitorName, PoolID: poolID, Type: "TCP", - Delay: defaultDelay, - Timeout: defaultTimeout, - MaxRetries: defaultMaxRetries, - MaxRetriesDown: defaultMaxRetriesDown, - } - - if monitorConfig != nil { - if monitorConfig.Delay != nil { - monitorCreateOpts.Delay = *monitorConfig.Delay - } - if monitorConfig.MaxRetries != nil { - monitorCreateOpts.MaxRetries = *monitorConfig.MaxRetries - } - if monitorConfig.MaxRetriesDown != nil { - monitorCreateOpts.MaxRetriesDown = *monitorConfig.MaxRetriesDown - } - if monitorConfig.Timeout != nil { - monitorCreateOpts.Timeout = *monitorConfig.Timeout + Delay: cfg.Delay, + Timeout: cfg.Timeout, + MaxRetries: cfg.MaxRetries, + MaxRetriesDown: cfg.MaxRetriesDown, + }) + if err != nil { + if capoerrors.IsNotImplementedError(err) { + record.Warnf(openStackCluster, "SkippedCreateMonitor", "Health Monitor is not created as it's not implemented with the current Octavia provider.") + return nil } - } - - newMonitor, err := s.loadbalancerClient.CreateMonitor(monitorCreateOpts) - - if capoerrors.IsNotImplementedError(err) { - record.Warnf(openStackCluster, "SkippedCreateMonitor", "Health Monitor is not created as it's not implemented with the current Octavia provider.") - return nil - } - if err != nil { record.Warnf(openStackCluster, "FailedCreateMonitor", "Failed to create monitor %s: %v", monitorName, err) return err } if _, err = s.waitForLoadBalancerActive(lbID); err != nil { - record.Warnf(openStackCluster, "FailedCreateMonitor", "Failed to create monitor %s with id %s: wait for load balancer active %s: %v", monitorName, newMonitor.ID, lbID, err) + record.Warnf(openStackCluster, "FailedCreateMonitor", "Failed to create monitor %s with id %s: wait for load balancer active %s: %v", monitorName, monitor.ID, lbID, err) return err } - record.Eventf(openStackCluster, "SuccessfulCreateMonitor", "Created monitor %s with id %s", monitorName, newMonitor.ID) + record.Eventf(openStackCluster, "SuccessfulCreateMonitor", "Created monitor %s with id %s", monitorName, monitor.ID) return nil } diff --git a/pkg/cloud/services/loadbalancer/loadbalancer_test.go b/pkg/cloud/services/loadbalancer/loadbalancer_test.go index b9470ce7a8..8594e18854 100644 --- a/pkg/cloud/services/loadbalancer/loadbalancer_test.go +++ b/pkg/cloud/services/loadbalancer/loadbalancer_test.go @@ -88,12 +88,14 @@ func Test_ReconcileLoadBalancer(t *testing.T) { } lbtests := []struct { name string + clusterSpec *infrav1.OpenStackCluster expectNetwork func(m *mock.MockNetworkClientMockRecorder) expectLoadBalancer func(m *mock.MockLbClientMockRecorder) wantError error }{ { - name: "reconcile loadbalancer in non active state should wait for active state", + name: "reconcile loadbalancer in non active state should wait for active state", + clusterSpec: openStackCluster, expectNetwork: func(*mock.MockNetworkClientMockRecorder) { // add network api call results here }, @@ -137,10 +139,15 @@ func Test_ReconcileLoadBalancer(t *testing.T) { } m.ListPools(pools.ListOpts{Name: poolList[0].Name}).Return(poolList, nil) + // create a monitor with values that match defaults to prevent update monitorList := []monitors.Monitor{ { - ID: "aaaaaaaa-bbbb-cccc-dddd-666666666666", - Name: "k8s-clusterapi-cluster-AAAAA-kubeapi-0", + ID: "aaaaaaaa-bbbb-cccc-dddd-666666666666", + Name: "k8s-clusterapi-cluster-AAAAA-kubeapi-0", + Delay: 10, + Timeout: 5, + MaxRetries: 5, + MaxRetriesDown: 3, }, } m.ListMonitors(monitors.ListOpts{Name: monitorList[0].Name}).Return(monitorList, nil) @@ -148,7 +155,8 @@ func Test_ReconcileLoadBalancer(t *testing.T) { wantError: nil, }, { - name: "reconcile loadbalancer in non active state should timeout", + name: "reconcile loadbalancer in non active state should timeout", + clusterSpec: openStackCluster, expectNetwork: func(*mock.MockNetworkClientMockRecorder) { // add network api call results here }, @@ -168,6 +176,299 @@ func Test_ReconcileLoadBalancer(t *testing.T) { }, wantError: fmt.Errorf("load balancer \"k8s-clusterapi-cluster-AAAAA-kubeapi\" with id aaaaaaaa-bbbb-cccc-dddd-333333333333 is not active after timeout: timed out waiting for the condition"), }, + { + name: "should update monitor when values are different than defaults", + clusterSpec: &infrav1.OpenStackCluster{ + Spec: infrav1.OpenStackClusterSpec{ + APIServerLoadBalancer: &infrav1.APIServerLoadBalancer{ + Enabled: ptr.To(true), + Monitor: &infrav1.APIServerLoadBalancerMonitor{ + Delay: 15, + Timeout: 8, + MaxRetries: 6, + MaxRetriesDown: 4, + }, + }, + DisableAPIServerFloatingIP: ptr.To(true), + ControlPlaneEndpoint: &clusterv1.APIEndpoint{ + Host: apiHostname, + Port: 6443, + }, + }, + Status: infrav1.OpenStackClusterStatus{ + ExternalNetwork: &infrav1.NetworkStatus{ + ID: "aaaaaaaa-bbbb-cccc-dddd-111111111111", + }, + Network: &infrav1.NetworkStatusWithSubnets{ + Subnets: []infrav1.Subnet{ + {ID: "aaaaaaaa-bbbb-cccc-dddd-222222222222"}, + }, + }, + }, + }, + expectNetwork: func(*mock.MockNetworkClientMockRecorder) { + // add network api call results here + }, + expectLoadBalancer: func(m *mock.MockLbClientMockRecorder) { + activeLB := loadbalancers.LoadBalancer{ + ID: "aaaaaaaa-bbbb-cccc-dddd-333333333333", + Name: "k8s-clusterapi-cluster-AAAAA-kubeapi", + ProvisioningStatus: "ACTIVE", + } + + // return existing loadbalancer in active state + lbList := []loadbalancers.LoadBalancer{activeLB} + m.ListLoadBalancers(loadbalancers.ListOpts{Name: activeLB.Name}).Return(lbList, nil) + + // return octavia versions + versions := []apiversions.APIVersion{ + {ID: "2.24"}, + {ID: "2.23"}, + {ID: "2.22"}, + } + m.ListOctaviaVersions().Return(versions, nil) + + listenerList := []listeners.Listener{ + { + ID: "aaaaaaaa-bbbb-cccc-dddd-444444444444", + Name: "k8s-clusterapi-cluster-AAAAA-kubeapi-0", + }, + } + m.ListListeners(listeners.ListOpts{Name: listenerList[0].Name}).Return(listenerList, nil) + + poolList := []pools.Pool{ + { + ID: "aaaaaaaa-bbbb-cccc-dddd-555555555555", + Name: "k8s-clusterapi-cluster-AAAAA-kubeapi-0", + }, + } + m.ListPools(pools.ListOpts{Name: poolList[0].Name}).Return(poolList, nil) + + // existing monitor has default values that need updating + existingMonitor := monitors.Monitor{ + ID: "aaaaaaaa-bbbb-cccc-dddd-666666666666", + Name: "k8s-clusterapi-cluster-AAAAA-kubeapi-0", + Delay: 10, + Timeout: 5, + MaxRetries: 5, + MaxRetriesDown: 3, + } + monitorList := []monitors.Monitor{existingMonitor} + m.ListMonitors(monitors.ListOpts{Name: monitorList[0].Name}).Return(monitorList, nil) + + // Expect update call with the new values + updateOpts := monitors.UpdateOpts{ + Delay: 15, + Timeout: 8, + MaxRetries: 6, + MaxRetriesDown: 4, + } + + updatedMonitor := existingMonitor + updatedMonitor.Delay = 15 + updatedMonitor.Timeout = 8 + updatedMonitor.MaxRetries = 6 + updatedMonitor.MaxRetriesDown = 4 + + m.UpdateMonitor(existingMonitor.ID, updateOpts).Return(&updatedMonitor, nil) + + // Expect wait for loadbalancer to be active after monitor update + m.GetLoadBalancer(activeLB.ID).Return(&activeLB, nil) + }, + wantError: nil, + }, + { + name: "should report error when monitor update fails", + clusterSpec: &infrav1.OpenStackCluster{ + Spec: infrav1.OpenStackClusterSpec{ + APIServerLoadBalancer: &infrav1.APIServerLoadBalancer{ + Enabled: ptr.To(true), + Monitor: &infrav1.APIServerLoadBalancerMonitor{ + Delay: 15, + Timeout: 8, + MaxRetries: 6, + MaxRetriesDown: 4, + }, + }, + DisableAPIServerFloatingIP: ptr.To(true), + ControlPlaneEndpoint: &clusterv1.APIEndpoint{ + Host: apiHostname, + Port: 6443, + }, + }, + Status: infrav1.OpenStackClusterStatus{ + ExternalNetwork: &infrav1.NetworkStatus{ + ID: "aaaaaaaa-bbbb-cccc-dddd-111111111111", + }, + Network: &infrav1.NetworkStatusWithSubnets{ + Subnets: []infrav1.Subnet{ + {ID: "aaaaaaaa-bbbb-cccc-dddd-222222222222"}, + }, + }, + }, + }, + expectNetwork: func(*mock.MockNetworkClientMockRecorder) { + // add network api call results here + }, + expectLoadBalancer: func(m *mock.MockLbClientMockRecorder) { + activeLB := loadbalancers.LoadBalancer{ + ID: "aaaaaaaa-bbbb-cccc-dddd-333333333333", + Name: "k8s-clusterapi-cluster-AAAAA-kubeapi", + ProvisioningStatus: "ACTIVE", + } + + // return existing loadbalancer in active state + lbList := []loadbalancers.LoadBalancer{activeLB} + m.ListLoadBalancers(loadbalancers.ListOpts{Name: activeLB.Name}).Return(lbList, nil) + + // return octavia versions + versions := []apiversions.APIVersion{ + {ID: "2.24"}, + {ID: "2.23"}, + {ID: "2.22"}, + } + m.ListOctaviaVersions().Return(versions, nil) + + listenerList := []listeners.Listener{ + { + ID: "aaaaaaaa-bbbb-cccc-dddd-444444444444", + Name: "k8s-clusterapi-cluster-AAAAA-kubeapi-0", + }, + } + m.ListListeners(listeners.ListOpts{Name: listenerList[0].Name}).Return(listenerList, nil) + + poolList := []pools.Pool{ + { + ID: "aaaaaaaa-bbbb-cccc-dddd-555555555555", + Name: "k8s-clusterapi-cluster-AAAAA-kubeapi-0", + }, + } + m.ListPools(pools.ListOpts{Name: poolList[0].Name}).Return(poolList, nil) + + // existing monitor has default values that need updating + existingMonitor := monitors.Monitor{ + ID: "aaaaaaaa-bbbb-cccc-dddd-666666666666", + Name: "k8s-clusterapi-cluster-AAAAA-kubeapi-0", + Delay: 10, + Timeout: 5, + MaxRetries: 5, + MaxRetriesDown: 3, + } + monitorList := []monitors.Monitor{existingMonitor} + m.ListMonitors(monitors.ListOpts{Name: monitorList[0].Name}).Return(monitorList, nil) + + // Expect update call with the new values but return an error + updateOpts := monitors.UpdateOpts{ + Delay: 15, + Timeout: 8, + MaxRetries: 6, + MaxRetriesDown: 4, + } + + updateError := fmt.Errorf("failed to update monitor") + m.UpdateMonitor(existingMonitor.ID, updateOpts).Return(nil, updateError) + }, + wantError: fmt.Errorf("failed to update monitor"), + }, + { + name: "should create monitor when it doesn't exist", + clusterSpec: &infrav1.OpenStackCluster{ + Spec: infrav1.OpenStackClusterSpec{ + APIServerLoadBalancer: &infrav1.APIServerLoadBalancer{ + Enabled: ptr.To(true), + Monitor: &infrav1.APIServerLoadBalancerMonitor{ + Delay: 15, + Timeout: 8, + MaxRetries: 6, + MaxRetriesDown: 4, + }, + }, + DisableAPIServerFloatingIP: ptr.To(true), + ControlPlaneEndpoint: &clusterv1.APIEndpoint{ + Host: apiHostname, + Port: 6443, + }, + }, + Status: infrav1.OpenStackClusterStatus{ + ExternalNetwork: &infrav1.NetworkStatus{ + ID: "aaaaaaaa-bbbb-cccc-dddd-111111111111", + }, + Network: &infrav1.NetworkStatusWithSubnets{ + Subnets: []infrav1.Subnet{ + {ID: "aaaaaaaa-bbbb-cccc-dddd-222222222222"}, + }, + }, + }, + }, + expectNetwork: func(*mock.MockNetworkClientMockRecorder) { + // add network api call results here + }, + expectLoadBalancer: func(m *mock.MockLbClientMockRecorder) { + activeLB := loadbalancers.LoadBalancer{ + ID: "aaaaaaaa-bbbb-cccc-dddd-333333333333", + Name: "k8s-clusterapi-cluster-AAAAA-kubeapi", + ProvisioningStatus: "ACTIVE", + } + + // return existing loadbalancer in active state + lbList := []loadbalancers.LoadBalancer{activeLB} + m.ListLoadBalancers(loadbalancers.ListOpts{Name: activeLB.Name}).Return(lbList, nil) + + // return octavia versions + versions := []apiversions.APIVersion{ + {ID: "2.24"}, + {ID: "2.23"}, + {ID: "2.22"}, + } + m.ListOctaviaVersions().Return(versions, nil) + + listenerList := []listeners.Listener{ + { + ID: "aaaaaaaa-bbbb-cccc-dddd-444444444444", + Name: "k8s-clusterapi-cluster-AAAAA-kubeapi-0", + }, + } + m.ListListeners(listeners.ListOpts{Name: listenerList[0].Name}).Return(listenerList, nil) + + poolList := []pools.Pool{ + { + ID: "aaaaaaaa-bbbb-cccc-dddd-555555555555", + Name: "k8s-clusterapi-cluster-AAAAA-kubeapi-0", + }, + } + m.ListPools(pools.ListOpts{Name: poolList[0].Name}).Return(poolList, nil) + + // No monitor exists yet + var emptyMonitorList []monitors.Monitor + m.ListMonitors(monitors.ListOpts{Name: "k8s-clusterapi-cluster-AAAAA-kubeapi-0"}).Return(emptyMonitorList, nil) + + // Expect create call with custom values + createOpts := monitors.CreateOpts{ + Name: "k8s-clusterapi-cluster-AAAAA-kubeapi-0", + PoolID: "aaaaaaaa-bbbb-cccc-dddd-555555555555", + Type: "TCP", + Delay: 15, + Timeout: 8, + MaxRetries: 6, + MaxRetriesDown: 4, + } + + createdMonitor := monitors.Monitor{ + ID: "aaaaaaaa-bbbb-cccc-dddd-666666666666", + Name: "k8s-clusterapi-cluster-AAAAA-kubeapi-0", + Delay: 15, + Timeout: 8, + MaxRetries: 6, + MaxRetriesDown: 4, + } + + m.CreateMonitor(createOpts).Return(&createdMonitor, nil) + + // Expect wait for loadbalancer to be active after monitor creation + m.GetLoadBalancer(activeLB.ID).Return(&activeLB, nil) + }, + wantError: nil, + }, } for _, tt := range lbtests { t.Run(tt.name, func(t *testing.T) { @@ -180,7 +481,7 @@ func Test_ReconcileLoadBalancer(t *testing.T) { tt.expectNetwork(mockScopeFactory.NetworkClient.EXPECT()) tt.expectLoadBalancer(mockScopeFactory.LbClient.EXPECT()) - _, err = lbs.ReconcileLoadBalancer(openStackCluster, "AAAAA", 0) + _, err = lbs.ReconcileLoadBalancer(tt.clusterSpec, "AAAAA", 0) if tt.wantError != nil { g.Expect(err).To(MatchError(tt.wantError)) } else { diff --git a/pkg/generated/applyconfiguration/api/v1beta1/apiserverloadbalancer.go b/pkg/generated/applyconfiguration/api/v1beta1/apiserverloadbalancer.go index 8a749eb46a..4d7442ef3b 100644 --- a/pkg/generated/applyconfiguration/api/v1beta1/apiserverloadbalancer.go +++ b/pkg/generated/applyconfiguration/api/v1beta1/apiserverloadbalancer.go @@ -21,14 +21,15 @@ package v1beta1 // APIServerLoadBalancerApplyConfiguration represents a declarative configuration of the APIServerLoadBalancer type for use // with apply. type APIServerLoadBalancerApplyConfiguration struct { - Enabled *bool `json:"enabled,omitempty"` - AdditionalPorts []int `json:"additionalPorts,omitempty"` - AllowedCIDRs []string `json:"allowedCIDRs,omitempty"` - Provider *string `json:"provider,omitempty"` - Network *NetworkParamApplyConfiguration `json:"network,omitempty"` - Subnets []SubnetParamApplyConfiguration `json:"subnets,omitempty"` - AvailabilityZone *string `json:"availabilityZone,omitempty"` - Flavor *string `json:"flavor,omitempty"` + Enabled *bool `json:"enabled,omitempty"` + AdditionalPorts []int `json:"additionalPorts,omitempty"` + AllowedCIDRs []string `json:"allowedCIDRs,omitempty"` + Provider *string `json:"provider,omitempty"` + Network *NetworkParamApplyConfiguration `json:"network,omitempty"` + Subnets []SubnetParamApplyConfiguration `json:"subnets,omitempty"` + AvailabilityZone *string `json:"availabilityZone,omitempty"` + Flavor *string `json:"flavor,omitempty"` + Monitor *APIServerLoadBalancerMonitorApplyConfiguration `json:"monitor,omitempty"` } // APIServerLoadBalancerApplyConfiguration constructs a declarative configuration of the APIServerLoadBalancer type for use with @@ -109,3 +110,11 @@ func (b *APIServerLoadBalancerApplyConfiguration) WithFlavor(value string) *APIS b.Flavor = &value return b } + +// WithMonitor sets the Monitor field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Monitor field is set to the value of the last call. +func (b *APIServerLoadBalancerApplyConfiguration) WithMonitor(value *APIServerLoadBalancerMonitorApplyConfiguration) *APIServerLoadBalancerApplyConfiguration { + b.Monitor = value + return b +} diff --git a/pkg/generated/applyconfiguration/api/v1beta1/apiserverloadbalancermonitor.go b/pkg/generated/applyconfiguration/api/v1beta1/apiserverloadbalancermonitor.go new file mode 100644 index 0000000000..fa8e0db7ea --- /dev/null +++ b/pkg/generated/applyconfiguration/api/v1beta1/apiserverloadbalancermonitor.go @@ -0,0 +1,66 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +// APIServerLoadBalancerMonitorApplyConfiguration represents a declarative configuration of the APIServerLoadBalancerMonitor type for use +// with apply. +type APIServerLoadBalancerMonitorApplyConfiguration struct { + Delay *int `json:"delay,omitempty"` + Timeout *int `json:"timeout,omitempty"` + MaxRetries *int `json:"maxRetries,omitempty"` + MaxRetriesDown *int `json:"maxRetriesDown,omitempty"` +} + +// APIServerLoadBalancerMonitorApplyConfiguration constructs a declarative configuration of the APIServerLoadBalancerMonitor type for use with +// apply. +func APIServerLoadBalancerMonitor() *APIServerLoadBalancerMonitorApplyConfiguration { + return &APIServerLoadBalancerMonitorApplyConfiguration{} +} + +// WithDelay sets the Delay field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Delay field is set to the value of the last call. +func (b *APIServerLoadBalancerMonitorApplyConfiguration) WithDelay(value int) *APIServerLoadBalancerMonitorApplyConfiguration { + b.Delay = &value + return b +} + +// WithTimeout sets the Timeout field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Timeout field is set to the value of the last call. +func (b *APIServerLoadBalancerMonitorApplyConfiguration) WithTimeout(value int) *APIServerLoadBalancerMonitorApplyConfiguration { + b.Timeout = &value + return b +} + +// WithMaxRetries sets the MaxRetries field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the MaxRetries field is set to the value of the last call. +func (b *APIServerLoadBalancerMonitorApplyConfiguration) WithMaxRetries(value int) *APIServerLoadBalancerMonitorApplyConfiguration { + b.MaxRetries = &value + return b +} + +// WithMaxRetriesDown sets the MaxRetriesDown field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the MaxRetriesDown field is set to the value of the last call. +func (b *APIServerLoadBalancerMonitorApplyConfiguration) WithMaxRetriesDown(value int) *APIServerLoadBalancerMonitorApplyConfiguration { + b.MaxRetriesDown = &value + return b +} diff --git a/pkg/generated/applyconfiguration/internal/internal.go b/pkg/generated/applyconfiguration/internal/internal.go index 46d86085d8..d406ba77cb 100644 --- a/pkg/generated/applyconfiguration/internal/internal.go +++ b/pkg/generated/applyconfiguration/internal/internal.go @@ -387,6 +387,9 @@ var schemaYAML = typed.YAMLObject(`types: - name: flavor type: scalar: string + - name: monitor + type: + namedType: io.k8s.sigs.cluster-api-provider-openstack.api.v1beta1.APIServerLoadBalancerMonitor - name: network type: namedType: io.k8s.sigs.cluster-api-provider-openstack.api.v1beta1.NetworkParam @@ -399,6 +402,21 @@ var schemaYAML = typed.YAMLObject(`types: elementType: namedType: io.k8s.sigs.cluster-api-provider-openstack.api.v1beta1.SubnetParam elementRelationship: atomic +- name: io.k8s.sigs.cluster-api-provider-openstack.api.v1beta1.APIServerLoadBalancerMonitor + map: + fields: + - name: delay + type: + scalar: numeric + - name: maxRetries + type: + scalar: numeric + - name: maxRetriesDown + type: + scalar: numeric + - name: timeout + type: + scalar: numeric - name: io.k8s.sigs.cluster-api-provider-openstack.api.v1beta1.AdditionalBlockDevice map: fields: diff --git a/pkg/generated/applyconfiguration/utils.go b/pkg/generated/applyconfiguration/utils.go index 3496afb337..dfa39a2abf 100644 --- a/pkg/generated/applyconfiguration/utils.go +++ b/pkg/generated/applyconfiguration/utils.go @@ -54,6 +54,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &apiv1beta1.AllocationPoolApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("APIServerLoadBalancer"): return &apiv1beta1.APIServerLoadBalancerApplyConfiguration{} + case v1beta1.SchemeGroupVersion.WithKind("APIServerLoadBalancerMonitor"): + return &apiv1beta1.APIServerLoadBalancerMonitorApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("Bastion"): return &apiv1beta1.BastionApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("BastionStatus"):