Allow API Loadbalancer Health Monitor configuration

Author: pbasov

api/v1beta1/types.go

@@ -879,6 +879,35 @@ type APIServerLoadBalancer struct {
 	// Flavor is the flavor name that will be used to create the APIServerLoadBalancer Spec.
 	//+optional
 	Flavor optional.String `json:"flavor,omitempty"`
+
+	// Monitor contains configuration for the load balancer health monitor.
+	//+optional
+	Monitor *APIServerLoadBalancerMonitor `json:"monitor,omitempty"`
+}
+
+// APIServerLoadBalancerMonitor contains configuration for the load balancer health monitor.
+type APIServerLoadBalancerMonitor struct {
+	// Delay is the time in seconds between sending probes to members. Default is 10.
+	//+optional
+	//+kubebuilder:validation:Minimum=1
+	Delay *int `json:"delay,omitempty"`
+
+	// Timeout is the maximum time in seconds for a monitor to wait for a connection to be established before it times out. Default is 5.
+	//+optional
+	//+kubebuilder:validation:Minimum=1
+	Timeout *int `json:"timeout,omitempty"`
+
+	// MaxRetries is the number of successful checks before changing the operating status of the member to ONLINE. Default is 5.
+	//+optional
+	//+kubebuilder:validation:Minimum=1
+	//+kubebuilder:validation:Maximum=10
+	MaxRetries *int `json:"maxRetries,omitempty"`
+
+	// MaxRetriesDown is the number of allowed check failures before changing the operating status of the member to ERROR. Default is 3.
+	//+optional
+	//+kubebuilder:validation:Minimum=1
+	//+kubebuilder:validation:Maximum=10
+	MaxRetriesDown *int `json:"maxRetriesDown,omitempty"`
 }
 
 func (s *APIServerLoadBalancer) IsZero() bool {

config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclusters.yaml

@@ -54,6 +54,7 @@ type LbClient interface {
 	DeletePoolMember(poolID string, lbMemberID string) error
 	CreateMonitor(opts monitors.CreateOptsBuilder) (*monitors.Monitor, error)
 	ListMonitors(opts monitors.ListOptsBuilder) ([]monitors.Monitor, error)
+	UpdateMonitor(id string, opts monitors.UpdateOptsBuilder) (*monitors.Monitor, error)
 	DeleteMonitor(id string) error
 	ListLoadBalancerProviders() ([]providers.Provider, error)
 	ListOctaviaVersions() ([]apiversions.APIVersion, error)
@@ -239,6 +240,15 @@ func (l lbClient) ListMonitors(opts monitors.ListOptsBuilder) ([]monitors.Monito
 	return monitors.ExtractMonitors(allPages)
 }
 
+func (l lbClient) UpdateMonitor(id string, opts monitors.UpdateOptsBuilder) (*monitors.Monitor, error) {
+	mc := metrics.NewMetricPrometheusContext("loadbalancer_healthmonitor", "update")
+	monitor, err := monitors.Update(context.TODO(), l.serviceClient, id, opts).Extract()
+	if mc.ObserveRequest(err) != nil {
+		return nil, err
+	}
+	return monitor, nil
+}
+
 func (l lbClient) DeleteMonitor(id string) error {
 	mc := metrics.NewMetricPrometheusContext("loadbalancer_healthmonitor", "delete")
 	err := monitors.Delete(context.TODO(), l.serviceClient, id).ExtractErr()

config/crd/bases/infrastructure.cluster.x-k8s.io_openstackclustertemplates.yaml

@@ -406,8 +406,7 @@ func (s *Service) reconcileAPILoadBalancerListener(lb *loadbalancers.LoadBalance
 	if err != nil {
 		return err
 	}
-
-	if err := s.getOrCreateMonitor(openStackCluster, lbPortObjectsName, pool.ID, lb.ID); err != nil {
+	if err := s.getOrUpdateMonitor(openStackCluster, lbPortObjectsName, pool.ID, lb.ID); err != nil {
 		return err
 	}
 
@@ -532,13 +531,83 @@ func (s *Service) getOrCreatePool(openStackCluster *infrav1.OpenStackCluster, po
 	return pool, nil
 }
 
-func (s *Service) getOrCreateMonitor(openStackCluster *infrav1.OpenStackCluster, monitorName, poolID, lbID string) error {
+func (s *Service) getOrUpdateMonitor(openStackCluster *infrav1.OpenStackCluster, monitorName, poolID, lbID string) error {
 	monitor, err := s.checkIfMonitorExists(monitorName)
 	if err != nil {
 		return err
 	}
 
+	monitorConfig := openStackCluster.Spec.APIServerLoadBalancer.Monitor
+
+	// Default values for monitor
+	const (
+		defaultDelay          = 10
+		defaultTimeout        = 5
+		defaultMaxRetries     = 5
+		defaultMaxRetriesDown = 3
+	)
+
 	if monitor != nil {
+		needsUpdate := false
+		monitorUpdateOpts := monitors.UpdateOpts{}
+
+		if (monitorConfig == nil || monitorConfig.Delay == nil) && monitor.Delay != defaultDelay {
+			s.scope.Logger().Info("Monitor delay needs update to default", "current", monitor.Delay, "default", defaultDelay)
+			monitorUpdateOpts.Delay = defaultDelay
+			needsUpdate = true
+		} else if monitorConfig != nil && monitorConfig.Delay != nil && monitor.Delay != *monitorConfig.Delay {
+			s.scope.Logger().Info("Monitor delay needs update", "current", monitor.Delay, "desired", *monitorConfig.Delay)
+			monitorUpdateOpts.Delay = *monitorConfig.Delay
+			needsUpdate = true
+		}
+
+		if (monitorConfig == nil || monitorConfig.Timeout == nil) && monitor.Timeout != defaultTimeout {
+			s.scope.Logger().Info("Monitor timeout needs update to default", "current", monitor.Timeout, "default", defaultTimeout)
+			monitorUpdateOpts.Timeout = defaultTimeout
+			needsUpdate = true
+		} else if monitorConfig != nil && monitorConfig.Timeout != nil && monitor.Timeout != *monitorConfig.Timeout {
+			s.scope.Logger().Info("Monitor timeout needs update", "current", monitor.Timeout, "desired", *monitorConfig.Timeout)
+			monitorUpdateOpts.Timeout = *monitorConfig.Timeout
+			needsUpdate = true
+		}
+
+		if (monitorConfig == nil || monitorConfig.MaxRetries == nil) && monitor.MaxRetries != defaultMaxRetries {
+			s.scope.Logger().Info("Monitor maxRetries needs update to default", "current", monitor.MaxRetries, "default", defaultMaxRetries)
+			monitorUpdateOpts.MaxRetries = defaultMaxRetries
+			needsUpdate = true
+		} else if monitorConfig != nil && monitorConfig.MaxRetries != nil && monitor.MaxRetries != *monitorConfig.MaxRetries {
+			s.scope.Logger().Info("Monitor maxRetries needs update", "current", monitor.MaxRetries, "desired", *monitorConfig.MaxRetries)
+			monitorUpdateOpts.MaxRetries = *monitorConfig.MaxRetries
+			needsUpdate = true
+		}
+
+		if (monitorConfig == nil || monitorConfig.MaxRetriesDown == nil) && monitor.MaxRetriesDown != defaultMaxRetriesDown {
+			s.scope.Logger().Info("Monitor maxRetriesDown needs update to default", "current", monitor.MaxRetriesDown, "default", defaultMaxRetriesDown)
+			monitorUpdateOpts.MaxRetriesDown = defaultMaxRetriesDown
+			needsUpdate = true
+		} else if monitorConfig != nil && monitorConfig.MaxRetriesDown != nil && monitor.MaxRetriesDown != *monitorConfig.MaxRetriesDown {
+			s.scope.Logger().Info("Monitor maxRetriesDown needs update", "current", monitor.MaxRetriesDown, "desired", *monitorConfig.MaxRetriesDown)
+			monitorUpdateOpts.MaxRetriesDown = *monitorConfig.MaxRetriesDown
+			needsUpdate = true
+		}
+
+		if needsUpdate {
+			s.scope.Logger().Info("Updating load balancer monitor", "loadBalancerID", lbID, "name", monitorName, "monitorID", monitor.ID)
+
+			updatedMonitor, err := s.loadbalancerClient.UpdateMonitor(monitor.ID, monitorUpdateOpts)
+			if err != nil {
+				record.Warnf(openStackCluster, "FailedUpdateMonitor", "Failed to update monitor %s with id %s: %v", monitorName, monitor.ID, err)
+				return err
+			}
+
+			if _, err = s.waitForLoadBalancerActive(lbID); err != nil {
+				record.Warnf(openStackCluster, "FailedUpdateMonitor", "Failed to update monitor %s with id %s: wait for load balancer active %s: %v", monitorName, monitor.ID, lbID, err)
+				return err
+			}
+
+			record.Eventf(openStackCluster, "SuccessfulUpdateMonitor", "Updated monitor %s with id %s", monitorName, updatedMonitor.ID)
+		}
+
 		return nil
 	}
 
@@ -548,13 +617,29 @@ func (s *Service) getOrCreateMonitor(openStackCluster *infrav1.OpenStackCluster,
 		Name:           monitorName,
 		PoolID:         poolID,
 		Type:           "TCP",
-		Delay:          10,
-		MaxRetries:     5,
-		MaxRetriesDown: 3,
-		Timeout:        5,
+		Delay:          defaultDelay,
+		Timeout:        defaultTimeout,
+		MaxRetries:     defaultMaxRetries,
+		MaxRetriesDown: defaultMaxRetriesDown,
+	}
+
+	if monitorConfig != nil {
+		if monitorConfig.Delay != nil {
+			monitorCreateOpts.Delay = *monitorConfig.Delay
+		}
+		if monitorConfig.MaxRetries != nil {
+			monitorCreateOpts.MaxRetries = *monitorConfig.MaxRetries
+		}
+		if monitorConfig.MaxRetriesDown != nil {
+			monitorCreateOpts.MaxRetriesDown = *monitorConfig.MaxRetriesDown
+		}
+		if monitorConfig.Timeout != nil {
+			monitorCreateOpts.Timeout = *monitorConfig.Timeout
+		}
 	}
-	monitor, err = s.loadbalancerClient.CreateMonitor(monitorCreateOpts)
-	// Skip creating monitor if it is not supported by Octavia provider
+
+	newMonitor, err := s.loadbalancerClient.CreateMonitor(monitorCreateOpts)
+
 	if capoerrors.IsNotImplementedError(err) {
 		record.Warnf(openStackCluster, "SkippedCreateMonitor", "Health Monitor is not created as it's not implemented with the current Octavia provider.")
 		return nil
@@ -566,11 +651,11 @@ func (s *Service) getOrCreateMonitor(openStackCluster *infrav1.OpenStackCluster,
 	}
 
 	if _, err = s.waitForLoadBalancerActive(lbID); err != nil {
-		record.Warnf(openStackCluster, "FailedCreateMonitor", "Failed to create monitor %s with id %s: wait for load balancer active %s: %v", monitorName, monitor.ID, lbID, err)
+		record.Warnf(openStackCluster, "FailedCreateMonitor", "Failed to create monitor %s with id %s: wait for load balancer active %s: %v", monitorName, newMonitor.ID, lbID, err)
 		return err
 	}
 
-	record.Eventf(openStackCluster, "SuccessfulCreateMonitor", "Created monitor %s with id %s", monitorName, monitor.ID)
+	record.Eventf(openStackCluster, "SuccessfulCreateMonitor", "Created monitor %s with id %s", monitorName, newMonitor.ID)
 	return nil
 }
 

pkg/clients/loadbalancer.go

@@ -145,6 +145,12 @@ func (*openStackClusterWebhook) ValidateUpdate(_ context.Context, oldObjRaw, new
 		newObj.Spec.APIServerLoadBalancer.AllowedCIDRs = []string{}
 	}
 
+	// Allow changes on APIServerLB monitors
+	if newObj.Spec.APIServerLoadBalancer != nil && oldObj.Spec.APIServerLoadBalancer != nil {
+		oldObj.Spec.APIServerLoadBalancer.Monitor = &infrav1.APIServerLoadBalancerMonitor{}
+		newObj.Spec.APIServerLoadBalancer.Monitor = &infrav1.APIServerLoadBalancerMonitor{}
+	}
+
 	// Allow changes to the availability zones.
 	oldObj.Spec.ControlPlaneAvailabilityZones = []string{}
 	newObj.Spec.ControlPlaneAvailabilityZones = []string{}