use IPs within custom service cidr range for registry and seaweedfs services (#238)

* use IPs within custom service cidr range for registry and seaweedfs services

* f

* factor out ClusterServiceCIDR function

Author: laverya

controllers/helm.go

@@ -17,6 +17,7 @@ import (
 	ectypes "github.com/replicatedhq/embedded-cluster-kinds/types"
 	"github.com/replicatedhq/embedded-cluster-operator/pkg/k8sutil"
 	"github.com/replicatedhq/embedded-cluster-operator/pkg/registry"
+	"github.com/replicatedhq/embedded-cluster-operator/pkg/util"
 )
 
 const DEFAULT_VENDOR_CHART_ORDER = 10
@@ -185,9 +186,16 @@ func updateInfraChartsFromInstall(ctx context.Context, in *v1beta1.Installation,
 				continue
 			}
 
-			serviceCIDR := k0sv1beta1.DefaultNetwork().ServiceCIDR
-			if clusterConfig.Spec != nil && clusterConfig.Spec.Network != nil {
-				serviceCIDR = clusterConfig.Spec.Network.ServiceCIDR
+			serviceCIDR := util.ClusterServiceCIDR(clusterConfig, in)
+			registryEndpoint, err := registry.GetRegistryServiceIP(serviceCIDR)
+			if err != nil {
+				log.Error(err, "failed to get registry endpoint", "chart", chart.Name)
+				continue
+			}
+
+			newVals, err := setHelmValue(chart.Values, "service.clusterIP", registryEndpoint)
+			if err != nil {
+				log.Error(err, "failed to set helm values service.clusterIP", "chart", chart.Name)
 			}
 
 			seaweedfsS3Endpoint, err := registry.GetSeaweedfsS3Endpoint(serviceCIDR)
@@ -196,9 +204,9 @@ func updateInfraChartsFromInstall(ctx context.Context, in *v1beta1.Installation,
 				continue
 			}
 
-			newVals, err := setHelmValue(chart.Values, "s3.regionEndpoint", seaweedfsS3Endpoint)
+			newVals, err = setHelmValue(newVals, "s3.regionEndpoint", seaweedfsS3Endpoint)
 			if err != nil {
-				log.Error(err, "failed to set helm values embeddedClusterID", "chart", chart.Name)
+				log.Error(err, "failed to set helm values s3.regionEndpoint", "chart", chart.Name)
 				continue
 			}
 

controllers/installation_controller.go

@@ -701,10 +701,7 @@ func (r *InstallationReconciler) ReconcileRegistry(ctx context.Context, in *v1be
 		return fmt.Errorf("failed to get cluster config: %w", err)
 	}
 
-	serviceCIDR := k0sv1beta1.DefaultNetwork().ServiceCIDR
-	if clusterConfig.Spec != nil && clusterConfig.Spec.Network != nil {
-		serviceCIDR = clusterConfig.Spec.Network.ServiceCIDR
-	}
+	serviceCIDR := util.ClusterServiceCIDR(clusterConfig, in)
 
 	err := registry.EnsureResources(ctx, in, r.Client, serviceCIDR)
 	if err != nil {

pkg/registry/registry.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	clusterv1beta1 "github.com/replicatedhq/embedded-cluster-kinds/apis/v1beta1"
+	"github.com/replicatedhq/embedded-cluster-operator/pkg/util"
 	corev1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -17,6 +18,10 @@ const (
 	// registryNamespace is the namespace where the Registry secret is stored.
 	// This namespace is defined in the chart in the release metadata.
 	registryNamespace = "registry"
+
+	// registryLowerBandIPIndex is the index of the registry service IP in the service CIDR.
+	// this is shared with the CLI as it is set on initial installation as well.
+	registryLowerBandIPIndex = 10
 )
 
 func EnsureResources(ctx context.Context, in *clusterv1beta1.Installation, cli client.Client, serviceCIDR string) error {
@@ -63,6 +68,14 @@ func RegistryNamespace() string {
 	return registryNamespace
 }
 
+func GetRegistryServiceIP(serviceCIDR string) (string, error) {
+	ip, err := util.GetLowerBandIP(serviceCIDR, registryLowerBandIPIndex)
+	if err != nil {
+		return "", fmt.Errorf("get lower band ip at index %d: %w", registryLowerBandIPIndex, err)
+	}
+	return ip.String(), nil
+}
+
 func ensureRegistryNamespace(ctx context.Context, cli client.Client) error {
 	obj := &corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{Name: registryNamespace},

pkg/util/cidr.go

@@ -0,0 +1,21 @@
+package util
+
+import (
+	"github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
+	v1beta12 "github.com/replicatedhq/embedded-cluster-kinds/apis/v1beta1"
+)
+
+// ClusterServiceCIDR determines the service CIDR for the cluster
+// if there is no custom service CIDR, return the default
+// if the service CIDR is set in the cluster config, use that - unless
+// if the service CIDR is set in the installation, use that because occasionally the cluster config is incorrect
+func ClusterServiceCIDR(clusterConfig v1beta1.ClusterConfig, in *v1beta12.Installation) string {
+	serviceCIDR := v1beta1.DefaultNetwork().ServiceCIDR
+	if clusterConfig.Spec != nil && clusterConfig.Spec.Network != nil {
+		serviceCIDR = clusterConfig.Spec.Network.ServiceCIDR
+	}
+	if in.Spec.Network != nil && in.Spec.Network.ServiceCIDR != "" {
+		serviceCIDR = in.Spec.Network.ServiceCIDR
+	}
+	return serviceCIDR
+}