feat: Add optional filtered node IP address config (#187)

This can be used to e.g. filter out CP endpoint IPs as used by
kube-vip. Without this there is the potential to assign the
kube-vip IP to the node address, leading to cluster networking
failures, as see when testing with DHCP subnets on Nutanix.

Author: jimmidyson

go.mod

@@ -15,6 +15,7 @@ require (
 	k8s.io/cloud-provider v0.30.2
 	k8s.io/component-base v0.30.2
 	k8s.io/klog/v2 v2.130.0
+	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
 )
 
 require (
@@ -110,7 +111,6 @@ require (
 	k8s.io/controller-manager v0.30.2 // indirect
 	k8s.io/kms v0.30.2 // indirect
 	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
-	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect

internal/testing/mock/constants.go

@@ -30,6 +30,7 @@ const (
 	MockVMNamePoweredOff                 = "mock-vm-poweredoff"
 	MockVMNameCategories                 = "mock-vm-categories"
 	MockVMNameNoAddresses                = "mock-vm-no-addresses"
+	MockVMNameFilteredNodeAddresses      = "mock-vm-filtered-node-addresses"
 	MockVMNamePoweredOnClusterCategories = "mock-vm-poweredon-cluster-categories"
 
 	MockNodeNameVMNotExisting = "mock-node-no-vm-exists"

internal/testing/mock/mock_environment.go

@@ -25,6 +25,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/fake"
+	"k8s.io/utils/ptr"
 
 	"github.com/nutanix-cloud-native/cloud-provider-nutanix/internal/constants"
 )
@@ -109,6 +110,21 @@ func CreateMockEnvironment(ctx context.Context, kClient *fake.Clientset) (*MockE
 		return nil, err
 	}
 
+	filteredAddressesVM := getDefaultVMSpec(MockVMNameFilteredNodeAddresses, cluster)
+	filteredAddressesVM.Status.Resources.NicList = []*prismClientV3.VMNicOutputStatus{{
+		IPEndpointList: []*prismClientV3.IPAddress{{
+			IP: ptr.To("127.100.100.1"),
+		}, {
+			IP: ptr.To("127.200.200.1"),
+		}, {
+			IP: ptr.To("127.300.300.1"),
+		}},
+	}}
+	filteredAddressesNode, err := createNodeForVM(ctx, kClient, filteredAddressesVM)
+	if err != nil {
+		return nil, err
+	}
+
 	nonExistingVMNode := &v1.Node{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: MockNodeNameVMNotExisting,
@@ -139,6 +155,7 @@ func CreateMockEnvironment(ctx context.Context, kClient *fake.Clientset) (*MockE
 			*categoriesVM.Metadata.UUID:                 categoriesVM,
 			*noAddressesVM.Metadata.UUID:                noAddressesVM,
 			*poweredOnVMClusterCategories.Metadata.UUID: poweredOnVMClusterCategories,
+			*filteredAddressesVM.Metadata.UUID:          filteredAddressesVM,
 		},
 		managedMockClusters: map[string]*prismClientV3.ClusterIntentResponse{
 			*cluster.Metadata.UUID:           cluster,
@@ -153,6 +170,7 @@ func CreateMockEnvironment(ctx context.Context, kClient *fake.Clientset) (*MockE
 			MockNodeNameVMNotExisting:            nonExistingVMNode,
 			MockNodeNameNoSystemUUID:             noSystemUUIDNode,
 			MockVMNamePoweredOnClusterCategories: poweredOnClusterCategoriesNode,
+			MockVMNameFilteredNodeAddresses:      filteredAddressesNode,
 		},
 	}, nil
 }

main.go

@@ -51,6 +51,7 @@ func main() {
 
 	command := app.NewCloudControllerManagerCommand(ccmOptions,
 		cloudInitializer, controllerInitializers, map[string]string{}, fss, wait.NeverStop)
+
 	code := cli.Run(command)
 	os.Exit(code)
 }

manifests/cm.yaml

@@ -19,5 +19,6 @@ data:
       "enableCustomLabeling": false,
       "topologyDiscovery": {
         "type": "Prism"
-      }
+      },
+      "ignoredNodeIPs": []
     }

pkg/provider/config/config.go

@@ -29,6 +29,7 @@ type Config struct {
 	PrismCentral         credentialTypes.NutanixPrismEndpoint `json:"prismCentral"`
 	TopologyDiscovery    TopologyDiscovery                    `json:"topologyDiscovery"`
 	EnableCustomLabeling bool                                 `json:"enableCustomLabeling"`
+	IgnoredNodeIPs       []string                             `json:"ignoredNodeIPs,omitempty"`
 }
 
 type TopologyDiscovery struct {

pkg/provider/manager.go

@@ -35,19 +35,28 @@ import (
 )
 
 type nutanixManager struct {
-	client        clientset.Interface
-	config        config.Config
-	nutanixClient interfaces.Client
+	client         clientset.Interface
+	config         config.Config
+	nutanixClient  interfaces.Client
+	ignoredNodeIPs map[string]struct{}
 }
 
 func newNutanixManager(config config.Config) (*nutanixManager, error) {
 	klog.V(1).Info("Creating new newNutanixManager")
+
+	// Initialize the ignoredNodeIPs map
+	ignoredNodeIPs := make(map[string]struct{}, len(config.IgnoredNodeIPs))
+	for _, ip := range config.IgnoredNodeIPs {
+		ignoredNodeIPs[ip] = struct{}{}
+	}
+
 	m := &nutanixManager{
 		config: config,
 		nutanixClient: &nutanixClient{
 			config:      config,
 			clientCache: prismclientv3.NewClientCache(prismclientv3.WithSessionAuth(true)),
 		},
+		ignoredNodeIPs: ignoredNodeIPs,
 	}
 	return m, nil
 }
@@ -262,26 +271,29 @@ func (n *nutanixManager) generateProviderID(ctx context.Context, vmUUID string)
 	return fmt.Sprintf("%s://%s", constants.ProviderName, strings.ToLower(vmUUID)), nil
 }
 
-func (n *nutanixManager) getNodeAddresses(ctx context.Context, vm *prismclientv3.VMIntentResponse) ([]v1.NodeAddress, error) {
+func (n *nutanixManager) getNodeAddresses(_ context.Context, vm *prismclientv3.VMIntentResponse) ([]v1.NodeAddress, error) {
 	if vm == nil {
 		return nil, fmt.Errorf("vm cannot be nil when getting node addresses")
 	}
-	addresses := make([]v1.NodeAddress, 0)
-	foundIPs := 0
+
+	var addresses []v1.NodeAddress
 	for _, nic := range vm.Status.Resources.NicList {
 		for _, ipEndpoint := range nic.IPEndpointList {
 			if ipEndpoint.IP != nil {
-				addresses = append(addresses, v1.NodeAddress{
-					Type:    v1.NodeInternalIP,
-					Address: *ipEndpoint.IP,
-				})
-				foundIPs++
+				// Ignore the IP address if it is one of the specified ignoredNodeIPs.
+				if _, ok := n.ignoredNodeIPs[*ipEndpoint.IP]; !ok {
+					addresses = append(addresses, v1.NodeAddress{
+						Type:    v1.NodeInternalIP,
+						Address: *ipEndpoint.IP,
+					})
+				}
 			}
 		}
 	}
-	if foundIPs == 0 {
+	if len(addresses) == 0 {
 		return addresses, fmt.Errorf("unable to determine network interfaces from VM with UUID %s", *vm.Metadata.UUID)
 	}
+
 	addresses = append(addresses, v1.NodeAddress{
 		Type:    v1.NodeHostName,
 		Address: *vm.Spec.Name,

pkg/provider/manager_test.go

@@ -59,9 +59,14 @@ var _ = Describe("Test Manager", func() {
 						ZoneCategory:   mock.MockDefaultZone,
 					},
 				},
+				IgnoredNodeIPs: []string{"127.100.100.1", "127.200.200.1"},
 			},
 			client:        kClient,
 			nutanixClient: nutanixClient,
+			ignoredNodeIPs: map[string]struct{}{
+				"127.100.100.1": struct{}{},
+				"127.200.200.1": struct{}{},
+			},
 		}
 	})
 
@@ -146,6 +151,15 @@ var _ = Describe("Test Manager", func() {
 				),
 			)
 		})
+
+		It("should filter node addresses if matching specified filtered addresses", func() {
+			vm := mockEnvironment.GetVM(ctx, mock.MockVMNameFilteredNodeAddresses)
+			Expect(vm).ToNot(BeNil())
+			addresses, err := m.getNodeAddresses(ctx, vm)
+			Expect(err).ShouldNot(HaveOccurred())
+			Expect(len(addresses)).To(Equal(2), "Received addresses: %v", addresses)
+			Expect(addresses).Should(ContainElement(v1.NodeAddress{Type: v1.NodeInternalIP, Address: "127.300.300.1"}))
+		})
 	})
 
 	Context("Test generateProviderID", func() {

test/e2e/go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/distribution/reference v0.5.0 // indirect
-	github.com/docker/docker v26.1.4+incompatible // indirect
+	github.com/docker/docker v26.1.5+incompatible // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect

test/e2e/go.sum

@@ -52,8 +52,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
 github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
-github.com/docker/docker v26.1.4+incompatible h1:vuTpXDuoga+Z38m1OZHzl7NKisKWaWlhjQk7IDPSLsU=
-github.com/docker/docker v26.1.4+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v26.1.5+incompatible h1:NEAxTwEjxV6VbBMBoGG3zPqbiJosIApZjxlbrG9q3/g=
+github.com/docker/docker v26.1.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
 github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=