Add support for static cluster

kubetest2/internal/deployers/eksapi/cluster.go

@@ -39,46 +39,60 @@ type Cluster struct {
 	cidr                     string
 }
 
-func (m *ClusterManager) createCluster(infra *Infrastructure, opts *deployerOptions) (*Cluster, error) {
-	input := eks.CreateClusterInput{
-		Name: aws.String(m.resourceID),
-		ResourcesVpcConfig: &ekstypes.VpcConfigRequest{
-			EndpointPrivateAccess: aws.Bool(true),
-			EndpointPublicAccess:  aws.Bool(true),
-			SubnetIds:             append(infra.subnetsPublic, infra.subnetsPrivate...),
-		},
-		RoleArn: aws.String(infra.clusterRole),
-		KubernetesNetworkConfig: &ekstypes.KubernetesNetworkConfigRequest{
-			IpFamily: ekstypes.IpFamily(opts.IPFamily),
-		},
-		Version: aws.String(opts.KubernetesVersion),
-	}
-	apiOpts, err := util.NewHTTPHeaderAPIOptions(opts.UpClusterHeaders)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create API options: %v", err)
+func (m *ClusterManager) getOrCreateCluster(infra *Infrastructure, opts *deployerOptions) (*Cluster, error) {
+	targetClusterName := &opts.StaticClusterName
+	if *targetClusterName == "" {
+		klog.Infof("No StaticClusterName specified creating new cluster...")
+		input := eks.CreateClusterInput{
+			Name: aws.String(m.resourceID),
+			ResourcesVpcConfig: &ekstypes.VpcConfigRequest{
+				EndpointPrivateAccess: aws.Bool(true),
+				EndpointPublicAccess:  aws.Bool(true),
+				SubnetIds:             append(infra.subnetsPublic, infra.subnetsPrivate...),
+			},
+			RoleArn: aws.String(infra.clusterRole),
+			KubernetesNetworkConfig: &ekstypes.KubernetesNetworkConfigRequest{
+				IpFamily: ekstypes.IpFamily(opts.IPFamily),
+			},
+			Version: aws.String(opts.KubernetesVersion),
+		}
+		apiOpts, err := util.NewHTTPHeaderAPIOptions(opts.UpClusterHeaders)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create API options: %v", err)
+		}
+		createOutput, err := m.clients.EKS().CreateCluster(context.TODO(), &input,
+			func(o *eks.Options) {
+				o.APIOptions = apiOpts
+			})
+		if err != nil {
+			return nil, fmt.Errorf("failed to create cluster: %v", err)
+		}
+		targetClusterName = createOutput.Cluster.Name
+	} else {
+		klog.Infof("reusing existing static cluster %s", *targetClusterName)
 	}
-	klog.Infof("creating cluster...")
-	createOutput, err := m.clients.EKS().CreateCluster(context.TODO(), &input,
-		func(o *eks.Options) {
-			o.APIOptions = apiOpts
-		})
-	if err != nil {
-		return nil, fmt.Errorf("failed to create cluster: %v", err)
+	cluster, waitErr := m.waitClusterReady(targetClusterName)
+	if waitErr != nil {
+		return nil, fmt.Errorf("failed to wait for cluster to become active: %v", waitErr)
 	}
+	return cluster, nil
+}
+
+func (m *ClusterManager) waitClusterReady(clusterName *string) (*Cluster, error) {
 	describeInput := eks.DescribeClusterInput{
-		Name: createOutput.Cluster.Name,
+		Name: clusterName,
 	}
-	klog.Infof("waiting for cluster to be active: %s", *createOutput.Cluster.Arn)
+	klog.Infof("waiting for cluster to be active: %s", *clusterName)
 	waitErr := eks.NewClusterActiveWaiter(m.clients.EKS()).Wait(context.TODO(), &describeInput, clusterCreationTimeout)
 	describeOutput, describeErr := m.clients.EKS().DescribeCluster(context.TODO(), &describeInput)
 	if describeErr != nil {
 		return nil, fmt.Errorf("failed to describe cluster after creation: %v", describeErr)
 	}
 	klog.Infof("cluster details after creation: %+v", describeOutput.Cluster)
 	if waitErr != nil {
-		return nil, fmt.Errorf("failed to wait for cluster to become active: %v", waitErr)
+		return nil, waitErr
 	}
-	klog.Infof("cluster is active: %s", *createOutput.Cluster.Arn)
+	klog.Infof("cluster is active: %s", *describeOutput.Cluster.Arn)
 	var cidr string
 	switch describeOutput.Cluster.KubernetesNetworkConfig.IpFamily {
 	case ekstypes.IpFamilyIpv4:

kubetest2/internal/deployers/eksapi/deployer.go

@@ -75,6 +75,7 @@ type deployerOptions struct {
 	Nodes                       int           `flag:"nodes" desc:"number of nodes to launch in cluster"`
 	NodeNameStrategy            string        `flag:"node-name-strategy" desc:"Specifies the naming strategy for node. Allowed values: ['SessionName', 'EC2PrivateDNSName'], default to EC2PrivateDNSName"`
 	Region                      string        `flag:"region" desc:"AWS region for EKS cluster"`
+	StaticClusterName           string        `flag:"static-cluster-name" desc:"Optional when re-use existing cluster and node group by querying the kubeconfig and run test"`
 	TuneVPCCNI                  bool          `flag:"tune-vpc-cni" desc:"Apply tuning parameters to the VPC CNI DaemonSet"`
 	UnmanagedNodes              bool          `flag:"unmanaged-nodes" desc:"Use an AutoScalingGroup instead of an EKS-managed nodegroup. Requires --ami"`
 	UpClusterHeaders            []string      `flag:"up-cluster-header" desc:"Additional header to add to eks:CreateCluster requests. Specified in the same format as curl's -H flag."`
@@ -163,12 +164,14 @@ func (d *deployer) Up() error {
 			return err
 		}
 	}
-	if infra, err := d.infraManager.createInfrastructureStack(&d.deployerOptions); err != nil {
-		return err
-	} else {
-		d.infra = infra
+	if d.deployerOptions.StaticClusterName == "" {
+		if infra, err := d.infraManager.createInfrastructureStack(&d.deployerOptions); err != nil {
+			return err
+		} else {
+			d.infra = infra
+		}
 	}
-	cluster, err := d.clusterManager.createCluster(d.infra, &d.deployerOptions)
+	cluster, err := d.clusterManager.getOrCreateCluster(d.infra, &d.deployerOptions)
 	if err != nil {
 		return err
 	}
@@ -181,6 +184,10 @@ func (d *deployer) Up() error {
 	if err != nil {
 		return err
 	}
+	if d.deployerOptions.StaticClusterName != "" {
+		klog.Infof("inited k8sclient, skip the rest resource creation for static cluster")
+		return nil
+	}
 	if d.UnmanagedNodes {
 		if err := createAWSAuthConfigMap(d.k8sClient, d.NodeNameStrategy, d.infra.nodeRole); err != nil {
 			return err
@@ -236,6 +243,16 @@ func (d *deployer) verifyUpFlags() error {
 		d.IPFamily = string(ekstypes.IpFamilyIpv4)
 		klog.Infof("Using default IP family: %s", d.IPFamily)
 	}
+	if d.NodeCreationTimeout == 0 {
+		d.NodeCreationTimeout = time.Minute * 20
+	}
+	if d.NodeReadyTimeout == 0 {
+		d.NodeReadyTimeout = time.Minute * 5
+	}
+	if d.StaticClusterName != "" {
+		klog.Infof("Skip configuration for static cluster")
+		return nil
+	}
 	if d.UnmanagedNodes {
 		if d.AMI == "" {
 			return fmt.Errorf("--ami must be specified for --unmanaged-nodes")
@@ -267,12 +284,6 @@ func (d *deployer) verifyUpFlags() error {
 			klog.Infof("Using default AMI type: %s", d.AMIType)
 		}
 	}
-	if d.NodeCreationTimeout == 0 {
-		d.NodeCreationTimeout = time.Minute * 20
-	}
-	if d.NodeReadyTimeout == 0 {
-		d.NodeReadyTimeout = time.Minute * 5
-	}
 	return nil
 }
 
@@ -307,6 +318,7 @@ func deleteResources(im *InfrastructureManager, cm *ClusterManager, nm *Nodegrou
 	// the EKS-managed cluster security group may be associated with a leaked ENI
 	// so we need to make sure we've deleted leaked ENIs before we delete the cluster
 	// otherwise, the cluster security group will be left behind and will block deletion of our VPC
+
 	if err := im.deleteLeakedENIs(); err != nil {
 		return err
 	}

kubetest2/internal/deployers/eksapi/infra.go

@@ -168,6 +168,7 @@ func (m *InfrastructureManager) createInfrastructureStack(opts *deployerOptions)
 		return nil, fmt.Errorf("failed to get infrastructure stack resources: %w", err)
 	}
 	klog.Infof("created infrastructure: %+v", infra)
+
 	return infra, nil
 }