CORS-3959, CORS-3864: CAPI-based AzureStack Installs

pkg/asset/machines/clusterapi.go

@@ -39,8 +39,10 @@ import (
 	awsdefaults "github.com/openshift/installer/pkg/types/aws/defaults"
 	azuretypes "github.com/openshift/installer/pkg/types/azure"
 	azuredefaults "github.com/openshift/installer/pkg/types/azure/defaults"
+	externaltypes "github.com/openshift/installer/pkg/types/external"
 	gcptypes "github.com/openshift/installer/pkg/types/gcp"
 	ibmcloudtypes "github.com/openshift/installer/pkg/types/ibmcloud"
+	nonetypes "github.com/openshift/installer/pkg/types/none"
 	nutanixtypes "github.com/openshift/installer/pkg/types/nutanix"
 	openstacktypes "github.com/openshift/installer/pkg/types/openstack"
 	powervstypes "github.com/openshift/installer/pkg/types/powervs"
@@ -500,8 +502,10 @@ func (c *ClusterAPI) Generate(ctx context.Context, dependencies asset.Parents) e
 		if err != nil {
 			return fmt.Errorf("failed to generate IBM Cloud VPC machine manifests: %w", err)
 		}
+	case externaltypes.Name, nonetypes.Name:
+		return nil
 	default:
-		// TODO: support other platforms
+		return fmt.Errorf("unrecognized platform: %q", ic.Platform.Name())
 	}
 
 	// Create the machine manifests.

pkg/asset/manifests/azure/cluster.go

@@ -2,19 +2,23 @@ package azure
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net"
+	"strings"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/ptr"
 	capz "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	azic "github.com/openshift/installer/pkg/asset/installconfig/azure"
+	capzash "github.com/openshift/installer/pkg/asset/manifests/azure/stack/v1beta1"
 	"github.com/openshift/installer/pkg/asset/manifests/capiutils"
 	"github.com/openshift/installer/pkg/asset/manifests/capiutils/cidr"
 	"github.com/openshift/installer/pkg/ipnet"
@@ -169,6 +173,8 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 		}
 	}
 
+	azEnv := string(installConfig.Azure.CloudName)
+
 	azureCluster := &capz.AzureCluster{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      clusterID.InfraID,
@@ -180,7 +186,7 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 				SubscriptionID:   session.Credentials.SubscriptionID,
 				Location:         installConfig.Config.Azure.Region,
 				AdditionalTags:   installConfig.Config.Platform.Azure.UserTags,
-				AzureEnvironment: string(installConfig.Azure.CloudName),
+				AzureEnvironment: azEnv,
 				IdentityRef: &corev1.ObjectReference{
 					APIVersion: capz.GroupVersion.String(),
 					Kind:       "AzureClusterIdentity",
@@ -234,9 +240,26 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 			},
 		},
 	}
-	azureCluster.SetGroupVersionKind(capz.GroupVersion.WithKind("AzureCluster"))
+
+	// We are maintaining a fork of CAPZ for azurestack. The only API difference
+	// is the ARMEndpoint field, so we can use the CAPZ cluster object, and if
+	// running on ASH convert to the fork API, and add the field.
+	var cluster client.Object
+	if !strings.EqualFold(azEnv, string(azure.StackCloud)) {
+		azureCluster.SetGroupVersionKind(capz.GroupVersion.WithKind("AzureCluster"))
+		cluster = azureCluster
+	} else {
+		var ashCluster capzash.AzureCluster
+		if err := deepCopy(azureCluster, &ashCluster); err != nil {
+			return nil, fmt.Errorf("failed to convert azureCluster to azure-stack cluster: %w", err)
+		}
+		ashCluster.Spec.ARMEndpoint = session.Environment.ServiceManagementEndpoint
+		ashCluster.SetGroupVersionKind(capzash.GroupVersion.WithKind("AzureCluster"))
+		cluster = &ashCluster
+	}
+
 	manifests = append(manifests, &asset.RuntimeFile{
-		Object: azureCluster,
+		Object: cluster,
 		File:   asset.File{Filename: "02_azure-cluster.yaml"},
 	})
 
@@ -385,3 +408,11 @@ func getNextAvailableIPForLoadBalancer(ctx context.Context, installConfig *insta
 	}
 	return "", fmt.Errorf("failed to get an IP that's available and in the given machine network: this error may be caused by lack of necessary permissions")
 }
+
+func deepCopy(src, dst interface{}) error {
+	bytes, err := json.Marshal(src)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(bytes, dst)
+}

pkg/asset/manifests/capiutils/helpers.go

@@ -3,8 +3,6 @@ package capiutils
 import (
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	"github.com/openshift/installer/pkg/ipnet"
-	"github.com/openshift/installer/pkg/types"
-	typesazure "github.com/openshift/installer/pkg/types/azure"
 )
 
 var (
@@ -22,11 +20,8 @@ func CIDRFromInstallConfig(installConfig *installconfig.InstallConfig) *ipnet.IP
 
 // IsEnabled returns true if the feature gate is enabled.
 func IsEnabled(installConfig *installconfig.InstallConfig) bool {
-	platform := installConfig.Config.Platform.Name()
-	if azure := installConfig.Config.Platform.Azure; azure != nil && azure.CloudName == typesazure.StackCloud {
-		platform = typesazure.StackTerraformName
-	}
-	return types.ClusterAPIFeatureGateEnabled(platform, installConfig.Config.EnabledFeatureGates())
+	// TODO(padillon): refactor to remove IsEnabled function.
+	return true
 }
 
 // GenerateBoostrapMachineName generates the Cluster API Machine used for bootstrapping

pkg/asset/manifests/clusterapi/cluster.go

@@ -33,8 +33,10 @@ import (
 	"github.com/openshift/installer/pkg/clusterapi"
 	awstypes "github.com/openshift/installer/pkg/types/aws"
 	azuretypes "github.com/openshift/installer/pkg/types/azure"
+	externaltypes "github.com/openshift/installer/pkg/types/external"
 	gcptypes "github.com/openshift/installer/pkg/types/gcp"
 	ibmcloudtypes "github.com/openshift/installer/pkg/types/ibmcloud"
+	nonetypes "github.com/openshift/installer/pkg/types/none"
 	nutanixtypes "github.com/openshift/installer/pkg/types/nutanix"
 	openstacktypes "github.com/openshift/installer/pkg/types/openstack"
 	powervstypes "github.com/openshift/installer/pkg/types/powervs"
@@ -143,6 +145,8 @@ func (c *Cluster) Generate(_ context.Context, dependencies asset.Parents) error
 		if err != nil {
 			return fmt.Errorf("failed to generate IBM Cloud VPC manifests: %w", err)
 		}
+	case externaltypes.Name, nonetypes.Name:
+		return nil
 	default:
 		return fmt.Errorf("unsupported platform %q", platform)
 	}

pkg/clusterapi/system.go

@@ -243,7 +243,7 @@ func (c *system) Run(ctx context.Context) error { //nolint:gocyclo
 					"AZURE_TENANT_ID":                   session.Credentials.TenantID,
 					"AZURE_SUBSCRIPTION_ID":             session.Credentials.SubscriptionID,
 					"AZURE_RESOURCE_MANAGER_ENDPOINT":   session.Environment.ResourceManagerEndpoint,
-					"AZURE_RESOURCE_MANAGER_AUDIENCE":   session.Environment.ServiceManagementEndpoint,
+					"AZURE_RESOURCE_MANAGER_AUDIENCE":   session.Environment.TokenAudience,
 				},
 			),
 		)

pkg/infrastructure/azure/azure.go

@@ -38,6 +38,12 @@ const (
 	retryCount       = 6
 	confidentialVMST = "ConfidentialVMSupported"
 	trustedLaunchST  = "TrustedLaunchsupported"
+
+	// stackAPIVersion is the Azure Stack compatible API version.
+	stackAPIVersion = "2019-06-01"
+
+	// stackComputeAPIVersion is the Azure Stack compatible API version for compute resources (VMs & images).
+	stackComputeAPIVersion = "2020-06-01"
 )
 
 // Provider implements Azure CAPI installation.
@@ -53,6 +59,8 @@ type Provider struct {
 	CloudConfiguration    cloud.Configuration
 	TokenCredential       azcore.TokenCredential
 	Tags                  map[string]*string
+	clientOptions         *arm.ClientOptions
+	computeClientOptions  *arm.ClientOptions
 }
 
 var _ clusterapi.InfraReadyProvider = (*Provider)(nil)
@@ -83,6 +91,8 @@ func (p Provider) ProvisionTimeout() time.Duration {
 func (*Provider) PublicGatherEndpoint() clusterapi.GatherEndpoint { return clusterapi.APILoadBalancer }
 
 // InfraReady is called once the installer infrastructure is ready.
+//
+//nolint:gocyclo //TODO(padillon): forthcoming marketplace image support should help reduce complexity here.
 func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput) error {
 	session, err := in.InstallConfig.Azure.Session()
 	if err != nil {
@@ -104,17 +114,27 @@ func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput
 	}
 	p.Tags = tags
 
+	opts := &arm.ClientOptions{
+		ClientOptions: policy.ClientOptions{
+			Cloud: cloudConfiguration,
+		},
+	}
+	computeClientOpts := opts
+	if platform.CloudName == aztypes.StackCloud {
+		opts.APIVersion = stackAPIVersion
+		computeClientOpts = &arm.ClientOptions{
+			ClientOptions: policy.ClientOptions{
+				Cloud:      cloudConfiguration,
+				APIVersion: stackComputeAPIVersion,
+			},
+		}
+	}
+	p.clientOptions = opts
+	p.computeClientOptions = computeClientOpts
+
 	// Creating a dummy nsg for existing vnets installation to appease the ingress operator.
 	if in.InstallConfig.Config.Azure.VirtualNetwork != "" {
-		networkClientFactory, err := armnetwork.NewClientFactory(
-			subscriptionID,
-			tokenCredential,
-			&arm.ClientOptions{
-				ClientOptions: policy.ClientOptions{
-					Cloud: cloudConfiguration,
-				},
-			},
-		)
+		networkClientFactory, err := armnetwork.NewClientFactory(subscriptionID, tokenCredential, p.clientOptions)
 		if err != nil {
 			return fmt.Errorf("failed to create azure network factory: %w", err)
 		}
@@ -197,7 +217,7 @@ func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput
 		Tags:               tags,
 		CustomerManagedKey: platform.CustomerManagedKey,
 		TokenCredential:    tokenCredential,
-		CloudConfiguration: cloudConfiguration,
+		ClientOpts:         p.clientOptions,
 	})
 	if err != nil {
 		return err
@@ -235,21 +255,21 @@ func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput
 			ImageLength:        imageLength,
 			StorageAccountName: storageAccountName,
 			StorageAccountKeys: storageAccountKeys,
-			CloudConfiguration: cloudConfiguration,
+			ClientOpts:         p.clientOptions,
 		})
 		if err != nil {
 			return err
 		}
 
 		// Create image gallery
 		createImageGalleryOutput, err := CreateImageGallery(ctx, &CreateImageGalleryInput{
-			SubscriptionID:     subscriptionID,
-			ResourceGroupName:  resourceGroupName,
-			GalleryName:        galleryName,
-			Region:             platform.Region,
-			Tags:               tags,
-			TokenCredential:    tokenCredential,
-			CloudConfiguration: cloudConfiguration,
+			SubscriptionID:    subscriptionID,
+			ResourceGroupName: resourceGroupName,
+			GalleryName:       galleryName,
+			Region:            platform.Region,
+			Tags:              tags,
+			TokenCredential:   tokenCredential,
+			ClientOpts:        p.clientOptions,
 		})
 		if err != nil {
 			return err
@@ -268,7 +288,7 @@ func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput
 			SKU:                  "basic",
 			Tags:                 tags,
 			TokenCredential:      tokenCredential,
-			CloudConfiguration:   cloudConfiguration,
+			ClientOpts:           p.clientOptions,
 			Architecture:         architecture,
 			OSType:               armcompute.OperatingSystemTypesLinux,
 			OSState:              armcompute.OperatingSystemStateTypesGeneralized,
@@ -297,7 +317,7 @@ func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput
 			SKU:                  "gen2",
 			Tags:                 tags,
 			TokenCredential:      tokenCredential,
-			CloudConfiguration:   cloudConfiguration,
+			ClientOpts:           p.clientOptions,
 			Architecture:         architecture,
 			OSType:               armcompute.OperatingSystemTypesLinux,
 			OSState:              armcompute.OperatingSystemStateTypesGeneralized,
@@ -341,13 +361,7 @@ func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput
 		}
 	}
 
-	networkClientFactory, err := armnetwork.NewClientFactory(subscriptionID, session.TokenCreds,
-		&arm.ClientOptions{
-			ClientOptions: policy.ClientOptions{
-				Cloud: cloudConfiguration,
-			},
-		},
-	)
+	networkClientFactory, err := armnetwork.NewClientFactory(subscriptionID, session.TokenCreds, p.clientOptions)
 	if err != nil {
 		return fmt.Errorf("error creating network client factory: %w", err)
 	}
@@ -422,7 +436,7 @@ func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput
 	p.NetworkClientFactory = networkClientFactory
 	p.lbBackendAddressPools = lbBaps
 
-	if err := createDNSEntries(ctx, in, extLBFQDN, resourceGroupName); err != nil {
+	if err := createDNSEntries(ctx, in, extLBFQDN, resourceGroupName, p.clientOptions); err != nil {
 		return fmt.Errorf("error creating DNS records: %w", err)
 	}
 
@@ -437,16 +451,9 @@ func (p *Provider) PostProvision(ctx context.Context, in clusterapi.PostProvisio
 		return fmt.Errorf("error retrieving Azure session: %w", err)
 	}
 	subscriptionID := ssn.Credentials.SubscriptionID
-	cloudConfiguration := ssn.CloudConfig
 
 	if in.InstallConfig.Config.PublicAPI() {
-		vmClient, err := armcompute.NewVirtualMachinesClient(subscriptionID, ssn.TokenCreds,
-			&arm.ClientOptions{
-				ClientOptions: policy.ClientOptions{
-					Cloud: cloudConfiguration,
-				},
-			},
-		)
+		vmClient, err := armcompute.NewVirtualMachinesClient(subscriptionID, ssn.TokenCreds, p.computeClientOptions)
 		if err != nil {
 			return fmt.Errorf("error creating vm client: %w", err)
 		}
@@ -535,14 +542,22 @@ func (p *Provider) PostDestroy(ctx context.Context, in clusterapi.PostDestroyerI
 		return fmt.Errorf("failed to get session: %w", err)
 	}
 
+	// Construct client options here, rather than relying on p.clientOptions,
+	// as PostDestroy can be called as part of destroy bootstrap, in which case
+	// p.clientOption would not be populated.
+	opts := &arm.ClientOptions{
+		ClientOptions: policy.ClientOptions{
+			Cloud: session.CloudConfig,
+		},
+	}
+	if in.Metadata.Azure.CloudName == aztypes.StackCloud {
+		opts.APIVersion = stackAPIVersion
+	}
+
 	networkClientFactory, err := armnetwork.NewClientFactory(
 		session.Credentials.SubscriptionID,
 		session.TokenCreds,
-		&arm.ClientOptions{
-			ClientOptions: policy.ClientOptions{
-				Cloud: session.CloudConfig,
-			},
-		},
+		opts,
 	)
 	if err != nil {
 		return fmt.Errorf("error creating network client factory: %w", err)
@@ -667,7 +682,6 @@ func (p Provider) Ignition(ctx context.Context, in clusterapi.IgnitionInput) ([]
 
 	bootstrapIgnData := in.BootstrapIgnData
 	subscriptionID := session.Credentials.SubscriptionID
-	cloudConfiguration := session.CloudConfig
 
 	ignitionContainerName := "ignition"
 	blobName := "bootstrap.ign"
@@ -698,7 +712,7 @@ func (p Provider) Ignition(ctx context.Context, in clusterapi.IgnitionInput) ([]
 			BlobURL:            blobURL,
 			StorageAccountName: p.StorageAccountName,
 			StorageAccountKeys: p.StorageAccountKeys,
-			CloudConfiguration: cloudConfiguration,
+			ClientOpts:         p.clientOptions,
 			BootstrapIgnData:   bootstrapIgnData,
 		})
 		if err != nil {
@@ -719,7 +733,7 @@ func (p Provider) Ignition(ctx context.Context, in clusterapi.IgnitionInput) ([]
 			BootstrapIgnData:   bootstrapIgnData,
 			ImageLength:        lengthBootstrapFile,
 			StorageAccountKeys: p.StorageAccountKeys,
-			CloudConfiguration: cloudConfiguration,
+			ClientOpts:         p.clientOptions,
 		})
 		if err != nil {
 			return nil, fmt.Errorf("failed to create PageBlob for ignition shim: %w", err)