Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions docs/usage/usage.md
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,6 @@ You can freely choose a private CIDR range.
* If a vnet name is given and cilium shoot clusters are created without a network overlay within one vnet make sure that the pod CIDR specified in `shoot.spec.networking.pods` is not overlapping with any other pod CIDR used in that vnet.
Overlapping pod CIDRs will lead to disfunctional shoot clusters.
* It's possible to place multiple shoot cluster into the same vnet
* **Important**: [As announced by Azure](https://azure.microsoft.com/en-us/updates?id=default-outbound-access-for-vms-in-azure-will-be-retired-transition-to-a-new-method-of-internet-access), starting September 30th, 2025, **new** virtual networks will require explicitly configured outbound connectivity.

The `networks.workers` section describes the CIDR for a subnet that is used for all shoot worker nodes, i.e., VMs which later run your applications.
The specified CIDR range must be contained in the VNet CIDR specified above, or the VNet CIDR of your already existing VNet.
Expand All @@ -206,8 +205,6 @@ You can freely choose this CIDR and it is your responsibility to properly design
In the `networks.serviceEndpoints[]` list you can specify the list of Azure service endpoints which shall be associated with the worker subnet. All available service endpoints and their technical names can be found in the (Azure Service Endpoint documentation](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-service-endpoints-overview).

The `networks.natGateway` section contains configuration for the Azure NatGateway which can be attached to the worker subnet of a Shoot cluster. Here are some key information about the usage of the NatGateway for a Shoot cluster:
- Starting with September 2025 NatGateway usage is no longer optional for **new** shoots. If not set it is automatically enabled. An exception is the case if you bring your own VNet, then it is your responsibility to ensure outbound access.
- The NatGateway can can still be disabled via `.networks.natGateway.enabled`. However, doing so only makes sense if there is an alternative way available to reach the control plane.
- If the NatGateway is not used then the egress connections initiated within the Shoot cluster will be nated via the LoadBalancer of the clusters (default Azure behaviour, see [here](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections#scenarios)).
- The NatGateway is currently **not** zone redundantly deployed. That mean the NatGateway of a Shoot cluster will always be in just one zone. This zone can be optionally selected via `.networks.natGateway.zone`.
- **Caution:** Modifying the `.networks.natGateway.zone` setting requires a recreation of the NatGateway and the managed public ip (automatically used if no own public ip is specified, see below). That mean you will most likely get a different public ip for egress connections.
Expand Down
76 changes: 0 additions & 76 deletions pkg/admission/mutator/shoot.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,11 @@ import (
extensionswebhook "github.com/gardener/gardener/extensions/pkg/webhook"
gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
gardencorev1beta1helper "github.com/gardener/gardener/pkg/apis/core/v1beta1/helper"
"github.com/gardener/gardener/pkg/utils"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/manager"

"github.com/gardener/gardener-extension-provider-azure/pkg/apis/azure/v1alpha1"
"github.com/gardener/gardener-extension-provider-azure/pkg/azure"
"github.com/gardener/gardener-extension-provider-azure/pkg/features"
)

// NewShootMutator returns a new instance of a shoot mutator.
Expand Down Expand Up @@ -85,11 +80,6 @@ func (s *shoot) Mutate(_ context.Context, newObj, oldObj client.Object) error {
return err
}

err = s.mutateInfrastructureNatConfig(shoot, oldShoot)
if err != nil {
return err
}

// Disable TCP to upstream DNS queries by default on Azure. DNS over TCP may cause performance issues on larger clusters.
if shoot.Spec.SystemComponents != nil {
if shoot.Spec.SystemComponents.NodeLocalDNS != nil {
Expand All @@ -104,72 +94,6 @@ func (s *shoot) Mutate(_ context.Context, newObj, oldObj client.Object) error {
return nil
}

// mutateInfrastructureNatConfig mutates the InfrastructureConfig to enable NAT-Gateway
// preserves nat config if it was already set
func (s *shoot) mutateInfrastructureNatConfig(shoot, oldShoot *gardencorev1beta1.Shoot) error {
if shoot.Spec.Provider.InfrastructureConfig == nil || shoot.Spec.Provider.InfrastructureConfig.Raw == nil {
return nil
}

infraConfig := v1alpha1.InfrastructureConfig{}
if _, _, err := s.decoder.Decode(shoot.Spec.Provider.InfrastructureConfig.Raw, nil, &infraConfig); err != nil {
return fmt.Errorf("failed to decode InfrastructureConfig: %w", err)
}

if !shouldMutateNatGateway(infraConfig, oldShoot) {
return nil
}

// add annotation for new shoot OR preserve annotation if it was already set
shoot.Annotations = utils.MergeStringMaps(shoot.Annotations, map[string]string{
azure.ShootMutateNatConfig: "true",
})

nat := infraConfig.Networks.NatGateway
zones := infraConfig.Networks.Zones

// Case 1: Non-zoned setup → enable NAT-Gateway if not explicitly set
if len(zones) == 0 && nat == nil {
infraConfig.Networks.NatGateway = &v1alpha1.NatGatewayConfig{Enabled: true}
}

// Case 2: Zoned setup → enable NAT-Gateway per zone if not explicitly set
for i := range zones {
if zones[i].NatGateway == nil {
zones[i].NatGateway = &v1alpha1.ZonedNatGatewayConfig{Enabled: true}
}
}

modifiedJSON, err := json.Marshal(infraConfig)
if err != nil {
return fmt.Errorf("failed to marshal modified InfrastructureConfig: %w", err)
}
shoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: modifiedJSON}

return nil
}

// shouldMutateNatGateway returns true if ForceNatGateway is enabled and either it's a
// new shoot or the old shoot has the annotation to mutate nat config.
func shouldMutateNatGateway(newInfraConfig v1alpha1.InfrastructureConfig, oldShoot *gardencorev1beta1.Shoot) bool {
if !features.ExtensionFeatureGate.Enabled(features.ForceNatGateway) {
return false
}
// don't mutate shoots with existing VNet
if newInfraConfig.Networks.VNet.Name != nil && newInfraConfig.Networks.VNet.ResourceGroup != nil {
return false
}
return oldShoot == nil || mutateNatConfigAnnotationExists(oldShoot)
}

func mutateNatConfigAnnotationExists(shoot *gardencorev1beta1.Shoot) bool {
if shoot.Annotations == nil {
return false
}
_, exists := shoot.Annotations[azure.ShootMutateNatConfig]
return exists
}

func (s *shoot) mutateNetworkConfig(shoot, oldShoot *gardencorev1beta1.Shoot) error {
// Skip if specs are matching
if oldShoot != nil && reflect.DeepEqual(shoot.Spec, oldShoot.Spec) {
Expand Down
217 changes: 0 additions & 217 deletions pkg/admission/mutator/shoot_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ package mutator_test

import (
"context"
"encoding/json"
"fmt"
"time"

extensionswebhook "github.com/gardener/gardener/extensions/pkg/webhook"
Expand All @@ -22,9 +20,7 @@ import (
"k8s.io/utils/ptr"

"github.com/gardener/gardener-extension-provider-azure/pkg/admission/mutator"
"github.com/gardener/gardener-extension-provider-azure/pkg/apis/azure/v1alpha1"
"github.com/gardener/gardener-extension-provider-azure/pkg/azure"
"github.com/gardener/gardener-extension-provider-azure/pkg/features"
)

var _ = Describe("Shoot mutator", func() {
Expand Down Expand Up @@ -221,218 +217,5 @@ var _ = Describe("Shoot mutator", func() {
Expect(*shoot.Spec.SystemComponents.NodeLocalDNS.ForceTCPToUpstreamDNS).To(BeFalse())
})
})

Context("Mutate shoot infrastructure config", func() {
var infraConfig *v1alpha1.InfrastructureConfig

Context("NAT-Gateway", func() {
BeforeEach(func() {
infraConfig = &v1alpha1.InfrastructureConfig{
Networks: v1alpha1.NetworkConfig{},
}
err := features.ExtensionFeatureGate.Set(fmt.Sprintf("%s=%s", features.ForceNatGateway, "true"))
Expect(err).NotTo(HaveOccurred(), "Failed to enable feature gate")
})

It("should prevent removing nat annotation", func() {
oldShoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(infraConfig)}
oldShoot.Annotations = map[string]string{azure.ShootMutateNatConfig: "true"}
shoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(infraConfig)}
shoot.Annotations = nil
err := shootMutator.Mutate(ctx, shoot, oldShoot)
Expect(err).NotTo(HaveOccurred())
Expect(shoot.Annotations).To(Equal(oldShoot.Annotations))
})

It("should not mutate if user brings VNET", func() {
infraConfig.Networks.NatGateway = nil
infraConfig.Networks.VNet.Name = ptr.To("my-vnet")
infraConfig.Networks.VNet.ResourceGroup = ptr.To("resource-group")
shoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(infraConfig)}
err := shootMutator.Mutate(ctx, shoot, nil)
Expect(err).NotTo(HaveOccurred())
Expect(shoot.Spec.Provider.InfrastructureConfig.Raw).To(Equal(encode(infraConfig)))
Expect(shoot.Annotations).To(BeNil())
})

It("should not mutate infrastructure config or annotations when old shoot is provided", func() {
infraConfig.Networks.NatGateway = nil
shoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(infraConfig)}
err := shootMutator.Mutate(ctx, shoot, shoot)
Expect(err).NotTo(HaveOccurred())
Expect(shoot.Spec.Provider.InfrastructureConfig.Raw).To(Equal(encode(infraConfig)))
Expect(shoot.Annotations).To(BeNil())
})

It("should mutate new shoot if NAT-Gateway is nil", func() {
infraConfig.Networks.NatGateway = nil
mutatedInfraConfig := infraConfig.DeepCopy()
mutatedInfraConfig.Networks.NatGateway = &v1alpha1.NatGatewayConfig{Enabled: true}
shoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(infraConfig)}
err := shootMutator.Mutate(ctx, shoot, nil)
Expect(err).NotTo(HaveOccurred())
Expect(shoot.Spec.Provider.InfrastructureConfig.Raw).To(Equal(encode(mutatedInfraConfig)))
Expect(shoot.Annotations).To(Not(BeNil()))
Expect(shoot.Annotations[azure.ShootMutateNatConfig]).To(Equal("true"))
})

It("should not mutate if feature gate is disabled", func() {
err := features.ExtensionFeatureGate.Set(fmt.Sprintf("%s=%s", features.ForceNatGateway, "false"))
Expect(err).NotTo(HaveOccurred(), "Failed to enable feature gate")
infraConfig.Networks.NatGateway = nil
shoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(infraConfig)}
err = shootMutator.Mutate(ctx, shoot, nil)
Expect(err).NotTo(HaveOccurred())
Expect(shoot.Spec.Provider.InfrastructureConfig.Raw).To(Equal(encode(infraConfig)))
Expect(shoot.Annotations).To(BeNil())
})

It("should not mutate if NAT-Gateway is disabled", func() {
infraConfig.Networks.NatGateway = &v1alpha1.NatGatewayConfig{Enabled: false}
shoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(infraConfig)}
err := shootMutator.Mutate(ctx, shoot, nil)
Expect(err).NotTo(HaveOccurred())
Expect(shoot.Spec.Provider.InfrastructureConfig.Raw).To(Equal(encode(infraConfig)))
Expect(shoot.Annotations).ToNot(BeNil())
Expect(shoot.Annotations[azure.ShootMutateNatConfig]).To(Equal("true"))
})

It("should mutate if NAT-Gateway is nil in a multi zones setup", func() {
infraConfig.Networks.Zones = []v1alpha1.Zone{
{
Name: 1,
},
{
Name: 2,
},
}
mutatedInfraConfig := infraConfig.DeepCopy()
mutatedInfraConfig.Networks.Zones[0].NatGateway = &v1alpha1.ZonedNatGatewayConfig{Enabled: true}
mutatedInfraConfig.Networks.Zones[1].NatGateway = &v1alpha1.ZonedNatGatewayConfig{Enabled: true}
shoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(infraConfig)}
err := shootMutator.Mutate(ctx, shoot, nil)
Expect(err).NotTo(HaveOccurred())
Expect(shoot.Spec.Provider.InfrastructureConfig.Raw).To(Equal(encode(mutatedInfraConfig)))
Expect(shoot.Annotations[azure.ShootMutateNatConfig]).To(Equal("true"))
})

It("should not mutate if NAT-Gateway is explicitly disabled in a multi zones setup", func() {
infraConfig.Networks.Zones = []v1alpha1.Zone{
{
Name: 1,
NatGateway: &v1alpha1.ZonedNatGatewayConfig{
Enabled: false,
},
},
{
Name: 2,
NatGateway: &v1alpha1.ZonedNatGatewayConfig{
Enabled: false,
},
},
}
shoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(infraConfig)}
err := shootMutator.Mutate(ctx, shoot, nil)
Expect(err).NotTo(HaveOccurred())
Expect(shoot.Spec.Provider.InfrastructureConfig.Raw).To(Equal(encode(infraConfig)))
Expect(shoot.Annotations[azure.ShootMutateNatConfig]).To(Equal("true"))
})

Context("Prevent unsetting NAT-Gateway", func() {
It("should keep old nat when trying to remove explicit NAT-Gateway and annotation is set", func() {
oldShoot.Annotations = map[string]string{azure.ShootMutateNatConfig: "true"}
oldInfra := infraConfig.DeepCopy()
oldInfra.Networks.NatGateway = &v1alpha1.NatGatewayConfig{Enabled: true}
infraConfig.Networks.NatGateway = nil
shoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(infraConfig)}
oldShoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(oldInfra)}
err := shootMutator.Mutate(ctx, shoot, oldShoot)
Expect(err).NotTo(HaveOccurred())
Expect(shoot.Spec.Provider.InfrastructureConfig.Raw).To(Equal(encode(oldInfra)))
})

It("should be able to unset nat if force-nat-annotation is not set", func() {
oldInfra := infraConfig.DeepCopy()
oldInfra.Networks.NatGateway = &v1alpha1.NatGatewayConfig{Enabled: true}
infraConfig.Networks.NatGateway = nil
shoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(infraConfig)}
oldShoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(oldInfra)}
err := shootMutator.Mutate(ctx, shoot, oldShoot)
Expect(err).NotTo(HaveOccurred())
Expect(shoot.Spec.Provider.InfrastructureConfig.Raw).To(Equal(encode(infraConfig)))
Expect(shoot.Annotations).To(BeNil())
})

It("should keep old explicitly set nats when trying to remove them in multi-zone setup", func() {
oldShoot.Annotations = map[string]string{azure.ShootMutateNatConfig: "true"}
infraConfig.Networks.Zones = []v1alpha1.Zone{{
Name: 1,
NatGateway: nil,
}}
oldInfra := infraConfig.DeepCopy()
oldInfra.Networks.Zones[0].NatGateway = &v1alpha1.ZonedNatGatewayConfig{Enabled: true}
shoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(infraConfig)}
oldShoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(oldInfra)}
err := shootMutator.Mutate(ctx, shoot, oldShoot)
Expect(err).NotTo(HaveOccurred())
Expect(shoot.Spec.Provider.InfrastructureConfig.Raw).To(Equal(encode(oldInfra)))
})

It("should allow unsetting NAT in multi zone setup if force-nat-annotation is not set", func() {
infraConfig.Networks.Zones = []v1alpha1.Zone{{
Name: 1,
NatGateway: nil,
}}
oldInfra := infraConfig.DeepCopy()
oldInfra.Networks.Zones[0].NatGateway = &v1alpha1.ZonedNatGatewayConfig{Enabled: true}
shoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(infraConfig)}
oldShoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(oldInfra)}
err := shootMutator.Mutate(ctx, shoot, oldShoot)
Expect(err).NotTo(HaveOccurred())
Expect(shoot.Spec.Provider.InfrastructureConfig.Raw).To(Equal(encode(infraConfig)))
Expect(shoot.Annotations).To(BeNil())
})

It("should allow removing nat if switching to multi-zone setup", func() {
oldInfra := infraConfig.DeepCopy()
oldInfra.Networks.NatGateway = &v1alpha1.NatGatewayConfig{Enabled: true}
oldInfra.Networks.Zones = []v1alpha1.Zone{}
infraConfig.Networks.NatGateway = nil
infraConfig.Networks.Zones = []v1alpha1.Zone{{
Name: 1,
NatGateway: &v1alpha1.ZonedNatGatewayConfig{Enabled: true},
}}
shoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(infraConfig)}
oldShoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(oldInfra)}
err := shootMutator.Mutate(ctx, shoot, oldShoot)
Expect(err).NotTo(HaveOccurred())
Expect(shoot.Spec.Provider.InfrastructureConfig.Raw).To(Equal(encode(infraConfig)))
})

It("should allow removing nat if switching to non-multi-zone setup", func() {
oldInfra := infraConfig.DeepCopy()
oldInfra.Networks.NatGateway = nil
oldInfra.Zoned = true
oldInfra.Networks.Zones = []v1alpha1.Zone{{
Name: 1,
NatGateway: &v1alpha1.ZonedNatGatewayConfig{Enabled: true},
}}
infraConfig.Networks.NatGateway = &v1alpha1.NatGatewayConfig{Enabled: true}
infraConfig.Networks.Zones = []v1alpha1.Zone{}
infraConfig.Zoned = true
shoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(infraConfig)}
oldShoot.Spec.Provider.InfrastructureConfig = &runtime.RawExtension{Raw: encode(oldInfra)}
err := shootMutator.Mutate(ctx, shoot, oldShoot)
Expect(err).NotTo(HaveOccurred())
Expect(shoot.Spec.Provider.InfrastructureConfig.Raw).To(Equal(encode(infraConfig)))
})
})
})
})
})
})

func encode(obj runtime.Object) []byte {
data, _ := json.Marshal(obj)
return data
}
4 changes: 0 additions & 4 deletions pkg/azure/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,6 @@ const (
// Deprecated: This annotation is deprecated and will only used for testing until the deprecation by Azure.
DisableDefaultOutboundAccessAnnotation = "azure.provider.extensions.gardener.cloud/disable-default-outbound-access"

// ShootMutateNatConfig is an annotation assigned to Shoots that are created while the ForceNatGateway feature gate is enabled.
// It indicates that the NAT configuration should be mutated to add a NAT gateway.
ShootMutateNatConfig = "azure.provider.extensions.gardener.cloud/nat-gateway-default-outbound"

// CloudControllerManagerImageName is the name of the cloud-controller-manager image.
CloudControllerManagerImageName = "cloud-controller-manager"
// CloudNodeManagerImageName is the name of the cloud-node-manager image.
Expand Down
7 changes: 0 additions & 7 deletions pkg/features/features.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,6 @@ const (
// ForceAvailabilitySetMigration controls whether the controller will force the migration of existing availability sets to virtual machine scale sets.
// alpha: v1.54.0
ForceAvailabilitySetMigration featuregate.Feature = "ForceAvailabilitySetMigration"
// ForceNatGateway controls whether the controller will force the creation of a NAT gateway for new shoot cluster if the NAT-Gateway is not explicitly set and a user does not bring his own VNet.
// Necessary because Azure deprecated default outbound access https://azure.microsoft.com/en-us/updates?id=default-outbound-access-for-vms-in-azure-will-be-retired-transition-to-a-new-method-of-internet-access
// alpha: v1.54.0
ForceNatGateway featuregate.Feature = "ForceNatGateway"
)

// ExtensionFeatureGate is the feature gate for the extension controllers.
Expand All @@ -39,7 +35,4 @@ func RegisterExtensionFeatureGate() {
runtime.Must(ExtensionFeatureGate.Add(map[featuregate.Feature]featuregate.FeatureSpec{
ForceAvailabilitySetMigration: {Default: false, PreRelease: featuregate.Alpha},
}))
runtime.Must(ExtensionFeatureGate.Add(map[featuregate.Feature]featuregate.FeatureSpec{
ForceNatGateway: {Default: false, PreRelease: featuregate.Alpha},
}))
}
Loading