No egress when NAT-Gateway is used

hack/api-reference/api.md

@@ -1471,6 +1471,17 @@ bool
 Only the subnet that was used prior to the migration should have this attribute set.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>NatGatewayName</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<p>NatGatewayName is enabled</p>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="azure.provider.extensions.gardener.cloud/v1alpha1.VNet">VNet

pkg/apis/azure/helper/helper.go

@@ -21,6 +21,7 @@ import (
 	"github.com/gardener/gardener-extension-provider-azure/pkg/azure"
 
 	v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants"
+	"github.com/gardener/gardener/pkg/extensions"
 	"k8s.io/utils/pointer"
 )
 
@@ -137,6 +138,33 @@ func IsVmoRequired(infrastructureStatus *api.InfrastructureStatus) bool {
 	return !infrastructureStatus.Zoned && len(infrastructureStatus.AvailabilitySets) == 0
 }
 
+// ShouldDeployEgressChart determines if chart should be deployed
+func ShouldDeployEgressChart(infraStatus *api.InfrastructureStatus, cluster *extensions.Cluster) bool {
+	case1 := !infraStatus.Zoned && len(infraStatus.AvailabilitySets) != 0
+	case2 := !infraStatus.Zoned && HasShootVmoAlphaAnnotation(cluster.Shoot.Annotations) && allNatEnabled(infraStatus.Networks.Subnets)
+	case3 := infraStatus.Zoned && allNatEnabled(getAllPurposeNodes(infraStatus.Networks.Subnets))
+	return !(case1 || case2 || case3)
+}
+
+func allNatEnabled(subnets []api.Subnet) bool {
+	for _, v := range subnets {
+		if !v.HasNatGateway() {
+			return false
+		}
+	}
+	return true
+}
+
+func getAllPurposeNodes(subnets []api.Subnet) []api.Subnet {
+	res := []api.Subnet{}
+	for _, v := range subnets {
+		if v.Purpose == api.PurposeNodes {
+			res = append(res, v)
+		}
+	}
+	return res
+}
+
 // HasShootVmoAlphaAnnotation determines if the passed Shoot annotations contain instruction to use VMO.
 func HasShootVmoAlphaAnnotation(shootAnnotations map[string]string) bool {
 	value, exists := shootAnnotations[azure.ShootVmoUsageAnnotation]

pkg/apis/azure/types_infrastructure.go

@@ -167,7 +167,13 @@ type Subnet struct {
 	Zone *string
 	// Migrated is set when the network layout is migrated from NetworkLayoutSingleSubnet to NetworkLayoutMultipleSubnet.
 	// Only the subnet that was used prior to the migration should have this attribute set.
-	Migrated bool
+	Migrated       bool
+	NatGatewayName *string
+}
+
+// HasNatGateway returns if NAT is enabled
+func (s Subnet) HasNatGateway() bool {
+	return s.NatGatewayName != nil
 }
 
 // AvailabilitySet contains information about the azure availability set

pkg/apis/azure/v1alpha1/types_infrastructure.go

@@ -187,6 +187,13 @@ type Subnet struct {
 	// Migrated is set when the network layout is migrated from NetworkLayoutSingleSubnet to NetworkLayoutMultipleSubnet.
 	// Only the subnet that was used prior to the migration should have this attribute set.
 	Migrated bool `json:"migrated,omitempty"`
+	// NatGatewayName is enabled
+	NatGatewayName *string
+}
+
+// HasNatGateway returns if NAT is enabled
+func (s Subnet) HasNatGateway() bool {
+	return s.NatGatewayName != nil
 }
 
 // AvailabilitySet contains information about the azure availability set

pkg/controller/controlplane/valuesprovider.go

@@ -697,7 +697,7 @@ func getControlPlaneShootChartValues(
 		"global": map[string]interface{}{
 			"vpaEnabled": gardencorev1beta1helper.ShootWantsVerticalPodAutoscaler(cluster.Shoot),
 		},
-		azure.AllowEgressName: map[string]interface{}{"enabled": infraStatus.Zoned || azureapihelper.IsVmoRequired(infraStatus)},
+		azure.AllowEgressName: map[string]interface{}{"enabled": azureapihelper.ShouldDeployEgressChart(infraStatus, cluster)},
 		azure.CloudControllerManagerName: map[string]interface{}{
 			"enabled":     true,
 			"pspDisabled": pspDisabled,

pkg/controller/controlplane/valuesprovider_test.go

@@ -563,6 +563,86 @@ var _ = Describe("ValuesProvider", func() {
 					azure.RemedyControllerName:       enabledTrue,
 				}))
 			})
+			It("should disable egress if all node subnets of zoned cluster have NAT gateway", func() {
+				infrastructureStatus.Zoned = true
+				infrastructureStatus.Networks.Subnets = []apisazure.Subnet{
+					{
+						Name:           "subnet-abcd1234-nodes",
+						Purpose:        "nodes",
+						NatGatewayName: pointer.String("name"),
+					},
+					{
+						Name:           "subnet-abcd5678-nodes",
+						Purpose:        "internal",
+						NatGatewayName: pointer.String("name"),
+					},
+				}
+				cp := generateControlPlane(controlPlaneConfig, infrastructureStatus)
+				values, err := vp.GetControlPlaneShootChartValues(ctx, cp, cluster, fakeSecretsManager, checksums)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(values[azure.AllowEgressName]).To(Equal(enabledFalse))
+			})
+			It("should enable egress if not all node subnets of zoned cluster have NAT gateway", func() {
+				infrastructureStatus.Zoned = true
+				infrastructureStatus.Networks.Subnets = []apisazure.Subnet{
+					{
+						Name:           "subnet-abcd1234-nodes",
+						Purpose:        "nodes",
+						NatGatewayName: pointer.String("name"),
+					},
+					{
+						Name:           "subnet-abcd5678-nodes",
+						Purpose:        "nodes",
+						NatGatewayName: nil,
+					},
+				}
+				cp := generateControlPlane(controlPlaneConfig, infrastructureStatus)
+				values, err := vp.GetControlPlaneShootChartValues(ctx, cp, cluster, fakeSecretsManager, checksums)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(values[azure.AllowEgressName]).To(Equal(enabledTrue))
+			})
+			It("should disable egress for cluster with vmss flex (vmo, non zoned) and NAT for subnets", func() {
+				infrastructureStatus.Zoned = false
+				infrastructureStatus.AvailabilitySets = []apisazure.AvailabilitySet{}
+				infrastructureStatus.Networks.Subnets = []apisazure.Subnet{
+					{
+						Name:           "subnet-abcd1234-nodes",
+						Purpose:        "nodes",
+						NatGatewayName: pointer.String("name"),
+					},
+					{
+						Name:           "subnet-abcd5678-nodes",
+						Purpose:        "nodes",
+						NatGatewayName: pointer.String("name"),
+					},
+				}
+				cluster = generateCluster(cidr, k8sVersionLessThan121, false, map[string]string{azure.ShootVmoUsageAnnotation: "true"})
+				cp := generateControlPlane(controlPlaneConfig, infrastructureStatus)
+				values, err := vp.GetControlPlaneShootChartValues(ctx, cp, cluster, fakeSecretsManager, checksums)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(values[azure.AllowEgressName]).To(Equal(enabledFalse))
+			})
+			It("should enable egress for cluster with vmss flex (vmo, non zoned) if not all subnets have NAT", func() {
+				infrastructureStatus.Zoned = false
+				infrastructureStatus.AvailabilitySets = []apisazure.AvailabilitySet{}
+				infrastructureStatus.Networks.Subnets = []apisazure.Subnet{
+					{
+						Name:           "subnet-abcd1234-nodes",
+						Purpose:        "nodes",
+						NatGatewayName: pointer.String("name"),
+					},
+					{
+						Name:           "subnet-abcd5678-nodes",
+						Purpose:        "nodes",
+						NatGatewayName: nil, // TODO purpose matters?
+					},
+				}
+				cluster = generateCluster(cidr, k8sVersionLessThan121, false, map[string]string{azure.ShootVmoUsageAnnotation: "true"})
+				cp := generateControlPlane(controlPlaneConfig, infrastructureStatus)
+				values, err := vp.GetControlPlaneShootChartValues(ctx, cp, cluster, fakeSecretsManager, checksums)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(values[azure.AllowEgressName]).To(Equal(enabledTrue))
+			})
 		})
 
 		Context("k8s >= 1.21", func() {

pkg/internal/infrastructure/terraform.go

@@ -236,7 +236,7 @@ func computeNetworkConfigSingleSubnetLayout(infra *extensionsv1alpha1.Infrastruc
 		networkCfg = make(map[string]interface{})
 		subnets    []map[string]interface{}
 	)
-	natGatewayConfig, err := generateNatGatewayValues(infra, config.Networks.NatGateway)
+	natGatewayConfig, err := generateNatGatewayValues(config.Networks.NatGateway)
 	if err != nil {
 		return nil, err
 	}
@@ -274,7 +274,22 @@ func computeNetworkConfigMultipleSubnetLayout(infra *extensionsv1alpha1.Infrastr
 	return networkCfg, nil
 }
 
-func generateNatGatewayValues(infra *extensionsv1alpha1.Infrastructure, nat *api.NatGatewayConfig) (map[string]interface{}, error) {
+func getSubnetsNatGatewayValues(config *api.InfrastructureConfig) []map[string]interface{} {
+	var (
+		subnets []map[string]interface{}
+	)
+
+	for _, zone := range config.Networks.Zones {
+		if zone.NatGateway == nil {
+			continue
+		} else {
+			subnets = append(subnets, generateZonedNatGatewayValues(zone.NatGateway, zone.Name))
+		}
+	}
+	return subnets
+}
+
+func generateNatGatewayValues(nat *api.NatGatewayConfig) (map[string]interface{}, error) {
 	natGatewayConfig := map[string]interface{}{
 		"enabled": false,
 	}
@@ -442,9 +457,9 @@ func ExtractTerraformState(ctx context.Context, tf terraformer.Terraformer, infr
 	return &tfState, nil
 }
 
-// StatusFromTerraformState computes an InfrastructureStatus from the given
+// statusFromTerraformStateWithoutNatGateway computes an InfrastructureStatus from the given
 // Terraform variables.
-func StatusFromTerraformState(config *api.InfrastructureConfig, tfState *TerraformState) *apiv1alpha1.InfrastructureStatus {
+func statusFromTerraformStateWithoutNatGateway(config *api.InfrastructureConfig, tfState *TerraformState) *apiv1alpha1.InfrastructureStatus {
 	infraState := apiv1alpha1.InfrastructureStatus{
 		TypeMeta: StatusTypeMeta,
 		ResourceGroup: apiv1alpha1.ResourceGroup{
@@ -505,17 +520,39 @@ func StatusFromTerraformState(config *api.InfrastructureConfig, tfState *Terrafo
 			Purpose:            apiv1alpha1.PurposeNodes,
 		})
 	}
-
 	return &infraState
 }
 
+func setNatGatewayConfigForSubnets(clusterName string, config *api.InfrastructureConfig, infraState *apiv1alpha1.InfrastructureStatus) {
+	subnetsNatGatewayValues := getSubnetsNatGatewayValues(config)
+	for _, subnet := range subnetsNatGatewayValues {
+		for j := range infraState.Networks.Subnets {
+			if fmt.Sprint(subnet["zone"]) == *infraState.Networks.Subnets[j].Zone {
+				enabled := subnet["enabled"].(bool)
+				if enabled {
+					name := getNatGatewayName(clusterName, infraState.Networks.Subnets[j])
+					infraState.Networks.Subnets[j].NatGatewayName = &name
+				}
+			}
+		}
+	}
+}
+
+func getNatGatewayName(clusterName string, subnet apiv1alpha1.Subnet) string {
+	name := fmt.Sprintf("%s-nat-gateway", clusterName)
+	if !subnet.Migrated {
+		name = fmt.Sprintf("%s-z%s", name, subnet.Name)
+	}
+	return name
+}
+
 // ComputeStatus computes the status based on the Terraformer and the given InfrastructureConfig.
 func ComputeStatus(ctx context.Context, tf terraformer.Terraformer, infra *extensionsv1alpha1.Infrastructure, config *api.InfrastructureConfig, cluster *controller.Cluster) (*apiv1alpha1.InfrastructureStatus, error) {
 	state, err := ExtractTerraformState(ctx, tf, infra, config, cluster)
 	if err != nil {
 		return nil, err
 	}
-	status := StatusFromTerraformState(config, state)
+	status := StatusFromTerraformState(cluster, config, state)
 
 	// Check if ACR access should be configured.
 	if config.Identity != nil && config.Identity.ACRAccess != nil && *config.Identity.ACRAccess && status.Identity != nil {
@@ -525,6 +562,14 @@ func ComputeStatus(ctx context.Context, tf terraformer.Terraformer, infra *exten
 	return status, nil
 }
 
+// StatusFromTerraformState computes an InfrastructureStatus from the given
+// Terraform variables.
+func StatusFromTerraformState(cluster *controller.Cluster, config *api.InfrastructureConfig, tfState *TerraformState) *apiv1alpha1.InfrastructureStatus {
+	status := statusFromTerraformStateWithoutNatGateway(config, tfState)
+	setNatGatewayConfigForSubnets(cluster.ObjectMeta.Name, config, status)
+	return status
+}
+
 type domainCounts struct {
 	faultDomains  int32
 	updateDomains int32