Skip to content

Commit 62dd1cf

Browse files
a7icursoragent
andcommitted
feat(service): add EIP allocation discovery by tags
Allow internet-facing NLB services to discover Elastic IP allocation IDs via EC2 tags instead of hardcoding eip-allocations in manifests. Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent ea15ea3 commit 62dd1cf

14 files changed

Lines changed: 496 additions & 8 deletions

File tree

controllers/service/service_controller.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ const (
4444

4545
func NewServiceReconciler(cloud services.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder,
4646
finalizerManager k8s.FinalizerManager, networkingManager networking.NetworkingManager, networkingSGManager networking.SecurityGroupManager,
47-
networkingSGReconciler networking.SecurityGroupReconciler, subnetsResolver networking.SubnetsResolver,
47+
networkingSGReconciler networking.SecurityGroupReconciler, subnetsResolver networking.SubnetsResolver, eipResolver networking.EIPResolver,
4848
vpcInfoProvider networking.VPCInfoProvider, elbv2TaggingManager elbv2deploy.TaggingManager, controllerConfig config.ControllerConfig,
4949
backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger, metricsCollector lbcmetrics.MetricCollector, reconcileCounters *metricsutil.ReconcileCounters,
5050
targetGroupCollector awsmetrics.TargetGroupCollector) *serviceReconciler {
@@ -53,7 +53,7 @@ func NewServiceReconciler(cloud services.Cloud, k8sClient client.Client, eventRe
5353
trackingProvider := tracking.NewDefaultProvider(serviceTagPrefix, controllerConfig.ClusterName)
5454
serviceUtils := service.NewServiceUtils(annotationParser, shared_constants.ServiceFinalizer, controllerConfig.ServiceConfig.LoadBalancerClass, controllerConfig.FeatureGates)
5555
enhancedBackendBuilder := service.NewDefaultEnhancedBackendBuilder(k8sClient, annotationParser, logger)
56-
modelBuilder := service.NewDefaultModelBuilder(annotationParser, subnetsResolver, vpcInfoProvider, cloud.VpcID(), trackingProvider,
56+
modelBuilder := service.NewDefaultModelBuilder(annotationParser, subnetsResolver, eipResolver, vpcInfoProvider, cloud.VpcID(), trackingProvider,
5757
elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags,
5858
controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.DefaultLoadBalancerScheme, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils,
5959
backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.EnableManageBackendSecurityGroupRules, controllerConfig.DisableRestrictedSGRules, logger, metricsCollector, controllerConfig.FeatureGates.Enabled(config.EnableTCPUDPListenerType), enhancedBackendBuilder)

docs/guide/service/annotations.md

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-interval](#healthcheck-interval) | integer | 10 | |
4949
| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-success-codes](#healthcheck-success-codes) | string | 200-399 | |
5050
| [service.beta.kubernetes.io/aws-load-balancer-eip-allocations](#eip-allocations) | stringList | | internet-facing lb only. Length must match the number of subnets |
51+
| [service.beta.kubernetes.io/aws-load-balancer-eip-allocations-discovery-tags](#eip-allocations-discovery-tags) | stringMap | | internet-facing lb only. Mutually exclusive with [eip-allocations](#eip-allocations) |
5152
| [service.beta.kubernetes.io/aws-load-balancer-private-ipv4-addresses](#private-ipv4-addresses) | stringList | | internal lb only. Length must match the number of subnets |
5253
| [service.beta.kubernetes.io/aws-load-balancer-ipv6-addresses](#ipv6-addresses) | stringList | | dualstack lb only. Length must match the number of subnets |
5354
| [service.beta.kubernetes.io/aws-load-balancer-target-group-attributes](#target-group-attributes) | stringMap | | |
@@ -178,6 +179,20 @@ on the load balancer.
178179
service.beta.kubernetes.io/aws-load-balancer-eip-allocations: eipalloc-xyz, eipalloc-zzz
179180
```
180181

182+
- <a name="eip-allocations-discovery-tags">`service.beta.kubernetes.io/aws-load-balancer-eip-allocations-discovery-tags`</a> discovers [elastic IP address](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/elastic-ip-addresses-eip.html) allocation IDs by EC2 resource tags instead of listing them explicitly.
183+
184+
!!!note
185+
- This configuration is optional, and you can use it to assign static IP addresses to your NLB without hardcoding allocation IDs
186+
- NLB must be internet-facing
187+
- Mutually exclusive with [aws-load-balancer-eip-allocations](#eip-allocations)
188+
- Tag one VPC-scoped EIP per load balancer subnet Availability Zone; the controller matches EIPs to subnets by AZ
189+
- Tags must uniquely identify the intended EIP set for the service
190+
191+
!!!example
192+
```
193+
service.beta.kubernetes.io/aws-load-balancer-eip-allocations-discovery-tags: pod=pod998,service=zorg,visibility=external
194+
```
195+
181196

182197
- <a name="private-ipv4-addresses">`service.beta.kubernetes.io/aws-load-balancer-private-ipv4-addresses`</a> specifies a list of private IPv4 addresses for an internal NLB.
183198

main.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,7 @@ func main() {
196196
controllerCFG.FeatureGates.Enabled(config.ALBSingleSubnet),
197197
controllerCFG.FeatureGates.Enabled(config.SubnetDiscoveryByReachability),
198198
ctrl.Log.WithName("subnets-resolver"))
199+
eipResolver := networking.NewDefaultEIPResolver(cloud.EC2())
199200
multiClusterManager := targetgroupbinding.NewMultiClusterManager(mgr.GetClient(), mgr.GetAPIReader(), ctrl.Log)
200201

201202
nodeInfoProvider := networking.NewDefaultNodeInfoProvider(cloud.EC2(), ctrl.Log)
@@ -221,7 +222,7 @@ func main() {
221222
controllerCFG, backendSGProvider, sgResolver, secretsManager, ctrl.Log.WithName("controllers").WithName("ingress"), lbcMetricsCollector, reconcileCounters,
222223
targetGroupCollector, tgArnMapper)
223224
svcReconciler := service.NewServiceReconciler(cloud, mgr.GetClient(), mgr.GetEventRecorderFor("service"),
224-
finalizerManager, networkingManager, sgManager, sgReconciler, subnetResolver, vpcInfoProvider, elbv2TaggingManager,
225+
finalizerManager, networkingManager, sgManager, sgReconciler, subnetResolver, eipResolver, vpcInfoProvider, elbv2TaggingManager,
225226
controllerCFG, backendSGProvider, sgResolver, ctrl.Log.WithName("controllers").WithName("service"), lbcMetricsCollector, reconcileCounters,
226227
targetGroupCollector)
227228

pkg/annotations/constants.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ const (
114114
SvcLBSuffixTargetGroupAttributes = "aws-load-balancer-target-group-attributes"
115115
SvcLBSuffixSubnets = "aws-load-balancer-subnets"
116116
SvcLBSuffixEIPAllocations = "aws-load-balancer-eip-allocations"
117+
SvcLBSuffixEIPAllocationsDiscoveryTags = "aws-load-balancer-eip-allocations-discovery-tags"
117118
SvcLBSuffixPrivateIpv4Addresses = "aws-load-balancer-private-ipv4-addresses"
118119
SvcLBSuffixIpv6Addresses = "aws-load-balancer-ipv6-addresses"
119120
SvcLBSuffixALPNPolicy = "aws-load-balancer-alpn-policy"

pkg/aws/services/ec2.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ type EC2 interface {
2828
// DescribeRouteTablesAsList wraps the DescribeRouteTablesWithContext API, which aggregates paged results into list.
2929
DescribeRouteTablesAsList(ctx context.Context, input *ec2.DescribeRouteTablesInput) ([]types.RouteTable, error)
3030

31+
// DescribeAddressesAsList wraps the DescribeAddresses API, which aggregates paged results into list.
32+
DescribeAddressesAsList(ctx context.Context, input *ec2.DescribeAddressesInput) ([]types.Address, error)
33+
3134
CreateTagsWithContext(ctx context.Context, input *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error)
3235
DeleteTagsWithContext(ctx context.Context, input *ec2.DeleteTagsInput) (*ec2.DeleteTagsOutput, error)
3336
CreateSecurityGroupWithContext(ctx context.Context, input *ec2.CreateSecurityGroupInput) (*ec2.CreateSecurityGroupOutput, error)
@@ -162,6 +165,18 @@ func (c *ec2Client) DescribeRouteTablesAsList(ctx context.Context, input *ec2.De
162165
return result, nil
163166
}
164167

168+
func (c *ec2Client) DescribeAddressesAsList(ctx context.Context, input *ec2.DescribeAddressesInput) ([]types.Address, error) {
169+
client, err := c.awsClientsProvider.GetEC2Client(ctx, "DescribeAddresses")
170+
if err != nil {
171+
return nil, err
172+
}
173+
output, err := client.DescribeAddresses(ctx, input)
174+
if err != nil {
175+
return nil, err
176+
}
177+
return output.Addresses, nil
178+
}
179+
165180
func (c *ec2Client) CreateTagsWithContext(ctx context.Context, input *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error) {
166181
client, err := c.awsClientsProvider.GetEC2Client(ctx, "CreateTags")
167182
if err != nil {

pkg/aws/services/ec2_mocks.go

Lines changed: 15 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

pkg/networking/eip_resolver.go

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
package networking
2+
3+
import (
4+
"context"
5+
"fmt"
6+
7+
awssdk "github.com/aws/aws-sdk-go-v2/aws"
8+
ec2sdk "github.com/aws/aws-sdk-go-v2/service/ec2"
9+
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
10+
"k8s.io/apimachinery/pkg/util/sets"
11+
"sigs.k8s.io/aws-load-balancer-controller/v3/pkg/aws/services"
12+
)
13+
14+
const ec2FilterNameDomain = "domain"
15+
16+
//go:generate mockgen -destination=eip_resolver_mocks.go -package=networking sigs.k8s.io/aws-load-balancer-controller/v3/pkg/networking EIPResolver
17+
18+
// EIPResolver resolves Elastic IP allocation IDs for NLB subnet mappings.
19+
type EIPResolver interface {
20+
ResolveForSubnets(ctx context.Context, tagFilters map[string]string, subnets []ec2types.Subnet) ([]string, error)
21+
}
22+
23+
// NewDefaultEIPResolver constructs a new defaultEIPResolver.
24+
func NewDefaultEIPResolver(ec2Client services.EC2) *defaultEIPResolver {
25+
return &defaultEIPResolver{
26+
ec2Client: ec2Client,
27+
}
28+
}
29+
30+
type defaultEIPResolver struct {
31+
ec2Client services.EC2
32+
}
33+
34+
var _ EIPResolver = &defaultEIPResolver{}
35+
36+
func (r *defaultEIPResolver) ResolveForSubnets(ctx context.Context, tagFilters map[string]string, subnets []ec2types.Subnet) ([]string, error) {
37+
if len(tagFilters) == 0 {
38+
return nil, fmt.Errorf("EIP discovery tags must not be empty")
39+
}
40+
if len(subnets) == 0 {
41+
return nil, fmt.Errorf("subnets must not be empty for EIP discovery")
42+
}
43+
44+
addresses, err := r.listAddressesByTagFilters(ctx, tagFilters)
45+
if err != nil {
46+
return nil, fmt.Errorf("failed to list EIPs by discovery tags: %w", err)
47+
}
48+
49+
addressesByAZ := make(map[string][]ec2types.Address)
50+
for _, addr := range addresses {
51+
az := awssdk.ToString(addr.NetworkBorderGroup)
52+
if az == "" {
53+
return nil, fmt.Errorf("discovered EIP %s has empty network border group", awssdk.ToString(addr.AllocationId))
54+
}
55+
addressesByAZ[az] = append(addressesByAZ[az], addr)
56+
}
57+
58+
allocationIDs := make([]string, 0, len(subnets))
59+
for _, subnet := range subnets {
60+
subnetAZ := awssdk.ToString(subnet.AvailabilityZone)
61+
addrsInAZ := addressesByAZ[subnetAZ]
62+
if len(addrsInAZ) == 0 {
63+
return nil, fmt.Errorf("no EIP found for subnet %s in availability zone %s matching discovery tags", awssdk.ToString(subnet.SubnetId), subnetAZ)
64+
}
65+
if len(addrsInAZ) > 1 {
66+
return nil, fmt.Errorf("multiple EIPs found for availability zone %s matching discovery tags", subnetAZ)
67+
}
68+
addr := addrsInAZ[0]
69+
if err := validateDiscoveredEIP(addr); err != nil {
70+
return nil, err
71+
}
72+
allocationIDs = append(allocationIDs, awssdk.ToString(addr.AllocationId))
73+
}
74+
75+
return allocationIDs, nil
76+
}
77+
78+
func validateDiscoveredEIP(addr ec2types.Address) error {
79+
if addr.AllocationId == nil || awssdk.ToString(addr.AllocationId) == "" {
80+
return fmt.Errorf("discovered EIP has empty allocation ID")
81+
}
82+
if addr.AssociationId != nil && awssdk.ToString(addr.AssociationId) != "" {
83+
owner := awssdk.ToString(addr.NetworkInterfaceOwnerId)
84+
if owner != "" && owner != "amazon-elb" {
85+
return fmt.Errorf("EIP %s is associated with another resource", awssdk.ToString(addr.AllocationId))
86+
}
87+
}
88+
return nil
89+
}
90+
91+
func (r *defaultEIPResolver) listAddressesByTagFilters(ctx context.Context, tagFilters map[string]string) ([]ec2types.Address, error) {
92+
req := &ec2sdk.DescribeAddressesInput{
93+
Filters: []ec2types.Filter{
94+
{
95+
Name: awssdk.String(ec2FilterNameDomain),
96+
Values: []string{"vpc"},
97+
},
98+
},
99+
}
100+
for _, key := range sets.StringKeySet(tagFilters).List() {
101+
req.Filters = append(req.Filters, ec2types.Filter{
102+
Name: awssdk.String("tag:" + key),
103+
Values: []string{tagFilters[key]},
104+
})
105+
}
106+
return r.ec2Client.DescribeAddressesAsList(ctx, req)
107+
}

pkg/networking/eip_resolver_mocks.go

Lines changed: 51 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)