Skip to content

Commit b805cc2

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #3329 from oliviassss/nlb-sg
Add support for NLB security group
2 parents 05e6f58 + 09b6030 commit b805cc2

12 files changed

Lines changed: 4390 additions & 246 deletions

‎controllers/service/service_controller.go‎

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"github.com/go-logr/logr"
88
"github.com/pkg/errors"
99
corev1 "k8s.io/api/core/v1"
10+
"k8s.io/apimachinery/pkg/types"
1011
"k8s.io/client-go/tools/record"
1112
"sigs.k8s.io/aws-load-balancer-controller/controllers/service/eventhandlers"
1213
"sigs.k8s.io/aws-load-balancer-controller/pkg/annotations"
@@ -37,14 +38,17 @@ const (
3738
func NewServiceReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorder record.EventRecorder,
3839
finalizerManager k8s.FinalizerManager, networkingSGManager networking.SecurityGroupManager,
3940
networkingSGReconciler networking.SecurityGroupReconciler, subnetsResolver networking.SubnetsResolver,
40-
vpcInfoProvider networking.VPCInfoProvider, controllerConfig config.ControllerConfig, logger logr.Logger) *serviceReconciler {
41+
vpcInfoProvider networking.VPCInfoProvider, controllerConfig config.ControllerConfig,
42+
backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, logger logr.Logger) *serviceReconciler {
4143

4244
annotationParser := annotations.NewSuffixAnnotationParser(serviceAnnotationPrefix)
4345
trackingProvider := tracking.NewDefaultProvider(serviceTagPrefix, controllerConfig.ClusterName)
4446
elbv2TaggingManager := elbv2.NewDefaultTaggingManager(cloud.ELBV2(), cloud.VpcID(), controllerConfig.FeatureGates, cloud.RGT(), logger)
4547
serviceUtils := service.NewServiceUtils(annotationParser, serviceFinalizer, controllerConfig.ServiceConfig.LoadBalancerClass, controllerConfig.FeatureGates)
4648
modelBuilder := service.NewDefaultModelBuilder(annotationParser, subnetsResolver, vpcInfoProvider, cloud.VpcID(), trackingProvider,
47-
elbv2TaggingManager, controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags, controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils)
49+
elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags,
50+
controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils,
51+
backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules)
4852
stackMarshaller := deploy.NewDefaultStackMarshaller()
4953
stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, controllerConfig, serviceTagPrefix, logger)
5054
return &serviceReconciler{
@@ -54,6 +58,7 @@ func NewServiceReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorde
5458
annotationParser: annotationParser,
5559
loadBalancerClass: controllerConfig.ServiceConfig.LoadBalancerClass,
5660
serviceUtils: serviceUtils,
61+
backendSGProvider: backendSGProvider,
5762

5863
modelBuilder: modelBuilder,
5964
stackMarshaller: stackMarshaller,
@@ -71,6 +76,7 @@ type serviceReconciler struct {
7176
annotationParser annotations.Parser
7277
loadBalancerClass string
7378
serviceUtils service.ServiceUtils
79+
backendSGProvider networking.BackendSGProvider
7480

7581
modelBuilder service.ModelBuilder
7682
stackMarshaller deploy.StackMarshaller
@@ -93,29 +99,29 @@ func (r *serviceReconciler) reconcile(ctx context.Context, req ctrl.Request) err
9399
if err := r.k8sClient.Get(ctx, req.NamespacedName, svc); err != nil {
94100
return client.IgnoreNotFound(err)
95101
}
96-
stack, lb, err := r.buildModel(ctx, svc)
102+
stack, lb, backendSGRequired, err := r.buildModel(ctx, svc)
97103
if err != nil {
98104
return err
99105
}
100106
if lb == nil {
101107
return r.cleanupLoadBalancerResources(ctx, svc, stack)
102108
}
103-
return r.reconcileLoadBalancerResources(ctx, svc, stack, lb)
109+
return r.reconcileLoadBalancerResources(ctx, svc, stack, lb, backendSGRequired)
104110
}
105111

106-
func (r *serviceReconciler) buildModel(ctx context.Context, svc *corev1.Service) (core.Stack, *elbv2model.LoadBalancer, error) {
107-
stack, lb, err := r.modelBuilder.Build(ctx, svc)
112+
func (r *serviceReconciler) buildModel(ctx context.Context, svc *corev1.Service) (core.Stack, *elbv2model.LoadBalancer, bool, error) {
113+
stack, lb, backendSGRequired, err := r.modelBuilder.Build(ctx, svc)
108114
if err != nil {
109115
r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedBuildModel, fmt.Sprintf("Failed build model due to %v", err))
110-
return nil, nil, err
116+
return nil, nil, false, err
111117
}
112118
stackJSON, err := r.stackMarshaller.Marshal(stack)
113119
if err != nil {
114120
r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedBuildModel, fmt.Sprintf("Failed build model due to %v", err))
115-
return nil, nil, err
121+
return nil, nil, false, err
116122
}
117123
r.logger.Info("successfully built model", "model", stackJSON)
118-
return stack, lb, nil
124+
return stack, lb, backendSGRequired, nil
119125
}
120126

121127
func (r *serviceReconciler) deployModel(ctx context.Context, svc *corev1.Service, stack core.Stack) error {
@@ -128,7 +134,8 @@ func (r *serviceReconciler) deployModel(ctx context.Context, svc *corev1.Service
128134
return nil
129135
}
130136

131-
func (r *serviceReconciler) reconcileLoadBalancerResources(ctx context.Context, svc *corev1.Service, stack core.Stack, lb *elbv2model.LoadBalancer) error {
137+
func (r *serviceReconciler) reconcileLoadBalancerResources(ctx context.Context, svc *corev1.Service, stack core.Stack,
138+
lb *elbv2model.LoadBalancer, backendSGRequired bool) error {
132139
if err := r.finalizerManager.AddFinalizers(ctx, svc, serviceFinalizer); err != nil {
133140
r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedAddFinalizer, fmt.Sprintf("Failed add finalizer due to %v", err))
134141
return err
@@ -142,6 +149,12 @@ func (r *serviceReconciler) reconcileLoadBalancerResources(ctx context.Context,
142149
return err
143150
}
144151

152+
if !backendSGRequired {
153+
if err := r.backendSGProvider.Release(ctx, networking.ResourceTypeService, []types.NamespacedName{k8s.NamespacedName(svc)}); err != nil {
154+
return err
155+
}
156+
}
157+
145158
if err = r.updateServiceStatus(ctx, lbDNS, svc); err != nil {
146159
r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedUpdateStatus, fmt.Sprintf("Failed update status due to %v", err))
147160
return err
@@ -156,6 +169,9 @@ func (r *serviceReconciler) cleanupLoadBalancerResources(ctx context.Context, sv
156169
if err != nil {
157170
return err
158171
}
172+
if err := r.backendSGProvider.Release(ctx, networking.ResourceTypeService, []types.NamespacedName{k8s.NamespacedName(svc)}); err != nil {
173+
return err
174+
}
159175
if err = r.cleanupServiceStatus(ctx, svc); err != nil {
160176
r.eventRecorder.Event(svc, corev1.EventTypeWarning, k8s.ServiceEventReasonFailedCleanupStatus, fmt.Sprintf("Failed update status due to %v", err))
161177
return err

‎main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ func main() {
117117
controllerCFG, backendSGProvider, sgResolver, ctrl.Log.WithName("controllers").WithName("ingress"))
118118
svcReconciler := service.NewServiceReconciler(cloud, mgr.GetClient(), mgr.GetEventRecorderFor("service"),
119119
finalizerManager, sgManager, sgReconciler, subnetResolver, vpcInfoProvider,
120-
controllerCFG, ctrl.Log.WithName("controllers").WithName("service"))
120+
controllerCFG, backendSGProvider, sgResolver, ctrl.Log.WithName("controllers").WithName("service"))
121121
tgbReconciler := elbv2controller.NewTargetGroupBindingReconciler(mgr.GetClient(), mgr.GetEventRecorderFor("targetGroupBinding"),
122122
finalizerManager, tgbResManager,
123123
controllerCFG, ctrl.Log.WithName("controllers").WithName("targetGroupBinding"))

‎pkg/annotations/constants.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,5 +82,6 @@ const (
8282
SvcLBSuffixALPNPolicy = "aws-load-balancer-alpn-policy"
8383
SvcLBSuffixTargetNodeLabels = "aws-load-balancer-target-node-labels"
8484
SvcLBSuffixLoadBalancerAttributes = "aws-load-balancer-attributes"
85+
SvcLBSuffixLoadBalancerSecurityGroups = "aws-load-balancer-security-groups"
8586
SvcLBSuffixManageSGRules = "aws-load-balancer-manage-backend-security-group-rules"
8687
)

‎pkg/config/feature_gates.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ const (
2020
EnableRGTAPI Feature = "EnableRGTAPI"
2121
SubnetsClusterTagCheck Feature = "SubnetsClusterTagCheck"
2222
NLBHealthCheckAdvancedConfig Feature = "NLBHealthCheckAdvancedConfig"
23+
NLBSecurityGroup Feature = "NLBSecurityGroup"
2324
)
2425

2526
type FeatureGates interface {
@@ -56,6 +57,7 @@ func NewFeatureGates() FeatureGates {
5657
EnableRGTAPI: false,
5758
SubnetsClusterTagCheck: true,
5859
NLBHealthCheckAdvancedConfig: true,
60+
NLBSecurityGroup: true,
5961
},
6062
}
6163
}

‎pkg/networking/backend_sg_provider.go‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
ec2sdk "github.com/aws/aws-sdk-go/service/ec2"
1717
"github.com/go-logr/logr"
1818
"github.com/pkg/errors"
19+
corev1 "k8s.io/api/core/v1"
1920
networking "k8s.io/api/networking/v1"
2021
"k8s.io/apimachinery/pkg/types"
2122
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services"
@@ -35,6 +36,7 @@ const (
3536

3637
explicitGroupFinalizerPrefix = "group.ingress.k8s.aws/"
3738
implicitGroupFinalizer = "ingress.k8s.aws/resources"
39+
serviceFinalizer = "service.k8s.aws/resources"
3840

3941
sgDescription = "[k8s] Shared Backend SecurityGroup for LoadBalancer"
4042
)
@@ -76,6 +78,15 @@ func NewBackendSGProvider(clusterName string, backendSG string, vpcID string,
7678
return false
7779
},
7880

81+
checkServiceFinalizersFunc: func(finalizers []string) bool {
82+
for _, fin := range finalizers {
83+
if fin == serviceFinalizer {
84+
return true
85+
}
86+
}
87+
return false
88+
},
89+
7990
defaultDeletionPollInterval: defaultSGDeletionPollInterval,
8091
defaultDeletionTimeout: defaultSGDeletionTimeout,
8192
}
@@ -101,6 +112,7 @@ type defaultBackendSGProvider struct {
101112
// controller deletes the backend SG.
102113
objectsMap sync.Map
103114

115+
checkServiceFinalizersFunc func([]string) bool
104116
checkIngressFinalizersFunc func([]string) bool
105117

106118
defaultDeletionPollInterval time.Duration
@@ -126,7 +138,7 @@ func (p *defaultBackendSGProvider) Release(ctx context.Context, resourceType Res
126138
}
127139
defer func() {
128140
for _, res := range inactiveResources {
129-
p.objectsMap.Delete(getObjectKey(resourceType, res))
141+
p.objectsMap.CompareAndDelete(getObjectKey(resourceType, res), false)
130142
}
131143
}()
132144
p.updateObjectsMap(ctx, resourceType, inactiveResources, false)
@@ -159,6 +171,9 @@ func (p *defaultBackendSGProvider) isBackendSGRequired(ctx context.Context) (boo
159171
if required, err := p.checkIngressListForUnmapped(ctx); required || err != nil {
160172
return required, err
161173
}
174+
if required, err := p.checkServiceListForUnmapped(ctx); required || err != nil {
175+
return required, err
176+
}
162177
return false, nil
163178
}
164179

@@ -178,6 +193,22 @@ func (p *defaultBackendSGProvider) checkIngressListForUnmapped(ctx context.Conte
178193
return false, nil
179194
}
180195

196+
func (p *defaultBackendSGProvider) checkServiceListForUnmapped(ctx context.Context) (bool, error) {
197+
svcList := &corev1.ServiceList{}
198+
if err := p.k8sClient.List(ctx, svcList); err != nil {
199+
return true, errors.Wrapf(err, "unable to list services")
200+
}
201+
for _, svc := range svcList.Items {
202+
if !p.checkServiceFinalizersFunc(svc.GetFinalizers()) {
203+
continue
204+
}
205+
if !p.existsInObjectMap(ResourceTypeService, k8s.NamespacedName(&svc)) {
206+
return true, nil
207+
}
208+
}
209+
return false, nil
210+
}
211+
181212
func (p *defaultBackendSGProvider) existsInObjectMap(resourceType ResourceType, resource types.NamespacedName) bool {
182213
if _, exists := p.objectsMap.Load(getObjectKey(resourceType, resource)); exists {
183214
return true

‎pkg/networking/backend_sg_provider_test.go‎

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -542,14 +542,6 @@ func Test_defaultBackendSGProvider_Release(t *testing.T) {
542542
},
543543
},
544544
inactiveIngresses: []*networking.Ingress{ing},
545-
deleteSGCalls: []deleteSecurityGroupWithContextCall{
546-
{
547-
req: &ec2sdk.DeleteSecurityGroupInput{
548-
GroupId: awssdk.String("sg-autogen"),
549-
},
550-
resp: &ec2sdk.DeleteSecurityGroupOutput{},
551-
},
552-
},
553545
},
554546
},
555547
{
@@ -727,15 +719,8 @@ func Test_defaultBackendSGProvider_Release(t *testing.T) {
727719
},
728720
},
729721
inactiveIngresses: []*networking.Ingress{ing},
730-
deleteSGCalls: []deleteSecurityGroupWithContextCall{
731-
{
732-
req: &ec2sdk.DeleteSecurityGroupInput{
733-
GroupId: awssdk.String("sg-autogen"),
734-
},
735-
resp: &ec2sdk.DeleteSecurityGroupOutput{},
736-
},
737-
},
738722
},
723+
wantErr: errors.New("unable to list services: failed"),
739724
},
740725
}
741726
for _, tt := range tests {
@@ -753,7 +738,7 @@ func Test_defaultBackendSGProvider_Release(t *testing.T) {
753738
}
754739
for _, item := range tt.fields.resourceMapItems {
755740
var resourceType ResourceType = ResourceTypeIngress
756-
if reflect.TypeOf(item).String() == "service" {
741+
if reflect.TypeOf(item.key).String() == "*v1.Service" {
757742
resourceType = ResourceTypeService
758743
}
759744
sgProvider.objectsMap.Store(getObjectKey(resourceType, k8s.NamespacedName(item.key)), item.value)

0 commit comments

Comments
 (0)