Merge pull request #3296 from michaelsaah/issue-3285-fix

don't block TGB reconciliation loop on failed SG ingress reconciliation

pkg/k8s/events.go

@@ -25,6 +25,7 @@ const (
 	TargetGroupBindingEventReasonFailedRemoveFinalizer  = "FailedRemoveFinalizer"
 	TargetGroupBindingEventReasonFailedUpdateStatus     = "FailedUpdateStatus"
 	TargetGroupBindingEventReasonFailedCleanup          = "FailedCleanup"
+	TargetGroupBindingEventReasonFailedNetworkReconcile = "FailedNetworkReconcile"
 	TargetGroupBindingEventReasonBackendNotFound        = "BackendNotFound"
 	TargetGroupBindingEventReasonSuccessfullyReconciled = "SuccessfullyReconciled"
 )

pkg/targetgroupbinding/networking_manager.go

@@ -6,6 +6,7 @@ import (
 	"net"
 	"strings"
 	"sync"
+	libErrors "errors"
 
 	awssdk "github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
@@ -199,11 +200,13 @@ func (m *defaultNetworkingManager) reconcileWithIngressPermissionsPerSG(ctx cont
 	aggregatedIngressPermissionsPerSG := m.computeAggregatedIngressPermissionsPerSG(ctx)
 
 	permissionSelector := labels.SelectorFromSet(labels.Set{tgbNetworkingIPPermissionLabelKey: tgbNetworkingIPPermissionLabelValue})
+	var sgReconciliationErrors []error
 	for sgID, permissions := range aggregatedIngressPermissionsPerSG {
 		if err := m.sgReconciler.ReconcileIngress(ctx, sgID, permissions,
 			networking.WithPermissionSelector(permissionSelector),
 			networking.WithAuthorizeOnly(!computedForAllTGBs)); err != nil {
-			return err
+			sgReconciliationErrors = append(sgReconciliationErrors, err)
+			continue
 		}
 	}
 
@@ -213,6 +216,11 @@ func (m *defaultNetworkingManager) reconcileWithIngressPermissionsPerSG(ctx cont
 		}
 	}
 
+	if len(sgReconciliationErrors) > 0 {
+		err := libErrors.Join(sgReconciliationErrors...)
+		return err
+	}
+
 	return nil
 }
 

pkg/targetgroupbinding/resource_manager.go

@@ -136,8 +136,10 @@ func (m *defaultResourceManager) reconcileWithIPTargetType(ctx context.Context,
 	notDrainingTargets, drainingTargets := partitionTargetsByDrainingStatus(targets)
 	matchedEndpointAndTargets, unmatchedEndpoints, unmatchedTargets := matchPodEndpointWithTargets(endpoints, notDrainingTargets)
 
+	needNetworkingRequeue := false
 	if err := m.networkingManager.ReconcileForPodEndpoints(ctx, tgb, endpoints); err != nil {
-		return err
+		m.eventRecorder.Event(tgb, corev1.EventTypeWarning, k8s.TargetGroupBindingEventReasonFailedNetworkReconcile, err.Error())
+		needNetworkingRequeue = true
 	}
 	if len(unmatchedTargets) > 0 {
 		if err := m.deregisterTargets(ctx, tgARN, unmatchedTargets); err != nil {
@@ -167,6 +169,10 @@ func (m *defaultResourceManager) reconcileWithIPTargetType(ctx context.Context,
 	}
 
 	_ = drainingTargets
+
+	if needNetworkingRequeue {
+		return runtime.NewRequeueNeeded("networking reconciliation")
+	}
 	return nil
 }