fix(k8sagent): retry watcher if it stops; fixes #416

honeycombio · Jun 3, 2024 · 0af592b · 0af592b
1 parent a0d5bd6
commit 0af592b
Showing 1 changed file with 22 additions and 13 deletions.
diff --git a/k8sagent/watcher.go b/k8sagent/watcher.go
@@ -157,17 +157,26 @@ func runInformer(
 
 	watchlist := &cache.ListWatch{ListFunc: listFunc, WatchFunc: watchFunc}
 	resyncPeriod := time.Minute
-	_, controller := cache.NewInformer(
-		watchlist,
-		&v1.Pod{},
-		resyncPeriod,
-		handler,
-	)
-
-	logrus.WithFields(logrus.Fields{
-		"labelSelector": labelSelector,
-		"namespace":     namespace,
-		"fieldSelector": fieldSelector,
-	}).Debug("Starting informer")
-	go controller.Run(wait.NeverStop)
+	go func() {
+		for {
+			_, controller := cache.NewInformer(
+				watchlist,
+				&v1.Pod{},
+				resyncPeriod,
+				handler,
+			)
+
+			logrus.WithFields(logrus.Fields{
+				"labelSelector": labelSelector,
+				"namespace":     namespace,
+				"fieldSelector": fieldSelector,
+			}).Debug("Starting informer")
+			controller.Run(wait.NeverStop)
+			logrus.WithFields(logrus.Fields{
+				"labelSelector": labelSelector,
+				"namespace":     namespace,
+				"fieldSelector": fieldSelector,
+			}).Warning("Informer unexpectedly stopped")
+		}
+	}()
 }