Ensure monitor loop exits early if pod shutdown is invoked and no pid…

… is found This helps some of the inconsistencies we're seeing in func tests. The qemu pid would get created and destroyed so fast that the monitor polling loop would never even see the pid at all. In this case, SIGTERM signal was being ignored. Signed-off-by: David Vossel <[email protected]>
portalgun-io · Apr 10, 2018 · d45a4bb · d45a4bb
1 parent 2ca42cd
commit d45a4bb
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 9 deletions.
diff --git a/pkg/virt-launcher/monitor.go b/pkg/virt-launcher/monitor.go
@@ -179,6 +179,16 @@ func NewProcessMonitor(commandPrefix string,
 	}
 }
 
+func (mon *monitor) isGracePeriodExpired() bool {
+	if mon.gracePeriodStartTime != 0 {
+		now := time.Now().UTC().Unix()
+		if (now - mon.gracePeriodStartTime) > int64(mon.gracePeriod) {
+			return true
+		}
+	}
+	return false
+}
+
 func (mon *monitor) refresh() {
 	if mon.isDone {
 		log.Log.Error("Called refresh after done!")
@@ -187,6 +197,8 @@ func (mon *monitor) refresh() {
 
 	log.Log.Debugf("Refreshing. CommandPrefix %s pid %d", mon.commandPrefix, mon.pid)
 
+	expired := mon.isGracePeriodExpired()
+
 	// is the process there?
 	if mon.pid == 0 {
 		var err error
@@ -200,6 +212,9 @@ func (mon *monitor) refresh() {
 			if mon.timeout > 0 && elapsed >= mon.timeout {
 				log.Log.Infof("%s not found after timeout", mon.commandPrefix)
 				mon.isDone = true
+			} else if expired {
+				log.Log.Infof("%s not found after grace period expired", mon.commandPrefix)
+				mon.isDone = true
 			}
 			return
 		}
@@ -219,12 +234,9 @@ func (mon *monitor) refresh() {
 		return
 	}
 
-	if mon.gracePeriodStartTime != 0 {
-		now := time.Now().UTC().Unix()
-		if (now - mon.gracePeriodStartTime) > int64(mon.gracePeriod) {
-			log.Log.Infof("Grace Period expired, shutting down.")
-			mon.shutdownCallback(mon.pid)
-		}
+	if expired {
+		log.Log.Infof("Grace Period expired, shutting down.")
+		mon.shutdownCallback(mon.pid)
 	}
 
 	return
@@ -252,9 +264,6 @@ func (mon *monitor) monitorLoop(startTimeout time.Duration, signalChan chan os.S
 			mon.refresh()
 		case s := <-signalChan:
 			log.Log.Infof("Received signal %d.", s)
-			if mon.pid == 0 {
-				continue
-			}
 
 			if mon.gracePeriodStartTime != 0 {
 				continue

diff --git a/pkg/virt-launcher/monitor_test.go b/pkg/virt-launcher/monitor_test.go
@@ -145,6 +145,30 @@ var _ = Describe("VirtLauncher", func() {
 				Expect(exited).To(Equal(true))
 			})
 
+			It("verify monitor loop exits when signal arrives and no pid is present", func() {
+				signalChannel := make(chan os.Signal, 1)
+				done := make(chan string)
+
+				go func() {
+					mon.monitorLoop(1*time.Second, signalChannel)
+					done <- "exit"
+				}()
+
+				time.Sleep(time.Second)
+
+				signalChannel <- syscall.SIGQUIT
+				noExitCheck := time.After(5 * time.Second)
+				exited := false
+
+				select {
+				case <-noExitCheck:
+				case <-done:
+					exited = true
+				}
+
+				Expect(exited).To(Equal(true))
+			})
+
 			It("verify graceful shutdown trigger works", func() {
 				signalChannel := make(chan os.Signal, 1)
 				done := make(chan string)