verify that live migration monitor cancels the migration when needed

Signed-off-by: Vladik Romanovsky <[email protected]>
portalgun-io · Mar 1, 2019 · 0587388 · 0587388
1 parent 5704f73
commit 0587388
Show file tree

Hide file tree

Showing 2 changed files with 83 additions and 5 deletions.
diff --git a/pkg/virt-launcher/virtwrap/manager.go b/pkg/virt-launcher/virtwrap/manager.go
@@ -418,6 +418,7 @@ func liveMigrationMonitor(vmi *v1.VirtualMachineInstance, dom cli.VirDomain, l *
 	// TODO:(vladikr) move to configMap
 	completionTimeoutPerGiB := int64(800)
 	acceptableCompletionTime := completionTimeoutPerGiB * getVMIMigrationDataSize(vmi)
+monitorLoop:
 	for {
 		stats, err := dom.GetJobInfo()
 		if err != nil {
@@ -446,7 +447,7 @@ func liveMigrationMonitor(vmi *v1.VirtualMachineInstance, dom cli.VirDomain, l *
 					logger.Reason(err).Error("failed to abort migration")
 				}
 				l.setMigrationResult(vmi, true, fmt.Sprintf("Live migration stuck for %d sec and has been aborted", progressDelay))
-				break
+				break monitorLoop
 			}
 
 			// check the overall migration time
@@ -459,21 +460,21 @@ func liveMigrationMonitor(vmi *v1.VirtualMachineInstance, dom cli.VirDomain, l *
 					logger.Reason(err).Error("failed to abort migration")
 				}
 				l.setMigrationResult(vmi, true, fmt.Sprintf("Live migration is not completed after %d sec and has been aborted", acceptableCompletionTime))
-				break
+				break monitorLoop
 			}
 
 		case libvirt.DOMAIN_JOB_NONE:
 			logger.Info("Migration job didn't start yet")
 		case libvirt.DOMAIN_JOB_COMPLETED:
 			logger.Info("Migration has beem completed")
-			break
+			break monitorLoop
 		case libvirt.DOMAIN_JOB_FAILED:
 			logger.Info("Migration job failed")
 			// migration failed
-			break
+			break monitorLoop
 		case libvirt.DOMAIN_JOB_CANCELLED:
 			logger.Info("Migration was canceled")
-			break
+			break monitorLoop
 		}
 		time.Sleep(500 * time.Millisecond)
 	}

diff --git a/pkg/virt-launcher/virtwrap/manager_test.go b/pkg/virt-launcher/virtwrap/manager_test.go
@@ -210,6 +210,83 @@ var _ = Describe("Manager", func() {
 			Expect(newspec).ToNot(BeNil())
 		})
 	})
+	Context("test migration monitor", func() {
+		It("migration should be canceled if it's not progressing", func() {
+			// Make sure that we always free the domain after use
+			mockDomain.EXPECT().Free().AnyTimes()
+			fake_jobinfo := &libvirt.DomainJobInfo{
+				Type:          libvirt.DOMAIN_JOB_UNBOUNDED,
+				DataRemaining: 32479827394,
+			}
+
+			vmi := newVMI(testNamespace, testVmName)
+			vmi.Status.MigrationState = &v1.VirtualMachineInstanceMigrationState{
+				MigrationUID: "111222333",
+			}
+			migrationConfig := &v1.MigrationConfig{
+				CompletionTimeoutPerGiB: 800,
+				ProgressTimeout:         2,
+			}
+
+			domainSpec := expectIsolationDetectionForVMI(vmi)
+			xml, err := xml.Marshal(domainSpec)
+			Expect(err).To(BeNil())
+			manager := &LibvirtDomainManager{
+				virConn:                mockConn,
+				virtShareDir:           "fake",
+				notifier:               nil,
+				lessPVCSpaceToleration: 0,
+			}
+			mockDomain.EXPECT().GetState().Return(libvirt.DOMAIN_RUNNING, 1, nil)
+			mockConn.EXPECT().LookupDomainByName(testDomainName).Return(mockDomain, nil)
+			mockDomain.EXPECT().GetJobInfo().AnyTimes().Return(fake_jobinfo, nil)
+			mockDomain.EXPECT().AbortJob()
+			mockDomain.EXPECT().GetXMLDesc(gomock.Eq(libvirt.DOMAIN_XML_MIGRATABLE)).Return(string(xml), nil)
+			mockDomain.EXPECT().GetXMLDesc(gomock.Eq(libvirt.DOMAIN_XML_INACTIVE)).Return(string(xml), nil)
+
+			liveMigrationMonitor(vmi, mockDomain, manager, migrationConfig)
+		})
+		It("migration should be canceled if timeout has been reached", func() {
+			// Make sure that we always free the domain after use
+			var migrationData = 32479827394
+			mockDomain.EXPECT().Free().AnyTimes()
+			fake_jobinfo := func() *libvirt.DomainJobInfo {
+				migrationData -= 125
+				return &libvirt.DomainJobInfo{
+					Type:          libvirt.DOMAIN_JOB_UNBOUNDED,
+					DataRemaining: uint64(migrationData),
+				}
+			}()
+
+			vmi := newVMI(testNamespace, testVmName)
+			vmi.Status.MigrationState = &v1.VirtualMachineInstanceMigrationState{
+				MigrationUID: "111222333",
+			}
+			migrationConfig := &v1.MigrationConfig{
+				CompletionTimeoutPerGiB: 3,
+				ProgressTimeout:         150,
+			}
+
+			domainSpec := expectIsolationDetectionForVMI(vmi)
+			xml, err := xml.Marshal(domainSpec)
+			Expect(err).To(BeNil())
+			manager := &LibvirtDomainManager{
+				virConn:                mockConn,
+				virtShareDir:           "fake",
+				notifier:               nil,
+				lessPVCSpaceToleration: 0,
+			}
+			mockDomain.EXPECT().GetState().Return(libvirt.DOMAIN_RUNNING, 1, nil)
+			mockConn.EXPECT().LookupDomainByName(testDomainName).Return(mockDomain, nil)
+			mockDomain.EXPECT().GetJobInfo().AnyTimes().Return(fake_jobinfo, nil)
+			mockDomain.EXPECT().AbortJob()
+			mockDomain.EXPECT().GetXMLDesc(gomock.Eq(libvirt.DOMAIN_XML_MIGRATABLE)).Return(string(xml), nil)
+			mockDomain.EXPECT().GetXMLDesc(gomock.Eq(libvirt.DOMAIN_XML_INACTIVE)).Return(string(xml), nil)
+
+			liveMigrationMonitor(vmi, mockDomain, manager, migrationConfig)
+		})
+
+	})
 
 	Context("on successful VirtualMachineInstance migrate", func() {
 		It("should prepare the target pod", func() {