Add unittest for readiness probe

Signed-off-by: Lukianov Artyom <[email protected]>

pkg/virt-controller/watch/application.go

@@ -80,24 +80,8 @@ func Execute() {
 
 	app.restClient = app.clientSet.RestClient()
 
-	readinessFunc := func(_ *restful.Request, response *restful.Response) {
-		res := map[string]interface{}{}
-
-		select {
-		case _, opened := <-app.readyChan:
-			if !opened {
-				res["apiserver"] = map[string]interface{}{"leader": "true"}
-				response.WriteHeaderAndJson(http.StatusOK, res, restful.MIME_JSON)
-				return
-			}
-		default:
-		}
-		res["apiserver"] = map[string]interface{}{"leader": "false", "error": "current pod is not leader"}
-		response.WriteHeaderAndJson(http.StatusServiceUnavailable, res, restful.MIME_JSON)
-		return
-	}
 	webService := rest.WebService
-	webService.Route(webService.GET("/leader").To(readinessFunc).Doc("Leader endpoint"))
+	webService.Route(webService.GET("/leader").To(app.readinessProbe).Doc("Leader endpoint"))
 	restful.Add(webService)
 
 	// Bootstrapping. From here on the initialization order is important
@@ -212,6 +196,22 @@ func (vca *VirtControllerApp) initReplicaSet() {
 	vca.rsController = NewVMReplicaSet(vca.vmInformer, vca.rsInformer, recorder, vca.clientSet, controller.BurstReplicas)
 }
 
+func (vca *VirtControllerApp) readinessProbe(_ *restful.Request, response *restful.Response) {
+	res := map[string]interface{}{}
+
+	select {
+	case _, opened := <-vca.readyChan:
+		if !opened {
+			res["apiserver"] = map[string]interface{}{"leader": "true"}
+			response.WriteHeaderAndJson(http.StatusOK, res, restful.MIME_JSON)
+			return
+		}
+	default:
+	}
+	res["apiserver"] = map[string]interface{}{"leader": "false", "error": "current pod is not leader"}
+	response.WriteHeaderAndJson(http.StatusServiceUnavailable, res, restful.MIME_JSON)
+}
+
 func (vca *VirtControllerApp) DefineFlags() {
 	flag.StringVar(&vca.host, "listen", "0.0.0.0", "Address and port where to listen on")
 	flag.IntVar(&vca.port, "port", 8182, "Port to listen on")

pkg/virt-controller/watch/application_test.go

@@ -0,0 +1,77 @@
+/*
+ * This file is part of the KubeVirt project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright 2017 Red Hat, Inc.
+ *
+ */
+
+package watch
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"net/http"
+
+	"net/http/httptest"
+	"net/url"
+
+	"github.com/emicklei/go-restful"
+
+	"kubevirt.io/kubevirt/pkg/rest"
+)
+
+func newValidGetRequest() *http.Request {
+	request, _ := http.NewRequest("GET", "/leader", nil)
+	return request
+}
+
+var _ = Describe("Application", func() {
+	var app VirtControllerApp = VirtControllerApp{}
+
+	Describe("Readiness probe", func() {
+		var recorder *httptest.ResponseRecorder
+		var request *http.Request
+		var handler http.Handler
+
+		BeforeEach(func() {
+			app.readyChan = make(chan bool, 1)
+
+			ws := new(restful.WebService)
+			ws.Produces(restful.MIME_JSON)
+			handler = http.Handler(restful.NewContainer().Add(ws))
+			ws.Route(ws.GET("/leader").Produces(rest.MIME_JSON).To(app.readinessProbe))
+
+			request = newValidGetRequest()
+			recorder = httptest.NewRecorder()
+		})
+
+		Context("with closed channel", func() {
+			It("should return 200", func() {
+				close(app.readyChan)
+				request.URL, _ = url.Parse("/leader")
+				handler.ServeHTTP(recorder, request)
+				Expect(recorder.Code).To(Equal(http.StatusOK))
+			})
+		})
+		Context("with opened channel", func() {
+			It("should return 503", func() {
+				request.URL, _ = url.Parse("/leader")
+				handler.ServeHTTP(recorder, request)
+				Expect(recorder.Code).To(Equal(http.StatusServiceUnavailable))
+			})
+		})
+	})
+})

tests/controller_leader_election_test.go

@@ -22,13 +22,16 @@ package tests_test
 import (
 	"encoding/json"
 	"flag"
+	"fmt"
 	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	k8sv1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
 
 	"kubevirt.io/kubevirt/pkg/kubecli"
@@ -48,14 +51,28 @@ var _ = Describe("LeaderElection", func() {
 
 	Context("Controller pod destroyed", func() {
 		It("should success to start VM", func() {
-			leaderPodName := getLeader()
-			Expect(virtClient.CoreV1().Pods(leaderelectionconfig.DefaultNamespace).Delete(leaderPodName, &metav1.DeleteOptions{})).To(BeNil())
+			newLeaderPod := getNewLeaderPod(virtClient)
+			Expect(newLeaderPod).NotTo(BeNil())
 
-			Eventually(getLeader, 30*time.Second, 5*time.Second).ShouldNot(Equal(leaderPodName))
+			// TODO: It can be race condition when newly deployed pod receive leadership, in this case we will need
+			// to reduce Deployment replica before destroy the pod and restore it after the test
+			Eventually(func() string {
+				leaderPodName := getLeader()
+
+				Expect(virtClient.CoreV1().Pods(leaderelectionconfig.DefaultNamespace).Delete(leaderPodName, &metav1.DeleteOptions{})).To(BeNil())
+
+				Eventually(getLeader, 30*time.Second, 5*time.Second).ShouldNot(Equal(leaderPodName))
 
-			Eventually(func() k8sv1.ConditionStatus {
 				leaderPod, err := virtClient.CoreV1().Pods(leaderelectionconfig.DefaultNamespace).Get(getLeader(), metav1.GetOptions{})
 				Expect(err).To(BeNil())
+
+				return leaderPod.Name
+			}, 90*time.Second, 5*time.Second).Should(Equal(newLeaderPod.Name))
+
+			Eventually(func() k8sv1.ConditionStatus {
+				leaderPod, err := virtClient.CoreV1().Pods(leaderelectionconfig.DefaultNamespace).Get(newLeaderPod.Name, metav1.GetOptions{})
+				Expect(err).To(BeNil())
+
 				for _, condition := range leaderPod.Status.Conditions {
 					if condition.Type == k8sv1.PodReady {
 						return condition.Status
@@ -70,7 +87,7 @@ var _ = Describe("LeaderElection", func() {
 			obj, err := virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do().Get()
 			Expect(err).To(BeNil())
 			tests.WaitForSuccessfulVMStart(obj)
-		}, 70)
+		}, 150)
 	})
 })
 
@@ -88,3 +105,18 @@ func getLeader() string {
 	}
 	return record.HolderIdentity
 }
+
+func getNewLeaderPod(virtClient kubecli.KubevirtClient) *k8sv1.Pod {
+	labelSelector, err := labels.Parse(fmt.Sprint("app=virt-controller"))
+	tests.PanicOnError(err)
+	fieldSelector := fields.ParseSelectorOrDie("status.phase=" + string(k8sv1.PodRunning))
+	controllerPods, err := virtClient.CoreV1().Pods(leaderelectionconfig.DefaultNamespace).List(
+		metav1.ListOptions{LabelSelector: labelSelector.String(), FieldSelector: fieldSelector.String()})
+	leaderPodName := getLeader()
+	for _, pod := range controllerPods.Items {
+		if pod.Name != leaderPodName {
+			return &pod
+		}
+	}
+	return nil
+}