From aabfccb76c4fe1f95f8fb395f742b211caed85f7 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 13 Dec 2017 11:35:25 -0500 Subject: [PATCH 01/37] Unix sock console connector script Signed-off-by: David Vossel --- cmd/virt-launcher/Dockerfile | 4 ++++ cmd/virt-launcher/sock-connector | 5 +++++ 2 files changed, 9 insertions(+) create mode 100644 cmd/virt-launcher/sock-connector diff --git a/cmd/virt-launcher/Dockerfile b/cmd/virt-launcher/Dockerfile index 94ec885c98a8..68ebd8e801c2 100644 --- a/cmd/virt-launcher/Dockerfile +++ b/cmd/virt-launcher/Dockerfile @@ -20,6 +20,10 @@ FROM fedora:26 MAINTAINER "The KubeVirt Project" +RUN dnf -y install socat && \ + dnf -y clean all + +COPY sock-connector /sock-connector COPY virt-launcher /virt-launcher ENTRYPOINT [ "/virt-launcher" ] diff --git a/cmd/virt-launcher/sock-connector b/cmd/virt-launcher/sock-connector new file mode 100644 index 000000000000..86c4381c986e --- /dev/null +++ b/cmd/virt-launcher/sock-connector @@ -0,0 +1,5 @@ +#!/bin/bash +stty -echo +#stty -isig +SOCKET=$1 +socat unix-connect:/$SOCKET stdio From b3e04b3fc0e9f627cce9e6147cf0732062cda4ed Mon Sep 17 00:00:00 2001 From: David Vossel Date: Tue, 2 Jan 2018 11:17:13 -0500 Subject: [PATCH 02/37] add unix serial options to domain api Signed-off-by: David Vossel --- pkg/virt-handler/virtwrap/api/schema.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/virt-handler/virtwrap/api/schema.go b/pkg/virt-handler/virtwrap/api/schema.go index 9c96bba2465e..b2813b5477af 100644 --- a/pkg/virt-handler/virtwrap/api/schema.go +++ b/pkg/virt-handler/virtwrap/api/schema.go @@ -254,12 +254,18 @@ type DiskSourceHost struct { type Serial struct { Type string `xml:"type,attr"` Target *SerialTarget `xml:"target,omitempty"` + Source *SerialSource `xml:"source,omitempty"` } type SerialTarget struct { Port *uint `xml:"port,attr,omitempty"` } +type SerialSource struct { + Mode string `xml:"mode,attr,omitempty"` + Path string `xml:"path,attr,omitempty"` +} + // END Serial ----------------------------- // BEGIN Console ----------------------------- From 3eaa0d770c4fab4b99890f7c771e09e77ea8c878 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Tue, 2 Jan 2018 11:22:38 -0500 Subject: [PATCH 03/37] add priviate vm shared dir Signed-off-by: David Vossel --- manifests/dev/libvirt.yaml.in | 7 +++++++ manifests/dev/virt-handler.yaml.in | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/manifests/dev/libvirt.yaml.in b/manifests/dev/libvirt.yaml.in index dfafd5eca0b4..f48cb4c5577a 100644 --- a/manifests/dev/libvirt.yaml.in +++ b/manifests/dev/libvirt.yaml.in @@ -39,6 +39,8 @@ spec: mountPath: /var/run/libvirt - name: virt-share-dir mountPath: /var/run/kubevirt + - name: virt-private-dir + mountPath: /var/run/kubevirt-private command: ["/libvirtd.sh"] - name: virtlogd image: {{ docker_prefix }}/libvirt-kubevirt:{{ docker_tag }} @@ -58,6 +60,8 @@ spec: mountPath: /var/run/libvirt - name: virt-share-dir mountPath: /var/run/kubevirt + - name: virt-private-dir + mountPath: /var/run/kubevirt-private command: ["/virt-dhcp"] volumes: - name: libvirt-data @@ -75,3 +79,6 @@ spec: - name: virt-share-dir hostPath: path: /var/run/kubevirt + - name: virt-private-dir + hostPath: + path: /var/run/kubevirt-private diff --git a/manifests/dev/virt-handler.yaml.in b/manifests/dev/virt-handler.yaml.in index b8f77e644618..4d0b863ff85c 100644 --- a/manifests/dev/virt-handler.yaml.in +++ b/manifests/dev/virt-handler.yaml.in @@ -36,6 +36,8 @@ spec: mountPath: /var/run/libvirt - name: virt-share-dir mountPath: /var/run/kubevirt + - name: virt-private-dir + mountPath: /var/run/kubevirt-private env: - name: NODE_NAME valueFrom: @@ -48,3 +50,6 @@ spec: - name: virt-share-dir hostPath: path: /var/run/kubevirt + - name: virt-private-dir + hostPath: + path: /var/run/kubevirt-private From e65c4116747a2ca84924130d87ff86ef3f1ac30d Mon Sep 17 00:00:00 2001 From: David Vossel Date: Tue, 2 Jan 2018 11:27:50 -0500 Subject: [PATCH 04/37] cleanup console unix sock after VM exits Signed-off-by: David Vossel --- pkg/virt-handler/vm.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/virt-handler/vm.go b/pkg/virt-handler/vm.go index 942ccc9a363a..2256bbf5221c 100644 --- a/pkg/virt-handler/vm.go +++ b/pkg/virt-handler/vm.go @@ -588,12 +588,25 @@ func (d *VirtualMachineController) injectDiskAuth(vm *v1.VirtualMachine) (map[st return secrets, nil } +func (d *VirtualMachineController) cleanupConsoleSockets(vm *v1.VirtualMachine) error { + // TODO this can go away once qemu is in the pods mount namespace. + namespace := vm.ObjectMeta.Namespace + name := vm.ObjectMeta.Name + unixPath := fmt.Sprintf("%s-private/%s/%s", d.virtShareDir, namespace, name) + return diskutils.RemoveFile(unixPath) +} + func (d *VirtualMachineController) processVmCleanup(vm *v1.VirtualMachine) error { err := d.domainManager.RemoveVMSecrets(vm) if err != nil { return err } + err = d.cleanupConsoleSockets(vm) + if err != nil { + return err + } + err = registrydisk.CleanupEphemeralDisks(vm) if err != nil { return err From a44ce47cc910fb35c67293f95da0a58fe8a06834 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 13 Dec 2017 11:40:51 -0500 Subject: [PATCH 05/37] add kubecli SerialConsole api Signed-off-by: David Vossel --- pkg/kubecli/generated_mock_kubevirt.go | 12 ++++ pkg/kubecli/kubecli.go | 2 +- pkg/kubecli/kubevirt.go | 5 ++ pkg/kubecli/vm.go | 98 +++++++++++++++++++++++++- 4 files changed, 115 insertions(+), 2 deletions(-) diff --git a/pkg/kubecli/generated_mock_kubevirt.go b/pkg/kubecli/generated_mock_kubevirt.go index 127540c8fee9..6e63877203df 100644 --- a/pkg/kubecli/generated_mock_kubevirt.go +++ b/pkg/kubecli/generated_mock_kubevirt.go @@ -4,6 +4,8 @@ package kubecli import ( + io "io" + gomock "github.com/golang/mock/gomock" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -589,6 +591,16 @@ func (_mr *_MockVMInterfaceRecorder) Patch(arg0, arg1, arg2 interface{}, arg3 .. return _mr.mock.ctrl.RecordCall(_mr.mock, "Patch", _s...) } +func (_m *MockVMInterface) SerialConsole(name string, device string, in io.Reader, out io.Writer) error { + ret := _m.ctrl.Call(_m, "SerialConsole", name, device, in, out) + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockVMInterfaceRecorder) SerialConsole(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "SerialConsole", arg0, arg1, arg2, arg3) +} + // Mock of ReplicaSetInterface interface type MockReplicaSetInterface struct { ctrl *gomock.Controller diff --git a/pkg/kubecli/kubecli.go b/pkg/kubecli/kubecli.go index e96a247626b5..bb887c5dbc94 100644 --- a/pkg/kubecli/kubecli.go +++ b/pkg/kubecli/kubecli.go @@ -63,7 +63,7 @@ func GetKubevirtClientFromFlags(master string, kubeconfig string) (KubevirtClien return nil, err } - return &kubevirt{restClient, coreClient}, nil + return &kubevirt{master, kubeconfig, restClient, coreClient}, nil } func GetKubevirtClient() (KubevirtClient, error) { diff --git a/pkg/kubecli/kubevirt.go b/pkg/kubecli/kubevirt.go index d8c868b3655b..cfdd639fac11 100644 --- a/pkg/kubecli/kubevirt.go +++ b/pkg/kubecli/kubevirt.go @@ -26,6 +26,8 @@ package kubecli */ import ( + "io" + k8smetav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -44,6 +46,8 @@ type KubevirtClient interface { } type kubevirt struct { + master string + kubeconfig string restClient *rest.RESTClient *kubernetes.Clientset } @@ -59,6 +63,7 @@ type VMInterface interface { Update(*v1.VirtualMachine) (*v1.VirtualMachine, error) Delete(name string, options *k8smetav1.DeleteOptions) error Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.VirtualMachine, err error) + SerialConsole(name string, device string, in io.Reader, out io.Writer) error } type ReplicaSetInterface interface { diff --git a/pkg/kubecli/vm.go b/pkg/kubecli/vm.go index ad4eca287249..25b8d99f7850 100644 --- a/pkg/kubecli/vm.go +++ b/pkg/kubecli/vm.go @@ -20,9 +20,21 @@ package kubecli import ( + goerror "errors" + "fmt" + "io" + + k8sv1 "k8s.io/api/core/v1" k8smetav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/remotecommand" "k8s.io/apimachinery/pkg/types" @@ -30,13 +42,97 @@ import ( ) func (k *kubevirt) VM(namespace string) VMInterface { - return &vms{k.restClient, namespace, "virtualmachines"} + return &vms{ + restClient: k.restClient, + clientSet: k.Clientset, + namespace: namespace, + resource: "virtualmachines", + master: k.master, + kubeconfig: k.kubeconfig, + } } type vms struct { restClient *rest.RESTClient + clientSet *kubernetes.Clientset namespace string resource string + master string + kubeconfig string +} + +func findPod(clientSet *kubernetes.Clientset, namespace string, name string) (string, error) { + fieldSelector := fields.ParseSelectorOrDie("status.phase==" + string(k8sv1.PodRunning)) + labelSelector, err := labels.Parse(fmt.Sprintf(v1.AppLabel+"=virt-launcher,"+v1.DomainLabel+" in (%s)", name)) + if err != nil { + return "", err + } + selector := k8smetav1.ListOptions{FieldSelector: fieldSelector.String(), LabelSelector: labelSelector.String()} + + podList, err := clientSet.CoreV1().Pods(namespace).List(selector) + if err != nil { + return "", err + } + + if len(podList.Items) == 0 { + return "", goerror.New("console connection failed. No VM pod is running") + } + return podList.Items[0].ObjectMeta.Name, nil +} + +func (v *vms) SerialConsole(name string, device string, in io.Reader, out io.Writer) error { + podName, err := findPod(v.clientSet, v.namespace, name) + if err != nil { + return err + } + + config, err := clientcmd.BuildConfigFromFlags(v.master, v.kubeconfig) + if err != nil { + return err + } + + gv := k8sv1.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/api" + config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs} + + restClient, err := restclient.RESTClientFor(config) + if err != nil { + return err + } + cmd := []string{"/sock-connector", fmt.Sprintf("/var/run/kubevirt-private/%s/%s/virt-%s", v.namespace, name, device)} + containerName := "compute" + req := restClient.Post(). + Resource("pods"). + Name(podName). + Namespace(v.namespace). + SubResource("exec"). + Param("container", containerName) + + req = req.VersionedParams(&k8sv1.PodExecOptions{ + Container: containerName, + Command: cmd, + Stdin: true, + Stdout: true, + Stderr: true, + TTY: true, + }, scheme.ParameterCodec) + + // execute request + method := "POST" + url := req.URL() + exec, err := remotecommand.NewSPDYExecutor(config, method, url) + if err != nil { + return err + } + + return exec.Stream(remotecommand.StreamOptions{ + Stdin: in, + Stdout: out, + Stderr: out, + Tty: false, + TerminalSizeQueue: nil, + }) } func (v *vms) Get(name string, options k8smetav1.GetOptions) (vm *v1.VirtualMachine, err error) { From 3ff57f3fcefe2dcb2f71e1e30fef9d552a1054a1 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 13 Dec 2017 11:41:45 -0500 Subject: [PATCH 06/37] Integrate new console support with virtctl Signed-off-by: David Vossel --- pkg/virtctl/console/console.go | 258 +++++++++------------------------ 1 file changed, 66 insertions(+), 192 deletions(-) diff --git a/pkg/virtctl/console/console.go b/pkg/virtctl/console/console.go index eb4505b8d5fb..a0abcff2b155 100644 --- a/pkg/virtctl/console/console.go +++ b/pkg/virtctl/console/console.go @@ -20,24 +20,17 @@ package console import ( - "bytes" "fmt" "io" "log" - "net/http" - "net/url" "os" "os/signal" - "time" - "sync" - - "github.com/gorilla/websocket" flag "github.com/spf13/pflag" "golang.org/x/crypto/ssh/terminal" "k8s.io/api/core/v1" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" + + "kubevirt.io/kubevirt/pkg/kubecli" ) type Console struct { @@ -75,132 +68,37 @@ func (c *Console) Run(flags *flag.FlagSet) int { } vm := flags.Arg(1) - config, err := clientcmd.BuildConfigFromFlags(server, kubeconfig) + virtCli, err := kubecli.GetKubevirtClientFromFlags(server, kubeconfig) if err != nil { log.Println(err) return 1 } - err = ConnectToConsole(config, namespace, vm, device, TerminalWebsocketCallback) + state, err := terminal.MakeRaw(int(os.Stdin.Fd())) if err != nil { - log.Println(err) + log.Printf("Make raw terminal failed: %s", err) return 1 } - return 0 -} - -func ConnectToConsole(config *rest.Config, namespace string, name string, console string, callback RoundTripCallback) error { - - // Create a round tripper with all necessary kubernetes security details - wrappedRoundTripper, err := RoundTripperFromConfig(config, callback) - if err != nil { - return err - } - - // Create the basic console request - req, err := RequestFromConfig(config, name, namespace, console) - if err != nil { - return err - } - - // Do the call and process the websocket connection with the callback - _, err = wrappedRoundTripper.RoundTrip(req) - - if err != nil { - return err - } - return nil -} - -func NewWebsocketCallback(in io.ReadCloser, out io.WriteCloser, stopChan chan struct{}) RoundTripCallback { - - return func(ws *websocket.Conn, resp *http.Response, err error) error { - - if err != nil { - if resp != nil && resp.StatusCode != http.StatusOK { - buf := new(bytes.Buffer) - buf.ReadFrom(resp.Body) - return fmt.Errorf("Can't connect to console (%d): %s\n", resp.StatusCode, buf.String()) - } - return fmt.Errorf("Can't connect to console: %s\n", err.Error()) - } - - writeStop := make(chan struct{}) - readStop := make(chan struct{}) - - go func() { - defer close(readStop) - for { - _, message, err := ws.ReadMessage() - if err != nil { - out.Write(message) - return - } - _, err = out.Write(message) - if err == io.EOF { - return - } - } - }() - - buf := make([]byte, 1024, 1024) - - // Synchronize writes for final close announcements - var writeMux sync.Mutex - writeProtected := func(messageType int, data []byte) error { - writeMux.Lock() - defer writeMux.Unlock() - return ws.WriteMessage(websocket.TextMessage, data) - } - - go func() { - defer close(writeStop) - for { - n, err := in.Read(buf) - if err != nil && err != io.EOF { - log.Println(err) - return - } + fmt.Fprint(os.Stderr, "Escape sequence is ^]") - // TODO move this to the TerminalWebsocketCallback - if buf[0] == 29 { - return - } + in := os.Stdin + out := os.Stdout - // If there is nothing more to write and we have reached EOF, return - if n == 0 && err == io.EOF { - return - } + stdinReader, stdinWriter := io.Pipe() + stdoutReader, stdoutWriter := io.Pipe() - err = writeProtected(websocket.TextMessage, buf[0:n]) - if err != nil && err != io.EOF { - log.Println(err) - return - } - } - }() - - select { - case <-stopChan: - case <-readStop: - case <-writeStop: - } - - err = writeProtected(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) - if err != nil { - return fmt.Errorf("Error on close announcement: %s", err.Error()) - } - select { - case <-readStop: - case <-time.After(time.Second): - } - return nil - } -} - -func TerminalWebsocketCallback(ws *websocket.Conn, resp *http.Response, connErr error) error { + // in -> stdinWriter | stdoutReader -> console + // out <- stdoutReader | stdoutWriter <- console + resChan := make(chan error) stopChan := make(chan struct{}, 1) + writeStop := make(chan struct{}) + readStop := make(chan struct{}) + + go func() { + err := virtCli.VM(namespace).SerialConsole(vm, device, stdinReader, stdoutWriter) + resChan <- err + }() go func() { interrupt := make(chan os.Signal, 1) @@ -209,82 +107,58 @@ func TerminalWebsocketCallback(ws *websocket.Conn, resp *http.Response, connErr close(stopChan) }() - // If there is no obvious connection error, set up the terminal - if connErr == nil { - state, err := terminal.MakeRaw(int(os.Stdin.Fd())) - if err != nil { - return fmt.Errorf("Make raw terminal failed: %s", err) + go func() { + defer close(readStop) + buf := make([]byte, 1024, 1024) + for { + // reading stdout from console + n, err := stdoutReader.Read(buf) + if err != nil && err != io.EOF { + return + } + if n == 0 && err == io.EOF { + return + } + // Writing to stdout + _, err = out.Write(buf[0:n]) + if err == io.EOF { + return + } } - defer terminal.Restore(int(os.Stdin.Fd()), state) - fmt.Fprint(os.Stderr, "Escape sequence is ^]") - } - - return NewWebsocketCallback(os.Stdin, os.Stdout, stopChan)(ws, resp, connErr) -} - -func RequestFromConfig(config *rest.Config, vm string, namespace string, device string) (*http.Request, error) { - - u, err := url.Parse(config.Host) - if err != nil { - return nil, err - } + }() - switch u.Scheme { - case "https": - u.Scheme = "wss" - case "http": - u.Scheme = "ws" - default: - return nil, fmt.Errorf("Unsupported Protocol %s", u.Scheme) - } + go func() { + defer close(writeStop) + buf := make([]byte, 1024, 1024) + for { + // reading from stdin + n, err := in.Read(buf) + if err != nil && err != io.EOF { + return + } + if n == 0 && err == io.EOF { + return + } + // Writing out to the console connection + _, err = stdinWriter.Write(buf[0:n]) + if err == io.EOF { + return + } + } + }() - u.Path = fmt.Sprintf("/apis/kubevirt.io/v1alpha1/namespaces/%s/virtualmachines/%s/console", namespace, vm) - if device != "" { - u.RawQuery = "console=" + device + select { + case <-stopChan: + case <-readStop: + case <-writeStop: + case err = <-resChan: } - req := &http.Request{ - Method: http.MethodGet, - URL: u, - } - - return req, nil -} -func RoundTripperFromConfig(config *rest.Config, callback RoundTripCallback) (http.RoundTripper, error) { + terminal.Restore(int(os.Stdin.Fd()), state) - // Configure TLS - tlsConfig, err := rest.TLSConfigFor(config) if err != nil { - return nil, err - } - - // Configure the websocket dialer - dialer := &websocket.Dialer{ - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: tlsConfig, - } - - // Create a roundtripper which will pass in the final underlying websocket connection to a callback - rt := &WebsocketRoundTripper{ - Do: callback, - Dialer: dialer, - } - - // Make sure we inherit all relevant security headers - return rest.HTTPWrappersForConfig(config, rt) -} - -type RoundTripCallback func(conn *websocket.Conn, resp *http.Response, err error) error - -type WebsocketRoundTripper struct { - Dialer *websocket.Dialer - Do RoundTripCallback -} - -func (d *WebsocketRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { - conn, resp, err := d.Dialer.Dial(r.URL.String(), r.Header) - if err == nil { - defer conn.Close() + log.Println(err) + return 1 } - return resp, d.Do(conn, resp, err) + return 0 } From 6036b1daf1d5463963c6440f2fb86c3c82ce13e0 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 13 Dec 2017 11:42:43 -0500 Subject: [PATCH 07/37] Add host mount for serial console unix file to virt-launcher pod Signed-off-by: David Vossel --- pkg/virt-controller/services/template.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/virt-controller/services/template.go b/pkg/virt-controller/services/template.go index 902a98e35342..d0ecd5f82d58 100644 --- a/pkg/virt-controller/services/template.go +++ b/pkg/virt-controller/services/template.go @@ -69,6 +69,10 @@ func (t *templateService) RenderLaunchManifest(vm *v1.VirtualMachine) (*kubev1.P gracePeriodSeconds = gracePeriodSeconds + int64(15) gracePeriodKillAfter := gracePeriodSeconds + int64(15) + // TODO remove mounting the privateDir once + // the qemu pid is in the pods mount namespace + privateDir := fmt.Sprintf("%s-private/%s/%s", t.virtShareDir, namespace, domain) + // VM target container container := kubev1.Container{ Name: "compute", @@ -87,6 +91,10 @@ func (t *templateService) RenderLaunchManifest(vm *v1.VirtualMachine) (*kubev1.P Name: "virt-share-dir", MountPath: t.virtShareDir, }, + { + Name: "virt-private-dir", + MountPath: privateDir, + }, }, ReadinessProbe: &kubev1.Probe{ Handler: kubev1.Handler{ @@ -118,6 +126,14 @@ func (t *templateService) RenderLaunchManifest(vm *v1.VirtualMachine) (*kubev1.P }, }, }) + volumes = append(volumes, kubev1.Volume{ + Name: "virt-private-dir", + VolumeSource: kubev1.VolumeSource{ + HostPath: &kubev1.HostPathVolumeSource{ + Path: privateDir, + }, + }, + }) containers = append(containers, container) // TODO use constants for labels From 612a26a981d1d549b28cc0849c33fcefd7cc32c8 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Tue, 2 Jan 2018 11:56:08 -0500 Subject: [PATCH 08/37] update functional tests to use new console api Signed-off-by: David Vossel --- tests/console_test.go | 4 +--- tests/utils.go | 6 ++---- tests/vm_monitoring_test.go | 4 +--- tests/vm_userdata_test.go | 4 +--- 4 files changed, 5 insertions(+), 13 deletions(-) diff --git a/tests/console_test.go b/tests/console_test.go index c38877b7bd50..4c5031075515 100644 --- a/tests/console_test.go +++ b/tests/console_test.go @@ -37,8 +37,6 @@ var _ = Describe("Console", func() { virtClient, err := kubecli.GetKubevirtClient() tests.PanicOnError(err) - virtConfig, err := kubecli.GetKubevirtClientConfig() - tests.PanicOnError(err) BeforeEach(func() { tests.BeforeTestCleanup() @@ -51,7 +49,7 @@ var _ = Describe("Console", func() { Expect(virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do().Error()).To(Succeed()) tests.WaitForSuccessfulVMStart(vm) - expecter, _, err := tests.NewConsoleExpecter(virtConfig, vm, "serial0", 10*time.Second) + expecter, _, err := tests.NewConsoleExpecter(virtClient, vm, "serial0", 10*time.Second) defer expecter.Close() Expect(err).ToNot(HaveOccurred()) diff --git a/tests/utils.go b/tests/utils.go index 20b8efede867..d568f55d71e5 100644 --- a/tests/utils.go +++ b/tests/utils.go @@ -41,12 +41,10 @@ import ( "io" "github.com/google/goexpect" - "k8s.io/client-go/rest" "kubevirt.io/kubevirt/pkg/api/v1" "kubevirt.io/kubevirt/pkg/kubecli" "kubevirt.io/kubevirt/pkg/log" - "kubevirt.io/kubevirt/pkg/virtctl/console" ) type EventType string @@ -742,13 +740,13 @@ func NewRandomReplicaSetFromVM(vm *v1.VirtualMachine, replicas int32) *v1.Virtua return rs } -func NewConsoleExpecter(config *rest.Config, vm *v1.VirtualMachine, consoleName string, timeout time.Duration, opts ...expect.Option) (expect.Expecter, <-chan error, error) { +func NewConsoleExpecter(virtCli kubecli.KubevirtClient, vm *v1.VirtualMachine, consoleName string, timeout time.Duration, opts ...expect.Option) (expect.Expecter, <-chan error, error) { vmReader, vmWriter := io.Pipe() expecterReader, expecterWriter := io.Pipe() resCh := make(chan error) stopChan := make(chan struct{}) go func() { - err := console.ConnectToConsole(config, vm.ObjectMeta.Namespace, vm.ObjectMeta.Name, consoleName, console.NewWebsocketCallback(vmReader, expecterWriter, stopChan)) + err := virtCli.VM(vm.ObjectMeta.Namespace).SerialConsole(vm.ObjectMeta.Name, consoleName, vmReader, expecterWriter) resCh <- err }() diff --git a/tests/vm_monitoring_test.go b/tests/vm_monitoring_test.go index cb096ab2b587..7f1f90410c12 100644 --- a/tests/vm_monitoring_test.go +++ b/tests/vm_monitoring_test.go @@ -40,8 +40,6 @@ var _ = Describe("Health Monitoring", func() { virtClient, err := kubecli.GetKubevirtClient() tests.PanicOnError(err) - virtConfig, err := kubecli.GetKubevirtClientConfig() - tests.PanicOnError(err) launchVM := func(vm *v1.VirtualMachine) { obj, err := virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do().Get() @@ -60,7 +58,7 @@ var _ = Describe("Health Monitoring", func() { Expect(err).ToNot(HaveOccurred()) launchVM(vm) - expecter, _, err := tests.NewConsoleExpecter(virtConfig, vm, "serial0", 10*time.Second) + expecter, _, err := tests.NewConsoleExpecter(virtClient, vm, "serial0", 10*time.Second) Expect(err).ToNot(HaveOccurred()) defer expecter.Close() diff --git a/tests/vm_userdata_test.go b/tests/vm_userdata_test.go index c2b461632344..96cf88a5cb90 100644 --- a/tests/vm_userdata_test.go +++ b/tests/vm_userdata_test.go @@ -43,8 +43,6 @@ var _ = Describe("CloudInit UserData", func() { virtClient, err := kubecli.GetKubevirtClient() tests.PanicOnError(err) - virtConfig, err := kubecli.GetKubevirtClientConfig() - tests.PanicOnError(err) LaunchVM := func(vm *v1.VirtualMachine) runtime.Object { obj, err := virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do().Get() @@ -57,7 +55,7 @@ var _ = Describe("CloudInit UserData", func() { Expect(ok).To(BeTrue(), "Object is not of type *v1.VM") tests.WaitForSuccessfulVMStart(obj) - expecter, _, err := tests.NewConsoleExpecter(virtConfig, vm, "serial0", 10*time.Second) + expecter, _, err := tests.NewConsoleExpecter(virtClient, vm, "serial0", 10*time.Second) defer expecter.Close() Expect(err).ToNot(HaveOccurred()) From a69dcb40a4d307121629407943dc0b9147734542 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 13 Dec 2017 11:44:24 -0500 Subject: [PATCH 09/37] Remove old virt-api console endpoint support Signed-off-by: David Vossel --- cmd/virt-handler/virt-handler.go | 3 - pkg/virt-api/api.go | 8 - pkg/virt-api/rest/console.go | 137 ---------------- pkg/virt-api/rest/console_test.go | 192 ---------------------- pkg/virt-handler/rest/console.go | 164 ------------------- pkg/virt-handler/rest/console_test.go | 223 -------------------------- 6 files changed, 727 deletions(-) delete mode 100644 pkg/virt-api/rest/console.go delete mode 100644 pkg/virt-api/rest/console_test.go delete mode 100644 pkg/virt-handler/rest/console.go delete mode 100644 pkg/virt-handler/rest/console_test.go diff --git a/cmd/virt-handler/virt-handler.go b/cmd/virt-handler/virt-handler.go index a0e9d3b8fecc..7a105950f83c 100644 --- a/cmd/virt-handler/virt-handler.go +++ b/cmd/virt-handler/virt-handler.go @@ -202,11 +202,8 @@ func (app *virtHandlerApp) Run() { // TODO add a http handler which provides health check - // Add websocket route to access consoles remotely - console := rest.NewConsoleResource(domainConn) migrationHostInfo := rest.NewMigrationHostInfo(isolation.NewSocketBasedIsolationDetector(app.VirtShareDir)) ws := new(restful.WebService) - ws.Route(ws.GET("/api/v1/namespaces/{namespace}/virtualmachines/{name}/console").To(console.Console)) ws.Route(ws.GET("/api/v1/namespaces/{namespace}/virtualmachines/{name}/migrationHostInfo").To(migrationHostInfo.MigrationHostInfo)) restful.DefaultContainer.Add(ws) server := &http.Server{Addr: app.Address(), Handler: restful.DefaultContainer} diff --git a/pkg/virt-api/api.go b/pkg/virt-api/api.go index 49fb47f84070..e806c667791c 100644 --- a/pkg/virt-api/api.go +++ b/pkg/virt-api/api.go @@ -87,14 +87,6 @@ func (app *VirtAPIApp) Compose() { // TODO: That os.File doesn't work as I expect. // I need end up with response_type="file", but I am getting response_type="os.File" - ws.Route(ws.GET(rest.ResourcePath(vmGVR) + rest.SubResourcePath("console")). - To(rest.NewConsoleResource(virtCli, virtCli.CoreV1()).Console). - Param(restful.QueryParameter("console", "Name of the serial console to connect to")). - Param(rest.NamespaceParam(ws)).Param(rest.NameParam(ws)). - Operation("console"). - Doc("Open a websocket connection to a serial console on the specified VM.")) - // TODO: Add 'Returns', but I don't know what return type to put there. - restful.Add(ws) ws.Route(ws.GET("/healthz"). diff --git a/pkg/virt-api/rest/console.go b/pkg/virt-api/rest/console.go deleted file mode 100644 index a3b2bb2432a0..000000000000 --- a/pkg/virt-api/rest/console.go +++ /dev/null @@ -1,137 +0,0 @@ -/* - * This file is part of the KubeVirt project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright 2017 Red Hat, Inc. - * - */ - -package rest - -import ( - "bytes" - "fmt" - "io" - "net/http" - - "github.com/emicklei/go-restful" - "github.com/gorilla/websocket" - k8sv1meta "k8s.io/apimachinery/pkg/apis/meta/v1" - k8scorev1 "k8s.io/client-go/kubernetes/typed/core/v1" - - "k8s.io/apimachinery/pkg/api/errors" - - "kubevirt.io/kubevirt/pkg/kubecli" - "kubevirt.io/kubevirt/pkg/log" -) - -var upgrader = websocket.Upgrader{ - ReadBufferSize: 1024, - WriteBufferSize: 1024, -} - -type Console struct { - virtClient kubecli.KubevirtClient - k8sClient k8scorev1.CoreV1Interface - VirtHandlerPort string -} - -func NewConsoleResource(virtClient kubecli.KubevirtClient, k8sClient k8scorev1.CoreV1Interface) *Console { - return &Console{virtClient: virtClient, k8sClient: k8sClient} -} - -func (t *Console) Console(request *restful.Request, response *restful.Response) { - console := request.QueryParameter("console") - vmName := request.PathParameter("name") - namespace := request.PathParameter("namespace") - - vm, err := t.virtClient.VM(namespace).Get(vmName, k8sv1meta.GetOptions{}) - if errors.IsNotFound(err) { - log.Log.V(3).Infof("VM '%s' does not exist", vmName) - response.WriteError(http.StatusNotFound, fmt.Errorf("VM does not exist")) - return - } - if err != nil { - log.Log.Reason(err).Errorf("Error fetching VM '%s'", vmName) - response.WriteError(http.StatusInternalServerError, err) - return - } - - logger := log.Log.Object(vm) - - if !vm.IsRunning() { - logger.V(3).Reason(err).Info("VM is not running") - response.WriteError(http.StatusBadRequest, fmt.Errorf("VM is not running")) - return - } - - virtHandlerCon := kubecli.NewVirtHandlerClient(t.virtClient).ForNode(vm.Status.NodeName) - uri, err := virtHandlerCon.ConsoleURI(vm) - if err != nil { - msg := fmt.Sprintf("Looking up the connection details for virt-handler on node %s failed", vm.Status.NodeName) - logger.Reason(err).Error(msg) - response.WriteError(http.StatusInternalServerError, fmt.Errorf(msg)) - return - } - - if t.VirtHandlerPort != "" { - uri.Hostname() - uri.Host = uri.Hostname() + ":" + t.VirtHandlerPort - } - uri.Scheme = "ws" - if console != "" { - uri.RawQuery = "console=" + console - } - handlerSocket, resp, err := websocket.DefaultDialer.Dial(uri.String(), nil) - if err != nil { - if resp != nil && resp.StatusCode != http.StatusOK { - buf := new(bytes.Buffer) - buf.ReadFrom(resp.Body) - err := fmt.Errorf("%s", buf.String()) - logger.With("statusCode", resp.StatusCode).Reason(err).Error("Failed to connect to virt-handler") - response.WriteError(resp.StatusCode, err) - } else { - logger.Reason(err).Error("Failed to connect to virt-handler") - response.WriteError(http.StatusInternalServerError, err) - } - return - } - defer handlerSocket.Close() - - clientSocket, err := upgrader.Upgrade(response.ResponseWriter, request.Request, nil) - if err != nil { - logger.Reason(err).Error("Failed to upgrade client websocket connection") - response.WriteError(http.StatusBadRequest, err) - return - } - defer clientSocket.Close() - - errorChan := make(chan error) - - go func() { - _, err := io.Copy(clientSocket.UnderlyingConn(), handlerSocket.UnderlyingConn()) - errorChan <- err - }() - - go func() { - _, err := io.Copy(handlerSocket.UnderlyingConn(), clientSocket.UnderlyingConn()) - errorChan <- err - }() - - err = <-errorChan - if err != nil { - logger.Reason(err).Error("Proxied Web Socket connection failed") - } - response.WriteHeader(http.StatusOK) -} diff --git a/pkg/virt-api/rest/console_test.go b/pkg/virt-api/rest/console_test.go deleted file mode 100644 index 2ecb814e1161..000000000000 --- a/pkg/virt-api/rest/console_test.go +++ /dev/null @@ -1,192 +0,0 @@ -/* - * This file is part of the KubeVirt project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright 2017 Red Hat, Inc. - * - */ - -package rest - -import ( - "fmt" - "io/ioutil" - "net/http" - "net/http/httptest" - "net/url" - "strings" - - "github.com/emicklei/go-restful" - "github.com/golang/mock/gomock" - "github.com/gorilla/websocket" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - k8sv1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - fake2 "k8s.io/client-go/kubernetes/fake" - k8scorev1 "k8s.io/client-go/kubernetes/typed/core/v1" - - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime/schema" - - "kubevirt.io/kubevirt/pkg/api/v1" - "kubevirt.io/kubevirt/pkg/kubecli" - "kubevirt.io/kubevirt/pkg/log" -) - -var _ = Describe("Console", func() { - - var ctrl *gomock.Controller - var virtClient *kubecli.MockKubevirtClient - var vmInterface *kubecli.MockVMInterface - var k8sClient k8scorev1.CoreV1Interface - var vm *v1.VirtualMachine - var node *k8sv1.Node - var virtHandlerPod *k8sv1.Pod - var server *httptest.Server - var dial func(vm string, console string) *websocket.Conn - var get func(vm string) (*http.Response, error) - - log.Log.SetIOWriter(GinkgoWriter) - - BeforeEach(func() { - ctrl = gomock.NewController(GinkgoT()) - virtClient = kubecli.NewMockKubevirtClient(ctrl) - vmInterface = kubecli.NewMockVMInterface(ctrl) - virtClient.EXPECT().VM(k8sv1.NamespaceDefault).Return(vmInterface) - - vm = v1.NewMinimalVM("testvm") - vm.Status.Phase = v1.Running - vm.Status.NodeName = "testnode" - - node = &k8sv1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testnode", - }, - } - virtHandlerPod = &k8sv1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "virt-handerler-xkfoiw", - Namespace: k8sv1.NamespaceDefault, - Labels: map[string]string{ - v1.AppLabel: "virt-handler", - }, - }, - Spec: k8sv1.PodSpec{ - NodeName: node.ObjectMeta.Name, - }, - } - k8sClient = fake2.NewSimpleClientset(node, virtHandlerPod).CoreV1() - - ws := new(restful.WebService) - handler := http.Handler(restful.NewContainer().Add(ws)) - - // Endpoint to test - consoleResource := NewConsoleResource(virtClient, k8sClient) - ws.Route(ws.GET("/virt-api/namespaces/{namespace}/virtualmachines/{name}/console").To(consoleResource.Console)) - - // Mock out virt-handler. Mirror the first message and exit. - ws.Route(ws.GET("/api/v1/namespaces/{namespace}/virtualmachines/{name}/console").To(func(request *restful.Request, response *restful.Response) { - defer GinkgoRecover() - ws, err := upgrader.Upgrade(response.ResponseWriter, request.Request, nil) - Expect(err).ToNot(HaveOccurred()) - defer ws.Close() - t, data, err := ws.ReadMessage() - Expect(err).ToNot(HaveOccurred()) - err = ws.WriteMessage(t, data) - Expect(err).ToNot(HaveOccurred()) - response.WriteHeader(http.StatusOK) - })) - - server = httptest.NewServer(handler) - - wsUrl, err := url.Parse(server.URL) - serverUrl, err := url.ParseRequestURI(server.URL) - Expect(err).ToNot(HaveOccurred()) - consoleResource.VirtHandlerPort = strings.Split(serverUrl.Host, ":")[1] - - dial = func(vm string, console string) *websocket.Conn { - wsUrl.Scheme = "ws" - wsUrl.Path = "/virt-api/namespaces/" + k8sv1.NamespaceDefault + "/virtualmachines/" + vm + "/console" - wsUrl.RawQuery = "console=" + console - c, _, err := websocket.DefaultDialer.Dial(wsUrl.String(), nil) - Expect(err).ToNot(HaveOccurred()) - return c - } - - get = func(vm string) (*http.Response, error) { - wsUrl.Scheme = "http" - wsUrl.Path = "/virt-api/namespaces/" + k8sv1.NamespaceDefault + "/virtualmachines/" + vm + "/console" - return http.DefaultClient.Get(wsUrl.String()) - } - Expect(err).ToNot(HaveOccurred()) - }) - - It("Should proxy message through virt-api", func() { - - vmInterface.EXPECT().Get("testvm", gomock.Any()).Return(vm, nil) - virtClient.EXPECT().CoreV1().Return(k8sClient) - ws := dial("testvm", "console0") - defer ws.Close() - ws.WriteMessage(websocket.TextMessage, []byte("hello echo!")) - t, data, err := ws.ReadMessage() - Expect(t).To(Equal(websocket.TextMessage)) - Expect(err).ToNot(HaveOccurred()) - Expect(string(data)).To(Equal("hello echo!")) - }) - - It("Should return 404 if the VM does not exist", func() { - vmInterface.EXPECT().Get("testvm", gomock.Any()).Return(vm, errors.NewNotFound(schema.GroupResource{}, "testvm")) - response, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(response.StatusCode).To(Equal(http.StatusNotFound)) - }) - - It("Should return 500 if looking up the VM failed", func() { - vmInterface.EXPECT().Get("testvm", gomock.Any()).Return(vm, errors.NewInternalError(fmt.Errorf("something is weird"))) - response, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) - Expect(body(response)).To(ContainSubstring("something is weird")) - }) - - It("Should return 400 if the VM is not running", func() { - vmInterface.EXPECT().Get("testvm", gomock.Any()).Return(vm, nil) - vm.Status.Phase = v1.Succeeded - response, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(response.StatusCode).To(Equal(http.StatusBadRequest)) - }) - - It("Should return 500 if we can't look up the node", func() { - k8sClient.Pods(k8sv1.NamespaceDefault).Delete(virtHandlerPod.GetObjectMeta().GetName(), nil) - vmInterface.EXPECT().Get("testvm", gomock.Any()).Return(vm, nil) - virtClient.EXPECT().CoreV1().Return(k8sClient) - vm.Status.NodeName = "nonexistentnode" - response, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) - Expect(body(response)).To(ContainSubstring("Looking up the connection details for virt-handler on node nonexistentnode failed")) - }) - - AfterEach(func() { - ctrl.Finish() - }) -}) - -func body(request *http.Response) string { - b, err := ioutil.ReadAll(request.Body) - Expect(err).ToNot(HaveOccurred()) - return string(b) -} diff --git a/pkg/virt-handler/rest/console.go b/pkg/virt-handler/rest/console.go deleted file mode 100644 index cd906f40831d..000000000000 --- a/pkg/virt-handler/rest/console.go +++ /dev/null @@ -1,164 +0,0 @@ -/* - * This file is part of the KubeVirt project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright 2017 Red Hat, Inc. - * - */ - -package rest - -import ( - "io" - "net/http" - - "github.com/emicklei/go-restful" - "github.com/gorilla/websocket" - "github.com/libvirt/libvirt-go" - - "kubevirt.io/kubevirt/pkg/api/v1" - "kubevirt.io/kubevirt/pkg/log" - "kubevirt.io/kubevirt/pkg/virt-handler/virtwrap/api" - "kubevirt.io/kubevirt/pkg/virt-handler/virtwrap/cli" - "kubevirt.io/kubevirt/pkg/virt-handler/virtwrap/errors" - "kubevirt.io/kubevirt/pkg/virt-handler/virtwrap/util" -) - -var upgrader = websocket.Upgrader{ - ReadBufferSize: 1024, - WriteBufferSize: 1024, -} - -type Console struct { - connection cli.Connection -} - -func NewConsoleResource(connection cli.Connection) *Console { - return &Console{connection: connection} -} - -func (t *Console) Console(request *restful.Request, response *restful.Response) { - console := request.QueryParameter("console") - vmName := request.PathParameter("name") - namespace := request.PathParameter("namespace") - vm := v1.NewVMReferenceFromNameWithNS(namespace, vmName) - logger := log.Log.Object(vm) - domain, err := t.connection.LookupDomainByName(api.VMNamespaceKeyFunc(vm)) - if err != nil { - if errors.IsNotFound(err) { - logger.Reason(err).Error("Domain not found.") - response.WriteError(http.StatusNotFound, err) - return - } else { - response.WriteError(http.StatusInternalServerError, err) - logger.Reason(err).Error("Failed to look up domain.") - return - } - } - defer domain.Free() - - // Fetch metadata to get the VM UID - spec, err := util.GetDomainSpecWithFlags(domain, libvirt.DOMAIN_XML_INACTIVE) - if err != nil { - response.WriteError(http.StatusInternalServerError, err) - logger.Reason(err).Error("Failed to look up domain UID.") - return - } - vm.GetObjectMeta().SetUID(spec.Metadata.KubeVirt.UID) - logger = log.Log.Object(vm) - - logger.Infof("Opening connection to console %s", console) - - consoleStream, err := t.connection.NewStream(0) - if err != nil { - logger.Reason(err).Error("Creating a consoleStream failed.") - response.WriteError(http.StatusInternalServerError, err) - return - } - defer consoleStream.Close() - - logger.V(3).Info("Stream created.") - - err = domain.OpenConsole(console, consoleStream.UnderlyingStream(), libvirt.DOMAIN_CONSOLE_FORCE) - if err != nil { - response.WriteError(http.StatusInternalServerError, err) - logger.Reason(err).Error("Failed to open console.") - return - } - logger.V(3).Info("Connection to console created.") - - errorChan := make(chan error) - - ws, err := upgrader.Upgrade(response.ResponseWriter, request.Request, nil) - if err != nil { - logger.Reason(err).Error("Failed to upgrade websocket connection.") - response.WriteError(http.StatusBadRequest, err) - return - } - defer ws.Close() - - wsReadWriter := &TextReadWriter{ws} - - go func() { - _, err := io.Copy(consoleStream, wsReadWriter) - errorChan <- err - }() - - go func() { - _, err := io.Copy(wsReadWriter, consoleStream) - errorChan <- err - }() - - err = <-errorChan - - if err != nil { - logger.Reason(err).Error("Proxying data between libvirt and the websocket failed.") - } - - logger.V(3).Info("Done.") - response.WriteHeader(http.StatusOK) -} - -type TextReadWriter struct { - *websocket.Conn -} - -func (s *TextReadWriter) Write(p []byte) (int, error) { - err := s.Conn.WriteMessage(websocket.TextMessage, p) - if err != nil { - return 0, s.err(err) - } - return len(p), nil -} - -func (s *TextReadWriter) Read(p []byte) (int, error) { - _, r, err := s.Conn.NextReader() - if err != nil { - return 0, s.err(err) - } - n, err := r.Read(p) - return n, s.err(err) -} - -func (s *TextReadWriter) err(err error) error { - if err == nil { - return nil - } - if e, ok := err.(*websocket.CloseError); ok { - if e.Code == websocket.CloseNormalClosure { - return io.EOF - } - } - return err -} diff --git a/pkg/virt-handler/rest/console_test.go b/pkg/virt-handler/rest/console_test.go deleted file mode 100644 index 23c52322f68f..000000000000 --- a/pkg/virt-handler/rest/console_test.go +++ /dev/null @@ -1,223 +0,0 @@ -/* - * This file is part of the KubeVirt project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright 2017 Red Hat, Inc. - * - */ - -package rest - -import ( - "bytes" - "net/http" - "net/http/httptest" - "net/url" - - "github.com/emicklei/go-restful" - "github.com/golang/mock/gomock" - "github.com/gorilla/websocket" - "github.com/libvirt/libvirt-go" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - k8sv1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/client-go/tools/record" - - "encoding/xml" - - "kubevirt.io/kubevirt/pkg/log" - "kubevirt.io/kubevirt/pkg/virt-handler/virtwrap/api" - "kubevirt.io/kubevirt/pkg/virt-handler/virtwrap/cli" -) - -var _ = Describe("Console", func() { - var handler http.Handler - - var mockConn *cli.MockConnection - var mockDomain *cli.MockVirDomain - var mockStream *cli.MockStream - var ctrl *gomock.Controller - var mockRecorder *record.FakeRecorder - var server *httptest.Server - var wsUrl *url.URL - var serverDone chan bool - - log.Log.SetIOWriter(GinkgoWriter) - - dial := func(vm string, console string) *websocket.Conn { - wsUrl.Scheme = "ws" - wsUrl.Path = "/api/v1/namespaces/" + k8sv1.NamespaceDefault + "/virtualmachines/" + vm + "/console" - wsUrl.RawQuery = "console=" + console - c, _, err := websocket.DefaultDialer.Dial(wsUrl.String(), nil) - Expect(err).ToNot(HaveOccurred()) - return c - } - - get := func(vm string) (*http.Response, error) { - wsUrl.Scheme = "http" - wsUrl.Path = "/api/v1/namespaces/" + k8sv1.NamespaceDefault + "/virtualmachines/" + vm + "/console" - return http.DefaultClient.Get(wsUrl.String()) - } - - BeforeEach(func() { - var err error - // Set up mocks - ctrl = gomock.NewController(GinkgoT()) - mockConn = cli.NewMockConnection(ctrl) - mockDomain = cli.NewMockVirDomain(ctrl) - mockStream = cli.NewMockStream(ctrl) - mockRecorder = record.NewFakeRecorder(10) - - // Set up web service - ws := new(restful.WebService) - handler = http.Handler(restful.NewContainer().Add(ws)) - - // Give us a chance to detect when the request is done. Otherwise we - // don't know when to check mock invokations - serverDone = make(chan bool) - waiter := func(request *restful.Request, response *restful.Response) { - NewConsoleResource(mockConn).Console(request, response) - close(serverDone) - } - ws.Route(ws.GET("/api/v1/namespaces/{namespace}/virtualmachines/{name}/console").To(waiter)) - server = httptest.NewServer(handler) - wsUrl, err = url.Parse(server.URL) - Expect(err).ToNot(HaveOccurred()) - }) - - It("should return 404 if VM does not exist", func() { - mockConn.EXPECT().LookupDomainByName("default_testvm").Return(nil, libvirt.Error{Code: libvirt.ERR_NO_DOMAIN}) - r, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(r.StatusCode).To(Equal(http.StatusNotFound)) - }) - - It("should return 500 if domain can't be looked up", func() { - mockConn.EXPECT().LookupDomainByName("default_testvm").Return(nil, libvirt.Error{Code: libvirt.ERR_INVALID_CONN}) - r, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(r.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - - Context("with existing domain", func() { - var specXML string - - BeforeEach(func() { - // Make sure that we always free the domain after use - mockDomain.EXPECT().Free() - spec := api.NewMinimalDomainSpec("default_testvm") - spec.Metadata.KubeVirt.UID = uuid.NewUUID() - data, err := xml.Marshal(spec) - specXML = string(data) - Expect(err).ToNot(HaveOccurred()) - }) - - It("should return 500 if metadata of domain can't be looked up", func() { - mockConn.EXPECT().LookupDomainByName("default_testvm").Return(mockDomain, nil) - mockDomain.EXPECT().GetXMLDesc(gomock.Eq(libvirt.DOMAIN_XML_INACTIVE)).Return("", libvirt.Error{Code: libvirt.ERR_INVALID_CONN}) - - r, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(r.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - It("should return 500 if creating a stream fails", func() { - mockConn.EXPECT().LookupDomainByName("default_testvm").Return(mockDomain, nil) - mockDomain.EXPECT().GetXMLDesc(gomock.Eq(libvirt.DOMAIN_XML_INACTIVE)).Return(specXML, nil) - mockConn.EXPECT().NewStream(libvirt.StreamFlags(0)).Return(nil, libvirt.Error{Code: libvirt.ERR_INVALID_CONN}) - r, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(r.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - It("should return 500 if opening a console connection fails", func() { - mockConn.EXPECT().LookupDomainByName("default_testvm").Return(mockDomain, nil) - mockDomain.EXPECT().GetXMLDesc(gomock.Eq(libvirt.DOMAIN_XML_INACTIVE)).Return(specXML, nil) - mockConn.EXPECT().NewStream(libvirt.StreamFlags(0)).Return(mockStream, nil) - stream := &libvirt.Stream{} - mockStream.EXPECT().UnderlyingStream().Return(stream) - mockStream.EXPECT().Close() - mockDomain.EXPECT().OpenConsole("", stream, libvirt.DomainConsoleFlags(libvirt.DOMAIN_CONSOLE_FORCE)).Return(libvirt.Error{Code: libvirt.ERR_INVALID_CONN}) - r, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(r.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - It("should return 400 if ws upgrade does not work", func() { - mockConn.EXPECT().LookupDomainByName("default_testvm").Return(mockDomain, nil) - mockDomain.EXPECT().GetXMLDesc(gomock.Eq(libvirt.DOMAIN_XML_INACTIVE)).Return(specXML, nil) - mockConn.EXPECT().NewStream(libvirt.StreamFlags(0)).Return(mockStream, nil) - stream := &libvirt.Stream{} - mockStream.EXPECT().UnderlyingStream().Return(stream) - mockStream.EXPECT().Close() - mockDomain.EXPECT().OpenConsole("", stream, libvirt.DomainConsoleFlags(libvirt.DOMAIN_CONSOLE_FORCE)).Return(nil) - r, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(r.StatusCode).To(Equal(http.StatusBadRequest)) - }) - It("should proxy websocket traffic", func() { - - var in []byte - var out []byte - inBuf := bytes.NewBuffer(in) - outBuf := bytes.NewBuffer(out) - outBuf.WriteString("hello client!") - stream := &fakeStream{in: inBuf, out: outBuf, s: &libvirt.Stream{}} - - mockConn.EXPECT().LookupDomainByName("default_testvm").Return(mockDomain, nil) - mockDomain.EXPECT().GetXMLDesc(gomock.Eq(libvirt.DOMAIN_XML_INACTIVE)).Return(specXML, nil) - mockConn.EXPECT().NewStream(libvirt.StreamFlags(0)).Return(stream, nil) - mockDomain.EXPECT().OpenConsole("console0", stream.s, libvirt.DomainConsoleFlags(libvirt.DOMAIN_CONSOLE_FORCE)).Return(nil) - - con := dial("testvm", "console0") - defer con.Close() - err := con.WriteMessage(websocket.TextMessage, []byte("hello console!")) - Expect(err).ToNot(HaveOccurred()) - - // FIXME, there is somewhere a buffer or timeout which delays the actual send - // Eventually(inBuf.String).Should(Equal("hello console!")) - - t, body, err := con.ReadMessage() - Expect(t).To(Equal(websocket.TextMessage)) - Expect(err).ToNot(HaveOccurred()) - Expect(body).To(Equal([]byte("hello client!"))) - }) - - }) - AfterEach(func() { - server.Close() - <-serverDone - ctrl.Finish() - }) -}) - -type fakeStream struct { - in *bytes.Buffer - out *bytes.Buffer - s *libvirt.Stream -} - -func (s *fakeStream) Write(p []byte) (n int, err error) { - return s.in.Write(p) -} - -func (s *fakeStream) Read(p []byte) (n int, err error) { - return s.out.Read(p) -} - -func (s *fakeStream) Close() (e error) { - return nil -} - -func (s *fakeStream) UnderlyingStream() *libvirt.Stream { - return s.s -} From 96070b37988b04750532bc764b5579a132abe045 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 13 Dec 2017 12:03:11 -0500 Subject: [PATCH 10/37] Add executable perm to sock-connector script Signed-off-by: David Vossel --- cmd/virt-launcher/sock-connector | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 cmd/virt-launcher/sock-connector diff --git a/cmd/virt-launcher/sock-connector b/cmd/virt-launcher/sock-connector old mode 100644 new mode 100755 From 3d410d6098966f356d1057b0f5cc418185e3af71 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 13 Dec 2017 13:16:44 -0500 Subject: [PATCH 11/37] Add support for ^] escape sequence and signal characters Signed-off-by: David Vossel --- cmd/virt-launcher/sock-connector | 3 +-- pkg/virtctl/console/console.go | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/virt-launcher/sock-connector b/cmd/virt-launcher/sock-connector index 86c4381c986e..bae34416c018 100755 --- a/cmd/virt-launcher/sock-connector +++ b/cmd/virt-launcher/sock-connector @@ -1,5 +1,4 @@ #!/bin/bash stty -echo -#stty -isig SOCKET=$1 -socat unix-connect:/$SOCKET stdio +socat unix-connect:/$SOCKET stdio,cfmakeraw diff --git a/pkg/virtctl/console/console.go b/pkg/virtctl/console/console.go index a0abcff2b155..fade3d35ddf6 100644 --- a/pkg/virtctl/console/console.go +++ b/pkg/virtctl/console/console.go @@ -139,6 +139,11 @@ func (c *Console) Run(flags *flag.FlagSet) int { if n == 0 && err == io.EOF { return } + + // the escape sequence + if buf[0] == 29 { + return + } // Writing out to the console connection _, err = stdinWriter.Write(buf[0:n]) if err == io.EOF { From c3b5f861ac9c1c7af54ec967668ea76b005e8611 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Tue, 2 Jan 2018 12:18:02 -0500 Subject: [PATCH 12/37] Add vnc support Signed-off-by: David Vossel --- cmd/virtctl/virtctl.go | 2 + pkg/kubecli/generated_mock_kubevirt.go | 10 ++ pkg/kubecli/kubevirt.go | 1 + pkg/kubecli/vm.go | 55 +++++++ pkg/virt-handler/virtwrap/api/schema.go | 1 + pkg/virtctl/console/console.go | 2 +- pkg/virtctl/vnc/vnc.go | 182 ++++++++++++++++++++++++ 7 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 pkg/virtctl/vnc/vnc.go diff --git a/cmd/virtctl/virtctl.go b/cmd/virtctl/virtctl.go index aebf3bc75dc7..2bc748a1f858 100644 --- a/cmd/virtctl/virtctl.go +++ b/cmd/virtctl/virtctl.go @@ -30,6 +30,7 @@ import ( "kubevirt.io/kubevirt/pkg/virtctl" "kubevirt.io/kubevirt/pkg/virtctl/console" "kubevirt.io/kubevirt/pkg/virtctl/spice" + "kubevirt.io/kubevirt/pkg/virtctl/vnc" ) func main() { @@ -41,6 +42,7 @@ func main() { "console": &console.Console{}, "options": &virtctl.Options{}, "spice": &spice.Spice{}, + "vnc": &vnc.Vnc{}, } if len(os.Args) > 1 { diff --git a/pkg/kubecli/generated_mock_kubevirt.go b/pkg/kubecli/generated_mock_kubevirt.go index 6e63877203df..f3d562c828d8 100644 --- a/pkg/kubecli/generated_mock_kubevirt.go +++ b/pkg/kubecli/generated_mock_kubevirt.go @@ -601,6 +601,16 @@ func (_mr *_MockVMInterfaceRecorder) SerialConsole(arg0, arg1, arg2, arg3 interf return _mr.mock.ctrl.RecordCall(_mr.mock, "SerialConsole", arg0, arg1, arg2, arg3) } +func (_m *MockVMInterface) Vnc(name string, in io.Reader, out io.Writer) error { + ret := _m.ctrl.Call(_m, "Vnc", name, in, out) + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockVMInterfaceRecorder) Vnc(arg0, arg1, arg2 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "Vnc", arg0, arg1, arg2) +} + // Mock of ReplicaSetInterface interface type MockReplicaSetInterface struct { ctrl *gomock.Controller diff --git a/pkg/kubecli/kubevirt.go b/pkg/kubecli/kubevirt.go index cfdd639fac11..4e3077d17b8a 100644 --- a/pkg/kubecli/kubevirt.go +++ b/pkg/kubecli/kubevirt.go @@ -64,6 +64,7 @@ type VMInterface interface { Delete(name string, options *k8smetav1.DeleteOptions) error Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.VirtualMachine, err error) SerialConsole(name string, device string, in io.Reader, out io.Writer) error + Vnc(name string, in io.Reader, out io.Writer) error } type ReplicaSetInterface interface { diff --git a/pkg/kubecli/vm.go b/pkg/kubecli/vm.go index 25b8d99f7850..79e210e2249e 100644 --- a/pkg/kubecli/vm.go +++ b/pkg/kubecli/vm.go @@ -80,6 +80,61 @@ func findPod(clientSet *kubernetes.Clientset, namespace string, name string) (st return podList.Items[0].ObjectMeta.Name, nil } +func (v *vms) Vnc(name string, in io.Reader, out io.Writer) error { + device := "vnc" + podName, err := findPod(v.clientSet, v.namespace, name) + if err != nil { + return err + } + + config, err := clientcmd.BuildConfigFromFlags(v.master, v.kubeconfig) + if err != nil { + return err + } + + gv := k8sv1.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/api" + config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs} + + restClient, err := restclient.RESTClientFor(config) + if err != nil { + return err + } + cmd := []string{"/sock-connector", fmt.Sprintf("/var/run/kubevirt-private/%s/%s/virt-%s", v.namespace, name, device)} + containerName := "compute" + req := restClient.Post(). + Resource("pods"). + Name(podName). + Namespace(v.namespace). + SubResource("exec"). + Param("container", containerName) + + req = req.VersionedParams(&k8sv1.PodExecOptions{ + Container: containerName, + Command: cmd, + Stdin: true, + Stdout: true, + Stderr: true, + TTY: true, + }, scheme.ParameterCodec) + + // execute request + method := "POST" + url := req.URL() + exec, err := remotecommand.NewSPDYExecutor(config, method, url) + if err != nil { + return err + } + + return exec.Stream(remotecommand.StreamOptions{ + Stdin: in, + Stdout: out, + Stderr: out, + Tty: false, + TerminalSizeQueue: nil, + }) +} func (v *vms) SerialConsole(name string, device string, in io.Reader, out io.Writer) error { podName, err := findPod(v.clientSet, v.namespace, name) if err != nil { diff --git a/pkg/virt-handler/virtwrap/api/schema.go b/pkg/virt-handler/virtwrap/api/schema.go index b2813b5477af..3d8d3ffb01bd 100644 --- a/pkg/virt-handler/virtwrap/api/schema.go +++ b/pkg/virt-handler/virtwrap/api/schema.go @@ -465,6 +465,7 @@ type Listen struct { Type string `xml:"type,attr"` Address string `xml:"address,attr,omitempty"` Network string `xml:"newtork,attr,omitempty"` + Socket string `xml:"socket,attr,omitempty"` } type Address struct { diff --git a/pkg/virtctl/console/console.go b/pkg/virtctl/console/console.go index fade3d35ddf6..e7d86c9b87ff 100644 --- a/pkg/virtctl/console/console.go +++ b/pkg/virtctl/console/console.go @@ -87,7 +87,7 @@ func (c *Console) Run(flags *flag.FlagSet) int { stdinReader, stdinWriter := io.Pipe() stdoutReader, stdoutWriter := io.Pipe() - // in -> stdinWriter | stdoutReader -> console + // in -> stdinWriter | stdinReader -> console // out <- stdoutReader | stdoutWriter <- console resChan := make(chan error) diff --git a/pkg/virtctl/vnc/vnc.go b/pkg/virtctl/vnc/vnc.go new file mode 100644 index 000000000000..4d9b2c01c84e --- /dev/null +++ b/pkg/virtctl/vnc/vnc.go @@ -0,0 +1,182 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright 2017 Red Hat, Inc. + * + */ + +package vnc + +import ( + "fmt" + "io" + "log" + "net" + "os" + "os/exec" + "os/signal" + + flag "github.com/spf13/pflag" + kubev1 "k8s.io/api/core/v1" + + "kubevirt.io/kubevirt/pkg/kubecli" +) + +const FLAG = "vnc" + +type Vnc struct{} + +func (o *Vnc) Run(flags *flag.FlagSet) int { + server, _ := flags.GetString("server") + kubeconfig, _ := flags.GetString("kubeconfig") + namespace, _ := flags.GetString("namespace") + if namespace == "" { + namespace = kubev1.NamespaceDefault + } + + if len(flags.Args()) != 2 { + log.Println("VM name is missing") + return 1 + } + vm := flags.Arg(1) + + virtCli, err := kubecli.GetKubevirtClientFromFlags(server, kubeconfig) + if err != nil { + log.Println(err) + return 1 + } + + // -> pipeInWriter -> pipeInReader + // remote-viewer -> unix sock connection + // <- pipeOutReader <- pipeOutWriter + pipeInReader, pipeInWriter := io.Pipe() + pipeOutReader, pipeOutWriter := io.Pipe() + + k8ResChan := make(chan error) + viewResChan := make(chan error) + stopChan := make(chan struct{}, 1) + writeStop := make(chan struct{}) + readStop := make(chan struct{}) + + // The local tcp server is used to proxy the podExec websock connection to remote-viewer + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + log.Printf("Can't listen on unix socket: %s", err.Error()) + return 1 + } + + port := ln.Addr().(*net.TCPAddr).Port + + // setup connection with VM + go func() { + err := virtCli.VM(namespace).Vnc(vm, pipeInReader, pipeOutWriter) + k8ResChan <- err + }() + + // execute remote viewer + go func() { + cmnd := exec.Command("remote-viewer", fmt.Sprintf("vnc://127.0.0.1:%d", port)) + err := cmnd.Run() + if err != nil { + log.Println(err) + } + viewResChan <- err + }() + + // wait for remote-viewer to connect to our local proxy server + fd, err := ln.Accept() + if err != nil { + log.Printf("Failed to accept unix sock connection. %s", err.Error()) + return 1 + } + defer fd.Close() + + log.Printf("remote-viewer connected") + go func() { + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, os.Interrupt) + <-interrupt + close(stopChan) + }() + + // write to FD <- pipeOutReader + go func() { + defer close(readStop) + buf := make([]byte, 4096, 4096) + for { + // reading qemu vnc server + n, err := pipeOutReader.Read(buf) + if err != nil && err != io.EOF { + return + } + if n == 0 && err == io.EOF { + return + } + + // Writing to remote viewer + _, err = fd.Write(buf[0:n]) + if err == io.EOF { + return + } + } + }() + + // read from FD -> pipeInWriter + go func() { + defer close(writeStop) + buf := make([]byte, 4096, 4096) + for { + // reading from remoteViewer + n, err := fd.Read(buf) + if err != nil && err != io.EOF { + return + } + if n == 0 && err == io.EOF { + return + } + + // Writing out to the qemu vnc server + _, err = pipeInWriter.Write(buf[0:n]) + if err == io.EOF { + return + } + } + }() + + select { + case <-stopChan: + case <-readStop: + case <-writeStop: + case err = <-k8ResChan: + case err = <-viewResChan: + } + + if err != nil { + log.Printf("Error encountered: %s", err.Error()) + return 1 + } + return 0 +} + +func (o *Vnc) Usage() string { + usage := "virtctl can connect via remote-viewer to a VM\n\n" + usage += "Examples:\n" + usage += "# Connect to testvm via remote-viewer\n" + usage += "./virtctl vnc testvm\n\n" + return usage +} +func (o *Vnc) FlagSet() *flag.FlagSet { + return flag.NewFlagSet(FLAG, flag.ExitOnError) +} From 6e9698ce4d1cc993cf53813d9b79364581b45023 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Fri, 15 Dec 2017 14:37:22 -0500 Subject: [PATCH 13/37] Common podExec helper code Signed-off-by: David Vossel --- pkg/kubecli/vm.go | 61 +++++++---------------------------------------- 1 file changed, 8 insertions(+), 53 deletions(-) diff --git a/pkg/kubecli/vm.go b/pkg/kubecli/vm.go index 79e210e2249e..55be949dd7ce 100644 --- a/pkg/kubecli/vm.go +++ b/pkg/kubecli/vm.go @@ -80,8 +80,8 @@ func findPod(clientSet *kubernetes.Clientset, namespace string, name string) (st return podList.Items[0].ObjectMeta.Name, nil } -func (v *vms) Vnc(name string, in io.Reader, out io.Writer) error { - device := "vnc" +func (v *vms) remoteExecHelper(name string, cmd []string, in io.Reader, out io.Writer) error { + podName, err := findPod(v.clientSet, v.namespace, name) if err != nil { return err @@ -101,7 +101,6 @@ func (v *vms) Vnc(name string, in io.Reader, out io.Writer) error { if err != nil { return err } - cmd := []string{"/sock-connector", fmt.Sprintf("/var/run/kubevirt-private/%s/%s/virt-%s", v.namespace, name, device)} containerName := "compute" req := restClient.Post(). Resource("pods"). @@ -135,59 +134,15 @@ func (v *vms) Vnc(name string, in io.Reader, out io.Writer) error { TerminalSizeQueue: nil, }) } -func (v *vms) SerialConsole(name string, device string, in io.Reader, out io.Writer) error { - podName, err := findPod(v.clientSet, v.namespace, name) - if err != nil { - return err - } - config, err := clientcmd.BuildConfigFromFlags(v.master, v.kubeconfig) - if err != nil { - return err - } - - gv := k8sv1.SchemeGroupVersion - config.GroupVersion = &gv - config.APIPath = "/api" - config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs} +func (v *vms) Vnc(name string, in io.Reader, out io.Writer) error { + cmd := []string{"/sock-connector", fmt.Sprintf("/var/run/kubevirt-private/%s/%s/virt-vnc", v.namespace, name)} + return v.remoteExecHelper(name, cmd, in, out) +} - restClient, err := restclient.RESTClientFor(config) - if err != nil { - return err - } +func (v *vms) SerialConsole(name string, device string, in io.Reader, out io.Writer) error { cmd := []string{"/sock-connector", fmt.Sprintf("/var/run/kubevirt-private/%s/%s/virt-%s", v.namespace, name, device)} - containerName := "compute" - req := restClient.Post(). - Resource("pods"). - Name(podName). - Namespace(v.namespace). - SubResource("exec"). - Param("container", containerName) - - req = req.VersionedParams(&k8sv1.PodExecOptions{ - Container: containerName, - Command: cmd, - Stdin: true, - Stdout: true, - Stderr: true, - TTY: true, - }, scheme.ParameterCodec) - - // execute request - method := "POST" - url := req.URL() - exec, err := remotecommand.NewSPDYExecutor(config, method, url) - if err != nil { - return err - } - - return exec.Stream(remotecommand.StreamOptions{ - Stdin: in, - Stdout: out, - Stderr: out, - Tty: false, - TerminalSizeQueue: nil, - }) + return v.remoteExecHelper(name, cmd, in, out) } func (v *vms) Get(name string, options k8smetav1.GetOptions) (vm *v1.VirtualMachine, err error) { From 4ae10e8d2889227f4c509d1b8256b9700f6fa47d Mon Sep 17 00:00:00 2001 From: David Vossel Date: Fri, 15 Dec 2017 14:38:01 -0500 Subject: [PATCH 14/37] Add VNC functional tests Signed-off-by: David Vossel --- tests/vnc_test.go | 89 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 tests/vnc_test.go diff --git a/tests/vnc_test.go b/tests/vnc_test.go new file mode 100644 index 000000000000..ed0abb24e705 --- /dev/null +++ b/tests/vnc_test.go @@ -0,0 +1,89 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright 2017 Red Hat, Inc. + * + */ + +package tests_test + +import ( + "flag" + "io" + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "kubevirt.io/kubevirt/pkg/kubecli" + "kubevirt.io/kubevirt/tests" +) + +var _ = Describe("Vmlifecycle", func() { + + flag.Parse() + + virtClient, err := kubecli.GetKubevirtClient() + tests.PanicOnError(err) + + BeforeEach(func() { + tests.BeforeTestCleanup() + }) + + Context("New VM with a spice connection given", func() { + It("should allow accessing the vnc device on the VM", func(done Done) { + vm := tests.NewRandomVM() + Expect(virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do().Error()).To(Succeed()) + tests.WaitForSuccessfulVMStart(vm) + + tests.WaitForSuccessfulVMStart(vm) + + pipeInReader, _ := io.Pipe() + pipeOutReader, pipeOutWriter := io.Pipe() + + k8ResChan := make(chan error) + readStop := make(chan string) + + go func() { + err := virtClient.VM(vm.ObjectMeta.Namespace).Vnc(vm.ObjectMeta.Name, pipeInReader, pipeOutWriter) + k8ResChan <- err + }() + // write to FD <- pipeOutReader + go func() { + buf := make([]byte, 1024, 1024) + // reading qemu vnc server + n, err := pipeOutReader.Read(buf) + if err != nil && err != io.EOF { + return + } + if n == 0 && err == io.EOF { + return + } + readStop <- strings.TrimSpace(string(buf[0:n])) + }() + + response := "" + + select { + case response = <-readStop: + case err = <-k8ResChan: + } + + Expect(response).To(Equal("RFB 003.008")) + Expect(err).To(BeNil()) + close(done) + }, 30) + }) +}) From 7fb6aa0cd2c9446c8328b0034e69d819450eef63 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Fri, 15 Dec 2017 22:40:13 -0500 Subject: [PATCH 15/37] Update deploy scripts to work without haproxy Signed-off-by: David Vossel --- cluster/deploy.sh | 1 + cluster/kubectl.sh | 7 +------ hack/build-go.sh | 2 +- hack/config-default.sh | 4 ++-- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/cluster/deploy.sh b/cluster/deploy.sh index f25ae9a152a7..33a7f34b23f7 100755 --- a/cluster/deploy.sh +++ b/cluster/deploy.sh @@ -64,6 +64,7 @@ fi ## Expose common services $KUBECTL expose deployment haproxy --port 8184 -l 'kubevirt.io=haproxy' -n kube-system --external-ip $master_ip +$KUBECTL expose pod kube-apiserver-master --port 6443 -l 'component=kube-apiserver' -n kube-system --external-ip $master_ip $KUBECTL expose deployment spice-proxy --port 3128 -l 'kubevirt.io=spice-proxy' -n kube-system --external-ip $master_ip # Deploy additional infra for testing diff --git a/cluster/kubectl.sh b/cluster/kubectl.sh index e6f24cd74ff1..38325b412ebe 100755 --- a/cluster/kubectl.sh +++ b/cluster/kubectl.sh @@ -27,11 +27,6 @@ then exit fi -if [ "$1" == "console" ] || [ "$1" == "spice" ]; then - cmd/virtctl/virtctl "$@" -s http://${master_ip}:8184 - exit -fi - # Print usage from virtctl and kubectl if [ "$1" == "--help" ] || [ "$1" == "-h" ] ; then cmd/virtctl/virtctl "$@" @@ -43,7 +38,7 @@ if [ -e ${KUBEVIRT_PATH}cluster/vagrant/.kubeconfig ] && shift ${KUBEVIRT_PATH}cluster/vagrant/.kubectl --kubeconfig=${KUBEVIRT_PATH}cluster/vagrant/.kubeconfig "$@" elif [ -e ${KUBEVIRT_PATH}cluster/vagrant/.kubectl ];then - ${KUBEVIRT_PATH}cluster/vagrant/.kubectl -s http://${master_ip}:8184 "$@" + ${KUBEVIRT_PATH}cluster/vagrant/.kubectl -s https://${master_ip}:${master_port} "$@" else echo "Did you already run '$SYNC_CONFIG' to deploy kubevirt?" fi diff --git a/hack/build-go.sh b/hack/build-go.sh index 6fd1478787e6..99f1408a5f34 100755 --- a/hack/build-go.sh +++ b/hack/build-go.sh @@ -40,7 +40,7 @@ if [ $# -eq 0 ]; then if [ "${target}" = "test" ]; then (cd pkg; go ${target} -v ./...) elif [ "${target}" = "functest" ]; then - (cd tests; go test -master=http://${master_ip}:${master_port} -timeout 30m ${FUNC_TEST_ARGS}) + (cd tests; go test -kubeconfig=../cluster/vagrant/.kubeconfig -timeout 30m ${FUNC_TEST_ARGS}) exit else (cd pkg; go $target ./...) diff --git a/hack/config-default.sh b/hack/config-default.sh index f91be8ddeb6e..669fb833b46d 100644 --- a/hack/config-default.sh +++ b/hack/config-default.sh @@ -3,6 +3,6 @@ docker_images="cmd/virt-controller cmd/virt-launcher cmd/virt-handler cmd/virt-a optional_docker_images="cmd/registry-disk-v1alpha images/fedora-atomic-registry-disk-demo" docker_prefix=kubevirt docker_tag=${DOCKER_TAG:-latest} -master_ip=192.168.200.2 -master_port=8184 +master_ip=192.168.121.121 +master_port=6443 network_provider=weave From 1737ad4a247c5e66dc4ddd4a3b74e5828ca76a44 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Fri, 15 Dec 2017 22:40:38 -0500 Subject: [PATCH 16/37] remove spice tests in favor of vnc Signed-off-by: David Vossel --- tests/spice_test.go | 246 -------------------------------------------- 1 file changed, 246 deletions(-) delete mode 100644 tests/spice_test.go diff --git a/tests/spice_test.go b/tests/spice_test.go deleted file mode 100644 index 4f4e72841dd1..000000000000 --- a/tests/spice_test.go +++ /dev/null @@ -1,246 +0,0 @@ -/* - * This file is part of the KubeVirt project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright 2017 Red Hat, Inc. - * - */ - -package tests_test - -import ( - "bufio" - "bytes" - "encoding/binary" - "flag" - "fmt" - "io" - "math/rand" - "net" - "strconv" - "strings" - "time" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - ini "gopkg.in/ini.v1" - - "kubevirt.io/kubevirt/pkg/api/v1" - "kubevirt.io/kubevirt/pkg/kubecli" - "kubevirt.io/kubevirt/pkg/rest" - "kubevirt.io/kubevirt/tests" -) - -var _ = Describe("Vmlifecycle", func() { - - flag.Parse() - - virtClient, err := kubecli.GetKubevirtClient() - tests.PanicOnError(err) - var vm *v1.VirtualMachine - - getVmNode := func() string { - obj, err := virtClient.RestClient().Get().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Name(vm.GetObjectMeta().GetName()).Do().Get() - Expect(err).ToNot(HaveOccurred()) - return obj.(*v1.VirtualMachine).Status.NodeName - } - - checkSpiceConnection := func() { - raw, err := virtClient.RestClient().Get().Resource("virtualmachines").SetHeader("Accept", rest.MIME_INI).SubResource("spice").Namespace(tests.NamespaceTestDefault).Name(vm.GetObjectMeta().GetName()).Do().Raw() - Expect(err).To(BeNil()) - spiceINI, err := ini.Load(raw) - Expect(err).NotTo(HaveOccurred()) - spice := v1.SpiceInfo{} - Expect(spiceINI.Section("virt-viewer").MapTo(&spice)).To(Succeed()) - - proxy := strings.TrimPrefix(spice.Proxy, "http://") - host := fmt.Sprintf("%s:%d", spice.Host, spice.Port) - - // Let's see if we can connect to the spice port through the proxy - conn, err := net.Dial("tcp", proxy) - Expect(err).To(BeNil()) - conn.Write([]byte("CONNECT " + host + " HTTP/1.1\r\n")) - conn.Write([]byte("Host: " + host + "\r\n")) - conn.Write([]byte("\r\n")) - line, err := bufio.NewReader(conn).ReadString('\n') - Expect(err).To(BeNil()) - Expect(strings.TrimSpace(line)).To(Equal("HTTP/1.1 200 Connection established")) - - // Let's send a spice handshake - conn.Write(newSpiceHandshake()) - - // Let's parse the response - var i int32 - x := make([]byte, 4, 4) - io.ReadFull(conn, x) - Expect(string(x)).To(Equal("REDQ")) // spice magic - binary.Read(conn, binary.LittleEndian, &i) - Expect(i).To(Equal(int32(2)), "Major version does not match.") - binary.Read(conn, binary.LittleEndian, &i) - Expect(i).To(Equal(int32(2)), "Minor version does not match.") - binary.Read(conn, binary.LittleEndian, &i) - Expect(i).To(BeNumerically(">", 4), "Message not long enough.") - binary.Read(conn, binary.LittleEndian, &i) - Expect(i).To(Equal(int32(0)), "Message status is not OK.") // 0 is equal to OK - } - - BeforeEach(func() { - vm = tests.NewRandomVM() - tests.BeforeTestCleanup() - }) - - Context("New VM with a spice connection given", func() { - - It("should return no connection details if VM does not exist", func(done Done) { - result := virtClient.RestClient().Get().Resource("virtualmachines").SubResource("spice").Namespace(tests.NamespaceTestDefault).Name("something-random").Do() - Expect(result.Error()).NotTo(BeNil()) - close(done) - }, 3) - - It("should return connection details for running VMs in ini format", func(done Done) { - // Create the VM - result := virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do() - obj, err := result.Get() - Expect(err).To(BeNil()) - - // Block until the VM is running - tests.WaitForSuccessfulVMStart(obj) - - raw, err := virtClient.RestClient().Get().Resource("virtualmachines").SetHeader("Accept", rest.MIME_INI).SubResource("spice").Namespace(tests.NamespaceTestDefault).Name(vm.GetObjectMeta().GetName()).Do().Raw() - spice, err := ini.Load(raw) - Expect(err).To(Not(HaveOccurred())) - - Expect(spice.Section("virt-viewer")).NotTo(BeNil()) - section := spice.Section("virt-viewer") - Expect(section.HasKey("type")).To(BeTrue()) - Expect(section.HasKey("host")).To(BeTrue()) - Expect(section.HasKey("port")).To(BeTrue()) - Expect(strconv.Atoi(section.Key("port").Value())).To(BeNumerically(">=", int32(5900))) - Expect(section.HasKey("proxy")).To(BeTrue()) - close(done) - }, 30) - - It("should return connection details for running VMs in json format", func(done Done) { - // Create the VM - result := virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do() - obj, err := result.Get() - Expect(err).To(BeNil()) - - // Block until the VM is running - tests.WaitForSuccessfulVMStart(obj) - - obj, err = virtClient.RestClient().Get().Resource("virtualmachines").SetHeader("Accept", rest.MIME_JSON).SubResource("spice").Namespace(tests.NamespaceTestDefault).Name(vm.GetObjectMeta().GetName()).Do().Get() - Expect(err).To(BeNil()) - spice := obj.(*v1.Spice).Info - Expect(spice.Type).To(Equal("spice")) - Expect(spice.Port).To(BeNumerically(">=", int32(5900))) - close(done) - }, 30) - - It("should allow accessing the spice device on the VM", func(done Done) { - // Create the VM - result := virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do() - obj, err := result.Get() - Expect(err).To(BeNil()) - - // Block until the VM is running - tests.WaitForSuccessfulVMStart(obj) - - checkSpiceConnection() - - close(done) - }, 30) - }) - - Context("Two new VMs scheduled on the same node with a spice graphics device", func() { - - It("should start without port clashes", func(done Done) { - // Create the VM - result := virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do() - obj, err := result.Get() - Expect(err).To(BeNil()) - - // Block until the VM is running - nodeName := tests.WaitForSuccessfulVMStart(obj) - - vm1 := tests.NewRandomVM() - vm1.Spec.NodeSelector = map[string]string{"kubernetes.io/hostname": nodeName} - result = virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm1).Do() - obj1, err := result.Get() - Expect(err).To(BeNil()) - - // Block until the VM is running - tests.WaitForSuccessfulVMStart(obj1) - - obj, err = virtClient.RestClient().Get().Resource("virtualmachines").SetHeader("Accept", rest.MIME_JSON).SubResource("spice").Namespace(tests.NamespaceTestDefault).Name(vm.GetObjectMeta().GetName()).Do().Get() - Expect(err).To(BeNil()) - obj1, err = virtClient.RestClient().Get().Resource("virtualmachines").SetHeader("Accept", rest.MIME_JSON).SubResource("spice").Namespace(tests.NamespaceTestDefault).Name(vm1.GetObjectMeta().GetName()).Do().Get() - Expect(err).To(BeNil()) - Expect(obj.(*v1.Spice).Info.Port).ToNot(BeNumerically("==", obj1.(*v1.Spice).Info.Port)) - close(done) - }, 30) - }) - - Context("Migrate VM with spice connection", func() { - - BeforeEach(func() { - if len(tests.GetReadyNodes()) < 2 { - Skip("To test migrations, at least two nodes need to be active") - } - // Create the VM - result := virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do() - obj, err := result.Get() - Expect(err).To(BeNil()) - - // Block until the VM is running - tests.WaitForSuccessfulVMStart(obj) - }) - - It("should allow accessing the spice device on the VM", func() { - sourceNode := getVmNode() - - migration := tests.NewRandomMigrationForVm(vm) - err = virtClient.RestClient().Post().Resource("migrations").Namespace(tests.NamespaceTestDefault).Body(migration).Do().Error() - Expect(err).ToNot(HaveOccurred()) - - Eventually(getVmNode, 30*time.Second, time.Second).ShouldNot(Equal(sourceNode)) - - Eventually(func() v1.VMPhase { - obj, err := virtClient.RestClient().Get().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Name(vm.GetObjectMeta().GetName()).Do().Get() - Expect(err).ToNot(HaveOccurred()) - fetchedVM := obj.(*v1.VirtualMachine) - return fetchedVM.Status.Phase - }, 10*time.Second, time.Second).Should(Equal(v1.Running)) - - checkSpiceConnection() - }) - }) -}) - -func newSpiceHandshake() []byte { - var b []byte - bb := bytes.NewBuffer(b) - bb.Write([]byte("REDQ")) // spice magic - binary.Write(bb, binary.LittleEndian, uint32(2)) // protocol major version - binary.Write(bb, binary.LittleEndian, uint32(2)) // protocol minor version - binary.Write(bb, binary.LittleEndian, uint32(22)) // message size - binary.Write(bb, binary.LittleEndian, uint32(rand.Int())) // session id - binary.Write(bb, binary.LittleEndian, uint8(3)) // channel type - binary.Write(bb, binary.LittleEndian, uint8(0)) // channel id - binary.Write(bb, binary.LittleEndian, uint32(1)) // number of common capabilities - binary.Write(bb, binary.LittleEndian, uint32(0)) // number of channel capabilities - binary.Write(bb, binary.LittleEndian, uint32(18)) // capabilities offset - binary.Write(bb, binary.LittleEndian, uint32(13)) // client common capabilities - return bb.Bytes() -} From 09a1a2fc1614f6323265fd8f656ab7a766cd9c2f Mon Sep 17 00:00:00 2001 From: David Vossel Date: Fri, 15 Dec 2017 22:43:00 -0500 Subject: [PATCH 17/37] Remove spice from virtctl Signed-off-by: David Vossel --- cmd/virtctl/virtctl.go | 4 +- pkg/virtctl/spice/spice.go | 150 ------------------------------------- 2 files changed, 1 insertion(+), 153 deletions(-) delete mode 100644 pkg/virtctl/spice/spice.go diff --git a/cmd/virtctl/virtctl.go b/cmd/virtctl/virtctl.go index 2bc748a1f858..975bb6a63b56 100644 --- a/cmd/virtctl/virtctl.go +++ b/cmd/virtctl/virtctl.go @@ -29,7 +29,6 @@ import ( "kubevirt.io/kubevirt/pkg/virtctl" "kubevirt.io/kubevirt/pkg/virtctl/console" - "kubevirt.io/kubevirt/pkg/virtctl/spice" "kubevirt.io/kubevirt/pkg/virtctl/vnc" ) @@ -41,7 +40,6 @@ func main() { registry := map[string]virtctl.App{ "console": &console.Console{}, "options": &virtctl.Options{}, - "spice": &spice.Spice{}, "vnc": &vnc.Vnc{}, } @@ -84,7 +82,7 @@ func Usage() { Basic Commands: console Connect to a serial console on a VM - spice Connect to a SPICE display of a VM + vnc Connect to a VNC display of a VM Use "virtctl --help" for more information about a given command. Use "virtctl options" for a list of global command-line options (applies to all commands). diff --git a/pkg/virtctl/spice/spice.go b/pkg/virtctl/spice/spice.go deleted file mode 100644 index 3b5ee8a24869..000000000000 --- a/pkg/virtctl/spice/spice.go +++ /dev/null @@ -1,150 +0,0 @@ -/* - * This file is part of the KubeVirt project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright 2017 Red Hat, Inc. - * - */ - -package spice - -import ( - "errors" - "fmt" - "io/ioutil" - "log" - "os" - "os/exec" - - flag "github.com/spf13/pflag" - kubev1 "k8s.io/api/core/v1" - "k8s.io/client-go/rest" - - "gopkg.in/ini.v1" - - "kubevirt.io/kubevirt/pkg/api/v1" - "kubevirt.io/kubevirt/pkg/kubecli" -) - -const FLAG = "spice" -const TEMP_PREFIX = "spice" - -type Spice struct { -} - -func DownloadSpice(namespace string, vm string, restClient *rest.RESTClient) (*v1.Spice, error) { - spice := &v1.Spice{} - err := restClient.Get(). - Resource("virtualmachines").SetHeader("Accept", "application/json"). - SubResource("spice"). - Namespace(namespace). - Name(vm).Do().Into(spice) - if err != nil { - return nil, errors.New(fmt.Sprintf("Can't fetch connection details: %s\n", err.Error())) - } - return spice, nil -} - -func (o *Spice) FlagSet() *flag.FlagSet { - - cf := flag.NewFlagSet(FLAG, flag.ExitOnError) - cf.BoolP("details", "d", false, "If present, print SPICE console to stdout, otherwise run remote-viewer") - cf.StringP("proxy", "p", "", "If given, will override any given proxy from the server") - return cf -} - -func (o *Spice) Run(flags *flag.FlagSet) int { - server, _ := flags.GetString("server") - kubeconfig, _ := flags.GetString("kubeconfig") - details, _ := flags.GetBool("details") - proxy, _ := flags.GetString("proxy") - namespace, _ := flags.GetString("namespace") - if namespace == "" { - namespace = kubev1.NamespaceDefault - } - - if len(flags.Args()) != 2 { - log.Println("VM name is missing") - return 1 - } - vm := flags.Arg(1) - - virtClient, err := kubecli.GetKubevirtClientFromFlags(server, kubeconfig) - - if err != nil { - log.Println(err) - return 1 - } - spice, err := DownloadSpice(namespace, vm, virtClient.RestClient()) - if err != nil { - log.Fatalf(err.Error()) - return 1 - } - if proxy != "" { - spice.Info.Proxy = proxy - } - cfg := ini.Empty() - err = ini.ReflectFrom(cfg, spice) - if err != nil { - log.Fatalf("Can't serialize spice struct to ini") - return 1 - } - if details { - _, err := cfg.WriteTo(os.Stdout) - if err != nil { - log.Fatalf("Failed to write to stdout") - return 1 - } - } else { - f, err := ioutil.TempFile("", TEMP_PREFIX) - - if err != nil { - log.Fatalf("Can't open file: %s", err.Error()) - return 1 - } - defer os.Remove(f.Name()) - defer f.Close() - - _, err = cfg.WriteTo(f) - if err != nil { - log.Fatalf("Can't write to file: %s", err.Error()) - return 1 - } - - f.Sync() - - cmnd := exec.Command("remote-viewer", f.Name()) - err = cmnd.Run() - - if err != nil { - log.Fatalf("Something goes wring with remote-viewer: %s", err.Error()) - return 1 - } - } - return 0 -} - -func (o *Spice) Usage() string { - usage := "virtctl can connect via remote-viewer to a VM, or can show SPICE connection details\n\n" - usage += "Examples:\n" - usage += "# Show SPICE connection details of the VM testvm\n" - usage += "./virtctl spice testvm --details\n\n" - usage += "# Connect to testvm via remote-viewer\n" - usage += "./virtctl spice testvm\n\n" - usage += "# Connect to testvm via remote-viewer using a proxy\n" - usage += "./virtctl spice testvm --proxy http://192.168.200.2:1234\n\n" - usage += "Options:\n" - usage += o.FlagSet().FlagUsages() - return usage -} From fd24ceba602dd1be36f4f0120c223efa09093514 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Fri, 15 Dec 2017 23:20:39 -0500 Subject: [PATCH 18/37] Fix master ip Signed-off-by: David Vossel --- hack/config-default.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/config-default.sh b/hack/config-default.sh index 669fb833b46d..ce96283ff46f 100644 --- a/hack/config-default.sh +++ b/hack/config-default.sh @@ -3,6 +3,6 @@ docker_images="cmd/virt-controller cmd/virt-launcher cmd/virt-handler cmd/virt-a optional_docker_images="cmd/registry-disk-v1alpha images/fedora-atomic-registry-disk-demo" docker_prefix=kubevirt docker_tag=${DOCKER_TAG:-latest} -master_ip=192.168.121.121 +master_ip=192.168.200.2 master_port=6443 network_provider=weave From 8a9108d048c578f5328bfe2294a66094176250f2 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Tue, 2 Jan 2018 12:33:12 -0500 Subject: [PATCH 19/37] Remove virt-api, HAproxy, and spice proxy Signed-off-by: David Vossel --- cluster/deploy.sh | 21 ------- cmd/virt-api/Dockerfile | 38 ------------ cmd/virt-api/virt-api.go | 36 ------------ hack/config-default.sh | 4 +- images/haproxy/Dockerfile | 36 ------------ images/haproxy/docker-entrypoint.sh | 23 -------- images/haproxy/haproxy.cfg | 36 ------------ images/spice-proxy/Dockerfile | 26 -------- manifests/dev/haproxy.yaml.in | 36 ------------ manifests/dev/squid.yaml.in | 23 -------- manifests/dev/virt-api.yaml.in | 44 -------------- manifests/release/kubevirt.yaml.in | 84 -------------------------- manifests/release/spice-proxy.yaml.in | 23 -------- pkg/virt-api/rest/spice.go | 85 --------------------------- 14 files changed, 2 insertions(+), 513 deletions(-) delete mode 100644 cmd/virt-api/Dockerfile delete mode 100644 cmd/virt-api/virt-api.go delete mode 100644 images/haproxy/Dockerfile delete mode 100755 images/haproxy/docker-entrypoint.sh delete mode 100644 images/haproxy/haproxy.cfg delete mode 100644 images/spice-proxy/Dockerfile delete mode 100644 manifests/dev/haproxy.yaml.in delete mode 100644 manifests/dev/squid.yaml.in delete mode 100644 manifests/dev/virt-api.yaml.in delete mode 100644 manifests/release/spice-proxy.yaml.in delete mode 100644 pkg/virt-api/rest/spice.go diff --git a/cluster/deploy.sh b/cluster/deploy.sh index 33a7f34b23f7..a852501d7700 100755 --- a/cluster/deploy.sh +++ b/cluster/deploy.sh @@ -23,20 +23,6 @@ KUBECTL=${KUBECTL:-kubectl} source hack/config.sh -spiceProxy() { -cat </dev/null || : @@ -58,15 +44,8 @@ if [ -z "$TARGET" ] || [ "$TARGET" = "vagrant-dev" ]; then $KUBECTL create -f manifests/dev -R $i elif [ "$TARGET" = "vagrant-release" ]; then $KUBECTL create -f manifests/release -R $i - ## Tell virt-api where to look for the spice proxy - spiceProxy | $KUBECTL patch deployment virt-api -n kube-system --patch "$(cat -)" fi -## Expose common services -$KUBECTL expose deployment haproxy --port 8184 -l 'kubevirt.io=haproxy' -n kube-system --external-ip $master_ip -$KUBECTL expose pod kube-apiserver-master --port 6443 -l 'component=kube-apiserver' -n kube-system --external-ip $master_ip -$KUBECTL expose deployment spice-proxy --port 3128 -l 'kubevirt.io=spice-proxy' -n kube-system --external-ip $master_ip - # Deploy additional infra for testing $KUBECTL create -f manifests/testing -R $i diff --git a/cmd/virt-api/Dockerfile b/cmd/virt-api/Dockerfile deleted file mode 100644 index ff138ca1b352..000000000000 --- a/cmd/virt-api/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -# -# This file is part of the KubeVirt project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Copyright 2017 Red Hat, Inc. -# - -FROM fedora:26 - -MAINTAINER "The KubeVirt Project" - -# Create non-root user -RUN useradd -u 1001 --create-home -s /bin/bash virt-api -WORKDIR /home/virt-api -USER 1001 - -# Configure swagger -RUN curl -OL https://github.com/swagger-api/swagger-ui/tarball/38f74164a7062edb5dc80ef2fdddda24f3f6eb85/swagger-ui.tar.gz \ - && mkdir swagger-ui && tar xf swagger-ui.tar.gz -C swagger-ui --strip-components 1 \ - && mkdir third_party \ - && mv swagger-ui/dist third_party/swagger-ui && rm -rf swagger-ui \ - && sed -e 's@"http://petstore.swagger.io/v2/swagger.json"@"/swaggerapi/"@' -i third_party/swagger-ui/index.html \ - && rm swagger-ui.tar.gz && rm -rf swagger-ui - -COPY virt-api /virt-api - -ENTRYPOINT [ "/virt-api" ] diff --git a/cmd/virt-api/virt-api.go b/cmd/virt-api/virt-api.go deleted file mode 100644 index 4b6e38c53d4f..000000000000 --- a/cmd/virt-api/virt-api.go +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is part of the KubeVirt project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright 2017 Red Hat, Inc. - * - */ - -package main - -import ( - klog "kubevirt.io/kubevirt/pkg/log" - "kubevirt.io/kubevirt/pkg/service" - "kubevirt.io/kubevirt/pkg/virt-api" -) - -func main() { - klog.InitializeLogging("virt-api") - - app := virt_api.VirtAPIApp{} - service.Setup(&app) - app.Compose() - app.ConfigureOpenAPIService() - app.Run() -} diff --git a/hack/config-default.sh b/hack/config-default.sh index ce96283ff46f..c6621f3bd43e 100644 --- a/hack/config-default.sh +++ b/hack/config-default.sh @@ -1,5 +1,5 @@ -binaries="cmd/virt-controller cmd/virt-launcher cmd/virt-handler cmd/virt-api cmd/virtctl cmd/fake-qemu-process cmd/virt-dhcp cmd/fake-dnsmasq-process" -docker_images="cmd/virt-controller cmd/virt-launcher cmd/virt-handler cmd/virt-api images/haproxy images/iscsi-demo-target-tgtd images/vm-killer images/libvirt-kubevirt images/spice-proxy cmd/virt-migrator cmd/registry-disk-v1alpha images/cirros-registry-disk-demo cmd/virt-dhcp" +binaries="cmd/virt-controller cmd/virt-launcher cmd/virt-handler cmd/virtctl cmd/fake-qemu-process cmd/virt-dhcp cmd/fake-dnsmasq-process" +docker_images="cmd/virt-controller cmd/virt-launcher cmd/virt-handler images/iscsi-demo-target-tgtd images/vm-killer images/libvirt-kubevirt cmd/virt-migrator cmd/registry-disk-v1alpha images/cirros-registry-disk-demo cmd/virt-dhcp" optional_docker_images="cmd/registry-disk-v1alpha images/fedora-atomic-registry-disk-demo" docker_prefix=kubevirt docker_tag=${DOCKER_TAG:-latest} diff --git a/images/haproxy/Dockerfile b/images/haproxy/Dockerfile deleted file mode 100644 index e1061a637ec0..000000000000 --- a/images/haproxy/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# -# This file is part of the KubeVirt project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Copyright 2017 Red Hat, Inc. -# - -FROM haproxy:1.6-alpine - -MAINTAINER "The KubeVirt Project" - -COPY haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg - -RUN sed -i "s|/run|/haproxy/run|g" /docker-entrypoint.sh -RUN cp /docker-entrypoint.sh /docker-entrypoint-orig.sh - -COPY docker-entrypoint.sh /docker-entrypoint.sh - -RUN addgroup -S haproxy && adduser -S -D -h /haproxy -s /bin/false -G haproxy -g haproxy haproxy - -RUN mkdir /haproxy/run && \ - chgrp -R 0 /haproxy/run && \ - chmod -R g=u /haproxy/run - -USER 1001 diff --git a/images/haproxy/docker-entrypoint.sh b/images/haproxy/docker-entrypoint.sh deleted file mode 100755 index fc7ca73a8b48..000000000000 --- a/images/haproxy/docker-entrypoint.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -# -# This file is part of the KubeVirt project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Copyright 2017 Red Hat, Inc. -# - -set -e -export TOKEN=`cat /var/run/secrets/kubernetes.io/serviceaccount/token` -export DNS=`cat /etc/resolv.conf |grep -i nameserver|head -n1|cut -d ' ' -f2` -/sbin/syslogd -O /dev/stdout && /docker-entrypoint-orig.sh $@ diff --git a/images/haproxy/haproxy.cfg b/images/haproxy/haproxy.cfg deleted file mode 100644 index e04aa7318e29..000000000000 --- a/images/haproxy/haproxy.cfg +++ /dev/null @@ -1,36 +0,0 @@ -resolvers kubernetes - nameserver skydns ${DNS}:53 - resolve_retries 10 - timeout retry 2s - hold valid 30s - -frontend virt_api - bind *:8184 - mode http - acl is_supported_method method POST PUT GET DELETE OPTIONS PATCH - acl is_openapi path_beg /swagger-2.0.0.pb-v1 - acl is_swagger_api path_beg /swagger-ui - acl is_swagger_spec path_beg /swaggerapi/apis/kubevirt.io/ - acl is_kubevirt path_beg /apis/kubevirt.io - http-request add-header Authorization Bearer\ %[env(TOKEN)] - timeout client 1m - use_backend srvs_apiserver if is_openapi - use_backend srvs_kubevirt if is_supported_method is_kubevirt - use_backend srvs_kubevirt if is_swagger_api - use_backend srvs_kubevirt if is_swagger_spec - default_backend srvs_apiserver - -backend srvs_kubevirt - mode http - timeout connect 10s - timeout server 1m - balance roundrobin - server host1 virt-api:8183 resolvers kubernetes - -backend srvs_apiserver - mode http - timeout connect 10s - timeout server 1m - balance roundrobin - server host1 ${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT} check ssl ca-file /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - diff --git a/images/spice-proxy/Dockerfile b/images/spice-proxy/Dockerfile deleted file mode 100644 index ea3fd535d58a..000000000000 --- a/images/spice-proxy/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -FROM fedora:26 - -MAINTAINER "The KubeVirt Project" - -EXPOSE 3128 - -RUN dnf -y install squid \ - && dnf -y clean all - -RUN sed -i -e "s/http_access deny CONNECT !SSL_ports/http_access deny CONNECT !Safe_ports/" /etc/squid/squid.conf -RUN echo "pid_filename /home/proxy/run/squid.pid" >> /etc/squid/squid.conf - -RUN useradd --create-home -s /bin/bash proxy -RUN chgrp -R 0 /var/log/squid && chmod -R g=u /var/log/squid -RUN cp /etc/squid/squid.conf /home/proxy && chown proxy /home/proxy/squid.conf -WORKDIR /home/proxy -RUN chgrp -R 0 /home/proxy && \ - chmod -R g=u /home/proxy - -RUN mkdir /home/proxy/run && \ - chgrp -R 0 /home/proxy/run && \ - chmod -R g=u /home/proxy/run - -USER 1001 - -CMD squid -NCd1 -f /home/proxy/squid.conf diff --git a/manifests/dev/haproxy.yaml.in b/manifests/dev/haproxy.yaml.in deleted file mode 100644 index 84cae2025e19..000000000000 --- a/manifests/dev/haproxy.yaml.in +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: haproxy - namespace: kube-system - labels: - kubevirt.io: "haproxy" -spec: - template: - metadata: - labels: - kubevirt.io: haproxy - spec: - serviceAccountName: kubevirt-infra - containers: - - name: haproxy - image: {{ docker_prefix }}/haproxy:{{ docker_tag }} - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8184 - name: "haproxy" - protocol: "TCP" - livenessProbe: - httpGet: - path: /apis/kubevirt.io/v1alpha1/healthz - port: 8184 - initialDelaySeconds: 5 - periodSeconds: 10 - readinessProbe: - httpGet: - path: /apis/kubevirt.io/v1alpha1/healthz - port: 8184 - initialDelaySeconds: 10 - periodSeconds: 20 - securityContext: - runAsNonRoot: true diff --git a/manifests/dev/squid.yaml.in b/manifests/dev/squid.yaml.in deleted file mode 100644 index 6833c65203f5..000000000000 --- a/manifests/dev/squid.yaml.in +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: spice-proxy - namespace: kube-system - labels: - kubevirt.io: "spice-proxy" -spec: - template: - metadata: - labels: - kubevirt.io: spice-proxy - spec: - containers: - - name: spice-proxy - image: {{ docker_prefix }}/spice-proxy:{{ docker_tag }} - imagePullPolicy: IfNotPresent - ports: - - containerPort: 3128 - name: "spice-proxy" - protocol: "TCP" - securityContext: - runAsNonRoot: true diff --git a/manifests/dev/virt-api.yaml.in b/manifests/dev/virt-api.yaml.in deleted file mode 100644 index 5a0b06055ba4..000000000000 --- a/manifests/dev/virt-api.yaml.in +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: virt-api - namespace: kube-system - labels: - kubevirt.io: "virt-api" -spec: - ports: - - port: 8183 - targetPort: virt-api - selector: - kubevirt.io: virt-api ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: virt-api - namespace: kube-system - labels: - kubevirt.io: "virt-api" -spec: - template: - metadata: - labels: - kubevirt.io: virt-api - spec: - serviceAccountName: kubevirt-infra - containers: - - name: virt-api - image: {{ docker_prefix }}/virt-api:{{ docker_tag }} - imagePullPolicy: IfNotPresent - command: - - "/virt-api" - - "--port" - - "8183" - - "--spice-proxy" - - "http://{{ master_ip }}:3128" - ports: - - containerPort: 8183 - name: "virt-api" - protocol: "TCP" - securityContext: - runAsNonRoot: true diff --git a/manifests/release/kubevirt.yaml.in b/manifests/release/kubevirt.yaml.in index 2f43f3a5c994..a3734a0b8666 100644 --- a/manifests/release/kubevirt.yaml.in +++ b/manifests/release/kubevirt.yaml.in @@ -142,90 +142,6 @@ spec: - vmrs - vmrss --- -# apiserver and virt-api proxy (will be unnecessary soon) -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: haproxy - namespace: kube-system - labels: - kubevirt.io: "haproxy" -spec: - template: - metadata: - labels: - kubevirt.io: haproxy - spec: - serviceAccountName: kubevirt-infra - containers: - - name: haproxy - image: {{ docker_prefix }}/haproxy:{{ docker_tag }} - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8184 - name: "haproxy" - protocol: "TCP" - livenessProbe: - httpGet: - path: /apis/kubevirt.io/v1alpha1/healthz - port: 8184 - initialDelaySeconds: 5 - periodSeconds: 10 - readinessProbe: - httpGet: - path: /apis/kubevirt.io/v1alpha1/healthz - port: 8184 - initialDelaySeconds: 10 - periodSeconds: 20 - securityContext: - runAsNonRoot: true ---- -# virt-api -apiVersion: v1 -kind: Service -metadata: - name: virt-api - namespace: kube-system - labels: - kubevirt.io: "virt-api" -spec: - ports: - - port: 8183 - targetPort: virt-api - selector: - kubevirt.io: virt-api ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: virt-api - namespace: kube-system - labels: - kubevirt.io: "virt-api" -spec: - template: - metadata: - labels: - kubevirt.io: virt-api - spec: - serviceAccountName: kubevirt-infra - containers: - - name: virt-api - image: {{ docker_prefix }}/virt-api:{{ docker_tag }} - imagePullPolicy: IfNotPresent - command: - - "/virt-api" - - "--port" - - "8183" - - "--spice-proxy" - - "$(SPICE_PROXY)" - ports: - - containerPort: 8183 - name: "virt-api" - protocol: "TCP" - securityContext: - runAsNonRoot: true ---- # kubevirt controller apiVersion: extensions/v1beta1 kind: Deployment diff --git a/manifests/release/spice-proxy.yaml.in b/manifests/release/spice-proxy.yaml.in deleted file mode 100644 index 6833c65203f5..000000000000 --- a/manifests/release/spice-proxy.yaml.in +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: spice-proxy - namespace: kube-system - labels: - kubevirt.io: "spice-proxy" -spec: - template: - metadata: - labels: - kubevirt.io: spice-proxy - spec: - containers: - - name: spice-proxy - image: {{ docker_prefix }}/spice-proxy:{{ docker_tag }} - imagePullPolicy: IfNotPresent - ports: - - containerPort: 3128 - name: "spice-proxy" - protocol: "TCP" - securityContext: - runAsNonRoot: true diff --git a/pkg/virt-api/rest/spice.go b/pkg/virt-api/rest/spice.go deleted file mode 100644 index cd1dbdf1f7f7..000000000000 --- a/pkg/virt-api/rest/spice.go +++ /dev/null @@ -1,85 +0,0 @@ -/* - * This file is part of the KubeVirt project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright 2017 Red Hat, Inc. - * - */ - -package rest - -import ( - "flag" - "fmt" - - "github.com/go-kit/kit/endpoint" - "golang.org/x/net/context" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/rest" - - "kubevirt.io/kubevirt/pkg/api/v1" - "kubevirt.io/kubevirt/pkg/middleware" - "kubevirt.io/kubevirt/pkg/rest/endpoints" -) - -var spiceProxy string - -func init() { - // TODO should be reloadable, use configmaps and update on every access? Watch a config file and reload? - flag.StringVar(&spiceProxy, "spice-proxy", "", "Spice proxy to use when spice access is requested") -} - -func NewSpiceEndpoint(cli *rest.RESTClient, gvr schema.GroupVersionResource) endpoint.Endpoint { - return func(ctx context.Context, payload interface{}) (interface{}, error) { - metadata := payload.(*endpoints.Metadata) - obj, err := cli.Get().Namespace(metadata.Namespace).Resource(gvr.Resource).Name(metadata.Name).Do().Get() - if err != nil { - return nil, middleware.NewInternalServerError(err) - } - - vm := obj.(*v1.VirtualMachine) - spice, err := spiceFromVM(vm) - if err != nil { - return nil, err - - } - - return spice, nil - } -} - -func spiceFromVM(vm *v1.VirtualMachine) (*v1.Spice, error) { - - if vm.Status.Phase != v1.Running { - return nil, middleware.NewResourceNotFoundError("VM is not running") - } - - // TODO allow specifying the spice device. For now select the first one. - for _, d := range vm.Status.Graphics { - if d.Type == "spice" { - spice := v1.NewSpice(vm.GetObjectMeta().GetNamespace(), vm.GetObjectMeta().GetName()) - spice.Info = v1.SpiceInfo{ - Type: "spice", - Host: d.Host, - Port: d.Port, - } - if spiceProxy != "" { - spice.Info.Proxy = fmt.Sprintf("%s", spiceProxy) - } - return spice, nil - } - } - - return nil, middleware.NewResourceNotFoundError("No spice device attached to the VM found.") -} From 3a8a0560dc68706acd29acd5dceb848690cc8e6b Mon Sep 17 00:00:00 2001 From: David Vossel Date: Tue, 2 Jan 2018 12:37:50 -0500 Subject: [PATCH 20/37] remove spice api endpoint Signed-off-by: David Vossel --- pkg/virt-api/api.go | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/pkg/virt-api/api.go b/pkg/virt-api/api.go index e806c667791c..5c9ceeb5ff72 100644 --- a/pkg/virt-api/api.go +++ b/pkg/virt-api/api.go @@ -6,7 +6,6 @@ import ( "github.com/emicklei/go-restful" "github.com/emicklei/go-restful-openapi" - kithttp "github.com/go-kit/kit/transport/http" openapispec "github.com/go-openapi/spec" flag "github.com/spf13/pflag" "golang.org/x/net/context" @@ -14,9 +13,6 @@ import ( "kubevirt.io/kubevirt/pkg/api/v1" "kubevirt.io/kubevirt/pkg/healthz" - "kubevirt.io/kubevirt/pkg/kubecli" - mime "kubevirt.io/kubevirt/pkg/rest" - "kubevirt.io/kubevirt/pkg/rest/endpoints" "kubevirt.io/kubevirt/pkg/rest/filter" "kubevirt.io/kubevirt/pkg/service" "kubevirt.io/kubevirt/pkg/virt-api/rest" @@ -63,30 +59,6 @@ func (app *VirtAPIApp) Compose() { log.Fatal(err) } - virtCli, err := kubecli.GetKubevirtClient() - if err != nil { - log.Fatal(err) - } - - // TODO, allow Encoder and Decoders per type and combine the endpoint logic - spice := endpoints.MakeGoRestfulWrapper(endpoints.NewHandlerBuilder().Get(). - Endpoint(rest.NewSpiceEndpoint(virtCli.RestClient(), vmGVR)).Encoder( - endpoints.NewMimeTypeAwareEncoder(endpoints.NewEncodeINIResponse(http.StatusOK), - map[string]kithttp.EncodeResponseFunc{ - mime.MIME_INI: endpoints.NewEncodeINIResponse(http.StatusOK), - mime.MIME_JSON: endpoints.NewEncodeJsonResponse(http.StatusOK), - mime.MIME_YAML: endpoints.NewEncodeYamlResponse(http.StatusOK), - })).Build(ctx)) - - ws.Route(ws.GET(rest.ResourcePath(vmGVR)+rest.SubResourcePath("spice")). - To(spice).Produces(mime.MIME_INI, mime.MIME_JSON, mime.MIME_YAML). - Param(rest.NamespaceParam(ws)).Param(rest.NameParam(ws)). - Operation("spice"). - Doc("Returns a remote-viewer configuration file. Run `man 1 remote-viewer` to learn more about the configuration format."). - Returns(http.StatusOK, "remote-viewer configuration file" /*os.File{}*/, nil)) - // TODO: That os.File doesn't work as I expect. - // I need end up with response_type="file", but I am getting response_type="os.File" - restful.Add(ws) ws.Route(ws.GET("/healthz"). From 77092f5f580dbec3d1e10eec30c38fc2cdb786cc Mon Sep 17 00:00:00 2001 From: David Vossel Date: Tue, 2 Jan 2018 12:45:03 -0500 Subject: [PATCH 21/37] update generated code Signed-off-by: David Vossel --- api/openapi-spec/swagger.json | 74 ------------------- .../virtwrap/api/deepcopy_generated.go | 25 +++++++ pkg/virt-handler/vm.go | 1 + 3 files changed, 26 insertions(+), 74 deletions(-) diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 768ba8fbd0e2..939967324e25 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -1176,80 +1176,6 @@ } } }, - "/apis/kubevirt.io/v1alpha1/namespaces/{namespace}/virtualmachines/{name}/console": { - "get": { - "description": "Open a websocket connection to a serial console on the specified VM.", - "summary": "Open a websocket connection to a serial console on the specified VM.", - "operationId": "console", - "parameters": [ - { - "type": "string", - "description": "Name of the serial console to connect to", - "name": "console", - "in": "query" - }, - { - "pattern": "[a-z0-9][a-z0-9\\-]*", - "type": "string", - "description": "Object name and auth scope, such as for teams and projects", - "name": "namespace", - "in": "path", - "required": true - }, - { - "pattern": "[a-z0-9][a-z0-9\\-]*", - "type": "string", - "description": "Name of the resource", - "name": "name", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK" - } - } - } - }, - "/apis/kubevirt.io/v1alpha1/namespaces/{namespace}/virtualmachines/{name}/spice": { - "get": { - "description": "Returns a remote-viewer configuration file. Run `man 1 remote-viewer` to learn more about the configuration format.", - "produces": [ - "text/plain", - "application/json", - "application/yaml" - ], - "summary": "Returns a remote-viewer configuration file. Run `man 1 remote-viewer` to learn more about the configuration format.", - "operationId": "spice", - "parameters": [ - { - "pattern": "[a-z0-9][a-z0-9\\-]*", - "type": "string", - "description": "Object name and auth scope, such as for teams and projects", - "name": "namespace", - "in": "path", - "required": true - }, - { - "pattern": "[a-z0-9][a-z0-9\\-]*", - "type": "string", - "description": "Name of the resource", - "name": "name", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "remote-viewer configuration file" - }, - "default": { - "description": "remote-viewer configuration file" - } - } - } - }, "/apis/kubevirt.io/v1alpha1/virtualmachinereplicasets": { "get": { "description": "Get a list of all VirtualMachineReplicaSet objects.", diff --git a/pkg/virt-handler/virtwrap/api/deepcopy_generated.go b/pkg/virt-handler/virtwrap/api/deepcopy_generated.go index f8fd399fdb07..9b62edb8e24d 100644 --- a/pkg/virt-handler/virtwrap/api/deepcopy_generated.go +++ b/pkg/virt-handler/virtwrap/api/deepcopy_generated.go @@ -1509,6 +1509,15 @@ func (in *Serial) DeepCopyInto(out *Serial) { (*in).DeepCopyInto(*out) } } + if in.Source != nil { + in, out := &in.Source, &out.Source + if *in == nil { + *out = nil + } else { + *out = new(SerialSource) + **out = **in + } + } return } @@ -1522,6 +1531,22 @@ func (in *Serial) DeepCopy() *Serial { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SerialSource) DeepCopyInto(out *SerialSource) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SerialSource. +func (in *SerialSource) DeepCopy() *SerialSource { + if in == nil { + return nil + } + out := new(SerialSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SerialTarget) DeepCopyInto(out *SerialTarget) { *out = *in diff --git a/pkg/virt-handler/vm.go b/pkg/virt-handler/vm.go index 2256bbf5221c..2645afcfac05 100644 --- a/pkg/virt-handler/vm.go +++ b/pkg/virt-handler/vm.go @@ -40,6 +40,7 @@ import ( "kubevirt.io/kubevirt/pkg/api/v1" "kubevirt.io/kubevirt/pkg/config-disk" "kubevirt.io/kubevirt/pkg/controller" + diskutils "kubevirt.io/kubevirt/pkg/ephemeral-disk-utils" "kubevirt.io/kubevirt/pkg/kubecli" "kubevirt.io/kubevirt/pkg/log" "kubevirt.io/kubevirt/pkg/registry-disk" From 838bf763d38f896cdcbc512564ab08253bcac2c9 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Tue, 2 Jan 2018 14:03:17 -0500 Subject: [PATCH 22/37] Add vnc and console converter code Signed-off-by: David Vossel --- pkg/virt-handler/virtwrap/api/converter.go | 37 +++++++++++++++++++ .../virtwrap/api/converter_test.go | 12 ++++-- pkg/virt-handler/virtwrap/api/defaults.go | 18 +-------- pkg/virt-handler/virtwrap/api/schema_test.go | 3 -- 4 files changed, 47 insertions(+), 23 deletions(-) diff --git a/pkg/virt-handler/virtwrap/api/converter.go b/pkg/virt-handler/virtwrap/api/converter.go index 036ab87ffcfc..99785a14f30c 100644 --- a/pkg/virt-handler/virtwrap/api/converter.go +++ b/pkg/virt-handler/virtwrap/api/converter.go @@ -324,6 +324,43 @@ func Convert_v1_VirtualMachine_To_api_Domain(vm *v1.VirtualMachine, domain *Doma } } + // Add mandatory console device + var serialPort uint = 0 + var serialType string = "serial" + domain.Spec.Devices.Consoles = []Console{ + { + Type: "pty", + Target: &ConsoleTarget{ + Type: &serialType, + Port: &serialPort, + }, + }, + } + + domain.Spec.Devices.Serials = []Serial{ + { + Type: "unix", + Target: &SerialTarget{ + Port: &serialPort, + }, + Source: &SerialSource{ + Mode: "bind", + Path: fmt.Sprintf("/var/run/kubevirt-private/%s/%s/virt-serial%d", vm.ObjectMeta.Namespace, vm.ObjectMeta.Name, serialPort), + }, + }, + } + + // Add mandatory vnc device + domain.Spec.Devices.Graphics = []Graphics{ + { + Listen: Listen{ + Type: "socket", + Socket: fmt.Sprintf("/var/run/kubevirt-private/%s/%s/virt-vnc", vm.ObjectMeta.Namespace, vm.ObjectMeta.Name), + }, + Type: "vnc", + }, + } + return nil } diff --git a/pkg/virt-handler/virtwrap/api/converter_test.go b/pkg/virt-handler/virtwrap/api/converter_test.go index 6541224c0f91..d96f3822a6fe 100644 --- a/pkg/virt-handler/virtwrap/api/converter_test.go +++ b/pkg/virt-handler/virtwrap/api/converter_test.go @@ -270,8 +270,8 @@ var _ = Describe("Converter", func() { - - + + @@ -338,7 +338,13 @@ var _ = Describe("Converter", func() { - + + + + + + + diff --git a/pkg/virt-handler/virtwrap/api/defaults.go b/pkg/virt-handler/virtwrap/api/defaults.go index b880271ccbb9..9c284ccacb6f 100644 --- a/pkg/virt-handler/virtwrap/api/defaults.go +++ b/pkg/virt-handler/virtwrap/api/defaults.go @@ -1,17 +1,6 @@ package api func SetDefaults_Devices(devices *Devices) { - // Add mandatory spice device - devices.Graphics = []Graphics{ - { - Port: -1, - Listen: Listen{ - Type: "address", - Address: "0.0.0.0", - }, - Type: "spice", - }, - } // Use vga as video device, since it is better than cirrus // and does not require guest drivers var heads uint = 1 @@ -25,12 +14,7 @@ func SetDefaults_Devices(devices *Devices) { }, }, } - // Add mandatory console device - devices.Consoles = []Console{ - { - Type: "pty", - }, - } + // For now connect every virtual machine to the default network devices.Interfaces = []Interface{{ Type: "network", diff --git a/pkg/virt-handler/virtwrap/api/schema_test.go b/pkg/virt-handler/virtwrap/api/schema_test.go index 4b08941ab2f1..26216179ffeb 100644 --- a/pkg/virt-handler/virtwrap/api/schema_test.go +++ b/pkg/virt-handler/virtwrap/api/schema_test.go @@ -46,9 +46,6 @@ var exampleXML = ` - - - From 33e37975767b71aa8344d7136f44f26b59848d70 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Tue, 2 Jan 2018 14:36:44 -0500 Subject: [PATCH 23/37] Add libvirt shared directory work around for unix sock ownership Signed-off-by: David Vossel --- pkg/virt-handler/vm.go | 49 ++++++++++++++++++++++++++++++++++--- pkg/virt-handler/vm_test.go | 6 +++++ 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/pkg/virt-handler/vm.go b/pkg/virt-handler/vm.go index 2645afcfac05..03da39edcd68 100644 --- a/pkg/virt-handler/vm.go +++ b/pkg/virt-handler/vm.go @@ -22,6 +22,8 @@ package virthandler import ( goerror "errors" "fmt" + "os" + "path/filepath" "reflect" "time" @@ -79,6 +81,7 @@ func NewController( domainInformer: domainInformer, watchdogInformer: watchdogInformer, gracefulShutdownInformer: gracefulShutdownInformer, + unixSockUser: "qemu", } vmInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -121,6 +124,14 @@ type VirtualMachineController struct { domainInformer cache.SharedInformer watchdogInformer cache.SharedIndexInformer gracefulShutdownInformer cache.SharedIndexInformer + // TODO Remove this once qemu process lives entirely in pod namespaces. + unixSockUser string +} + +// TODO Remove this once qemu process lives in pod namespace +// This function is only used by testing framework. +func (d *VirtualMachineController) overrideUnixSockUser(user string) { + d.unixSockUser = user } // Determines if a domain's grace period has expired during shutdown. @@ -589,21 +600,48 @@ func (d *VirtualMachineController) injectDiskAuth(vm *v1.VirtualMachine) (map[st return secrets, nil } -func (d *VirtualMachineController) cleanupConsoleSockets(vm *v1.VirtualMachine) error { - // TODO this can go away once qemu is in the pods mount namespace. +// TODO this function should go away once qemu is in the pods mount namespace. +func (d *VirtualMachineController) cleanupUnixSockets(vm *v1.VirtualMachine) error { namespace := vm.ObjectMeta.Namespace name := vm.ObjectMeta.Name unixPath := fmt.Sprintf("%s-private/%s/%s", d.virtShareDir, namespace, name) return diskutils.RemoveFile(unixPath) } +// TODO this function should go away once qemu is in the pods mount namespace. +func (d *VirtualMachineController) initializeUnixSockets(vm *v1.VirtualMachine) error { + namespace := vm.ObjectMeta.Namespace + name := vm.ObjectMeta.Name + unixPathVnc := fmt.Sprintf("%s-private/%s/%s/virt-vnc", d.virtShareDir, namespace, name) + unixPathConsole := fmt.Sprintf("%s-private/%s/%s/virt-serial0", d.virtShareDir, namespace, name) + + err := os.MkdirAll(filepath.Dir(unixPathVnc), 0755) + if err != nil { + return err + } + err = os.MkdirAll(filepath.Dir(unixPathConsole), 0755) + if err != nil { + return err + } + err = diskutils.SetFileOwnership(d.unixSockUser, filepath.Dir(unixPathVnc)) + if err != nil { + return err + } + err = diskutils.SetFileOwnership(d.unixSockUser, filepath.Dir(unixPathConsole)) + if err != nil { + return err + } + + return nil +} + func (d *VirtualMachineController) processVmCleanup(vm *v1.VirtualMachine) error { err := d.domainManager.RemoveVMSecrets(vm) if err != nil { return err } - err = d.cleanupConsoleSockets(vm) + err = d.cleanupUnixSockets(vm) if err != nil { return err } @@ -691,6 +729,11 @@ func (d *VirtualMachineController) processVmUpdate(vm *v1.VirtualMachine) error return err } + err = d.initializeUnixSockets(vm) + if err != nil { + return err + } + // TODO MigrationNodeName should be a pointer if vm.Status.MigrationNodeName != "" { // Only sync if the VM is not marked as migrating. diff --git a/pkg/virt-handler/vm_test.go b/pkg/virt-handler/vm_test.go index 26c5f93efc2d..a5bda4cb1544 100644 --- a/pkg/virt-handler/vm_test.go +++ b/pkg/virt-handler/vm_test.go @@ -22,6 +22,7 @@ package virthandler import ( "io/ioutil" "os" + "os/user" "time" "github.com/golang/mock/gomock" @@ -117,6 +118,11 @@ var _ = Describe("VM", func() { watchdogInformer, gracefulShutdownInformer) + owner, err := user.Current() + if err != nil { + panic(err) + } + controller.overrideUnixSockUser(owner.Username) mockQueue = testutils.NewMockWorkQueue(controller.Queue) controller.Queue = mockQueue From 3792ce785e40b7456edb91b93148d5abe5b339cc Mon Sep 17 00:00:00 2001 From: David Vossel Date: Tue, 2 Jan 2018 14:56:58 -0500 Subject: [PATCH 24/37] remove device option from virtctl's console support Signed-off-by: David Vossel --- pkg/virtctl/console/console.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/virtctl/console/console.go b/pkg/virtctl/console/console.go index e7d86c9b87ff..b2e5f4d6fdf6 100644 --- a/pkg/virtctl/console/console.go +++ b/pkg/virtctl/console/console.go @@ -38,7 +38,7 @@ type Console struct { func (c *Console) FlagSet() *flag.FlagSet { cf := flag.NewFlagSet("console", flag.ExitOnError) - cf.StringP("device", "d", "", "Console to connect to") + cf.StringP("device", "d", "serial0", "Console to connect to") return cf } @@ -46,8 +46,8 @@ func (c *Console) FlagSet() *flag.FlagSet { func (c *Console) Usage() string { usage := "Connect to a serial console on a VM:\n\n" usage += "Examples:\n" - usage += "# Connect to the console 'serial0' on the VM 'myvm':\n" - usage += "virtctl console myvm --device serial0\n\n" + usage += "# Connect to the console on VM 'myvm':\n" + usage += "virtctl console myvm\n\n" usage += "Options:\n" usage += c.FlagSet().FlagUsages() return usage @@ -58,7 +58,7 @@ func (c *Console) Run(flags *flag.FlagSet) int { server, _ := flags.GetString("server") kubeconfig, _ := flags.GetString("kubeconfig") namespace, _ := flags.GetString("namespace") - device, _ := flags.GetString("device") + device := "serial0" if namespace == "" { namespace = v1.NamespaceDefault } From 41126ec526ee136f5250e2cf8db42140114ce9fc Mon Sep 17 00:00:00 2001 From: David Vossel Date: Tue, 2 Jan 2018 17:26:23 -0500 Subject: [PATCH 25/37] Make disk util remove recursive Signed-off-by: David Vossel --- pkg/ephemeral-disk-utils/utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/ephemeral-disk-utils/utils.go b/pkg/ephemeral-disk-utils/utils.go index c58538fd5894..d5e76878ab42 100644 --- a/pkg/ephemeral-disk-utils/utils.go +++ b/pkg/ephemeral-disk-utils/utils.go @@ -34,11 +34,11 @@ import ( ) func RemoveFile(path string) error { - err := os.Remove(path) + err := os.RemoveAll(path) if err != nil && os.IsNotExist(err) { return nil } else if err != nil { - log.Log.Reason(err).Errorf("failed to remove cloud-init temporary data file %s", path) + log.Log.Reason(err).Errorf("failed to remove %s", path) return err } return nil From 92611d734821ef8b7e518cd4f89a10afe9f527d4 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Tue, 2 Jan 2018 17:27:24 -0500 Subject: [PATCH 26/37] update domain xml graphic and console related fields Signed-off-by: David Vossel --- pkg/virt-handler/virtwrap/api/converter.go | 2 +- .../virtwrap/api/deepcopy_generated.go | 89 +++++++++++++++---- .../virtwrap/api/deepcopy_test.go | 2 +- pkg/virt-handler/virtwrap/api/schema.go | 24 +++-- 4 files changed, 89 insertions(+), 28 deletions(-) diff --git a/pkg/virt-handler/virtwrap/api/converter.go b/pkg/virt-handler/virtwrap/api/converter.go index 99785a14f30c..739cfd86e6cd 100644 --- a/pkg/virt-handler/virtwrap/api/converter.go +++ b/pkg/virt-handler/virtwrap/api/converter.go @@ -353,7 +353,7 @@ func Convert_v1_VirtualMachine_To_api_Domain(vm *v1.VirtualMachine, domain *Doma // Add mandatory vnc device domain.Spec.Devices.Graphics = []Graphics{ { - Listen: Listen{ + Listen: &GraphicsListen{ Type: "socket", Socket: fmt.Sprintf("/var/run/kubevirt-private/%s/%s/virt-vnc", vm.ObjectMeta.Namespace, vm.ObjectMeta.Name), }, diff --git a/pkg/virt-handler/virtwrap/api/deepcopy_generated.go b/pkg/virt-handler/virtwrap/api/deepcopy_generated.go index 9b62edb8e24d..6609bd2ae622 100644 --- a/pkg/virt-handler/virtwrap/api/deepcopy_generated.go +++ b/pkg/virt-handler/virtwrap/api/deepcopy_generated.go @@ -277,6 +277,24 @@ func (in *Console) DeepCopyInto(out *Console) { (*in).DeepCopyInto(*out) } } + if in.Source != nil { + in, out := &in.Source, &out.Source + if *in == nil { + *out = nil + } else { + *out = new(ConsoleSource) + **out = **in + } + } + if in.Alias != nil { + in, out := &in.Alias, &out.Alias + if *in == nil { + *out = nil + } else { + *out = new(Alias) + **out = **in + } + } return } @@ -290,6 +308,22 @@ func (in *Console) DeepCopy() *Console { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConsoleSource) DeepCopyInto(out *ConsoleSource) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConsoleSource. +func (in *ConsoleSource) DeepCopy() *ConsoleSource { + if in == nil { + return nil + } + out := new(ConsoleSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ConsoleTarget) DeepCopyInto(out *ConsoleTarget) { *out = *in @@ -388,7 +422,9 @@ func (in *Devices) DeepCopyInto(out *Devices) { if in.Graphics != nil { in, out := &in.Graphics, &out.Graphics *out = make([]Graphics, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } if in.Ballooning != nil { in, out := &in.Ballooning, &out.Ballooning @@ -1044,7 +1080,15 @@ func (in *GracePeriodMetadata) DeepCopy() *GracePeriodMetadata { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Graphics) DeepCopyInto(out *Graphics) { *out = *in - out.Listen = in.Listen + if in.Listen != nil { + in, out := &in.Listen, &out.Listen + if *in == nil { + *out = nil + } else { + *out = new(GraphicsListen) + **out = **in + } + } return } @@ -1058,6 +1102,22 @@ func (in *Graphics) DeepCopy() *Graphics { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GraphicsListen) DeepCopyInto(out *GraphicsListen) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GraphicsListen. +func (in *GraphicsListen) DeepCopy() *GraphicsListen { + if in == nil { + return nil + } + out := new(GraphicsListen) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Interface) DeepCopyInto(out *Interface) { *out = *in @@ -1221,22 +1281,6 @@ func (in *LinkState) DeepCopy() *LinkState { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Listen) DeepCopyInto(out *Listen) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Listen. -func (in *Listen) DeepCopy() *Listen { - if in == nil { - return nil - } - out := new(Listen) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Loader) DeepCopyInto(out *Loader) { *out = *in @@ -1518,6 +1562,15 @@ func (in *Serial) DeepCopyInto(out *Serial) { **out = **in } } + if in.Alias != nil { + in, out := &in.Alias, &out.Alias + if *in == nil { + *out = nil + } else { + *out = new(Alias) + **out = **in + } + } return } diff --git a/pkg/virt-handler/virtwrap/api/deepcopy_test.go b/pkg/virt-handler/virtwrap/api/deepcopy_test.go index 0fc7a747fc7e..d93a2fbb9305 100644 --- a/pkg/virt-handler/virtwrap/api/deepcopy_test.go +++ b/pkg/virt-handler/virtwrap/api/deepcopy_test.go @@ -72,7 +72,7 @@ var _ = Describe("Generated deepcopy functions", func() { &Video{}, &VideoModel{}, &Graphics{}, - &Listen{}, + &GraphicsListen{}, &Address{}, &Ballooning{}, &RandomGenerator{}, diff --git a/pkg/virt-handler/virtwrap/api/schema.go b/pkg/virt-handler/virtwrap/api/schema.go index 3d8d3ffb01bd..e1a7d96f472d 100644 --- a/pkg/virt-handler/virtwrap/api/schema.go +++ b/pkg/virt-handler/virtwrap/api/schema.go @@ -255,6 +255,7 @@ type Serial struct { Type string `xml:"type,attr"` Target *SerialTarget `xml:"target,omitempty"` Source *SerialSource `xml:"source,omitempty"` + Alias *Alias `xml:"alias,omitmepty"` } type SerialTarget struct { @@ -273,6 +274,8 @@ type SerialSource struct { type Console struct { Type string `xml:"type,attr"` Target *ConsoleTarget `xml:"target,omitempty"` + Source *ConsoleSource `xml:"source,omitempty"` + Alias *Alias `xml:"alias,omitmepty"` } type ConsoleTarget struct { @@ -280,6 +283,11 @@ type ConsoleTarget struct { Port *uint `xml:"port,attr,omitempty"` } +type ConsoleSource struct { + Mode string `xml:"mode,attr,omitempty"` + Path string `xml:"path,attr,omitempty"` +} + // END Serial ----------------------------- // BEGIN Inteface ----------------------------- @@ -452,16 +460,16 @@ type VideoModel struct { } type Graphics struct { - AutoPort string `xml:"autoPort,attr,omitempty"` - DefaultMode string `xml:"defaultMode,attr,omitempty"` - Listen Listen `xml:"listen,omitempty"` - PasswdValidTo string `xml:"passwdValidTo,attr,omitempty"` - Port int32 `xml:"port,attr,omitempty"` - TLSPort int `xml:"tlsPort,attr,omitempty"` - Type string `xml:"type,attr"` + AutoPort string `xml:"autoPort,attr,omitempty"` + DefaultMode string `xml:"defaultMode,attr,omitempty"` + Listen *GraphicsListen `xml:"listen,omitempty"` + PasswdValidTo string `xml:"passwdValidTo,attr,omitempty"` + Port int32 `xml:"port,attr,omitempty"` + TLSPort int `xml:"tlsPort,attr,omitempty"` + Type string `xml:"type,attr"` } -type Listen struct { +type GraphicsListen struct { Type string `xml:"type,attr"` Address string `xml:"address,attr,omitempty"` Network string `xml:"newtork,attr,omitempty"` From 2b8137582b320a53f6f612139d279c8d9762fc44 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 3 Jan 2018 15:05:40 -0500 Subject: [PATCH 27/37] fix migration support with new vnc and console redesign - set cache=none for iscsi devices - move private dir initialization to virt-launcher Signed-off-by: David Vossel --- cmd/virt-launcher/Dockerfile | 2 + cmd/virt-launcher/virt-launcher.go | 5 +++ pkg/virt-handler/virtwrap/api/converter.go | 1 + .../virtwrap/api/converter_test.go | 12 +++--- pkg/virt-handler/vm.go | 43 ------------------- pkg/virt-handler/vm_test.go | 6 --- pkg/virt-launcher/monitor.go | 24 +++++++++++ 7 files changed, 38 insertions(+), 55 deletions(-) diff --git a/cmd/virt-launcher/Dockerfile b/cmd/virt-launcher/Dockerfile index 68ebd8e801c2..b3dc738a53f2 100644 --- a/cmd/virt-launcher/Dockerfile +++ b/cmd/virt-launcher/Dockerfile @@ -21,6 +21,8 @@ FROM fedora:26 MAINTAINER "The KubeVirt Project" RUN dnf -y install socat && \ + groupadd --gid 107 qemu && \ + useradd --uid 107 --gid 107 qemu && \ dnf -y clean all COPY sock-connector /sock-connector diff --git a/cmd/virt-launcher/virt-launcher.go b/cmd/virt-launcher/virt-launcher.go index bf38ea66d3e5..d3cd034ebb88 100644 --- a/cmd/virt-launcher/virt-launcher.go +++ b/cmd/virt-launcher/virt-launcher.go @@ -89,6 +89,11 @@ func main() { panic(err) } + err = virtlauncher.InitializePrivateDirectories(filepath.Join("/var/run/kubevirt-private", *namespace, *name)) + if err != nil { + panic(err) + } + watchdogFile := watchdog.WatchdogFileFromNamespaceName(*virtShareDir, *namespace, *name) err = watchdog.WatchdogFileUpdate(watchdogFile) if err != nil { diff --git a/pkg/virt-handler/virtwrap/api/converter.go b/pkg/virt-handler/virtwrap/api/converter.go index 739cfd86e6cd..b222e321111b 100644 --- a/pkg/virt-handler/virtwrap/api/converter.go +++ b/pkg/virt-handler/virtwrap/api/converter.go @@ -75,6 +75,7 @@ func Convert_v1_ISCSIVolumeSource_To_api_Disk(source *k8sv1.ISCSIVolumeSource, d disk.Type = "network" disk.Driver.Type = "raw" + disk.Driver.Cache = "none" disk.Source.Name = fmt.Sprintf("%s/%d", source.IQN, source.Lun) disk.Source.Protocol = "iscsi" diff --git a/pkg/virt-handler/virtwrap/api/converter_test.go b/pkg/virt-handler/virtwrap/api/converter_test.go index d96f3822a6fe..e4c0b002adce 100644 --- a/pkg/virt-handler/virtwrap/api/converter_test.go +++ b/pkg/virt-handler/virtwrap/api/converter_test.go @@ -278,7 +278,7 @@ var _ = Describe("Converter", func() { - + @@ -299,7 +299,7 @@ var _ = Describe("Converter", func() { - + @@ -307,7 +307,7 @@ var _ = Describe("Converter", func() { - + @@ -315,7 +315,7 @@ var _ = Describe("Converter", func() { - + @@ -324,7 +324,7 @@ var _ = Describe("Converter", func() { - + @@ -332,7 +332,7 @@ var _ = Describe("Converter", func() { - + diff --git a/pkg/virt-handler/vm.go b/pkg/virt-handler/vm.go index 03da39edcd68..f87c9dfb8ab0 100644 --- a/pkg/virt-handler/vm.go +++ b/pkg/virt-handler/vm.go @@ -22,8 +22,6 @@ package virthandler import ( goerror "errors" "fmt" - "os" - "path/filepath" "reflect" "time" @@ -81,7 +79,6 @@ func NewController( domainInformer: domainInformer, watchdogInformer: watchdogInformer, gracefulShutdownInformer: gracefulShutdownInformer, - unixSockUser: "qemu", } vmInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -124,14 +121,6 @@ type VirtualMachineController struct { domainInformer cache.SharedInformer watchdogInformer cache.SharedIndexInformer gracefulShutdownInformer cache.SharedIndexInformer - // TODO Remove this once qemu process lives entirely in pod namespaces. - unixSockUser string -} - -// TODO Remove this once qemu process lives in pod namespace -// This function is only used by testing framework. -func (d *VirtualMachineController) overrideUnixSockUser(user string) { - d.unixSockUser = user } // Determines if a domain's grace period has expired during shutdown. @@ -608,33 +597,6 @@ func (d *VirtualMachineController) cleanupUnixSockets(vm *v1.VirtualMachine) err return diskutils.RemoveFile(unixPath) } -// TODO this function should go away once qemu is in the pods mount namespace. -func (d *VirtualMachineController) initializeUnixSockets(vm *v1.VirtualMachine) error { - namespace := vm.ObjectMeta.Namespace - name := vm.ObjectMeta.Name - unixPathVnc := fmt.Sprintf("%s-private/%s/%s/virt-vnc", d.virtShareDir, namespace, name) - unixPathConsole := fmt.Sprintf("%s-private/%s/%s/virt-serial0", d.virtShareDir, namespace, name) - - err := os.MkdirAll(filepath.Dir(unixPathVnc), 0755) - if err != nil { - return err - } - err = os.MkdirAll(filepath.Dir(unixPathConsole), 0755) - if err != nil { - return err - } - err = diskutils.SetFileOwnership(d.unixSockUser, filepath.Dir(unixPathVnc)) - if err != nil { - return err - } - err = diskutils.SetFileOwnership(d.unixSockUser, filepath.Dir(unixPathConsole)) - if err != nil { - return err - } - - return nil -} - func (d *VirtualMachineController) processVmCleanup(vm *v1.VirtualMachine) error { err := d.domainManager.RemoveVMSecrets(vm) if err != nil { @@ -729,11 +691,6 @@ func (d *VirtualMachineController) processVmUpdate(vm *v1.VirtualMachine) error return err } - err = d.initializeUnixSockets(vm) - if err != nil { - return err - } - // TODO MigrationNodeName should be a pointer if vm.Status.MigrationNodeName != "" { // Only sync if the VM is not marked as migrating. diff --git a/pkg/virt-handler/vm_test.go b/pkg/virt-handler/vm_test.go index a5bda4cb1544..26c5f93efc2d 100644 --- a/pkg/virt-handler/vm_test.go +++ b/pkg/virt-handler/vm_test.go @@ -22,7 +22,6 @@ package virthandler import ( "io/ioutil" "os" - "os/user" "time" "github.com/golang/mock/gomock" @@ -118,11 +117,6 @@ var _ = Describe("VM", func() { watchdogInformer, gracefulShutdownInformer) - owner, err := user.Current() - if err != nil { - panic(err) - } - controller.overrideUnixSockUser(owner.Username) mockQueue = testutils.NewMockWorkQueue(controller.Queue) controller.Queue = mockQueue diff --git a/pkg/virt-launcher/monitor.go b/pkg/virt-launcher/monitor.go index a351d1f7f191..e1fbc241c5f6 100644 --- a/pkg/virt-launcher/monitor.go +++ b/pkg/virt-launcher/monitor.go @@ -106,6 +106,30 @@ func GracefulShutdownTriggerInitiate(triggerFile string) error { return nil } +func InitializePrivateDirectories(baseDir string) error { + unixPathVnc := filepath.Join(baseDir, "virt-vnc") + unixPathConsole := filepath.Join(baseDir, "virt-serial0") + + err := os.MkdirAll(filepath.Dir(unixPathVnc), 0755) + if err != nil { + return err + } + err = os.MkdirAll(filepath.Dir(unixPathConsole), 0755) + if err != nil { + return err + } + err = diskutils.SetFileOwnership("qemu", filepath.Dir(unixPathVnc)) + if err != nil { + return err + } + err = diskutils.SetFileOwnership("qemu", filepath.Dir(unixPathConsole)) + if err != nil { + return err + } + + return nil +} + func InitializeSharedDirectories(baseDir string) error { err := os.MkdirAll(watchdog.WatchdogFileDirectory(baseDir), 0755) if err != nil { From fa26caeb57aa53d517b434de6a5a20597cf828e1 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 3 Jan 2018 15:39:50 -0500 Subject: [PATCH 28/37] Remove usage of --core for cluster/kubectl.sh Signed-off-by: David Vossel --- Makefile | 2 +- automation/test.sh | 2 +- cluster/kubectl.sh | 6 +----- cluster/vm-isolation-check.sh | 2 +- docs/debugging.md | 14 -------------- hack/config-default.sh | 1 - hack/config.sh | 4 ++-- 7 files changed, 6 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index ffb69b1bda09..b60eed4e8711 100644 --- a/Makefile +++ b/Makefile @@ -93,7 +93,7 @@ vagrant-sync-optional: ./cluster/vagrant/sync_build.sh 'build optional' vagrant-deploy: vagrant-sync-config vagrant-sync-build - export KUBECTL="cluster/kubectl.sh --core" && ./cluster/deploy.sh + export KUBECTL="cluster/kubectl.sh" && ./cluster/deploy.sh .release-functest: make functest > .release-functest 2>&1 diff --git a/automation/test.sh b/automation/test.sh index a83d567229d0..a3019e4f293c 100644 --- a/automation/test.sh +++ b/automation/test.sh @@ -25,7 +25,7 @@ set -ex -kubectl() { cluster/kubectl.sh --core "$@"; } +kubectl() { cluster/kubectl.sh "$@"; } if [ "$TARGET" = "vagrant-dev" ]; then cat > hack/config-local.sh < Date: Wed, 3 Jan 2018 16:14:13 -0500 Subject: [PATCH 29/37] use io.copy for vnc/console logic in virtctl Signed-off-by: David Vossel --- pkg/virtctl/console/console.go | 17 +--------------- pkg/virtctl/vnc/vnc.go | 36 ++-------------------------------- 2 files changed, 3 insertions(+), 50 deletions(-) diff --git a/pkg/virtctl/console/console.go b/pkg/virtctl/console/console.go index b2e5f4d6fdf6..c1c2a747e06e 100644 --- a/pkg/virtctl/console/console.go +++ b/pkg/virtctl/console/console.go @@ -109,22 +109,7 @@ func (c *Console) Run(flags *flag.FlagSet) int { go func() { defer close(readStop) - buf := make([]byte, 1024, 1024) - for { - // reading stdout from console - n, err := stdoutReader.Read(buf) - if err != nil && err != io.EOF { - return - } - if n == 0 && err == io.EOF { - return - } - // Writing to stdout - _, err = out.Write(buf[0:n]) - if err == io.EOF { - return - } - } + io.Copy(out, stdoutReader) }() go func() { diff --git a/pkg/virtctl/vnc/vnc.go b/pkg/virtctl/vnc/vnc.go index 4d9b2c01c84e..1669419de4af 100644 --- a/pkg/virtctl/vnc/vnc.go +++ b/pkg/virtctl/vnc/vnc.go @@ -114,45 +114,13 @@ func (o *Vnc) Run(flags *flag.FlagSet) int { // write to FD <- pipeOutReader go func() { defer close(readStop) - buf := make([]byte, 4096, 4096) - for { - // reading qemu vnc server - n, err := pipeOutReader.Read(buf) - if err != nil && err != io.EOF { - return - } - if n == 0 && err == io.EOF { - return - } - - // Writing to remote viewer - _, err = fd.Write(buf[0:n]) - if err == io.EOF { - return - } - } + io.Copy(fd, pipeOutReader) }() // read from FD -> pipeInWriter go func() { defer close(writeStop) - buf := make([]byte, 4096, 4096) - for { - // reading from remoteViewer - n, err := fd.Read(buf) - if err != nil && err != io.EOF { - return - } - if n == 0 && err == io.EOF { - return - } - - // Writing out to the qemu vnc server - _, err = pipeInWriter.Write(buf[0:n]) - if err == io.EOF { - return - } - } + io.Copy(pipeInWriter, fd) }() select { From 7120bd5308b91d6320bc0edd9ab98effdc87a516 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 3 Jan 2018 16:21:11 -0500 Subject: [PATCH 30/37] use VNC instead of Vnc when naming functions and objects Signed-off-by: David Vossel --- cmd/virtctl/virtctl.go | 2 +- pkg/kubecli/generated_mock_kubevirt.go | 8 ++++---- pkg/kubecli/kubevirt.go | 2 +- pkg/kubecli/vm.go | 2 +- pkg/virt-launcher/monitor.go | 6 +++--- pkg/virtctl/vnc/vnc.go | 10 +++++----- tests/vnc_test.go | 2 +- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cmd/virtctl/virtctl.go b/cmd/virtctl/virtctl.go index 975bb6a63b56..ecddf1a10b95 100644 --- a/cmd/virtctl/virtctl.go +++ b/cmd/virtctl/virtctl.go @@ -40,7 +40,7 @@ func main() { registry := map[string]virtctl.App{ "console": &console.Console{}, "options": &virtctl.Options{}, - "vnc": &vnc.Vnc{}, + "vnc": &vnc.VNC{}, } if len(os.Args) > 1 { diff --git a/pkg/kubecli/generated_mock_kubevirt.go b/pkg/kubecli/generated_mock_kubevirt.go index f3d562c828d8..c7c958d5f855 100644 --- a/pkg/kubecli/generated_mock_kubevirt.go +++ b/pkg/kubecli/generated_mock_kubevirt.go @@ -601,14 +601,14 @@ func (_mr *_MockVMInterfaceRecorder) SerialConsole(arg0, arg1, arg2, arg3 interf return _mr.mock.ctrl.RecordCall(_mr.mock, "SerialConsole", arg0, arg1, arg2, arg3) } -func (_m *MockVMInterface) Vnc(name string, in io.Reader, out io.Writer) error { - ret := _m.ctrl.Call(_m, "Vnc", name, in, out) +func (_m *MockVMInterface) VNC(name string, in io.Reader, out io.Writer) error { + ret := _m.ctrl.Call(_m, "VNC", name, in, out) ret0, _ := ret[0].(error) return ret0 } -func (_mr *_MockVMInterfaceRecorder) Vnc(arg0, arg1, arg2 interface{}) *gomock.Call { - return _mr.mock.ctrl.RecordCall(_mr.mock, "Vnc", arg0, arg1, arg2) +func (_mr *_MockVMInterfaceRecorder) VNC(arg0, arg1, arg2 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "VNC", arg0, arg1, arg2) } // Mock of ReplicaSetInterface interface diff --git a/pkg/kubecli/kubevirt.go b/pkg/kubecli/kubevirt.go index 4e3077d17b8a..6ff7df10ab47 100644 --- a/pkg/kubecli/kubevirt.go +++ b/pkg/kubecli/kubevirt.go @@ -64,7 +64,7 @@ type VMInterface interface { Delete(name string, options *k8smetav1.DeleteOptions) error Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.VirtualMachine, err error) SerialConsole(name string, device string, in io.Reader, out io.Writer) error - Vnc(name string, in io.Reader, out io.Writer) error + VNC(name string, in io.Reader, out io.Writer) error } type ReplicaSetInterface interface { diff --git a/pkg/kubecli/vm.go b/pkg/kubecli/vm.go index 55be949dd7ce..ac86d189b7b8 100644 --- a/pkg/kubecli/vm.go +++ b/pkg/kubecli/vm.go @@ -135,7 +135,7 @@ func (v *vms) remoteExecHelper(name string, cmd []string, in io.Reader, out io.W }) } -func (v *vms) Vnc(name string, in io.Reader, out io.Writer) error { +func (v *vms) VNC(name string, in io.Reader, out io.Writer) error { cmd := []string{"/sock-connector", fmt.Sprintf("/var/run/kubevirt-private/%s/%s/virt-vnc", v.namespace, name)} return v.remoteExecHelper(name, cmd, in, out) } diff --git a/pkg/virt-launcher/monitor.go b/pkg/virt-launcher/monitor.go index e1fbc241c5f6..ecffa9074f37 100644 --- a/pkg/virt-launcher/monitor.go +++ b/pkg/virt-launcher/monitor.go @@ -107,10 +107,10 @@ func GracefulShutdownTriggerInitiate(triggerFile string) error { } func InitializePrivateDirectories(baseDir string) error { - unixPathVnc := filepath.Join(baseDir, "virt-vnc") + unixPathVNC := filepath.Join(baseDir, "virt-vnc") unixPathConsole := filepath.Join(baseDir, "virt-serial0") - err := os.MkdirAll(filepath.Dir(unixPathVnc), 0755) + err := os.MkdirAll(filepath.Dir(unixPathVNC), 0755) if err != nil { return err } @@ -118,7 +118,7 @@ func InitializePrivateDirectories(baseDir string) error { if err != nil { return err } - err = diskutils.SetFileOwnership("qemu", filepath.Dir(unixPathVnc)) + err = diskutils.SetFileOwnership("qemu", filepath.Dir(unixPathVNC)) if err != nil { return err } diff --git a/pkg/virtctl/vnc/vnc.go b/pkg/virtctl/vnc/vnc.go index 1669419de4af..634f5adef81e 100644 --- a/pkg/virtctl/vnc/vnc.go +++ b/pkg/virtctl/vnc/vnc.go @@ -36,9 +36,9 @@ import ( const FLAG = "vnc" -type Vnc struct{} +type VNC struct{} -func (o *Vnc) Run(flags *flag.FlagSet) int { +func (o *VNC) Run(flags *flag.FlagSet) int { server, _ := flags.GetString("server") kubeconfig, _ := flags.GetString("kubeconfig") namespace, _ := flags.GetString("namespace") @@ -81,7 +81,7 @@ func (o *Vnc) Run(flags *flag.FlagSet) int { // setup connection with VM go func() { - err := virtCli.VM(namespace).Vnc(vm, pipeInReader, pipeOutWriter) + err := virtCli.VM(namespace).VNC(vm, pipeInReader, pipeOutWriter) k8ResChan <- err }() @@ -138,13 +138,13 @@ func (o *Vnc) Run(flags *flag.FlagSet) int { return 0 } -func (o *Vnc) Usage() string { +func (o *VNC) Usage() string { usage := "virtctl can connect via remote-viewer to a VM\n\n" usage += "Examples:\n" usage += "# Connect to testvm via remote-viewer\n" usage += "./virtctl vnc testvm\n\n" return usage } -func (o *Vnc) FlagSet() *flag.FlagSet { +func (o *VNC) FlagSet() *flag.FlagSet { return flag.NewFlagSet(FLAG, flag.ExitOnError) } diff --git a/tests/vnc_test.go b/tests/vnc_test.go index ed0abb24e705..bf369d3c415f 100644 --- a/tests/vnc_test.go +++ b/tests/vnc_test.go @@ -57,7 +57,7 @@ var _ = Describe("Vmlifecycle", func() { readStop := make(chan string) go func() { - err := virtClient.VM(vm.ObjectMeta.Namespace).Vnc(vm.ObjectMeta.Name, pipeInReader, pipeOutWriter) + err := virtClient.VM(vm.ObjectMeta.Namespace).VNC(vm.ObjectMeta.Name, pipeInReader, pipeOutWriter) k8ResChan <- err }() // write to FD <- pipeOutReader From b1f3b62e948f6e9cda4e48c9c934f91507330120 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 3 Jan 2018 16:32:54 -0500 Subject: [PATCH 31/37] update vnc test Signed-off-by: David Vossel --- pkg/kubecli/vm.go | 1 + tests/vnc_test.go | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/pkg/kubecli/vm.go b/pkg/kubecli/vm.go index ac86d189b7b8..14e6750deb8d 100644 --- a/pkg/kubecli/vm.go +++ b/pkg/kubecli/vm.go @@ -39,6 +39,7 @@ import ( "k8s.io/apimachinery/pkg/types" "kubevirt.io/kubevirt/pkg/api/v1" + "kubevirt.io/kubevirt/pkg/log" ) func (k *kubevirt) VM(namespace string) VMInterface { diff --git a/tests/vnc_test.go b/tests/vnc_test.go index bf369d3c415f..b34e1c76cf03 100644 --- a/tests/vnc_test.go +++ b/tests/vnc_test.go @@ -28,6 +28,7 @@ import ( . "github.com/onsi/gomega" "kubevirt.io/kubevirt/pkg/kubecli" + "kubevirt.io/kubevirt/pkg/log" "kubevirt.io/kubevirt/tests" ) @@ -66,9 +67,11 @@ var _ = Describe("Vmlifecycle", func() { // reading qemu vnc server n, err := pipeOutReader.Read(buf) if err != nil && err != io.EOF { + log.Log.Reason(err).Error("error while reading from vnc socket.") return } if n == 0 && err == io.EOF { + log.Log.Error("zero bytes read from vnc socket.") return } readStop <- strings.TrimSpace(string(buf[0:n])) @@ -81,6 +84,9 @@ var _ = Describe("Vmlifecycle", func() { case err = <-k8ResChan: } + // This is the response capture by wireshark when the VNC server is contacted. + // This verifies that the test is able to establish a connection with VNC and + // communicate. Expect(response).To(Equal("RFB 003.008")) Expect(err).To(BeNil()) close(done) From 44e5f61c39aeee586e24513436b09d477d40d3bd Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 3 Jan 2018 16:48:16 -0500 Subject: [PATCH 32/37] Remove graphics section from VM status Signed-off-by: David Vossel --- api/openapi-spec/swagger.json | 26 ------------------------- pkg/api/v1/deepcopy_generated.go | 21 -------------------- pkg/api/v1/deepcopy_test.go | 1 - pkg/api/v1/schema_test.go | 4 +--- pkg/api/v1/types.go | 8 -------- pkg/api/v1/types_swagger_generated.go | 5 ----- pkg/kubecli/vm.go | 1 - pkg/virt-handler/vm.go | 28 --------------------------- pkg/virt-handler/vm_test.go | 3 --- tests/utils.go | 3 +-- 10 files changed, 2 insertions(+), 98 deletions(-) diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 939967324e25..c855061ac0c7 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -2911,25 +2911,6 @@ } } }, - "v1.VirtualMachineGraphics": { - "required": [ - "type", - "host", - "port" - ], - "properties": { - "host": { - "type": "string" - }, - "port": { - "type": "integer", - "format": "int32" - }, - "type": { - "type": "string" - } - } - }, "v1.VirtualMachineList": { "description": "VirtualMachineList is a list of VirtualMachines\n+k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object", "required": [ @@ -3046,13 +3027,6 @@ "$ref": "#/definitions/v1.VirtualMachineCondition" } }, - "graphics": { - "description": "Graphics represent the details of available graphical consoles.", - "type": "array", - "items": { - "$ref": "#/definitions/v1.VirtualMachineGraphics" - } - }, "migrationNodeName": { "description": "MigrationNodeName is the node where the VM is live migrating to.", "type": "string" diff --git a/pkg/api/v1/deepcopy_generated.go b/pkg/api/v1/deepcopy_generated.go index 3b521384f8fb..e21d3e3a9ab0 100644 --- a/pkg/api/v1/deepcopy_generated.go +++ b/pkg/api/v1/deepcopy_generated.go @@ -1221,22 +1221,6 @@ func (in *VirtualMachineCondition) DeepCopy() *VirtualMachineCondition { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *VirtualMachineGraphics) DeepCopyInto(out *VirtualMachineGraphics) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachineGraphics. -func (in *VirtualMachineGraphics) DeepCopy() *VirtualMachineGraphics { - if in == nil { - return nil - } - out := new(VirtualMachineGraphics) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VirtualMachineList) DeepCopyInto(out *VirtualMachineList) { *out = *in @@ -1393,11 +1377,6 @@ func (in *VirtualMachineStatus) DeepCopyInto(out *VirtualMachineStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.Graphics != nil { - in, out := &in.Graphics, &out.Graphics - *out = make([]VirtualMachineGraphics, len(*in)) - copy(*out, *in) - } return } diff --git a/pkg/api/v1/deepcopy_test.go b/pkg/api/v1/deepcopy_test.go index 276f9de37420..bb941f61e923 100644 --- a/pkg/api/v1/deepcopy_test.go +++ b/pkg/api/v1/deepcopy_test.go @@ -51,7 +51,6 @@ var _ = Describe("Generated deepcopy functions", func() { &VirtualMachineSpec{}, &Affinity{}, &VirtualMachineStatus{}, - &VirtualMachineGraphics{}, &VirtualMachineCondition{}, &Spice{}, &SpiceInfo{}, diff --git a/pkg/api/v1/schema_test.go b/pkg/api/v1/schema_test.go index 97bd6f1d82b6..a121dd0194ea 100644 --- a/pkg/api/v1/schema_test.go +++ b/pkg/api/v1/schema_test.go @@ -177,9 +177,7 @@ var exampleJSON = `{ } ] }, - "status": { - "graphics": null - } + "status": {} }` var _ = Describe("Schema", func() { diff --git a/pkg/api/v1/types.go b/pkg/api/v1/types.go index c1c31e439b6b..323c0bf2516a 100644 --- a/pkg/api/v1/types.go +++ b/pkg/api/v1/types.go @@ -149,14 +149,6 @@ type VirtualMachineStatus struct { Conditions []VirtualMachineCondition `json:"conditions,omitempty"` // Phase is the status of the VM in kubernetes world. It is not the VM status, but partially correlates to it. Phase VMPhase `json:"phase,omitempty"` - // Graphics represent the details of available graphical consoles. - Graphics []VirtualMachineGraphics `json:"graphics" optional:"true"` -} - -type VirtualMachineGraphics struct { - Type string `json:"type"` - Host string `json:"host"` - Port int32 `json:"port"` } // Required to satisfy Object interface diff --git a/pkg/api/v1/types_swagger_generated.go b/pkg/api/v1/types_swagger_generated.go index f1da0727ffbb..347ca7efacf9 100644 --- a/pkg/api/v1/types_swagger_generated.go +++ b/pkg/api/v1/types_swagger_generated.go @@ -41,14 +41,9 @@ func (VirtualMachineStatus) SwaggerDoc() map[string]string { "migrationNodeName": "MigrationNodeName is the node where the VM is live migrating to.", "conditions": "Conditions are specific points in VM's pod runtime.", "phase": "Phase is the status of the VM in kubernetes world. It is not the VM status, but partially correlates to it.", - "graphics": "Graphics represent the details of available graphical consoles.", } } -func (VirtualMachineGraphics) SwaggerDoc() map[string]string { - return map[string]string{} -} - func (VirtualMachineCondition) SwaggerDoc() map[string]string { return map[string]string{} } diff --git a/pkg/kubecli/vm.go b/pkg/kubecli/vm.go index 14e6750deb8d..ac86d189b7b8 100644 --- a/pkg/kubecli/vm.go +++ b/pkg/kubecli/vm.go @@ -39,7 +39,6 @@ import ( "k8s.io/apimachinery/pkg/types" "kubevirt.io/kubevirt/pkg/api/v1" - "kubevirt.io/kubevirt/pkg/log" ) func (k *kubevirt) VM(namespace string) VMInterface { diff --git a/pkg/virt-handler/vm.go b/pkg/virt-handler/vm.go index f87c9dfb8ab0..c4e624b123c7 100644 --- a/pkg/virt-handler/vm.go +++ b/pkg/virt-handler/vm.go @@ -206,38 +206,10 @@ func (d *VirtualMachineController) updateVMStatus(vm *v1.VirtualMachine, domain } oldStatus := vm.DeepCopy().Status - // Make sure that we always deal with an empty instance for later equality checks - if oldStatus.Graphics == nil { - oldStatus.Graphics = []v1.VirtualMachineGraphics{} - } // Calculate the new VM state based on what libvirt reported d.setVmPhaseForStatusReason(domain, vm) - vm.Status.Graphics = []v1.VirtualMachineGraphics{} - - // Update devices if device status changed - // TODO needs caching, better position or init fetch - if domain != nil { - nodeIP, err := d.getVMNodeAddress(vm) - if err != nil { - return err - } - - vm.Status.Graphics = []v1.VirtualMachineGraphics{} - for _, src := range domain.Spec.Devices.Graphics { - if (src.Type != "spice" && src.Type != "vnc") || src.Port == -1 { - continue - } - dst := v1.VirtualMachineGraphics{ - Type: src.Type, - Host: nodeIP, - Port: src.Port, - } - vm.Status.Graphics = append(vm.Status.Graphics, dst) - } - } - d.checkFailure(vm, syncError, "Synchronizing with the Domain failed.") if !reflect.DeepEqual(oldStatus, vm.Status) { diff --git a/pkg/virt-handler/vm_test.go b/pkg/virt-handler/vm_test.go index 26c5f93efc2d..c87ba486a19c 100644 --- a/pkg/virt-handler/vm_test.go +++ b/pkg/virt-handler/vm_test.go @@ -263,7 +263,6 @@ var _ = Describe("VM", func() { vm := v1.NewMinimalVM("testvm") vm.ObjectMeta.ResourceVersion = "1" vm.Status.Phase = v1.Scheduled - vm.Status.Graphics = []v1.VirtualMachineGraphics{} mockWatchdog.CreateFile(vm) domain := api.NewMinimalDomain("testvm") @@ -278,7 +277,6 @@ var _ = Describe("VM", func() { vm := v1.NewMinimalVM("testvm") vm.ObjectMeta.ResourceVersion = "1" vm.Status.Phase = v1.Scheduled - vm.Status.Graphics = []v1.VirtualMachineGraphics{} updatedVM := vm.DeepCopy() updatedVM.Status.Phase = v1.Running @@ -322,7 +320,6 @@ var _ = Describe("VM", func() { It("should remove an error condition if a synchronization run succeeds", func() { vm := v1.NewMinimalVM("testvm") vm.ObjectMeta.ResourceVersion = "1" - vm.Status.Graphics = []v1.VirtualMachineGraphics{} vm.Status.Phase = v1.Scheduled vm.Status.Conditions = []v1.VirtualMachineCondition{ { diff --git a/tests/utils.go b/tests/utils.go index d568f55d71e5..492b96112a98 100644 --- a/tests/utils.go +++ b/tests/utils.go @@ -680,8 +680,7 @@ func WaitForSuccessfulVMStartWithTimeout(vm runtime.Object, seconds int) (nodeNa nodeName = fetchedVM.Status.NodeName // wait on both phase and graphics - if len(fetchedVM.Status.Graphics) == 1 && - fetchedVM.Status.Phase == v1.Running { + if fetchedVM.Status.Phase == v1.Running { return true } return false From 941881dc08f40db93d1bc7bb2a7190031e98e974 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 3 Jan 2018 17:14:48 -0500 Subject: [PATCH 33/37] Document source of race condition Signed-off-by: David Vossel --- pkg/virt-handler/vm.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/virt-handler/vm.go b/pkg/virt-handler/vm.go index c4e624b123c7..734e86e8d146 100644 --- a/pkg/virt-handler/vm.go +++ b/pkg/virt-handler/vm.go @@ -566,6 +566,7 @@ func (d *VirtualMachineController) cleanupUnixSockets(vm *v1.VirtualMachine) err namespace := vm.ObjectMeta.Namespace name := vm.ObjectMeta.Name unixPath := fmt.Sprintf("%s-private/%s/%s", d.virtShareDir, namespace, name) + // when this is removed, it will fix issue #626 return diskutils.RemoveFile(unixPath) } From c51015b2a5c6e6ba3a64fe41aa732996dbb0fd40 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Wed, 3 Jan 2018 17:36:15 -0500 Subject: [PATCH 34/37] Add additional error information for vmc/console remote execution Signed-off-by: David Vossel --- pkg/kubecli/vm.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/kubecli/vm.go b/pkg/kubecli/vm.go index ac86d189b7b8..c26bd473a433 100644 --- a/pkg/kubecli/vm.go +++ b/pkg/kubecli/vm.go @@ -84,12 +84,12 @@ func (v *vms) remoteExecHelper(name string, cmd []string, in io.Reader, out io.W podName, err := findPod(v.clientSet, v.namespace, name) if err != nil { - return err + return fmt.Errorf("unable to find matching pod for remote execution: %v", err) } config, err := clientcmd.BuildConfigFromFlags(v.master, v.kubeconfig) if err != nil { - return err + return fmt.Errorf("unable to build api config for remote execution: %v", err) } gv := k8sv1.SchemeGroupVersion @@ -99,7 +99,7 @@ func (v *vms) remoteExecHelper(name string, cmd []string, in io.Reader, out io.W restClient, err := restclient.RESTClientFor(config) if err != nil { - return err + return fmt.Errorf("unable to create restClient for remote execution: %v", err) } containerName := "compute" req := restClient.Post(). @@ -123,7 +123,7 @@ func (v *vms) remoteExecHelper(name string, cmd []string, in io.Reader, out io.W url := req.URL() exec, err := remotecommand.NewSPDYExecutor(config, method, url) if err != nil { - return err + return fmt.Errorf("remote execution failed: %v", err) } return exec.Stream(remotecommand.StreamOptions{ From 50c5e3507c04a0fefa0f1df71c776d95f7e88371 Mon Sep 17 00:00:00 2001 From: David Vossel Date: Thu, 4 Jan 2018 08:53:02 -0500 Subject: [PATCH 35/37] s/spice/vnc for func test description Signed-off-by: David Vossel --- tests/vnc_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/vnc_test.go b/tests/vnc_test.go index b34e1c76cf03..9862f923c293 100644 --- a/tests/vnc_test.go +++ b/tests/vnc_test.go @@ -43,7 +43,7 @@ var _ = Describe("Vmlifecycle", func() { tests.BeforeTestCleanup() }) - Context("New VM with a spice connection given", func() { + Context("New VM with a vnc connection given", func() { It("should allow accessing the vnc device on the VM", func(done Done) { vm := tests.NewRandomVM() Expect(virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do().Error()).To(Succeed()) From 5a6553301830f850916df6c130adaa955c57d19f Mon Sep 17 00:00:00 2001 From: David Vossel Date: Thu, 4 Jan 2018 09:11:14 -0500 Subject: [PATCH 36/37] add kubevirt-private volume to release manifests Signed-off-by: David Vossel --- manifests/dev/libvirt.yaml.in | 2 -- manifests/release/kubevirt.yaml.in | 10 ++++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/manifests/dev/libvirt.yaml.in b/manifests/dev/libvirt.yaml.in index f48cb4c5577a..a7bf9d0c9cfa 100644 --- a/manifests/dev/libvirt.yaml.in +++ b/manifests/dev/libvirt.yaml.in @@ -60,8 +60,6 @@ spec: mountPath: /var/run/libvirt - name: virt-share-dir mountPath: /var/run/kubevirt - - name: virt-private-dir - mountPath: /var/run/kubevirt-private command: ["/virt-dhcp"] volumes: - name: libvirt-data diff --git a/manifests/release/kubevirt.yaml.in b/manifests/release/kubevirt.yaml.in index a3734a0b8666..4b52ff6512b3 100644 --- a/manifests/release/kubevirt.yaml.in +++ b/manifests/release/kubevirt.yaml.in @@ -229,6 +229,8 @@ spec: mountPath: /var/run/libvirt - name: virt-share-dir mountPath: /var/run/kubevirt + - name: virt-private-dir + mountPath: /var/run/kubevirt-private env: - name: NODE_NAME valueFrom: @@ -241,6 +243,9 @@ spec: - name: virt-share-dir hostPath: path: /var/run/kubevirt + - name: virt-private-dir + hostPath: + path: /var/run/kubevirt-private --- # libvirt daemon set apiVersion: extensions/v1beta1 @@ -284,6 +289,8 @@ spec: mountPath: /var/run/libvirt - name: virt-share-dir mountPath: /var/run/kubevirt + - name: virt-private-dir + mountPath: /var/run/kubevirt-private command: ["/libvirtd.sh"] - name: virtlogd image: {{ docker_prefix }}/libvirt-kubevirt:{{ docker_tag }} @@ -308,3 +315,6 @@ spec: - name: virt-share-dir hostPath: path: /var/run/kubevirt + - name: virt-private-dir + hostPath: + path: /var/run/kubevirt-private From c3957712632eae26a959fd28cf1773caad430c5e Mon Sep 17 00:00:00 2001 From: David Vossel Date: Thu, 4 Jan 2018 10:38:59 -0500 Subject: [PATCH 37/37] report io.Copy errors for virtctl vnc/console Signed-off-by: David Vossel --- pkg/virtctl/console/console.go | 13 +++++++------ pkg/virtctl/vnc/vnc.go | 16 ++++++++-------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/pkg/virtctl/console/console.go b/pkg/virtctl/console/console.go index c1c2a747e06e..010c7f9df205 100644 --- a/pkg/virtctl/console/console.go +++ b/pkg/virtctl/console/console.go @@ -92,8 +92,8 @@ func (c *Console) Run(flags *flag.FlagSet) int { resChan := make(chan error) stopChan := make(chan struct{}, 1) - writeStop := make(chan struct{}) - readStop := make(chan struct{}) + writeStop := make(chan error) + readStop := make(chan error) go func() { err := virtCli.VM(namespace).SerialConsole(vm, device, stdinReader, stdoutWriter) @@ -108,8 +108,8 @@ func (c *Console) Run(flags *flag.FlagSet) int { }() go func() { - defer close(readStop) - io.Copy(out, stdoutReader) + _, err := io.Copy(out, stdoutReader) + readStop <- err }() go func() { @@ -119,6 +119,7 @@ func (c *Console) Run(flags *flag.FlagSet) int { // reading from stdin n, err := in.Read(buf) if err != nil && err != io.EOF { + writeStop <- err return } if n == 0 && err == io.EOF { @@ -139,8 +140,8 @@ func (c *Console) Run(flags *flag.FlagSet) int { select { case <-stopChan: - case <-readStop: - case <-writeStop: + case err = <-readStop: + case err = <-writeStop: case err = <-resChan: } diff --git a/pkg/virtctl/vnc/vnc.go b/pkg/virtctl/vnc/vnc.go index 634f5adef81e..5e8b00a41c50 100644 --- a/pkg/virtctl/vnc/vnc.go +++ b/pkg/virtctl/vnc/vnc.go @@ -67,8 +67,8 @@ func (o *VNC) Run(flags *flag.FlagSet) int { k8ResChan := make(chan error) viewResChan := make(chan error) stopChan := make(chan struct{}, 1) - writeStop := make(chan struct{}) - readStop := make(chan struct{}) + writeStop := make(chan error) + readStop := make(chan error) // The local tcp server is used to proxy the podExec websock connection to remote-viewer ln, err := net.Listen("tcp", "127.0.0.1:0") @@ -113,20 +113,20 @@ func (o *VNC) Run(flags *flag.FlagSet) int { // write to FD <- pipeOutReader go func() { - defer close(readStop) - io.Copy(fd, pipeOutReader) + _, err := io.Copy(fd, pipeOutReader) + readStop <- err }() // read from FD -> pipeInWriter go func() { - defer close(writeStop) - io.Copy(pipeInWriter, fd) + _, err := io.Copy(pipeInWriter, fd) + writeStop <- err }() select { case <-stopChan: - case <-readStop: - case <-writeStop: + case err = <-readStop: + case err = <-writeStop: case err = <-k8ResChan: case err = <-viewResChan: }