diff --git a/Makefile b/Makefile index ffb69b1bda09..b60eed4e8711 100644 --- a/Makefile +++ b/Makefile @@ -93,7 +93,7 @@ vagrant-sync-optional: ./cluster/vagrant/sync_build.sh 'build optional' vagrant-deploy: vagrant-sync-config vagrant-sync-build - export KUBECTL="cluster/kubectl.sh --core" && ./cluster/deploy.sh + export KUBECTL="cluster/kubectl.sh" && ./cluster/deploy.sh .release-functest: make functest > .release-functest 2>&1 diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 768ba8fbd0e2..c855061ac0c7 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -1176,80 +1176,6 @@ } } }, - "/apis/kubevirt.io/v1alpha1/namespaces/{namespace}/virtualmachines/{name}/console": { - "get": { - "description": "Open a websocket connection to a serial console on the specified VM.", - "summary": "Open a websocket connection to a serial console on the specified VM.", - "operationId": "console", - "parameters": [ - { - "type": "string", - "description": "Name of the serial console to connect to", - "name": "console", - "in": "query" - }, - { - "pattern": "[a-z0-9][a-z0-9\\-]*", - "type": "string", - "description": "Object name and auth scope, such as for teams and projects", - "name": "namespace", - "in": "path", - "required": true - }, - { - "pattern": "[a-z0-9][a-z0-9\\-]*", - "type": "string", - "description": "Name of the resource", - "name": "name", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "OK" - } - } - } - }, - "/apis/kubevirt.io/v1alpha1/namespaces/{namespace}/virtualmachines/{name}/spice": { - "get": { - "description": "Returns a remote-viewer configuration file. Run `man 1 remote-viewer` to learn more about the configuration format.", - "produces": [ - "text/plain", - "application/json", - "application/yaml" - ], - "summary": "Returns a remote-viewer configuration file. Run `man 1 remote-viewer` to learn more about the configuration format.", - "operationId": "spice", - "parameters": [ - { - "pattern": "[a-z0-9][a-z0-9\\-]*", - "type": "string", - "description": "Object name and auth scope, such as for teams and projects", - "name": "namespace", - "in": "path", - "required": true - }, - { - "pattern": "[a-z0-9][a-z0-9\\-]*", - "type": "string", - "description": "Name of the resource", - "name": "name", - "in": "path", - "required": true - } - ], - "responses": { - "200": { - "description": "remote-viewer configuration file" - }, - "default": { - "description": "remote-viewer configuration file" - } - } - } - }, "/apis/kubevirt.io/v1alpha1/virtualmachinereplicasets": { "get": { "description": "Get a list of all VirtualMachineReplicaSet objects.", @@ -2985,25 +2911,6 @@ } } }, - "v1.VirtualMachineGraphics": { - "required": [ - "type", - "host", - "port" - ], - "properties": { - "host": { - "type": "string" - }, - "port": { - "type": "integer", - "format": "int32" - }, - "type": { - "type": "string" - } - } - }, "v1.VirtualMachineList": { "description": "VirtualMachineList is a list of VirtualMachines\n+k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object", "required": [ @@ -3120,13 +3027,6 @@ "$ref": "#/definitions/v1.VirtualMachineCondition" } }, - "graphics": { - "description": "Graphics represent the details of available graphical consoles.", - "type": "array", - "items": { - "$ref": "#/definitions/v1.VirtualMachineGraphics" - } - }, "migrationNodeName": { "description": "MigrationNodeName is the node where the VM is live migrating to.", "type": "string" diff --git a/automation/test.sh b/automation/test.sh index c8eea9679636..ccff51491419 100644 --- a/automation/test.sh +++ b/automation/test.sh @@ -25,7 +25,7 @@ set -ex -kubectl() { cluster/kubectl.sh --core "$@"; } +kubectl() { cluster/kubectl.sh "$@"; } if [ "$TARGET" = "vagrant-dev" ]; then cat > hack/config-local.sh </dev/null || : @@ -58,14 +44,8 @@ if [ -z "$TARGET" ] || [ "$TARGET" = "vagrant-dev" ]; then $KUBECTL create -f manifests/dev -R $i elif [ "$TARGET" = "vagrant-release" ]; then $KUBECTL create -f manifests/release -R $i - ## Tell virt-api where to look for the spice proxy - spiceProxy | $KUBECTL patch deployment virt-api -n kube-system --patch "$(cat -)" fi -## Expose common services -$KUBECTL expose deployment haproxy --port 8184 -l 'kubevirt.io=haproxy' -n kube-system --external-ip $master_ip -$KUBECTL expose deployment spice-proxy --port 3128 -l 'kubevirt.io=spice-proxy' -n kube-system --external-ip $master_ip - # Deploy additional infra for testing $KUBECTL create -f manifests/testing -R $i diff --git a/cluster/kubectl.sh b/cluster/kubectl.sh index e6f24cd74ff1..9d2fb10d53a7 100755 --- a/cluster/kubectl.sh +++ b/cluster/kubectl.sh @@ -27,23 +27,14 @@ then exit fi -if [ "$1" == "console" ] || [ "$1" == "spice" ]; then - cmd/virtctl/virtctl "$@" -s http://${master_ip}:8184 - exit -fi - # Print usage from virtctl and kubectl if [ "$1" == "--help" ] || [ "$1" == "-h" ] ; then cmd/virtctl/virtctl "$@" fi if [ -e ${KUBEVIRT_PATH}cluster/vagrant/.kubeconfig ] && - [ -e ${KUBEVIRT_PATH}cluster/vagrant/.kubectl ] && - [ "x$1" == "x--core" ]; then - shift + [ -e ${KUBEVIRT_PATH}cluster/vagrant/.kubectl ]; then ${KUBEVIRT_PATH}cluster/vagrant/.kubectl --kubeconfig=${KUBEVIRT_PATH}cluster/vagrant/.kubeconfig "$@" -elif [ -e ${KUBEVIRT_PATH}cluster/vagrant/.kubectl ];then - ${KUBEVIRT_PATH}cluster/vagrant/.kubectl -s http://${master_ip}:8184 "$@" else echo "Did you already run '$SYNC_CONFIG' to deploy kubevirt?" fi diff --git a/cluster/vm-isolation-check.sh b/cluster/vm-isolation-check.sh index 61eceaef9350..a900a8e65ea0 100755 --- a/cluster/vm-isolation-check.sh +++ b/cluster/vm-isolation-check.sh @@ -53,7 +53,7 @@ then exit 1 fi -NODE=$(cluster/kubectl.sh --core get pods -o json -l kubevirt.io/domain=${VM_NAME} | jq '.items[].spec.nodeName' -r) +NODE=$(cluster/kubectl.sh get pods -o json -l kubevirt.io/domain=${VM_NAME} | jq '.items[].spec.nodeName' -r) if [ -z $NODE ]; then echo "Could not detect the VM." diff --git a/cmd/virt-api/Dockerfile b/cmd/virt-api/Dockerfile deleted file mode 100644 index ff138ca1b352..000000000000 --- a/cmd/virt-api/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -# -# This file is part of the KubeVirt project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Copyright 2017 Red Hat, Inc. -# - -FROM fedora:26 - -MAINTAINER "The KubeVirt Project" - -# Create non-root user -RUN useradd -u 1001 --create-home -s /bin/bash virt-api -WORKDIR /home/virt-api -USER 1001 - -# Configure swagger -RUN curl -OL https://github.com/swagger-api/swagger-ui/tarball/38f74164a7062edb5dc80ef2fdddda24f3f6eb85/swagger-ui.tar.gz \ - && mkdir swagger-ui && tar xf swagger-ui.tar.gz -C swagger-ui --strip-components 1 \ - && mkdir third_party \ - && mv swagger-ui/dist third_party/swagger-ui && rm -rf swagger-ui \ - && sed -e 's@"http://petstore.swagger.io/v2/swagger.json"@"/swaggerapi/"@' -i third_party/swagger-ui/index.html \ - && rm swagger-ui.tar.gz && rm -rf swagger-ui - -COPY virt-api /virt-api - -ENTRYPOINT [ "/virt-api" ] diff --git a/cmd/virt-api/virt-api.go b/cmd/virt-api/virt-api.go deleted file mode 100644 index 4b6e38c53d4f..000000000000 --- a/cmd/virt-api/virt-api.go +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is part of the KubeVirt project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright 2017 Red Hat, Inc. - * - */ - -package main - -import ( - klog "kubevirt.io/kubevirt/pkg/log" - "kubevirt.io/kubevirt/pkg/service" - "kubevirt.io/kubevirt/pkg/virt-api" -) - -func main() { - klog.InitializeLogging("virt-api") - - app := virt_api.VirtAPIApp{} - service.Setup(&app) - app.Compose() - app.ConfigureOpenAPIService() - app.Run() -} diff --git a/cmd/virt-handler/virt-handler.go b/cmd/virt-handler/virt-handler.go index a0e9d3b8fecc..7a105950f83c 100644 --- a/cmd/virt-handler/virt-handler.go +++ b/cmd/virt-handler/virt-handler.go @@ -202,11 +202,8 @@ func (app *virtHandlerApp) Run() { // TODO add a http handler which provides health check - // Add websocket route to access consoles remotely - console := rest.NewConsoleResource(domainConn) migrationHostInfo := rest.NewMigrationHostInfo(isolation.NewSocketBasedIsolationDetector(app.VirtShareDir)) ws := new(restful.WebService) - ws.Route(ws.GET("/api/v1/namespaces/{namespace}/virtualmachines/{name}/console").To(console.Console)) ws.Route(ws.GET("/api/v1/namespaces/{namespace}/virtualmachines/{name}/migrationHostInfo").To(migrationHostInfo.MigrationHostInfo)) restful.DefaultContainer.Add(ws) server := &http.Server{Addr: app.Address(), Handler: restful.DefaultContainer} diff --git a/cmd/virt-launcher/Dockerfile b/cmd/virt-launcher/Dockerfile index 94ec885c98a8..b3dc738a53f2 100644 --- a/cmd/virt-launcher/Dockerfile +++ b/cmd/virt-launcher/Dockerfile @@ -20,6 +20,12 @@ FROM fedora:26 MAINTAINER "The KubeVirt Project" +RUN dnf -y install socat && \ + groupadd --gid 107 qemu && \ + useradd --uid 107 --gid 107 qemu && \ + dnf -y clean all + +COPY sock-connector /sock-connector COPY virt-launcher /virt-launcher ENTRYPOINT [ "/virt-launcher" ] diff --git a/cmd/virt-launcher/sock-connector b/cmd/virt-launcher/sock-connector new file mode 100755 index 000000000000..bae34416c018 --- /dev/null +++ b/cmd/virt-launcher/sock-connector @@ -0,0 +1,4 @@ +#!/bin/bash +stty -echo +SOCKET=$1 +socat unix-connect:/$SOCKET stdio,cfmakeraw diff --git a/cmd/virt-launcher/virt-launcher.go b/cmd/virt-launcher/virt-launcher.go index bf38ea66d3e5..d3cd034ebb88 100644 --- a/cmd/virt-launcher/virt-launcher.go +++ b/cmd/virt-launcher/virt-launcher.go @@ -89,6 +89,11 @@ func main() { panic(err) } + err = virtlauncher.InitializePrivateDirectories(filepath.Join("/var/run/kubevirt-private", *namespace, *name)) + if err != nil { + panic(err) + } + watchdogFile := watchdog.WatchdogFileFromNamespaceName(*virtShareDir, *namespace, *name) err = watchdog.WatchdogFileUpdate(watchdogFile) if err != nil { diff --git a/cmd/virtctl/virtctl.go b/cmd/virtctl/virtctl.go index aebf3bc75dc7..ecddf1a10b95 100644 --- a/cmd/virtctl/virtctl.go +++ b/cmd/virtctl/virtctl.go @@ -29,7 +29,7 @@ import ( "kubevirt.io/kubevirt/pkg/virtctl" "kubevirt.io/kubevirt/pkg/virtctl/console" - "kubevirt.io/kubevirt/pkg/virtctl/spice" + "kubevirt.io/kubevirt/pkg/virtctl/vnc" ) func main() { @@ -40,7 +40,7 @@ func main() { registry := map[string]virtctl.App{ "console": &console.Console{}, "options": &virtctl.Options{}, - "spice": &spice.Spice{}, + "vnc": &vnc.VNC{}, } if len(os.Args) > 1 { @@ -82,7 +82,7 @@ func Usage() { Basic Commands: console Connect to a serial console on a VM - spice Connect to a SPICE display of a VM + vnc Connect to a VNC display of a VM Use "virtctl --help" for more information about a given command. Use "virtctl options" for a list of global command-line options (applies to all commands). diff --git a/docs/debugging.md b/docs/debugging.md index 0376305d67e8..55180f45fc33 100644 --- a/docs/debugging.md +++ b/docs/debugging.md @@ -1,23 +1,9 @@ # Debugging -When debugging KubeVirt in the development environment, you have to be aware -that we have a proxy for ThirdPartyResource related preprocessing in front of -the apiserver. By default the `cluster/kubectl.sh` script assumes that you want -to talk to the cluster through that proxy. So if - ```bash cluster/kubectl.sh version ``` -is run, `cluster/kubectl.sh` connects to the proxy. However when something is -not right with the proxy, or KubeVirt is not even yet deployed, it is possible -to connect to the apiserver directly by adding `--core` as the first argument. -So - -```bash -cluster/kubectl.sh --core version -``` - will try to connect to the apiserver. ## Retrieving Logs diff --git a/hack/build-go.sh b/hack/build-go.sh index 6fd1478787e6..99f1408a5f34 100755 --- a/hack/build-go.sh +++ b/hack/build-go.sh @@ -40,7 +40,7 @@ if [ $# -eq 0 ]; then if [ "${target}" = "test" ]; then (cd pkg; go ${target} -v ./...) elif [ "${target}" = "functest" ]; then - (cd tests; go test -master=http://${master_ip}:${master_port} -timeout 30m ${FUNC_TEST_ARGS}) + (cd tests; go test -kubeconfig=../cluster/vagrant/.kubeconfig -timeout 30m ${FUNC_TEST_ARGS}) exit else (cd pkg; go $target ./...) diff --git a/hack/config-default.sh b/hack/config-default.sh index f91be8ddeb6e..78edb34171d9 100644 --- a/hack/config-default.sh +++ b/hack/config-default.sh @@ -1,8 +1,7 @@ -binaries="cmd/virt-controller cmd/virt-launcher cmd/virt-handler cmd/virt-api cmd/virtctl cmd/fake-qemu-process cmd/virt-dhcp cmd/fake-dnsmasq-process" -docker_images="cmd/virt-controller cmd/virt-launcher cmd/virt-handler cmd/virt-api images/haproxy images/iscsi-demo-target-tgtd images/vm-killer images/libvirt-kubevirt images/spice-proxy cmd/virt-migrator cmd/registry-disk-v1alpha images/cirros-registry-disk-demo cmd/virt-dhcp" +binaries="cmd/virt-controller cmd/virt-launcher cmd/virt-handler cmd/virtctl cmd/fake-qemu-process cmd/virt-dhcp cmd/fake-dnsmasq-process" +docker_images="cmd/virt-controller cmd/virt-launcher cmd/virt-handler images/iscsi-demo-target-tgtd images/vm-killer images/libvirt-kubevirt cmd/virt-migrator cmd/registry-disk-v1alpha images/cirros-registry-disk-demo cmd/virt-dhcp" optional_docker_images="cmd/registry-disk-v1alpha images/fedora-atomic-registry-disk-demo" docker_prefix=kubevirt docker_tag=${DOCKER_TAG:-latest} master_ip=192.168.200.2 -master_port=8184 network_provider=weave diff --git a/hack/config.sh b/hack/config.sh index 6a869d77f658..9f7be5a79ce6 100644 --- a/hack/config.sh +++ b/hack/config.sh @@ -1,5 +1,5 @@ unset binaries docker_images docker_prefix docker_tag manifest_templates \ - master_ip master_port network_provider + master_ip network_provider source ${KUBEVIRT_PATH}hack/config-default.sh @@ -8,4 +8,4 @@ source ${KUBEVIRT_PATH}hack/config-default.sh test -f "hack/config-local.sh" && source hack/config-local.sh export binaries docker_images docker_prefix docker_tag manifest_templates \ - master_ip master_port network_provider + master_ip network_provider diff --git a/images/haproxy/Dockerfile b/images/haproxy/Dockerfile deleted file mode 100644 index e1061a637ec0..000000000000 --- a/images/haproxy/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# -# This file is part of the KubeVirt project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Copyright 2017 Red Hat, Inc. -# - -FROM haproxy:1.6-alpine - -MAINTAINER "The KubeVirt Project" - -COPY haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg - -RUN sed -i "s|/run|/haproxy/run|g" /docker-entrypoint.sh -RUN cp /docker-entrypoint.sh /docker-entrypoint-orig.sh - -COPY docker-entrypoint.sh /docker-entrypoint.sh - -RUN addgroup -S haproxy && adduser -S -D -h /haproxy -s /bin/false -G haproxy -g haproxy haproxy - -RUN mkdir /haproxy/run && \ - chgrp -R 0 /haproxy/run && \ - chmod -R g=u /haproxy/run - -USER 1001 diff --git a/images/haproxy/docker-entrypoint.sh b/images/haproxy/docker-entrypoint.sh deleted file mode 100755 index fc7ca73a8b48..000000000000 --- a/images/haproxy/docker-entrypoint.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -# -# This file is part of the KubeVirt project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Copyright 2017 Red Hat, Inc. -# - -set -e -export TOKEN=`cat /var/run/secrets/kubernetes.io/serviceaccount/token` -export DNS=`cat /etc/resolv.conf |grep -i nameserver|head -n1|cut -d ' ' -f2` -/sbin/syslogd -O /dev/stdout && /docker-entrypoint-orig.sh $@ diff --git a/images/haproxy/haproxy.cfg b/images/haproxy/haproxy.cfg deleted file mode 100644 index e04aa7318e29..000000000000 --- a/images/haproxy/haproxy.cfg +++ /dev/null @@ -1,36 +0,0 @@ -resolvers kubernetes - nameserver skydns ${DNS}:53 - resolve_retries 10 - timeout retry 2s - hold valid 30s - -frontend virt_api - bind *:8184 - mode http - acl is_supported_method method POST PUT GET DELETE OPTIONS PATCH - acl is_openapi path_beg /swagger-2.0.0.pb-v1 - acl is_swagger_api path_beg /swagger-ui - acl is_swagger_spec path_beg /swaggerapi/apis/kubevirt.io/ - acl is_kubevirt path_beg /apis/kubevirt.io - http-request add-header Authorization Bearer\ %[env(TOKEN)] - timeout client 1m - use_backend srvs_apiserver if is_openapi - use_backend srvs_kubevirt if is_supported_method is_kubevirt - use_backend srvs_kubevirt if is_swagger_api - use_backend srvs_kubevirt if is_swagger_spec - default_backend srvs_apiserver - -backend srvs_kubevirt - mode http - timeout connect 10s - timeout server 1m - balance roundrobin - server host1 virt-api:8183 resolvers kubernetes - -backend srvs_apiserver - mode http - timeout connect 10s - timeout server 1m - balance roundrobin - server host1 ${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT} check ssl ca-file /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - diff --git a/images/spice-proxy/Dockerfile b/images/spice-proxy/Dockerfile deleted file mode 100644 index ea3fd535d58a..000000000000 --- a/images/spice-proxy/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -FROM fedora:26 - -MAINTAINER "The KubeVirt Project" - -EXPOSE 3128 - -RUN dnf -y install squid \ - && dnf -y clean all - -RUN sed -i -e "s/http_access deny CONNECT !SSL_ports/http_access deny CONNECT !Safe_ports/" /etc/squid/squid.conf -RUN echo "pid_filename /home/proxy/run/squid.pid" >> /etc/squid/squid.conf - -RUN useradd --create-home -s /bin/bash proxy -RUN chgrp -R 0 /var/log/squid && chmod -R g=u /var/log/squid -RUN cp /etc/squid/squid.conf /home/proxy && chown proxy /home/proxy/squid.conf -WORKDIR /home/proxy -RUN chgrp -R 0 /home/proxy && \ - chmod -R g=u /home/proxy - -RUN mkdir /home/proxy/run && \ - chgrp -R 0 /home/proxy/run && \ - chmod -R g=u /home/proxy/run - -USER 1001 - -CMD squid -NCd1 -f /home/proxy/squid.conf diff --git a/manifests/dev/haproxy.yaml.in b/manifests/dev/haproxy.yaml.in deleted file mode 100644 index 84cae2025e19..000000000000 --- a/manifests/dev/haproxy.yaml.in +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: haproxy - namespace: kube-system - labels: - kubevirt.io: "haproxy" -spec: - template: - metadata: - labels: - kubevirt.io: haproxy - spec: - serviceAccountName: kubevirt-infra - containers: - - name: haproxy - image: {{ docker_prefix }}/haproxy:{{ docker_tag }} - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8184 - name: "haproxy" - protocol: "TCP" - livenessProbe: - httpGet: - path: /apis/kubevirt.io/v1alpha1/healthz - port: 8184 - initialDelaySeconds: 5 - periodSeconds: 10 - readinessProbe: - httpGet: - path: /apis/kubevirt.io/v1alpha1/healthz - port: 8184 - initialDelaySeconds: 10 - periodSeconds: 20 - securityContext: - runAsNonRoot: true diff --git a/manifests/dev/libvirt.yaml.in b/manifests/dev/libvirt.yaml.in index dfafd5eca0b4..a7bf9d0c9cfa 100644 --- a/manifests/dev/libvirt.yaml.in +++ b/manifests/dev/libvirt.yaml.in @@ -39,6 +39,8 @@ spec: mountPath: /var/run/libvirt - name: virt-share-dir mountPath: /var/run/kubevirt + - name: virt-private-dir + mountPath: /var/run/kubevirt-private command: ["/libvirtd.sh"] - name: virtlogd image: {{ docker_prefix }}/libvirt-kubevirt:{{ docker_tag }} @@ -75,3 +77,6 @@ spec: - name: virt-share-dir hostPath: path: /var/run/kubevirt + - name: virt-private-dir + hostPath: + path: /var/run/kubevirt-private diff --git a/manifests/dev/squid.yaml.in b/manifests/dev/squid.yaml.in deleted file mode 100644 index 6833c65203f5..000000000000 --- a/manifests/dev/squid.yaml.in +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: spice-proxy - namespace: kube-system - labels: - kubevirt.io: "spice-proxy" -spec: - template: - metadata: - labels: - kubevirt.io: spice-proxy - spec: - containers: - - name: spice-proxy - image: {{ docker_prefix }}/spice-proxy:{{ docker_tag }} - imagePullPolicy: IfNotPresent - ports: - - containerPort: 3128 - name: "spice-proxy" - protocol: "TCP" - securityContext: - runAsNonRoot: true diff --git a/manifests/dev/virt-api.yaml.in b/manifests/dev/virt-api.yaml.in deleted file mode 100644 index 5a0b06055ba4..000000000000 --- a/manifests/dev/virt-api.yaml.in +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: virt-api - namespace: kube-system - labels: - kubevirt.io: "virt-api" -spec: - ports: - - port: 8183 - targetPort: virt-api - selector: - kubevirt.io: virt-api ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: virt-api - namespace: kube-system - labels: - kubevirt.io: "virt-api" -spec: - template: - metadata: - labels: - kubevirt.io: virt-api - spec: - serviceAccountName: kubevirt-infra - containers: - - name: virt-api - image: {{ docker_prefix }}/virt-api:{{ docker_tag }} - imagePullPolicy: IfNotPresent - command: - - "/virt-api" - - "--port" - - "8183" - - "--spice-proxy" - - "http://{{ master_ip }}:3128" - ports: - - containerPort: 8183 - name: "virt-api" - protocol: "TCP" - securityContext: - runAsNonRoot: true diff --git a/manifests/dev/virt-handler.yaml.in b/manifests/dev/virt-handler.yaml.in index b8f77e644618..4d0b863ff85c 100644 --- a/manifests/dev/virt-handler.yaml.in +++ b/manifests/dev/virt-handler.yaml.in @@ -36,6 +36,8 @@ spec: mountPath: /var/run/libvirt - name: virt-share-dir mountPath: /var/run/kubevirt + - name: virt-private-dir + mountPath: /var/run/kubevirt-private env: - name: NODE_NAME valueFrom: @@ -48,3 +50,6 @@ spec: - name: virt-share-dir hostPath: path: /var/run/kubevirt + - name: virt-private-dir + hostPath: + path: /var/run/kubevirt-private diff --git a/manifests/release/kubevirt.yaml.in b/manifests/release/kubevirt.yaml.in index 2f43f3a5c994..4b52ff6512b3 100644 --- a/manifests/release/kubevirt.yaml.in +++ b/manifests/release/kubevirt.yaml.in @@ -142,90 +142,6 @@ spec: - vmrs - vmrss --- -# apiserver and virt-api proxy (will be unnecessary soon) -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: haproxy - namespace: kube-system - labels: - kubevirt.io: "haproxy" -spec: - template: - metadata: - labels: - kubevirt.io: haproxy - spec: - serviceAccountName: kubevirt-infra - containers: - - name: haproxy - image: {{ docker_prefix }}/haproxy:{{ docker_tag }} - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8184 - name: "haproxy" - protocol: "TCP" - livenessProbe: - httpGet: - path: /apis/kubevirt.io/v1alpha1/healthz - port: 8184 - initialDelaySeconds: 5 - periodSeconds: 10 - readinessProbe: - httpGet: - path: /apis/kubevirt.io/v1alpha1/healthz - port: 8184 - initialDelaySeconds: 10 - periodSeconds: 20 - securityContext: - runAsNonRoot: true ---- -# virt-api -apiVersion: v1 -kind: Service -metadata: - name: virt-api - namespace: kube-system - labels: - kubevirt.io: "virt-api" -spec: - ports: - - port: 8183 - targetPort: virt-api - selector: - kubevirt.io: virt-api ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: virt-api - namespace: kube-system - labels: - kubevirt.io: "virt-api" -spec: - template: - metadata: - labels: - kubevirt.io: virt-api - spec: - serviceAccountName: kubevirt-infra - containers: - - name: virt-api - image: {{ docker_prefix }}/virt-api:{{ docker_tag }} - imagePullPolicy: IfNotPresent - command: - - "/virt-api" - - "--port" - - "8183" - - "--spice-proxy" - - "$(SPICE_PROXY)" - ports: - - containerPort: 8183 - name: "virt-api" - protocol: "TCP" - securityContext: - runAsNonRoot: true ---- # kubevirt controller apiVersion: extensions/v1beta1 kind: Deployment @@ -313,6 +229,8 @@ spec: mountPath: /var/run/libvirt - name: virt-share-dir mountPath: /var/run/kubevirt + - name: virt-private-dir + mountPath: /var/run/kubevirt-private env: - name: NODE_NAME valueFrom: @@ -325,6 +243,9 @@ spec: - name: virt-share-dir hostPath: path: /var/run/kubevirt + - name: virt-private-dir + hostPath: + path: /var/run/kubevirt-private --- # libvirt daemon set apiVersion: extensions/v1beta1 @@ -368,6 +289,8 @@ spec: mountPath: /var/run/libvirt - name: virt-share-dir mountPath: /var/run/kubevirt + - name: virt-private-dir + mountPath: /var/run/kubevirt-private command: ["/libvirtd.sh"] - name: virtlogd image: {{ docker_prefix }}/libvirt-kubevirt:{{ docker_tag }} @@ -392,3 +315,6 @@ spec: - name: virt-share-dir hostPath: path: /var/run/kubevirt + - name: virt-private-dir + hostPath: + path: /var/run/kubevirt-private diff --git a/manifests/release/spice-proxy.yaml.in b/manifests/release/spice-proxy.yaml.in deleted file mode 100644 index 6833c65203f5..000000000000 --- a/manifests/release/spice-proxy.yaml.in +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: spice-proxy - namespace: kube-system - labels: - kubevirt.io: "spice-proxy" -spec: - template: - metadata: - labels: - kubevirt.io: spice-proxy - spec: - containers: - - name: spice-proxy - image: {{ docker_prefix }}/spice-proxy:{{ docker_tag }} - imagePullPolicy: IfNotPresent - ports: - - containerPort: 3128 - name: "spice-proxy" - protocol: "TCP" - securityContext: - runAsNonRoot: true diff --git a/pkg/api/v1/deepcopy_generated.go b/pkg/api/v1/deepcopy_generated.go index 3b521384f8fb..e21d3e3a9ab0 100644 --- a/pkg/api/v1/deepcopy_generated.go +++ b/pkg/api/v1/deepcopy_generated.go @@ -1221,22 +1221,6 @@ func (in *VirtualMachineCondition) DeepCopy() *VirtualMachineCondition { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *VirtualMachineGraphics) DeepCopyInto(out *VirtualMachineGraphics) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VirtualMachineGraphics. -func (in *VirtualMachineGraphics) DeepCopy() *VirtualMachineGraphics { - if in == nil { - return nil - } - out := new(VirtualMachineGraphics) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VirtualMachineList) DeepCopyInto(out *VirtualMachineList) { *out = *in @@ -1393,11 +1377,6 @@ func (in *VirtualMachineStatus) DeepCopyInto(out *VirtualMachineStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.Graphics != nil { - in, out := &in.Graphics, &out.Graphics - *out = make([]VirtualMachineGraphics, len(*in)) - copy(*out, *in) - } return } diff --git a/pkg/api/v1/deepcopy_test.go b/pkg/api/v1/deepcopy_test.go index 276f9de37420..bb941f61e923 100644 --- a/pkg/api/v1/deepcopy_test.go +++ b/pkg/api/v1/deepcopy_test.go @@ -51,7 +51,6 @@ var _ = Describe("Generated deepcopy functions", func() { &VirtualMachineSpec{}, &Affinity{}, &VirtualMachineStatus{}, - &VirtualMachineGraphics{}, &VirtualMachineCondition{}, &Spice{}, &SpiceInfo{}, diff --git a/pkg/api/v1/schema_test.go b/pkg/api/v1/schema_test.go index 97bd6f1d82b6..a121dd0194ea 100644 --- a/pkg/api/v1/schema_test.go +++ b/pkg/api/v1/schema_test.go @@ -177,9 +177,7 @@ var exampleJSON = `{ } ] }, - "status": { - "graphics": null - } + "status": {} }` var _ = Describe("Schema", func() { diff --git a/pkg/api/v1/types.go b/pkg/api/v1/types.go index c1c31e439b6b..323c0bf2516a 100644 --- a/pkg/api/v1/types.go +++ b/pkg/api/v1/types.go @@ -149,14 +149,6 @@ type VirtualMachineStatus struct { Conditions []VirtualMachineCondition `json:"conditions,omitempty"` // Phase is the status of the VM in kubernetes world. It is not the VM status, but partially correlates to it. Phase VMPhase `json:"phase,omitempty"` - // Graphics represent the details of available graphical consoles. - Graphics []VirtualMachineGraphics `json:"graphics" optional:"true"` -} - -type VirtualMachineGraphics struct { - Type string `json:"type"` - Host string `json:"host"` - Port int32 `json:"port"` } // Required to satisfy Object interface diff --git a/pkg/api/v1/types_swagger_generated.go b/pkg/api/v1/types_swagger_generated.go index f1da0727ffbb..347ca7efacf9 100644 --- a/pkg/api/v1/types_swagger_generated.go +++ b/pkg/api/v1/types_swagger_generated.go @@ -41,14 +41,9 @@ func (VirtualMachineStatus) SwaggerDoc() map[string]string { "migrationNodeName": "MigrationNodeName is the node where the VM is live migrating to.", "conditions": "Conditions are specific points in VM's pod runtime.", "phase": "Phase is the status of the VM in kubernetes world. It is not the VM status, but partially correlates to it.", - "graphics": "Graphics represent the details of available graphical consoles.", } } -func (VirtualMachineGraphics) SwaggerDoc() map[string]string { - return map[string]string{} -} - func (VirtualMachineCondition) SwaggerDoc() map[string]string { return map[string]string{} } diff --git a/pkg/ephemeral-disk-utils/utils.go b/pkg/ephemeral-disk-utils/utils.go index c58538fd5894..d5e76878ab42 100644 --- a/pkg/ephemeral-disk-utils/utils.go +++ b/pkg/ephemeral-disk-utils/utils.go @@ -34,11 +34,11 @@ import ( ) func RemoveFile(path string) error { - err := os.Remove(path) + err := os.RemoveAll(path) if err != nil && os.IsNotExist(err) { return nil } else if err != nil { - log.Log.Reason(err).Errorf("failed to remove cloud-init temporary data file %s", path) + log.Log.Reason(err).Errorf("failed to remove %s", path) return err } return nil diff --git a/pkg/kubecli/generated_mock_kubevirt.go b/pkg/kubecli/generated_mock_kubevirt.go index 127540c8fee9..c7c958d5f855 100644 --- a/pkg/kubecli/generated_mock_kubevirt.go +++ b/pkg/kubecli/generated_mock_kubevirt.go @@ -4,6 +4,8 @@ package kubecli import ( + io "io" + gomock "github.com/golang/mock/gomock" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" @@ -589,6 +591,26 @@ func (_mr *_MockVMInterfaceRecorder) Patch(arg0, arg1, arg2 interface{}, arg3 .. return _mr.mock.ctrl.RecordCall(_mr.mock, "Patch", _s...) } +func (_m *MockVMInterface) SerialConsole(name string, device string, in io.Reader, out io.Writer) error { + ret := _m.ctrl.Call(_m, "SerialConsole", name, device, in, out) + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockVMInterfaceRecorder) SerialConsole(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "SerialConsole", arg0, arg1, arg2, arg3) +} + +func (_m *MockVMInterface) VNC(name string, in io.Reader, out io.Writer) error { + ret := _m.ctrl.Call(_m, "VNC", name, in, out) + ret0, _ := ret[0].(error) + return ret0 +} + +func (_mr *_MockVMInterfaceRecorder) VNC(arg0, arg1, arg2 interface{}) *gomock.Call { + return _mr.mock.ctrl.RecordCall(_mr.mock, "VNC", arg0, arg1, arg2) +} + // Mock of ReplicaSetInterface interface type MockReplicaSetInterface struct { ctrl *gomock.Controller diff --git a/pkg/kubecli/kubecli.go b/pkg/kubecli/kubecli.go index e96a247626b5..bb887c5dbc94 100644 --- a/pkg/kubecli/kubecli.go +++ b/pkg/kubecli/kubecli.go @@ -63,7 +63,7 @@ func GetKubevirtClientFromFlags(master string, kubeconfig string) (KubevirtClien return nil, err } - return &kubevirt{restClient, coreClient}, nil + return &kubevirt{master, kubeconfig, restClient, coreClient}, nil } func GetKubevirtClient() (KubevirtClient, error) { diff --git a/pkg/kubecli/kubevirt.go b/pkg/kubecli/kubevirt.go index d8c868b3655b..6ff7df10ab47 100644 --- a/pkg/kubecli/kubevirt.go +++ b/pkg/kubecli/kubevirt.go @@ -26,6 +26,8 @@ package kubecli */ import ( + "io" + k8smetav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -44,6 +46,8 @@ type KubevirtClient interface { } type kubevirt struct { + master string + kubeconfig string restClient *rest.RESTClient *kubernetes.Clientset } @@ -59,6 +63,8 @@ type VMInterface interface { Update(*v1.VirtualMachine) (*v1.VirtualMachine, error) Delete(name string, options *k8smetav1.DeleteOptions) error Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1.VirtualMachine, err error) + SerialConsole(name string, device string, in io.Reader, out io.Writer) error + VNC(name string, in io.Reader, out io.Writer) error } type ReplicaSetInterface interface { diff --git a/pkg/kubecli/vm.go b/pkg/kubecli/vm.go index ad4eca287249..c26bd473a433 100644 --- a/pkg/kubecli/vm.go +++ b/pkg/kubecli/vm.go @@ -20,9 +20,21 @@ package kubecli import ( + goerror "errors" + "fmt" + "io" + + k8sv1 "k8s.io/api/core/v1" k8smetav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/remotecommand" "k8s.io/apimachinery/pkg/types" @@ -30,13 +42,107 @@ import ( ) func (k *kubevirt) VM(namespace string) VMInterface { - return &vms{k.restClient, namespace, "virtualmachines"} + return &vms{ + restClient: k.restClient, + clientSet: k.Clientset, + namespace: namespace, + resource: "virtualmachines", + master: k.master, + kubeconfig: k.kubeconfig, + } } type vms struct { restClient *rest.RESTClient + clientSet *kubernetes.Clientset namespace string resource string + master string + kubeconfig string +} + +func findPod(clientSet *kubernetes.Clientset, namespace string, name string) (string, error) { + fieldSelector := fields.ParseSelectorOrDie("status.phase==" + string(k8sv1.PodRunning)) + labelSelector, err := labels.Parse(fmt.Sprintf(v1.AppLabel+"=virt-launcher,"+v1.DomainLabel+" in (%s)", name)) + if err != nil { + return "", err + } + selector := k8smetav1.ListOptions{FieldSelector: fieldSelector.String(), LabelSelector: labelSelector.String()} + + podList, err := clientSet.CoreV1().Pods(namespace).List(selector) + if err != nil { + return "", err + } + + if len(podList.Items) == 0 { + return "", goerror.New("console connection failed. No VM pod is running") + } + return podList.Items[0].ObjectMeta.Name, nil +} + +func (v *vms) remoteExecHelper(name string, cmd []string, in io.Reader, out io.Writer) error { + + podName, err := findPod(v.clientSet, v.namespace, name) + if err != nil { + return fmt.Errorf("unable to find matching pod for remote execution: %v", err) + } + + config, err := clientcmd.BuildConfigFromFlags(v.master, v.kubeconfig) + if err != nil { + return fmt.Errorf("unable to build api config for remote execution: %v", err) + } + + gv := k8sv1.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/api" + config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs} + + restClient, err := restclient.RESTClientFor(config) + if err != nil { + return fmt.Errorf("unable to create restClient for remote execution: %v", err) + } + containerName := "compute" + req := restClient.Post(). + Resource("pods"). + Name(podName). + Namespace(v.namespace). + SubResource("exec"). + Param("container", containerName) + + req = req.VersionedParams(&k8sv1.PodExecOptions{ + Container: containerName, + Command: cmd, + Stdin: true, + Stdout: true, + Stderr: true, + TTY: true, + }, scheme.ParameterCodec) + + // execute request + method := "POST" + url := req.URL() + exec, err := remotecommand.NewSPDYExecutor(config, method, url) + if err != nil { + return fmt.Errorf("remote execution failed: %v", err) + } + + return exec.Stream(remotecommand.StreamOptions{ + Stdin: in, + Stdout: out, + Stderr: out, + Tty: false, + TerminalSizeQueue: nil, + }) +} + +func (v *vms) VNC(name string, in io.Reader, out io.Writer) error { + cmd := []string{"/sock-connector", fmt.Sprintf("/var/run/kubevirt-private/%s/%s/virt-vnc", v.namespace, name)} + return v.remoteExecHelper(name, cmd, in, out) +} + +func (v *vms) SerialConsole(name string, device string, in io.Reader, out io.Writer) error { + cmd := []string{"/sock-connector", fmt.Sprintf("/var/run/kubevirt-private/%s/%s/virt-%s", v.namespace, name, device)} + return v.remoteExecHelper(name, cmd, in, out) } func (v *vms) Get(name string, options k8smetav1.GetOptions) (vm *v1.VirtualMachine, err error) { diff --git a/pkg/virt-api/api.go b/pkg/virt-api/api.go index 49fb47f84070..5c9ceeb5ff72 100644 --- a/pkg/virt-api/api.go +++ b/pkg/virt-api/api.go @@ -6,7 +6,6 @@ import ( "github.com/emicklei/go-restful" "github.com/emicklei/go-restful-openapi" - kithttp "github.com/go-kit/kit/transport/http" openapispec "github.com/go-openapi/spec" flag "github.com/spf13/pflag" "golang.org/x/net/context" @@ -14,9 +13,6 @@ import ( "kubevirt.io/kubevirt/pkg/api/v1" "kubevirt.io/kubevirt/pkg/healthz" - "kubevirt.io/kubevirt/pkg/kubecli" - mime "kubevirt.io/kubevirt/pkg/rest" - "kubevirt.io/kubevirt/pkg/rest/endpoints" "kubevirt.io/kubevirt/pkg/rest/filter" "kubevirt.io/kubevirt/pkg/service" "kubevirt.io/kubevirt/pkg/virt-api/rest" @@ -63,38 +59,6 @@ func (app *VirtAPIApp) Compose() { log.Fatal(err) } - virtCli, err := kubecli.GetKubevirtClient() - if err != nil { - log.Fatal(err) - } - - // TODO, allow Encoder and Decoders per type and combine the endpoint logic - spice := endpoints.MakeGoRestfulWrapper(endpoints.NewHandlerBuilder().Get(). - Endpoint(rest.NewSpiceEndpoint(virtCli.RestClient(), vmGVR)).Encoder( - endpoints.NewMimeTypeAwareEncoder(endpoints.NewEncodeINIResponse(http.StatusOK), - map[string]kithttp.EncodeResponseFunc{ - mime.MIME_INI: endpoints.NewEncodeINIResponse(http.StatusOK), - mime.MIME_JSON: endpoints.NewEncodeJsonResponse(http.StatusOK), - mime.MIME_YAML: endpoints.NewEncodeYamlResponse(http.StatusOK), - })).Build(ctx)) - - ws.Route(ws.GET(rest.ResourcePath(vmGVR)+rest.SubResourcePath("spice")). - To(spice).Produces(mime.MIME_INI, mime.MIME_JSON, mime.MIME_YAML). - Param(rest.NamespaceParam(ws)).Param(rest.NameParam(ws)). - Operation("spice"). - Doc("Returns a remote-viewer configuration file. Run `man 1 remote-viewer` to learn more about the configuration format."). - Returns(http.StatusOK, "remote-viewer configuration file" /*os.File{}*/, nil)) - // TODO: That os.File doesn't work as I expect. - // I need end up with response_type="file", but I am getting response_type="os.File" - - ws.Route(ws.GET(rest.ResourcePath(vmGVR) + rest.SubResourcePath("console")). - To(rest.NewConsoleResource(virtCli, virtCli.CoreV1()).Console). - Param(restful.QueryParameter("console", "Name of the serial console to connect to")). - Param(rest.NamespaceParam(ws)).Param(rest.NameParam(ws)). - Operation("console"). - Doc("Open a websocket connection to a serial console on the specified VM.")) - // TODO: Add 'Returns', but I don't know what return type to put there. - restful.Add(ws) ws.Route(ws.GET("/healthz"). diff --git a/pkg/virt-api/rest/console.go b/pkg/virt-api/rest/console.go deleted file mode 100644 index a3b2bb2432a0..000000000000 --- a/pkg/virt-api/rest/console.go +++ /dev/null @@ -1,137 +0,0 @@ -/* - * This file is part of the KubeVirt project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright 2017 Red Hat, Inc. - * - */ - -package rest - -import ( - "bytes" - "fmt" - "io" - "net/http" - - "github.com/emicklei/go-restful" - "github.com/gorilla/websocket" - k8sv1meta "k8s.io/apimachinery/pkg/apis/meta/v1" - k8scorev1 "k8s.io/client-go/kubernetes/typed/core/v1" - - "k8s.io/apimachinery/pkg/api/errors" - - "kubevirt.io/kubevirt/pkg/kubecli" - "kubevirt.io/kubevirt/pkg/log" -) - -var upgrader = websocket.Upgrader{ - ReadBufferSize: 1024, - WriteBufferSize: 1024, -} - -type Console struct { - virtClient kubecli.KubevirtClient - k8sClient k8scorev1.CoreV1Interface - VirtHandlerPort string -} - -func NewConsoleResource(virtClient kubecli.KubevirtClient, k8sClient k8scorev1.CoreV1Interface) *Console { - return &Console{virtClient: virtClient, k8sClient: k8sClient} -} - -func (t *Console) Console(request *restful.Request, response *restful.Response) { - console := request.QueryParameter("console") - vmName := request.PathParameter("name") - namespace := request.PathParameter("namespace") - - vm, err := t.virtClient.VM(namespace).Get(vmName, k8sv1meta.GetOptions{}) - if errors.IsNotFound(err) { - log.Log.V(3).Infof("VM '%s' does not exist", vmName) - response.WriteError(http.StatusNotFound, fmt.Errorf("VM does not exist")) - return - } - if err != nil { - log.Log.Reason(err).Errorf("Error fetching VM '%s'", vmName) - response.WriteError(http.StatusInternalServerError, err) - return - } - - logger := log.Log.Object(vm) - - if !vm.IsRunning() { - logger.V(3).Reason(err).Info("VM is not running") - response.WriteError(http.StatusBadRequest, fmt.Errorf("VM is not running")) - return - } - - virtHandlerCon := kubecli.NewVirtHandlerClient(t.virtClient).ForNode(vm.Status.NodeName) - uri, err := virtHandlerCon.ConsoleURI(vm) - if err != nil { - msg := fmt.Sprintf("Looking up the connection details for virt-handler on node %s failed", vm.Status.NodeName) - logger.Reason(err).Error(msg) - response.WriteError(http.StatusInternalServerError, fmt.Errorf(msg)) - return - } - - if t.VirtHandlerPort != "" { - uri.Hostname() - uri.Host = uri.Hostname() + ":" + t.VirtHandlerPort - } - uri.Scheme = "ws" - if console != "" { - uri.RawQuery = "console=" + console - } - handlerSocket, resp, err := websocket.DefaultDialer.Dial(uri.String(), nil) - if err != nil { - if resp != nil && resp.StatusCode != http.StatusOK { - buf := new(bytes.Buffer) - buf.ReadFrom(resp.Body) - err := fmt.Errorf("%s", buf.String()) - logger.With("statusCode", resp.StatusCode).Reason(err).Error("Failed to connect to virt-handler") - response.WriteError(resp.StatusCode, err) - } else { - logger.Reason(err).Error("Failed to connect to virt-handler") - response.WriteError(http.StatusInternalServerError, err) - } - return - } - defer handlerSocket.Close() - - clientSocket, err := upgrader.Upgrade(response.ResponseWriter, request.Request, nil) - if err != nil { - logger.Reason(err).Error("Failed to upgrade client websocket connection") - response.WriteError(http.StatusBadRequest, err) - return - } - defer clientSocket.Close() - - errorChan := make(chan error) - - go func() { - _, err := io.Copy(clientSocket.UnderlyingConn(), handlerSocket.UnderlyingConn()) - errorChan <- err - }() - - go func() { - _, err := io.Copy(handlerSocket.UnderlyingConn(), clientSocket.UnderlyingConn()) - errorChan <- err - }() - - err = <-errorChan - if err != nil { - logger.Reason(err).Error("Proxied Web Socket connection failed") - } - response.WriteHeader(http.StatusOK) -} diff --git a/pkg/virt-api/rest/console_test.go b/pkg/virt-api/rest/console_test.go deleted file mode 100644 index 2ecb814e1161..000000000000 --- a/pkg/virt-api/rest/console_test.go +++ /dev/null @@ -1,192 +0,0 @@ -/* - * This file is part of the KubeVirt project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright 2017 Red Hat, Inc. - * - */ - -package rest - -import ( - "fmt" - "io/ioutil" - "net/http" - "net/http/httptest" - "net/url" - "strings" - - "github.com/emicklei/go-restful" - "github.com/golang/mock/gomock" - "github.com/gorilla/websocket" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - k8sv1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - fake2 "k8s.io/client-go/kubernetes/fake" - k8scorev1 "k8s.io/client-go/kubernetes/typed/core/v1" - - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime/schema" - - "kubevirt.io/kubevirt/pkg/api/v1" - "kubevirt.io/kubevirt/pkg/kubecli" - "kubevirt.io/kubevirt/pkg/log" -) - -var _ = Describe("Console", func() { - - var ctrl *gomock.Controller - var virtClient *kubecli.MockKubevirtClient - var vmInterface *kubecli.MockVMInterface - var k8sClient k8scorev1.CoreV1Interface - var vm *v1.VirtualMachine - var node *k8sv1.Node - var virtHandlerPod *k8sv1.Pod - var server *httptest.Server - var dial func(vm string, console string) *websocket.Conn - var get func(vm string) (*http.Response, error) - - log.Log.SetIOWriter(GinkgoWriter) - - BeforeEach(func() { - ctrl = gomock.NewController(GinkgoT()) - virtClient = kubecli.NewMockKubevirtClient(ctrl) - vmInterface = kubecli.NewMockVMInterface(ctrl) - virtClient.EXPECT().VM(k8sv1.NamespaceDefault).Return(vmInterface) - - vm = v1.NewMinimalVM("testvm") - vm.Status.Phase = v1.Running - vm.Status.NodeName = "testnode" - - node = &k8sv1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testnode", - }, - } - virtHandlerPod = &k8sv1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "virt-handerler-xkfoiw", - Namespace: k8sv1.NamespaceDefault, - Labels: map[string]string{ - v1.AppLabel: "virt-handler", - }, - }, - Spec: k8sv1.PodSpec{ - NodeName: node.ObjectMeta.Name, - }, - } - k8sClient = fake2.NewSimpleClientset(node, virtHandlerPod).CoreV1() - - ws := new(restful.WebService) - handler := http.Handler(restful.NewContainer().Add(ws)) - - // Endpoint to test - consoleResource := NewConsoleResource(virtClient, k8sClient) - ws.Route(ws.GET("/virt-api/namespaces/{namespace}/virtualmachines/{name}/console").To(consoleResource.Console)) - - // Mock out virt-handler. Mirror the first message and exit. - ws.Route(ws.GET("/api/v1/namespaces/{namespace}/virtualmachines/{name}/console").To(func(request *restful.Request, response *restful.Response) { - defer GinkgoRecover() - ws, err := upgrader.Upgrade(response.ResponseWriter, request.Request, nil) - Expect(err).ToNot(HaveOccurred()) - defer ws.Close() - t, data, err := ws.ReadMessage() - Expect(err).ToNot(HaveOccurred()) - err = ws.WriteMessage(t, data) - Expect(err).ToNot(HaveOccurred()) - response.WriteHeader(http.StatusOK) - })) - - server = httptest.NewServer(handler) - - wsUrl, err := url.Parse(server.URL) - serverUrl, err := url.ParseRequestURI(server.URL) - Expect(err).ToNot(HaveOccurred()) - consoleResource.VirtHandlerPort = strings.Split(serverUrl.Host, ":")[1] - - dial = func(vm string, console string) *websocket.Conn { - wsUrl.Scheme = "ws" - wsUrl.Path = "/virt-api/namespaces/" + k8sv1.NamespaceDefault + "/virtualmachines/" + vm + "/console" - wsUrl.RawQuery = "console=" + console - c, _, err := websocket.DefaultDialer.Dial(wsUrl.String(), nil) - Expect(err).ToNot(HaveOccurred()) - return c - } - - get = func(vm string) (*http.Response, error) { - wsUrl.Scheme = "http" - wsUrl.Path = "/virt-api/namespaces/" + k8sv1.NamespaceDefault + "/virtualmachines/" + vm + "/console" - return http.DefaultClient.Get(wsUrl.String()) - } - Expect(err).ToNot(HaveOccurred()) - }) - - It("Should proxy message through virt-api", func() { - - vmInterface.EXPECT().Get("testvm", gomock.Any()).Return(vm, nil) - virtClient.EXPECT().CoreV1().Return(k8sClient) - ws := dial("testvm", "console0") - defer ws.Close() - ws.WriteMessage(websocket.TextMessage, []byte("hello echo!")) - t, data, err := ws.ReadMessage() - Expect(t).To(Equal(websocket.TextMessage)) - Expect(err).ToNot(HaveOccurred()) - Expect(string(data)).To(Equal("hello echo!")) - }) - - It("Should return 404 if the VM does not exist", func() { - vmInterface.EXPECT().Get("testvm", gomock.Any()).Return(vm, errors.NewNotFound(schema.GroupResource{}, "testvm")) - response, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(response.StatusCode).To(Equal(http.StatusNotFound)) - }) - - It("Should return 500 if looking up the VM failed", func() { - vmInterface.EXPECT().Get("testvm", gomock.Any()).Return(vm, errors.NewInternalError(fmt.Errorf("something is weird"))) - response, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) - Expect(body(response)).To(ContainSubstring("something is weird")) - }) - - It("Should return 400 if the VM is not running", func() { - vmInterface.EXPECT().Get("testvm", gomock.Any()).Return(vm, nil) - vm.Status.Phase = v1.Succeeded - response, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(response.StatusCode).To(Equal(http.StatusBadRequest)) - }) - - It("Should return 500 if we can't look up the node", func() { - k8sClient.Pods(k8sv1.NamespaceDefault).Delete(virtHandlerPod.GetObjectMeta().GetName(), nil) - vmInterface.EXPECT().Get("testvm", gomock.Any()).Return(vm, nil) - virtClient.EXPECT().CoreV1().Return(k8sClient) - vm.Status.NodeName = "nonexistentnode" - response, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) - Expect(body(response)).To(ContainSubstring("Looking up the connection details for virt-handler on node nonexistentnode failed")) - }) - - AfterEach(func() { - ctrl.Finish() - }) -}) - -func body(request *http.Response) string { - b, err := ioutil.ReadAll(request.Body) - Expect(err).ToNot(HaveOccurred()) - return string(b) -} diff --git a/pkg/virt-api/rest/spice.go b/pkg/virt-api/rest/spice.go deleted file mode 100644 index cd1dbdf1f7f7..000000000000 --- a/pkg/virt-api/rest/spice.go +++ /dev/null @@ -1,85 +0,0 @@ -/* - * This file is part of the KubeVirt project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright 2017 Red Hat, Inc. - * - */ - -package rest - -import ( - "flag" - "fmt" - - "github.com/go-kit/kit/endpoint" - "golang.org/x/net/context" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/rest" - - "kubevirt.io/kubevirt/pkg/api/v1" - "kubevirt.io/kubevirt/pkg/middleware" - "kubevirt.io/kubevirt/pkg/rest/endpoints" -) - -var spiceProxy string - -func init() { - // TODO should be reloadable, use configmaps and update on every access? Watch a config file and reload? - flag.StringVar(&spiceProxy, "spice-proxy", "", "Spice proxy to use when spice access is requested") -} - -func NewSpiceEndpoint(cli *rest.RESTClient, gvr schema.GroupVersionResource) endpoint.Endpoint { - return func(ctx context.Context, payload interface{}) (interface{}, error) { - metadata := payload.(*endpoints.Metadata) - obj, err := cli.Get().Namespace(metadata.Namespace).Resource(gvr.Resource).Name(metadata.Name).Do().Get() - if err != nil { - return nil, middleware.NewInternalServerError(err) - } - - vm := obj.(*v1.VirtualMachine) - spice, err := spiceFromVM(vm) - if err != nil { - return nil, err - - } - - return spice, nil - } -} - -func spiceFromVM(vm *v1.VirtualMachine) (*v1.Spice, error) { - - if vm.Status.Phase != v1.Running { - return nil, middleware.NewResourceNotFoundError("VM is not running") - } - - // TODO allow specifying the spice device. For now select the first one. - for _, d := range vm.Status.Graphics { - if d.Type == "spice" { - spice := v1.NewSpice(vm.GetObjectMeta().GetNamespace(), vm.GetObjectMeta().GetName()) - spice.Info = v1.SpiceInfo{ - Type: "spice", - Host: d.Host, - Port: d.Port, - } - if spiceProxy != "" { - spice.Info.Proxy = fmt.Sprintf("%s", spiceProxy) - } - return spice, nil - } - } - - return nil, middleware.NewResourceNotFoundError("No spice device attached to the VM found.") -} diff --git a/pkg/virt-controller/services/template.go b/pkg/virt-controller/services/template.go index 902a98e35342..d0ecd5f82d58 100644 --- a/pkg/virt-controller/services/template.go +++ b/pkg/virt-controller/services/template.go @@ -69,6 +69,10 @@ func (t *templateService) RenderLaunchManifest(vm *v1.VirtualMachine) (*kubev1.P gracePeriodSeconds = gracePeriodSeconds + int64(15) gracePeriodKillAfter := gracePeriodSeconds + int64(15) + // TODO remove mounting the privateDir once + // the qemu pid is in the pods mount namespace + privateDir := fmt.Sprintf("%s-private/%s/%s", t.virtShareDir, namespace, domain) + // VM target container container := kubev1.Container{ Name: "compute", @@ -87,6 +91,10 @@ func (t *templateService) RenderLaunchManifest(vm *v1.VirtualMachine) (*kubev1.P Name: "virt-share-dir", MountPath: t.virtShareDir, }, + { + Name: "virt-private-dir", + MountPath: privateDir, + }, }, ReadinessProbe: &kubev1.Probe{ Handler: kubev1.Handler{ @@ -118,6 +126,14 @@ func (t *templateService) RenderLaunchManifest(vm *v1.VirtualMachine) (*kubev1.P }, }, }) + volumes = append(volumes, kubev1.Volume{ + Name: "virt-private-dir", + VolumeSource: kubev1.VolumeSource{ + HostPath: &kubev1.HostPathVolumeSource{ + Path: privateDir, + }, + }, + }) containers = append(containers, container) // TODO use constants for labels diff --git a/pkg/virt-handler/rest/console.go b/pkg/virt-handler/rest/console.go deleted file mode 100644 index cd906f40831d..000000000000 --- a/pkg/virt-handler/rest/console.go +++ /dev/null @@ -1,164 +0,0 @@ -/* - * This file is part of the KubeVirt project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright 2017 Red Hat, Inc. - * - */ - -package rest - -import ( - "io" - "net/http" - - "github.com/emicklei/go-restful" - "github.com/gorilla/websocket" - "github.com/libvirt/libvirt-go" - - "kubevirt.io/kubevirt/pkg/api/v1" - "kubevirt.io/kubevirt/pkg/log" - "kubevirt.io/kubevirt/pkg/virt-handler/virtwrap/api" - "kubevirt.io/kubevirt/pkg/virt-handler/virtwrap/cli" - "kubevirt.io/kubevirt/pkg/virt-handler/virtwrap/errors" - "kubevirt.io/kubevirt/pkg/virt-handler/virtwrap/util" -) - -var upgrader = websocket.Upgrader{ - ReadBufferSize: 1024, - WriteBufferSize: 1024, -} - -type Console struct { - connection cli.Connection -} - -func NewConsoleResource(connection cli.Connection) *Console { - return &Console{connection: connection} -} - -func (t *Console) Console(request *restful.Request, response *restful.Response) { - console := request.QueryParameter("console") - vmName := request.PathParameter("name") - namespace := request.PathParameter("namespace") - vm := v1.NewVMReferenceFromNameWithNS(namespace, vmName) - logger := log.Log.Object(vm) - domain, err := t.connection.LookupDomainByName(api.VMNamespaceKeyFunc(vm)) - if err != nil { - if errors.IsNotFound(err) { - logger.Reason(err).Error("Domain not found.") - response.WriteError(http.StatusNotFound, err) - return - } else { - response.WriteError(http.StatusInternalServerError, err) - logger.Reason(err).Error("Failed to look up domain.") - return - } - } - defer domain.Free() - - // Fetch metadata to get the VM UID - spec, err := util.GetDomainSpecWithFlags(domain, libvirt.DOMAIN_XML_INACTIVE) - if err != nil { - response.WriteError(http.StatusInternalServerError, err) - logger.Reason(err).Error("Failed to look up domain UID.") - return - } - vm.GetObjectMeta().SetUID(spec.Metadata.KubeVirt.UID) - logger = log.Log.Object(vm) - - logger.Infof("Opening connection to console %s", console) - - consoleStream, err := t.connection.NewStream(0) - if err != nil { - logger.Reason(err).Error("Creating a consoleStream failed.") - response.WriteError(http.StatusInternalServerError, err) - return - } - defer consoleStream.Close() - - logger.V(3).Info("Stream created.") - - err = domain.OpenConsole(console, consoleStream.UnderlyingStream(), libvirt.DOMAIN_CONSOLE_FORCE) - if err != nil { - response.WriteError(http.StatusInternalServerError, err) - logger.Reason(err).Error("Failed to open console.") - return - } - logger.V(3).Info("Connection to console created.") - - errorChan := make(chan error) - - ws, err := upgrader.Upgrade(response.ResponseWriter, request.Request, nil) - if err != nil { - logger.Reason(err).Error("Failed to upgrade websocket connection.") - response.WriteError(http.StatusBadRequest, err) - return - } - defer ws.Close() - - wsReadWriter := &TextReadWriter{ws} - - go func() { - _, err := io.Copy(consoleStream, wsReadWriter) - errorChan <- err - }() - - go func() { - _, err := io.Copy(wsReadWriter, consoleStream) - errorChan <- err - }() - - err = <-errorChan - - if err != nil { - logger.Reason(err).Error("Proxying data between libvirt and the websocket failed.") - } - - logger.V(3).Info("Done.") - response.WriteHeader(http.StatusOK) -} - -type TextReadWriter struct { - *websocket.Conn -} - -func (s *TextReadWriter) Write(p []byte) (int, error) { - err := s.Conn.WriteMessage(websocket.TextMessage, p) - if err != nil { - return 0, s.err(err) - } - return len(p), nil -} - -func (s *TextReadWriter) Read(p []byte) (int, error) { - _, r, err := s.Conn.NextReader() - if err != nil { - return 0, s.err(err) - } - n, err := r.Read(p) - return n, s.err(err) -} - -func (s *TextReadWriter) err(err error) error { - if err == nil { - return nil - } - if e, ok := err.(*websocket.CloseError); ok { - if e.Code == websocket.CloseNormalClosure { - return io.EOF - } - } - return err -} diff --git a/pkg/virt-handler/rest/console_test.go b/pkg/virt-handler/rest/console_test.go deleted file mode 100644 index 23c52322f68f..000000000000 --- a/pkg/virt-handler/rest/console_test.go +++ /dev/null @@ -1,223 +0,0 @@ -/* - * This file is part of the KubeVirt project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright 2017 Red Hat, Inc. - * - */ - -package rest - -import ( - "bytes" - "net/http" - "net/http/httptest" - "net/url" - - "github.com/emicklei/go-restful" - "github.com/golang/mock/gomock" - "github.com/gorilla/websocket" - "github.com/libvirt/libvirt-go" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - k8sv1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/uuid" - "k8s.io/client-go/tools/record" - - "encoding/xml" - - "kubevirt.io/kubevirt/pkg/log" - "kubevirt.io/kubevirt/pkg/virt-handler/virtwrap/api" - "kubevirt.io/kubevirt/pkg/virt-handler/virtwrap/cli" -) - -var _ = Describe("Console", func() { - var handler http.Handler - - var mockConn *cli.MockConnection - var mockDomain *cli.MockVirDomain - var mockStream *cli.MockStream - var ctrl *gomock.Controller - var mockRecorder *record.FakeRecorder - var server *httptest.Server - var wsUrl *url.URL - var serverDone chan bool - - log.Log.SetIOWriter(GinkgoWriter) - - dial := func(vm string, console string) *websocket.Conn { - wsUrl.Scheme = "ws" - wsUrl.Path = "/api/v1/namespaces/" + k8sv1.NamespaceDefault + "/virtualmachines/" + vm + "/console" - wsUrl.RawQuery = "console=" + console - c, _, err := websocket.DefaultDialer.Dial(wsUrl.String(), nil) - Expect(err).ToNot(HaveOccurred()) - return c - } - - get := func(vm string) (*http.Response, error) { - wsUrl.Scheme = "http" - wsUrl.Path = "/api/v1/namespaces/" + k8sv1.NamespaceDefault + "/virtualmachines/" + vm + "/console" - return http.DefaultClient.Get(wsUrl.String()) - } - - BeforeEach(func() { - var err error - // Set up mocks - ctrl = gomock.NewController(GinkgoT()) - mockConn = cli.NewMockConnection(ctrl) - mockDomain = cli.NewMockVirDomain(ctrl) - mockStream = cli.NewMockStream(ctrl) - mockRecorder = record.NewFakeRecorder(10) - - // Set up web service - ws := new(restful.WebService) - handler = http.Handler(restful.NewContainer().Add(ws)) - - // Give us a chance to detect when the request is done. Otherwise we - // don't know when to check mock invokations - serverDone = make(chan bool) - waiter := func(request *restful.Request, response *restful.Response) { - NewConsoleResource(mockConn).Console(request, response) - close(serverDone) - } - ws.Route(ws.GET("/api/v1/namespaces/{namespace}/virtualmachines/{name}/console").To(waiter)) - server = httptest.NewServer(handler) - wsUrl, err = url.Parse(server.URL) - Expect(err).ToNot(HaveOccurred()) - }) - - It("should return 404 if VM does not exist", func() { - mockConn.EXPECT().LookupDomainByName("default_testvm").Return(nil, libvirt.Error{Code: libvirt.ERR_NO_DOMAIN}) - r, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(r.StatusCode).To(Equal(http.StatusNotFound)) - }) - - It("should return 500 if domain can't be looked up", func() { - mockConn.EXPECT().LookupDomainByName("default_testvm").Return(nil, libvirt.Error{Code: libvirt.ERR_INVALID_CONN}) - r, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(r.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - - Context("with existing domain", func() { - var specXML string - - BeforeEach(func() { - // Make sure that we always free the domain after use - mockDomain.EXPECT().Free() - spec := api.NewMinimalDomainSpec("default_testvm") - spec.Metadata.KubeVirt.UID = uuid.NewUUID() - data, err := xml.Marshal(spec) - specXML = string(data) - Expect(err).ToNot(HaveOccurred()) - }) - - It("should return 500 if metadata of domain can't be looked up", func() { - mockConn.EXPECT().LookupDomainByName("default_testvm").Return(mockDomain, nil) - mockDomain.EXPECT().GetXMLDesc(gomock.Eq(libvirt.DOMAIN_XML_INACTIVE)).Return("", libvirt.Error{Code: libvirt.ERR_INVALID_CONN}) - - r, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(r.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - It("should return 500 if creating a stream fails", func() { - mockConn.EXPECT().LookupDomainByName("default_testvm").Return(mockDomain, nil) - mockDomain.EXPECT().GetXMLDesc(gomock.Eq(libvirt.DOMAIN_XML_INACTIVE)).Return(specXML, nil) - mockConn.EXPECT().NewStream(libvirt.StreamFlags(0)).Return(nil, libvirt.Error{Code: libvirt.ERR_INVALID_CONN}) - r, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(r.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - It("should return 500 if opening a console connection fails", func() { - mockConn.EXPECT().LookupDomainByName("default_testvm").Return(mockDomain, nil) - mockDomain.EXPECT().GetXMLDesc(gomock.Eq(libvirt.DOMAIN_XML_INACTIVE)).Return(specXML, nil) - mockConn.EXPECT().NewStream(libvirt.StreamFlags(0)).Return(mockStream, nil) - stream := &libvirt.Stream{} - mockStream.EXPECT().UnderlyingStream().Return(stream) - mockStream.EXPECT().Close() - mockDomain.EXPECT().OpenConsole("", stream, libvirt.DomainConsoleFlags(libvirt.DOMAIN_CONSOLE_FORCE)).Return(libvirt.Error{Code: libvirt.ERR_INVALID_CONN}) - r, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(r.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - It("should return 400 if ws upgrade does not work", func() { - mockConn.EXPECT().LookupDomainByName("default_testvm").Return(mockDomain, nil) - mockDomain.EXPECT().GetXMLDesc(gomock.Eq(libvirt.DOMAIN_XML_INACTIVE)).Return(specXML, nil) - mockConn.EXPECT().NewStream(libvirt.StreamFlags(0)).Return(mockStream, nil) - stream := &libvirt.Stream{} - mockStream.EXPECT().UnderlyingStream().Return(stream) - mockStream.EXPECT().Close() - mockDomain.EXPECT().OpenConsole("", stream, libvirt.DomainConsoleFlags(libvirt.DOMAIN_CONSOLE_FORCE)).Return(nil) - r, err := get("testvm") - Expect(err).ToNot(HaveOccurred()) - Expect(r.StatusCode).To(Equal(http.StatusBadRequest)) - }) - It("should proxy websocket traffic", func() { - - var in []byte - var out []byte - inBuf := bytes.NewBuffer(in) - outBuf := bytes.NewBuffer(out) - outBuf.WriteString("hello client!") - stream := &fakeStream{in: inBuf, out: outBuf, s: &libvirt.Stream{}} - - mockConn.EXPECT().LookupDomainByName("default_testvm").Return(mockDomain, nil) - mockDomain.EXPECT().GetXMLDesc(gomock.Eq(libvirt.DOMAIN_XML_INACTIVE)).Return(specXML, nil) - mockConn.EXPECT().NewStream(libvirt.StreamFlags(0)).Return(stream, nil) - mockDomain.EXPECT().OpenConsole("console0", stream.s, libvirt.DomainConsoleFlags(libvirt.DOMAIN_CONSOLE_FORCE)).Return(nil) - - con := dial("testvm", "console0") - defer con.Close() - err := con.WriteMessage(websocket.TextMessage, []byte("hello console!")) - Expect(err).ToNot(HaveOccurred()) - - // FIXME, there is somewhere a buffer or timeout which delays the actual send - // Eventually(inBuf.String).Should(Equal("hello console!")) - - t, body, err := con.ReadMessage() - Expect(t).To(Equal(websocket.TextMessage)) - Expect(err).ToNot(HaveOccurred()) - Expect(body).To(Equal([]byte("hello client!"))) - }) - - }) - AfterEach(func() { - server.Close() - <-serverDone - ctrl.Finish() - }) -}) - -type fakeStream struct { - in *bytes.Buffer - out *bytes.Buffer - s *libvirt.Stream -} - -func (s *fakeStream) Write(p []byte) (n int, err error) { - return s.in.Write(p) -} - -func (s *fakeStream) Read(p []byte) (n int, err error) { - return s.out.Read(p) -} - -func (s *fakeStream) Close() (e error) { - return nil -} - -func (s *fakeStream) UnderlyingStream() *libvirt.Stream { - return s.s -} diff --git a/pkg/virt-handler/virtwrap/api/converter.go b/pkg/virt-handler/virtwrap/api/converter.go index 036ab87ffcfc..b222e321111b 100644 --- a/pkg/virt-handler/virtwrap/api/converter.go +++ b/pkg/virt-handler/virtwrap/api/converter.go @@ -75,6 +75,7 @@ func Convert_v1_ISCSIVolumeSource_To_api_Disk(source *k8sv1.ISCSIVolumeSource, d disk.Type = "network" disk.Driver.Type = "raw" + disk.Driver.Cache = "none" disk.Source.Name = fmt.Sprintf("%s/%d", source.IQN, source.Lun) disk.Source.Protocol = "iscsi" @@ -324,6 +325,43 @@ func Convert_v1_VirtualMachine_To_api_Domain(vm *v1.VirtualMachine, domain *Doma } } + // Add mandatory console device + var serialPort uint = 0 + var serialType string = "serial" + domain.Spec.Devices.Consoles = []Console{ + { + Type: "pty", + Target: &ConsoleTarget{ + Type: &serialType, + Port: &serialPort, + }, + }, + } + + domain.Spec.Devices.Serials = []Serial{ + { + Type: "unix", + Target: &SerialTarget{ + Port: &serialPort, + }, + Source: &SerialSource{ + Mode: "bind", + Path: fmt.Sprintf("/var/run/kubevirt-private/%s/%s/virt-serial%d", vm.ObjectMeta.Namespace, vm.ObjectMeta.Name, serialPort), + }, + }, + } + + // Add mandatory vnc device + domain.Spec.Devices.Graphics = []Graphics{ + { + Listen: &GraphicsListen{ + Type: "socket", + Socket: fmt.Sprintf("/var/run/kubevirt-private/%s/%s/virt-vnc", vm.ObjectMeta.Namespace, vm.ObjectMeta.Name), + }, + Type: "vnc", + }, + } + return nil } diff --git a/pkg/virt-handler/virtwrap/api/converter_test.go b/pkg/virt-handler/virtwrap/api/converter_test.go index 6541224c0f91..e4c0b002adce 100644 --- a/pkg/virt-handler/virtwrap/api/converter_test.go +++ b/pkg/virt-handler/virtwrap/api/converter_test.go @@ -270,15 +270,15 @@ var _ = Describe("Converter", func() { - - + + - + @@ -299,7 +299,7 @@ var _ = Describe("Converter", func() { - + @@ -307,7 +307,7 @@ var _ = Describe("Converter", func() { - + @@ -315,7 +315,7 @@ var _ = Describe("Converter", func() { - + @@ -324,7 +324,7 @@ var _ = Describe("Converter", func() { - + @@ -332,13 +332,19 @@ var _ = Describe("Converter", func() { - + - + + + + + + + diff --git a/pkg/virt-handler/virtwrap/api/deepcopy_generated.go b/pkg/virt-handler/virtwrap/api/deepcopy_generated.go index f8fd399fdb07..6609bd2ae622 100644 --- a/pkg/virt-handler/virtwrap/api/deepcopy_generated.go +++ b/pkg/virt-handler/virtwrap/api/deepcopy_generated.go @@ -277,6 +277,24 @@ func (in *Console) DeepCopyInto(out *Console) { (*in).DeepCopyInto(*out) } } + if in.Source != nil { + in, out := &in.Source, &out.Source + if *in == nil { + *out = nil + } else { + *out = new(ConsoleSource) + **out = **in + } + } + if in.Alias != nil { + in, out := &in.Alias, &out.Alias + if *in == nil { + *out = nil + } else { + *out = new(Alias) + **out = **in + } + } return } @@ -290,6 +308,22 @@ func (in *Console) DeepCopy() *Console { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConsoleSource) DeepCopyInto(out *ConsoleSource) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConsoleSource. +func (in *ConsoleSource) DeepCopy() *ConsoleSource { + if in == nil { + return nil + } + out := new(ConsoleSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ConsoleTarget) DeepCopyInto(out *ConsoleTarget) { *out = *in @@ -388,7 +422,9 @@ func (in *Devices) DeepCopyInto(out *Devices) { if in.Graphics != nil { in, out := &in.Graphics, &out.Graphics *out = make([]Graphics, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } if in.Ballooning != nil { in, out := &in.Ballooning, &out.Ballooning @@ -1044,7 +1080,15 @@ func (in *GracePeriodMetadata) DeepCopy() *GracePeriodMetadata { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Graphics) DeepCopyInto(out *Graphics) { *out = *in - out.Listen = in.Listen + if in.Listen != nil { + in, out := &in.Listen, &out.Listen + if *in == nil { + *out = nil + } else { + *out = new(GraphicsListen) + **out = **in + } + } return } @@ -1058,6 +1102,22 @@ func (in *Graphics) DeepCopy() *Graphics { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GraphicsListen) DeepCopyInto(out *GraphicsListen) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GraphicsListen. +func (in *GraphicsListen) DeepCopy() *GraphicsListen { + if in == nil { + return nil + } + out := new(GraphicsListen) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Interface) DeepCopyInto(out *Interface) { *out = *in @@ -1221,22 +1281,6 @@ func (in *LinkState) DeepCopy() *LinkState { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Listen) DeepCopyInto(out *Listen) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Listen. -func (in *Listen) DeepCopy() *Listen { - if in == nil { - return nil - } - out := new(Listen) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Loader) DeepCopyInto(out *Loader) { *out = *in @@ -1509,6 +1553,24 @@ func (in *Serial) DeepCopyInto(out *Serial) { (*in).DeepCopyInto(*out) } } + if in.Source != nil { + in, out := &in.Source, &out.Source + if *in == nil { + *out = nil + } else { + *out = new(SerialSource) + **out = **in + } + } + if in.Alias != nil { + in, out := &in.Alias, &out.Alias + if *in == nil { + *out = nil + } else { + *out = new(Alias) + **out = **in + } + } return } @@ -1522,6 +1584,22 @@ func (in *Serial) DeepCopy() *Serial { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SerialSource) DeepCopyInto(out *SerialSource) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SerialSource. +func (in *SerialSource) DeepCopy() *SerialSource { + if in == nil { + return nil + } + out := new(SerialSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SerialTarget) DeepCopyInto(out *SerialTarget) { *out = *in diff --git a/pkg/virt-handler/virtwrap/api/deepcopy_test.go b/pkg/virt-handler/virtwrap/api/deepcopy_test.go index 0fc7a747fc7e..d93a2fbb9305 100644 --- a/pkg/virt-handler/virtwrap/api/deepcopy_test.go +++ b/pkg/virt-handler/virtwrap/api/deepcopy_test.go @@ -72,7 +72,7 @@ var _ = Describe("Generated deepcopy functions", func() { &Video{}, &VideoModel{}, &Graphics{}, - &Listen{}, + &GraphicsListen{}, &Address{}, &Ballooning{}, &RandomGenerator{}, diff --git a/pkg/virt-handler/virtwrap/api/defaults.go b/pkg/virt-handler/virtwrap/api/defaults.go index b880271ccbb9..9c284ccacb6f 100644 --- a/pkg/virt-handler/virtwrap/api/defaults.go +++ b/pkg/virt-handler/virtwrap/api/defaults.go @@ -1,17 +1,6 @@ package api func SetDefaults_Devices(devices *Devices) { - // Add mandatory spice device - devices.Graphics = []Graphics{ - { - Port: -1, - Listen: Listen{ - Type: "address", - Address: "0.0.0.0", - }, - Type: "spice", - }, - } // Use vga as video device, since it is better than cirrus // and does not require guest drivers var heads uint = 1 @@ -25,12 +14,7 @@ func SetDefaults_Devices(devices *Devices) { }, }, } - // Add mandatory console device - devices.Consoles = []Console{ - { - Type: "pty", - }, - } + // For now connect every virtual machine to the default network devices.Interfaces = []Interface{{ Type: "network", diff --git a/pkg/virt-handler/virtwrap/api/schema.go b/pkg/virt-handler/virtwrap/api/schema.go index 9c96bba2465e..e1a7d96f472d 100644 --- a/pkg/virt-handler/virtwrap/api/schema.go +++ b/pkg/virt-handler/virtwrap/api/schema.go @@ -254,12 +254,19 @@ type DiskSourceHost struct { type Serial struct { Type string `xml:"type,attr"` Target *SerialTarget `xml:"target,omitempty"` + Source *SerialSource `xml:"source,omitempty"` + Alias *Alias `xml:"alias,omitmepty"` } type SerialTarget struct { Port *uint `xml:"port,attr,omitempty"` } +type SerialSource struct { + Mode string `xml:"mode,attr,omitempty"` + Path string `xml:"path,attr,omitempty"` +} + // END Serial ----------------------------- // BEGIN Console ----------------------------- @@ -267,6 +274,8 @@ type SerialTarget struct { type Console struct { Type string `xml:"type,attr"` Target *ConsoleTarget `xml:"target,omitempty"` + Source *ConsoleSource `xml:"source,omitempty"` + Alias *Alias `xml:"alias,omitmepty"` } type ConsoleTarget struct { @@ -274,6 +283,11 @@ type ConsoleTarget struct { Port *uint `xml:"port,attr,omitempty"` } +type ConsoleSource struct { + Mode string `xml:"mode,attr,omitempty"` + Path string `xml:"path,attr,omitempty"` +} + // END Serial ----------------------------- // BEGIN Inteface ----------------------------- @@ -446,19 +460,20 @@ type VideoModel struct { } type Graphics struct { - AutoPort string `xml:"autoPort,attr,omitempty"` - DefaultMode string `xml:"defaultMode,attr,omitempty"` - Listen Listen `xml:"listen,omitempty"` - PasswdValidTo string `xml:"passwdValidTo,attr,omitempty"` - Port int32 `xml:"port,attr,omitempty"` - TLSPort int `xml:"tlsPort,attr,omitempty"` - Type string `xml:"type,attr"` + AutoPort string `xml:"autoPort,attr,omitempty"` + DefaultMode string `xml:"defaultMode,attr,omitempty"` + Listen *GraphicsListen `xml:"listen,omitempty"` + PasswdValidTo string `xml:"passwdValidTo,attr,omitempty"` + Port int32 `xml:"port,attr,omitempty"` + TLSPort int `xml:"tlsPort,attr,omitempty"` + Type string `xml:"type,attr"` } -type Listen struct { +type GraphicsListen struct { Type string `xml:"type,attr"` Address string `xml:"address,attr,omitempty"` Network string `xml:"newtork,attr,omitempty"` + Socket string `xml:"socket,attr,omitempty"` } type Address struct { diff --git a/pkg/virt-handler/virtwrap/api/schema_test.go b/pkg/virt-handler/virtwrap/api/schema_test.go index 4b08941ab2f1..26216179ffeb 100644 --- a/pkg/virt-handler/virtwrap/api/schema_test.go +++ b/pkg/virt-handler/virtwrap/api/schema_test.go @@ -46,9 +46,6 @@ var exampleXML = ` - - - diff --git a/pkg/virt-handler/vm.go b/pkg/virt-handler/vm.go index 942ccc9a363a..734e86e8d146 100644 --- a/pkg/virt-handler/vm.go +++ b/pkg/virt-handler/vm.go @@ -40,6 +40,7 @@ import ( "kubevirt.io/kubevirt/pkg/api/v1" "kubevirt.io/kubevirt/pkg/config-disk" "kubevirt.io/kubevirt/pkg/controller" + diskutils "kubevirt.io/kubevirt/pkg/ephemeral-disk-utils" "kubevirt.io/kubevirt/pkg/kubecli" "kubevirt.io/kubevirt/pkg/log" "kubevirt.io/kubevirt/pkg/registry-disk" @@ -205,38 +206,10 @@ func (d *VirtualMachineController) updateVMStatus(vm *v1.VirtualMachine, domain } oldStatus := vm.DeepCopy().Status - // Make sure that we always deal with an empty instance for later equality checks - if oldStatus.Graphics == nil { - oldStatus.Graphics = []v1.VirtualMachineGraphics{} - } // Calculate the new VM state based on what libvirt reported d.setVmPhaseForStatusReason(domain, vm) - vm.Status.Graphics = []v1.VirtualMachineGraphics{} - - // Update devices if device status changed - // TODO needs caching, better position or init fetch - if domain != nil { - nodeIP, err := d.getVMNodeAddress(vm) - if err != nil { - return err - } - - vm.Status.Graphics = []v1.VirtualMachineGraphics{} - for _, src := range domain.Spec.Devices.Graphics { - if (src.Type != "spice" && src.Type != "vnc") || src.Port == -1 { - continue - } - dst := v1.VirtualMachineGraphics{ - Type: src.Type, - Host: nodeIP, - Port: src.Port, - } - vm.Status.Graphics = append(vm.Status.Graphics, dst) - } - } - d.checkFailure(vm, syncError, "Synchronizing with the Domain failed.") if !reflect.DeepEqual(oldStatus, vm.Status) { @@ -588,12 +561,26 @@ func (d *VirtualMachineController) injectDiskAuth(vm *v1.VirtualMachine) (map[st return secrets, nil } +// TODO this function should go away once qemu is in the pods mount namespace. +func (d *VirtualMachineController) cleanupUnixSockets(vm *v1.VirtualMachine) error { + namespace := vm.ObjectMeta.Namespace + name := vm.ObjectMeta.Name + unixPath := fmt.Sprintf("%s-private/%s/%s", d.virtShareDir, namespace, name) + // when this is removed, it will fix issue #626 + return diskutils.RemoveFile(unixPath) +} + func (d *VirtualMachineController) processVmCleanup(vm *v1.VirtualMachine) error { err := d.domainManager.RemoveVMSecrets(vm) if err != nil { return err } + err = d.cleanupUnixSockets(vm) + if err != nil { + return err + } + err = registrydisk.CleanupEphemeralDisks(vm) if err != nil { return err diff --git a/pkg/virt-handler/vm_test.go b/pkg/virt-handler/vm_test.go index 26c5f93efc2d..c87ba486a19c 100644 --- a/pkg/virt-handler/vm_test.go +++ b/pkg/virt-handler/vm_test.go @@ -263,7 +263,6 @@ var _ = Describe("VM", func() { vm := v1.NewMinimalVM("testvm") vm.ObjectMeta.ResourceVersion = "1" vm.Status.Phase = v1.Scheduled - vm.Status.Graphics = []v1.VirtualMachineGraphics{} mockWatchdog.CreateFile(vm) domain := api.NewMinimalDomain("testvm") @@ -278,7 +277,6 @@ var _ = Describe("VM", func() { vm := v1.NewMinimalVM("testvm") vm.ObjectMeta.ResourceVersion = "1" vm.Status.Phase = v1.Scheduled - vm.Status.Graphics = []v1.VirtualMachineGraphics{} updatedVM := vm.DeepCopy() updatedVM.Status.Phase = v1.Running @@ -322,7 +320,6 @@ var _ = Describe("VM", func() { It("should remove an error condition if a synchronization run succeeds", func() { vm := v1.NewMinimalVM("testvm") vm.ObjectMeta.ResourceVersion = "1" - vm.Status.Graphics = []v1.VirtualMachineGraphics{} vm.Status.Phase = v1.Scheduled vm.Status.Conditions = []v1.VirtualMachineCondition{ { diff --git a/pkg/virt-launcher/monitor.go b/pkg/virt-launcher/monitor.go index a351d1f7f191..ecffa9074f37 100644 --- a/pkg/virt-launcher/monitor.go +++ b/pkg/virt-launcher/monitor.go @@ -106,6 +106,30 @@ func GracefulShutdownTriggerInitiate(triggerFile string) error { return nil } +func InitializePrivateDirectories(baseDir string) error { + unixPathVNC := filepath.Join(baseDir, "virt-vnc") + unixPathConsole := filepath.Join(baseDir, "virt-serial0") + + err := os.MkdirAll(filepath.Dir(unixPathVNC), 0755) + if err != nil { + return err + } + err = os.MkdirAll(filepath.Dir(unixPathConsole), 0755) + if err != nil { + return err + } + err = diskutils.SetFileOwnership("qemu", filepath.Dir(unixPathVNC)) + if err != nil { + return err + } + err = diskutils.SetFileOwnership("qemu", filepath.Dir(unixPathConsole)) + if err != nil { + return err + } + + return nil +} + func InitializeSharedDirectories(baseDir string) error { err := os.MkdirAll(watchdog.WatchdogFileDirectory(baseDir), 0755) if err != nil { diff --git a/pkg/virtctl/console/console.go b/pkg/virtctl/console/console.go index eb4505b8d5fb..010c7f9df205 100644 --- a/pkg/virtctl/console/console.go +++ b/pkg/virtctl/console/console.go @@ -20,24 +20,17 @@ package console import ( - "bytes" "fmt" "io" "log" - "net/http" - "net/url" "os" "os/signal" - "time" - "sync" - - "github.com/gorilla/websocket" flag "github.com/spf13/pflag" "golang.org/x/crypto/ssh/terminal" "k8s.io/api/core/v1" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" + + "kubevirt.io/kubevirt/pkg/kubecli" ) type Console struct { @@ -45,7 +38,7 @@ type Console struct { func (c *Console) FlagSet() *flag.FlagSet { cf := flag.NewFlagSet("console", flag.ExitOnError) - cf.StringP("device", "d", "", "Console to connect to") + cf.StringP("device", "d", "serial0", "Console to connect to") return cf } @@ -53,8 +46,8 @@ func (c *Console) FlagSet() *flag.FlagSet { func (c *Console) Usage() string { usage := "Connect to a serial console on a VM:\n\n" usage += "Examples:\n" - usage += "# Connect to the console 'serial0' on the VM 'myvm':\n" - usage += "virtctl console myvm --device serial0\n\n" + usage += "# Connect to the console on VM 'myvm':\n" + usage += "virtctl console myvm\n\n" usage += "Options:\n" usage += c.FlagSet().FlagUsages() return usage @@ -65,7 +58,7 @@ func (c *Console) Run(flags *flag.FlagSet) int { server, _ := flags.GetString("server") kubeconfig, _ := flags.GetString("kubeconfig") namespace, _ := flags.GetString("namespace") - device, _ := flags.GetString("device") + device := "serial0" if namespace == "" { namespace = v1.NamespaceDefault } @@ -75,132 +68,37 @@ func (c *Console) Run(flags *flag.FlagSet) int { } vm := flags.Arg(1) - config, err := clientcmd.BuildConfigFromFlags(server, kubeconfig) + virtCli, err := kubecli.GetKubevirtClientFromFlags(server, kubeconfig) if err != nil { log.Println(err) return 1 } - err = ConnectToConsole(config, namespace, vm, device, TerminalWebsocketCallback) + state, err := terminal.MakeRaw(int(os.Stdin.Fd())) if err != nil { - log.Println(err) + log.Printf("Make raw terminal failed: %s", err) return 1 } - return 0 -} - -func ConnectToConsole(config *rest.Config, namespace string, name string, console string, callback RoundTripCallback) error { - - // Create a round tripper with all necessary kubernetes security details - wrappedRoundTripper, err := RoundTripperFromConfig(config, callback) - if err != nil { - return err - } - - // Create the basic console request - req, err := RequestFromConfig(config, name, namespace, console) - if err != nil { - return err - } - - // Do the call and process the websocket connection with the callback - _, err = wrappedRoundTripper.RoundTrip(req) - - if err != nil { - return err - } - return nil -} - -func NewWebsocketCallback(in io.ReadCloser, out io.WriteCloser, stopChan chan struct{}) RoundTripCallback { + fmt.Fprint(os.Stderr, "Escape sequence is ^]") - return func(ws *websocket.Conn, resp *http.Response, err error) error { + in := os.Stdin + out := os.Stdout - if err != nil { - if resp != nil && resp.StatusCode != http.StatusOK { - buf := new(bytes.Buffer) - buf.ReadFrom(resp.Body) - return fmt.Errorf("Can't connect to console (%d): %s\n", resp.StatusCode, buf.String()) - } - return fmt.Errorf("Can't connect to console: %s\n", err.Error()) - } - - writeStop := make(chan struct{}) - readStop := make(chan struct{}) - - go func() { - defer close(readStop) - for { - _, message, err := ws.ReadMessage() - if err != nil { - out.Write(message) - return - } - _, err = out.Write(message) - if err == io.EOF { - return - } - } - }() - - buf := make([]byte, 1024, 1024) - - // Synchronize writes for final close announcements - var writeMux sync.Mutex - writeProtected := func(messageType int, data []byte) error { - writeMux.Lock() - defer writeMux.Unlock() - return ws.WriteMessage(websocket.TextMessage, data) - } - - go func() { - defer close(writeStop) - for { - n, err := in.Read(buf) - if err != nil && err != io.EOF { - log.Println(err) - return - } - - // TODO move this to the TerminalWebsocketCallback - if buf[0] == 29 { - return - } - - // If there is nothing more to write and we have reached EOF, return - if n == 0 && err == io.EOF { - return - } - - err = writeProtected(websocket.TextMessage, buf[0:n]) - if err != nil && err != io.EOF { - log.Println(err) - return - } - } - }() - - select { - case <-stopChan: - case <-readStop: - case <-writeStop: - } - - err = writeProtected(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) - if err != nil { - return fmt.Errorf("Error on close announcement: %s", err.Error()) - } - select { - case <-readStop: - case <-time.After(time.Second): - } - return nil - } -} + stdinReader, stdinWriter := io.Pipe() + stdoutReader, stdoutWriter := io.Pipe() -func TerminalWebsocketCallback(ws *websocket.Conn, resp *http.Response, connErr error) error { + // in -> stdinWriter | stdinReader -> console + // out <- stdoutReader | stdoutWriter <- console + resChan := make(chan error) stopChan := make(chan struct{}, 1) + writeStop := make(chan error) + readStop := make(chan error) + + go func() { + err := virtCli.VM(namespace).SerialConsole(vm, device, stdinReader, stdoutWriter) + resChan <- err + }() go func() { interrupt := make(chan os.Signal, 1) @@ -209,82 +107,49 @@ func TerminalWebsocketCallback(ws *websocket.Conn, resp *http.Response, connErr close(stopChan) }() - // If there is no obvious connection error, set up the terminal - if connErr == nil { - state, err := terminal.MakeRaw(int(os.Stdin.Fd())) - if err != nil { - return fmt.Errorf("Make raw terminal failed: %s", err) - } - defer terminal.Restore(int(os.Stdin.Fd()), state) - fmt.Fprint(os.Stderr, "Escape sequence is ^]") - } - - return NewWebsocketCallback(os.Stdin, os.Stdout, stopChan)(ws, resp, connErr) -} - -func RequestFromConfig(config *rest.Config, vm string, namespace string, device string) (*http.Request, error) { + go func() { + _, err := io.Copy(out, stdoutReader) + readStop <- err + }() - u, err := url.Parse(config.Host) - if err != nil { - return nil, err - } + go func() { + defer close(writeStop) + buf := make([]byte, 1024, 1024) + for { + // reading from stdin + n, err := in.Read(buf) + if err != nil && err != io.EOF { + writeStop <- err + return + } + if n == 0 && err == io.EOF { + return + } - switch u.Scheme { - case "https": - u.Scheme = "wss" - case "http": - u.Scheme = "ws" - default: - return nil, fmt.Errorf("Unsupported Protocol %s", u.Scheme) - } + // the escape sequence + if buf[0] == 29 { + return + } + // Writing out to the console connection + _, err = stdinWriter.Write(buf[0:n]) + if err == io.EOF { + return + } + } + }() - u.Path = fmt.Sprintf("/apis/kubevirt.io/v1alpha1/namespaces/%s/virtualmachines/%s/console", namespace, vm) - if device != "" { - u.RawQuery = "console=" + device - } - req := &http.Request{ - Method: http.MethodGet, - URL: u, + select { + case <-stopChan: + case err = <-readStop: + case err = <-writeStop: + case err = <-resChan: } - return req, nil -} - -func RoundTripperFromConfig(config *rest.Config, callback RoundTripCallback) (http.RoundTripper, error) { + terminal.Restore(int(os.Stdin.Fd()), state) - // Configure TLS - tlsConfig, err := rest.TLSConfigFor(config) if err != nil { - return nil, err - } - - // Configure the websocket dialer - dialer := &websocket.Dialer{ - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: tlsConfig, - } - - // Create a roundtripper which will pass in the final underlying websocket connection to a callback - rt := &WebsocketRoundTripper{ - Do: callback, - Dialer: dialer, - } - - // Make sure we inherit all relevant security headers - return rest.HTTPWrappersForConfig(config, rt) -} - -type RoundTripCallback func(conn *websocket.Conn, resp *http.Response, err error) error - -type WebsocketRoundTripper struct { - Dialer *websocket.Dialer - Do RoundTripCallback -} - -func (d *WebsocketRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { - conn, resp, err := d.Dialer.Dial(r.URL.String(), r.Header) - if err == nil { - defer conn.Close() + log.Println(err) + return 1 } - return resp, d.Do(conn, resp, err) + return 0 } diff --git a/pkg/virtctl/spice/spice.go b/pkg/virtctl/spice/spice.go deleted file mode 100644 index 3b5ee8a24869..000000000000 --- a/pkg/virtctl/spice/spice.go +++ /dev/null @@ -1,150 +0,0 @@ -/* - * This file is part of the KubeVirt project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright 2017 Red Hat, Inc. - * - */ - -package spice - -import ( - "errors" - "fmt" - "io/ioutil" - "log" - "os" - "os/exec" - - flag "github.com/spf13/pflag" - kubev1 "k8s.io/api/core/v1" - "k8s.io/client-go/rest" - - "gopkg.in/ini.v1" - - "kubevirt.io/kubevirt/pkg/api/v1" - "kubevirt.io/kubevirt/pkg/kubecli" -) - -const FLAG = "spice" -const TEMP_PREFIX = "spice" - -type Spice struct { -} - -func DownloadSpice(namespace string, vm string, restClient *rest.RESTClient) (*v1.Spice, error) { - spice := &v1.Spice{} - err := restClient.Get(). - Resource("virtualmachines").SetHeader("Accept", "application/json"). - SubResource("spice"). - Namespace(namespace). - Name(vm).Do().Into(spice) - if err != nil { - return nil, errors.New(fmt.Sprintf("Can't fetch connection details: %s\n", err.Error())) - } - return spice, nil -} - -func (o *Spice) FlagSet() *flag.FlagSet { - - cf := flag.NewFlagSet(FLAG, flag.ExitOnError) - cf.BoolP("details", "d", false, "If present, print SPICE console to stdout, otherwise run remote-viewer") - cf.StringP("proxy", "p", "", "If given, will override any given proxy from the server") - return cf -} - -func (o *Spice) Run(flags *flag.FlagSet) int { - server, _ := flags.GetString("server") - kubeconfig, _ := flags.GetString("kubeconfig") - details, _ := flags.GetBool("details") - proxy, _ := flags.GetString("proxy") - namespace, _ := flags.GetString("namespace") - if namespace == "" { - namespace = kubev1.NamespaceDefault - } - - if len(flags.Args()) != 2 { - log.Println("VM name is missing") - return 1 - } - vm := flags.Arg(1) - - virtClient, err := kubecli.GetKubevirtClientFromFlags(server, kubeconfig) - - if err != nil { - log.Println(err) - return 1 - } - spice, err := DownloadSpice(namespace, vm, virtClient.RestClient()) - if err != nil { - log.Fatalf(err.Error()) - return 1 - } - if proxy != "" { - spice.Info.Proxy = proxy - } - cfg := ini.Empty() - err = ini.ReflectFrom(cfg, spice) - if err != nil { - log.Fatalf("Can't serialize spice struct to ini") - return 1 - } - if details { - _, err := cfg.WriteTo(os.Stdout) - if err != nil { - log.Fatalf("Failed to write to stdout") - return 1 - } - } else { - f, err := ioutil.TempFile("", TEMP_PREFIX) - - if err != nil { - log.Fatalf("Can't open file: %s", err.Error()) - return 1 - } - defer os.Remove(f.Name()) - defer f.Close() - - _, err = cfg.WriteTo(f) - if err != nil { - log.Fatalf("Can't write to file: %s", err.Error()) - return 1 - } - - f.Sync() - - cmnd := exec.Command("remote-viewer", f.Name()) - err = cmnd.Run() - - if err != nil { - log.Fatalf("Something goes wring with remote-viewer: %s", err.Error()) - return 1 - } - } - return 0 -} - -func (o *Spice) Usage() string { - usage := "virtctl can connect via remote-viewer to a VM, or can show SPICE connection details\n\n" - usage += "Examples:\n" - usage += "# Show SPICE connection details of the VM testvm\n" - usage += "./virtctl spice testvm --details\n\n" - usage += "# Connect to testvm via remote-viewer\n" - usage += "./virtctl spice testvm\n\n" - usage += "# Connect to testvm via remote-viewer using a proxy\n" - usage += "./virtctl spice testvm --proxy http://192.168.200.2:1234\n\n" - usage += "Options:\n" - usage += o.FlagSet().FlagUsages() - return usage -} diff --git a/pkg/virtctl/vnc/vnc.go b/pkg/virtctl/vnc/vnc.go new file mode 100644 index 000000000000..5e8b00a41c50 --- /dev/null +++ b/pkg/virtctl/vnc/vnc.go @@ -0,0 +1,150 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright 2017 Red Hat, Inc. + * + */ + +package vnc + +import ( + "fmt" + "io" + "log" + "net" + "os" + "os/exec" + "os/signal" + + flag "github.com/spf13/pflag" + kubev1 "k8s.io/api/core/v1" + + "kubevirt.io/kubevirt/pkg/kubecli" +) + +const FLAG = "vnc" + +type VNC struct{} + +func (o *VNC) Run(flags *flag.FlagSet) int { + server, _ := flags.GetString("server") + kubeconfig, _ := flags.GetString("kubeconfig") + namespace, _ := flags.GetString("namespace") + if namespace == "" { + namespace = kubev1.NamespaceDefault + } + + if len(flags.Args()) != 2 { + log.Println("VM name is missing") + return 1 + } + vm := flags.Arg(1) + + virtCli, err := kubecli.GetKubevirtClientFromFlags(server, kubeconfig) + if err != nil { + log.Println(err) + return 1 + } + + // -> pipeInWriter -> pipeInReader + // remote-viewer -> unix sock connection + // <- pipeOutReader <- pipeOutWriter + pipeInReader, pipeInWriter := io.Pipe() + pipeOutReader, pipeOutWriter := io.Pipe() + + k8ResChan := make(chan error) + viewResChan := make(chan error) + stopChan := make(chan struct{}, 1) + writeStop := make(chan error) + readStop := make(chan error) + + // The local tcp server is used to proxy the podExec websock connection to remote-viewer + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + log.Printf("Can't listen on unix socket: %s", err.Error()) + return 1 + } + + port := ln.Addr().(*net.TCPAddr).Port + + // setup connection with VM + go func() { + err := virtCli.VM(namespace).VNC(vm, pipeInReader, pipeOutWriter) + k8ResChan <- err + }() + + // execute remote viewer + go func() { + cmnd := exec.Command("remote-viewer", fmt.Sprintf("vnc://127.0.0.1:%d", port)) + err := cmnd.Run() + if err != nil { + log.Println(err) + } + viewResChan <- err + }() + + // wait for remote-viewer to connect to our local proxy server + fd, err := ln.Accept() + if err != nil { + log.Printf("Failed to accept unix sock connection. %s", err.Error()) + return 1 + } + defer fd.Close() + + log.Printf("remote-viewer connected") + go func() { + interrupt := make(chan os.Signal, 1) + signal.Notify(interrupt, os.Interrupt) + <-interrupt + close(stopChan) + }() + + // write to FD <- pipeOutReader + go func() { + _, err := io.Copy(fd, pipeOutReader) + readStop <- err + }() + + // read from FD -> pipeInWriter + go func() { + _, err := io.Copy(pipeInWriter, fd) + writeStop <- err + }() + + select { + case <-stopChan: + case err = <-readStop: + case err = <-writeStop: + case err = <-k8ResChan: + case err = <-viewResChan: + } + + if err != nil { + log.Printf("Error encountered: %s", err.Error()) + return 1 + } + return 0 +} + +func (o *VNC) Usage() string { + usage := "virtctl can connect via remote-viewer to a VM\n\n" + usage += "Examples:\n" + usage += "# Connect to testvm via remote-viewer\n" + usage += "./virtctl vnc testvm\n\n" + return usage +} +func (o *VNC) FlagSet() *flag.FlagSet { + return flag.NewFlagSet(FLAG, flag.ExitOnError) +} diff --git a/tests/console_test.go b/tests/console_test.go index c38877b7bd50..4c5031075515 100644 --- a/tests/console_test.go +++ b/tests/console_test.go @@ -37,8 +37,6 @@ var _ = Describe("Console", func() { virtClient, err := kubecli.GetKubevirtClient() tests.PanicOnError(err) - virtConfig, err := kubecli.GetKubevirtClientConfig() - tests.PanicOnError(err) BeforeEach(func() { tests.BeforeTestCleanup() @@ -51,7 +49,7 @@ var _ = Describe("Console", func() { Expect(virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do().Error()).To(Succeed()) tests.WaitForSuccessfulVMStart(vm) - expecter, _, err := tests.NewConsoleExpecter(virtConfig, vm, "serial0", 10*time.Second) + expecter, _, err := tests.NewConsoleExpecter(virtClient, vm, "serial0", 10*time.Second) defer expecter.Close() Expect(err).ToNot(HaveOccurred()) diff --git a/tests/spice_test.go b/tests/spice_test.go deleted file mode 100644 index 4f4e72841dd1..000000000000 --- a/tests/spice_test.go +++ /dev/null @@ -1,246 +0,0 @@ -/* - * This file is part of the KubeVirt project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright 2017 Red Hat, Inc. - * - */ - -package tests_test - -import ( - "bufio" - "bytes" - "encoding/binary" - "flag" - "fmt" - "io" - "math/rand" - "net" - "strconv" - "strings" - "time" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - ini "gopkg.in/ini.v1" - - "kubevirt.io/kubevirt/pkg/api/v1" - "kubevirt.io/kubevirt/pkg/kubecli" - "kubevirt.io/kubevirt/pkg/rest" - "kubevirt.io/kubevirt/tests" -) - -var _ = Describe("Vmlifecycle", func() { - - flag.Parse() - - virtClient, err := kubecli.GetKubevirtClient() - tests.PanicOnError(err) - var vm *v1.VirtualMachine - - getVmNode := func() string { - obj, err := virtClient.RestClient().Get().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Name(vm.GetObjectMeta().GetName()).Do().Get() - Expect(err).ToNot(HaveOccurred()) - return obj.(*v1.VirtualMachine).Status.NodeName - } - - checkSpiceConnection := func() { - raw, err := virtClient.RestClient().Get().Resource("virtualmachines").SetHeader("Accept", rest.MIME_INI).SubResource("spice").Namespace(tests.NamespaceTestDefault).Name(vm.GetObjectMeta().GetName()).Do().Raw() - Expect(err).To(BeNil()) - spiceINI, err := ini.Load(raw) - Expect(err).NotTo(HaveOccurred()) - spice := v1.SpiceInfo{} - Expect(spiceINI.Section("virt-viewer").MapTo(&spice)).To(Succeed()) - - proxy := strings.TrimPrefix(spice.Proxy, "http://") - host := fmt.Sprintf("%s:%d", spice.Host, spice.Port) - - // Let's see if we can connect to the spice port through the proxy - conn, err := net.Dial("tcp", proxy) - Expect(err).To(BeNil()) - conn.Write([]byte("CONNECT " + host + " HTTP/1.1\r\n")) - conn.Write([]byte("Host: " + host + "\r\n")) - conn.Write([]byte("\r\n")) - line, err := bufio.NewReader(conn).ReadString('\n') - Expect(err).To(BeNil()) - Expect(strings.TrimSpace(line)).To(Equal("HTTP/1.1 200 Connection established")) - - // Let's send a spice handshake - conn.Write(newSpiceHandshake()) - - // Let's parse the response - var i int32 - x := make([]byte, 4, 4) - io.ReadFull(conn, x) - Expect(string(x)).To(Equal("REDQ")) // spice magic - binary.Read(conn, binary.LittleEndian, &i) - Expect(i).To(Equal(int32(2)), "Major version does not match.") - binary.Read(conn, binary.LittleEndian, &i) - Expect(i).To(Equal(int32(2)), "Minor version does not match.") - binary.Read(conn, binary.LittleEndian, &i) - Expect(i).To(BeNumerically(">", 4), "Message not long enough.") - binary.Read(conn, binary.LittleEndian, &i) - Expect(i).To(Equal(int32(0)), "Message status is not OK.") // 0 is equal to OK - } - - BeforeEach(func() { - vm = tests.NewRandomVM() - tests.BeforeTestCleanup() - }) - - Context("New VM with a spice connection given", func() { - - It("should return no connection details if VM does not exist", func(done Done) { - result := virtClient.RestClient().Get().Resource("virtualmachines").SubResource("spice").Namespace(tests.NamespaceTestDefault).Name("something-random").Do() - Expect(result.Error()).NotTo(BeNil()) - close(done) - }, 3) - - It("should return connection details for running VMs in ini format", func(done Done) { - // Create the VM - result := virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do() - obj, err := result.Get() - Expect(err).To(BeNil()) - - // Block until the VM is running - tests.WaitForSuccessfulVMStart(obj) - - raw, err := virtClient.RestClient().Get().Resource("virtualmachines").SetHeader("Accept", rest.MIME_INI).SubResource("spice").Namespace(tests.NamespaceTestDefault).Name(vm.GetObjectMeta().GetName()).Do().Raw() - spice, err := ini.Load(raw) - Expect(err).To(Not(HaveOccurred())) - - Expect(spice.Section("virt-viewer")).NotTo(BeNil()) - section := spice.Section("virt-viewer") - Expect(section.HasKey("type")).To(BeTrue()) - Expect(section.HasKey("host")).To(BeTrue()) - Expect(section.HasKey("port")).To(BeTrue()) - Expect(strconv.Atoi(section.Key("port").Value())).To(BeNumerically(">=", int32(5900))) - Expect(section.HasKey("proxy")).To(BeTrue()) - close(done) - }, 30) - - It("should return connection details for running VMs in json format", func(done Done) { - // Create the VM - result := virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do() - obj, err := result.Get() - Expect(err).To(BeNil()) - - // Block until the VM is running - tests.WaitForSuccessfulVMStart(obj) - - obj, err = virtClient.RestClient().Get().Resource("virtualmachines").SetHeader("Accept", rest.MIME_JSON).SubResource("spice").Namespace(tests.NamespaceTestDefault).Name(vm.GetObjectMeta().GetName()).Do().Get() - Expect(err).To(BeNil()) - spice := obj.(*v1.Spice).Info - Expect(spice.Type).To(Equal("spice")) - Expect(spice.Port).To(BeNumerically(">=", int32(5900))) - close(done) - }, 30) - - It("should allow accessing the spice device on the VM", func(done Done) { - // Create the VM - result := virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do() - obj, err := result.Get() - Expect(err).To(BeNil()) - - // Block until the VM is running - tests.WaitForSuccessfulVMStart(obj) - - checkSpiceConnection() - - close(done) - }, 30) - }) - - Context("Two new VMs scheduled on the same node with a spice graphics device", func() { - - It("should start without port clashes", func(done Done) { - // Create the VM - result := virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do() - obj, err := result.Get() - Expect(err).To(BeNil()) - - // Block until the VM is running - nodeName := tests.WaitForSuccessfulVMStart(obj) - - vm1 := tests.NewRandomVM() - vm1.Spec.NodeSelector = map[string]string{"kubernetes.io/hostname": nodeName} - result = virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm1).Do() - obj1, err := result.Get() - Expect(err).To(BeNil()) - - // Block until the VM is running - tests.WaitForSuccessfulVMStart(obj1) - - obj, err = virtClient.RestClient().Get().Resource("virtualmachines").SetHeader("Accept", rest.MIME_JSON).SubResource("spice").Namespace(tests.NamespaceTestDefault).Name(vm.GetObjectMeta().GetName()).Do().Get() - Expect(err).To(BeNil()) - obj1, err = virtClient.RestClient().Get().Resource("virtualmachines").SetHeader("Accept", rest.MIME_JSON).SubResource("spice").Namespace(tests.NamespaceTestDefault).Name(vm1.GetObjectMeta().GetName()).Do().Get() - Expect(err).To(BeNil()) - Expect(obj.(*v1.Spice).Info.Port).ToNot(BeNumerically("==", obj1.(*v1.Spice).Info.Port)) - close(done) - }, 30) - }) - - Context("Migrate VM with spice connection", func() { - - BeforeEach(func() { - if len(tests.GetReadyNodes()) < 2 { - Skip("To test migrations, at least two nodes need to be active") - } - // Create the VM - result := virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do() - obj, err := result.Get() - Expect(err).To(BeNil()) - - // Block until the VM is running - tests.WaitForSuccessfulVMStart(obj) - }) - - It("should allow accessing the spice device on the VM", func() { - sourceNode := getVmNode() - - migration := tests.NewRandomMigrationForVm(vm) - err = virtClient.RestClient().Post().Resource("migrations").Namespace(tests.NamespaceTestDefault).Body(migration).Do().Error() - Expect(err).ToNot(HaveOccurred()) - - Eventually(getVmNode, 30*time.Second, time.Second).ShouldNot(Equal(sourceNode)) - - Eventually(func() v1.VMPhase { - obj, err := virtClient.RestClient().Get().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Name(vm.GetObjectMeta().GetName()).Do().Get() - Expect(err).ToNot(HaveOccurred()) - fetchedVM := obj.(*v1.VirtualMachine) - return fetchedVM.Status.Phase - }, 10*time.Second, time.Second).Should(Equal(v1.Running)) - - checkSpiceConnection() - }) - }) -}) - -func newSpiceHandshake() []byte { - var b []byte - bb := bytes.NewBuffer(b) - bb.Write([]byte("REDQ")) // spice magic - binary.Write(bb, binary.LittleEndian, uint32(2)) // protocol major version - binary.Write(bb, binary.LittleEndian, uint32(2)) // protocol minor version - binary.Write(bb, binary.LittleEndian, uint32(22)) // message size - binary.Write(bb, binary.LittleEndian, uint32(rand.Int())) // session id - binary.Write(bb, binary.LittleEndian, uint8(3)) // channel type - binary.Write(bb, binary.LittleEndian, uint8(0)) // channel id - binary.Write(bb, binary.LittleEndian, uint32(1)) // number of common capabilities - binary.Write(bb, binary.LittleEndian, uint32(0)) // number of channel capabilities - binary.Write(bb, binary.LittleEndian, uint32(18)) // capabilities offset - binary.Write(bb, binary.LittleEndian, uint32(13)) // client common capabilities - return bb.Bytes() -} diff --git a/tests/utils.go b/tests/utils.go index 20b8efede867..492b96112a98 100644 --- a/tests/utils.go +++ b/tests/utils.go @@ -41,12 +41,10 @@ import ( "io" "github.com/google/goexpect" - "k8s.io/client-go/rest" "kubevirt.io/kubevirt/pkg/api/v1" "kubevirt.io/kubevirt/pkg/kubecli" "kubevirt.io/kubevirt/pkg/log" - "kubevirt.io/kubevirt/pkg/virtctl/console" ) type EventType string @@ -682,8 +680,7 @@ func WaitForSuccessfulVMStartWithTimeout(vm runtime.Object, seconds int) (nodeNa nodeName = fetchedVM.Status.NodeName // wait on both phase and graphics - if len(fetchedVM.Status.Graphics) == 1 && - fetchedVM.Status.Phase == v1.Running { + if fetchedVM.Status.Phase == v1.Running { return true } return false @@ -742,13 +739,13 @@ func NewRandomReplicaSetFromVM(vm *v1.VirtualMachine, replicas int32) *v1.Virtua return rs } -func NewConsoleExpecter(config *rest.Config, vm *v1.VirtualMachine, consoleName string, timeout time.Duration, opts ...expect.Option) (expect.Expecter, <-chan error, error) { +func NewConsoleExpecter(virtCli kubecli.KubevirtClient, vm *v1.VirtualMachine, consoleName string, timeout time.Duration, opts ...expect.Option) (expect.Expecter, <-chan error, error) { vmReader, vmWriter := io.Pipe() expecterReader, expecterWriter := io.Pipe() resCh := make(chan error) stopChan := make(chan struct{}) go func() { - err := console.ConnectToConsole(config, vm.ObjectMeta.Namespace, vm.ObjectMeta.Name, consoleName, console.NewWebsocketCallback(vmReader, expecterWriter, stopChan)) + err := virtCli.VM(vm.ObjectMeta.Namespace).SerialConsole(vm.ObjectMeta.Name, consoleName, vmReader, expecterWriter) resCh <- err }() diff --git a/tests/vm_monitoring_test.go b/tests/vm_monitoring_test.go index cb096ab2b587..7f1f90410c12 100644 --- a/tests/vm_monitoring_test.go +++ b/tests/vm_monitoring_test.go @@ -40,8 +40,6 @@ var _ = Describe("Health Monitoring", func() { virtClient, err := kubecli.GetKubevirtClient() tests.PanicOnError(err) - virtConfig, err := kubecli.GetKubevirtClientConfig() - tests.PanicOnError(err) launchVM := func(vm *v1.VirtualMachine) { obj, err := virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do().Get() @@ -60,7 +58,7 @@ var _ = Describe("Health Monitoring", func() { Expect(err).ToNot(HaveOccurred()) launchVM(vm) - expecter, _, err := tests.NewConsoleExpecter(virtConfig, vm, "serial0", 10*time.Second) + expecter, _, err := tests.NewConsoleExpecter(virtClient, vm, "serial0", 10*time.Second) Expect(err).ToNot(HaveOccurred()) defer expecter.Close() diff --git a/tests/vm_userdata_test.go b/tests/vm_userdata_test.go index c2b461632344..96cf88a5cb90 100644 --- a/tests/vm_userdata_test.go +++ b/tests/vm_userdata_test.go @@ -43,8 +43,6 @@ var _ = Describe("CloudInit UserData", func() { virtClient, err := kubecli.GetKubevirtClient() tests.PanicOnError(err) - virtConfig, err := kubecli.GetKubevirtClientConfig() - tests.PanicOnError(err) LaunchVM := func(vm *v1.VirtualMachine) runtime.Object { obj, err := virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do().Get() @@ -57,7 +55,7 @@ var _ = Describe("CloudInit UserData", func() { Expect(ok).To(BeTrue(), "Object is not of type *v1.VM") tests.WaitForSuccessfulVMStart(obj) - expecter, _, err := tests.NewConsoleExpecter(virtConfig, vm, "serial0", 10*time.Second) + expecter, _, err := tests.NewConsoleExpecter(virtClient, vm, "serial0", 10*time.Second) defer expecter.Close() Expect(err).ToNot(HaveOccurred()) diff --git a/tests/vnc_test.go b/tests/vnc_test.go new file mode 100644 index 000000000000..9862f923c293 --- /dev/null +++ b/tests/vnc_test.go @@ -0,0 +1,95 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright 2017 Red Hat, Inc. + * + */ + +package tests_test + +import ( + "flag" + "io" + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "kubevirt.io/kubevirt/pkg/kubecli" + "kubevirt.io/kubevirt/pkg/log" + "kubevirt.io/kubevirt/tests" +) + +var _ = Describe("Vmlifecycle", func() { + + flag.Parse() + + virtClient, err := kubecli.GetKubevirtClient() + tests.PanicOnError(err) + + BeforeEach(func() { + tests.BeforeTestCleanup() + }) + + Context("New VM with a vnc connection given", func() { + It("should allow accessing the vnc device on the VM", func(done Done) { + vm := tests.NewRandomVM() + Expect(virtClient.RestClient().Post().Resource("virtualmachines").Namespace(tests.NamespaceTestDefault).Body(vm).Do().Error()).To(Succeed()) + tests.WaitForSuccessfulVMStart(vm) + + tests.WaitForSuccessfulVMStart(vm) + + pipeInReader, _ := io.Pipe() + pipeOutReader, pipeOutWriter := io.Pipe() + + k8ResChan := make(chan error) + readStop := make(chan string) + + go func() { + err := virtClient.VM(vm.ObjectMeta.Namespace).VNC(vm.ObjectMeta.Name, pipeInReader, pipeOutWriter) + k8ResChan <- err + }() + // write to FD <- pipeOutReader + go func() { + buf := make([]byte, 1024, 1024) + // reading qemu vnc server + n, err := pipeOutReader.Read(buf) + if err != nil && err != io.EOF { + log.Log.Reason(err).Error("error while reading from vnc socket.") + return + } + if n == 0 && err == io.EOF { + log.Log.Error("zero bytes read from vnc socket.") + return + } + readStop <- strings.TrimSpace(string(buf[0:n])) + }() + + response := "" + + select { + case response = <-readStop: + case err = <-k8ResChan: + } + + // This is the response capture by wireshark when the VNC server is contacted. + // This verifies that the test is able to establish a connection with VNC and + // communicate. + Expect(response).To(Equal("RFB 003.008")) + Expect(err).To(BeNil()) + close(done) + }, 30) + }) +})