ci: Bump kubevirtci an k8s to 1.23 (#221)

The last kubevirtci does not work well with the functests at ovs-cni.
This change copy the ssh to node mechanism from kubernetes-nmstate that
is not to work fine, it also bump k8s to 1.23.

Signed-off-by: Quique Llorente <[email protected]>

cluster/cli.sh

@@ -14,7 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-source ./cluster/kubevirtci.sh
-kubevirtci::install
+SCRIPTS_PATH="$(dirname "$(realpath "$0")")"
+source ${SCRIPTS_PATH}/cluster.sh
+cluster::install
 
-$(kubevirtci::path)/cluster-up/cli.sh "$@"
+$(cluster::path)/cluster-up/cli.sh "$@"

cluster/cluster.sh

@@ -0,0 +1,58 @@
+# Copyright 2018-2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-'k8s-1.23'}
+export KUBEVIRTCI_TAG='2202081102-e8daf721'
+
+KUBEVIRTCI_REPO='https://github.com/kubevirt/kubevirtci.git'
+# The CLUSTER_PATH var is used in cluster folder and points to the _kubevirtci where the cluster is deployed from.
+CLUSTER_PATH=${CLUSTER_PATH:-"${PWD}/_kubevirtci/"}
+
+function cluster::_get_repo() {
+    git --git-dir ${CLUSTER_PATH}/.git remote get-url origin
+}
+
+function cluster::_get_tag() {
+    git -C ${CLUSTER_PATH} describe --tags
+}
+
+function cluster::install() {
+    # Remove cloned kubevirtci repository if it does not match the requested one
+    if [ -d ${CLUSTER_PATH} ]; then
+        if [ $(cluster::_get_repo) != ${KUBEVIRTCI_REPO} -o $(cluster::_get_tag) != ${KUBEVIRTCI_TAG} ]; then
+            rm -rf ${CLUSTER_PATH}
+        fi
+    fi
+
+    if [ ! -d ${CLUSTER_PATH} ]; then
+        git clone https://github.com/kubevirt/kubevirtci.git ${CLUSTER_PATH}
+        (
+            cd ${CLUSTER_PATH}
+            git checkout ${KUBEVIRTCI_TAG}
+        )
+    fi
+}
+
+function cluster::path() {
+    echo -n ${CLUSTER_PATH}
+}
+
+function cluster::kubeconfig() {
+    if [ ${KUBEVIRT_PROVIDER} != "external" ]; then
+        echo -n ${CLUSTER_PATH}/_ci-configs/${KUBEVIRT_PROVIDER}/.kubeconfig
+    else
+        [[ -n $KUBECONFIG ]] || (echo "missing KUBECONFIG"; exit 1)
+        echo -n ${KUBECONFIG}
+    fi
+}

cluster/down.sh

@@ -16,7 +16,8 @@
 
 set -ex
 
-source ./cluster/kubevirtci.sh
-kubevirtci::install
+SCRIPTS_PATH="$(dirname "$(realpath "$0")")"
+source ${SCRIPTS_PATH}/cluster.sh
+cluster::install
 
-$(kubevirtci::path)/cluster-up/down.sh
+$(cluster::path)/cluster-up/down.sh

cluster/kubectl.sh

@@ -14,7 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-source ./cluster/kubevirtci.sh
-kubevirtci::install
+SCRIPTS_PATH="$(dirname "$(realpath "$0")")"
+source ${SCRIPTS_PATH}/cluster.sh
+cluster::install
 
-$(kubevirtci::path)/cluster-up/kubectl.sh "$@"
+$(cluster::path)/cluster-up/kubectl.sh "$@"

cluster/ssh.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+SCRIPTS_PATH="$(dirname "$(realpath "$0")")"
+source ${SCRIPTS_PATH}/cluster.sh
+cluster::install
+
+$(cluster::path)/cluster-up/cli.sh ssh "$@"

cluster/sync.sh

@@ -16,8 +16,8 @@
 
 set -ex
 
-source ./cluster/kubevirtci.sh
-kubevirtci::install
+source ./cluster/cluster.sh
+cluster::install
 
 function check_deleted() {
     OUTPUT=$(./cluster/kubectl.sh get --ignore-not-found $1)

cluster/up.sh

@@ -16,18 +16,15 @@
 
 set -ex
 
-source ./cluster/kubevirtci.sh
-kubevirtci::install
+source ./cluster/cluster.sh
+cluster::install
 
-$(kubevirtci::path)/cluster-up/up.sh
+$(cluster::path)/cluster-up/up.sh
 
 echo 'Installing Open vSwitch on nodes'
 for node in $(./cluster/kubectl.sh get nodes --no-headers | awk '{print $1}'); do
-    ./cluster/cli.sh ssh ${node} -- sudo yum install -y  \
-     dpdk \
-     https://cbs.centos.org/kojifiles/packages/openvswitch-selinux-extra-policy/1.0/28.el8/noarch/openvswitch-selinux-extra-policy-1.0-28.el8.noarch.rpm \
-     https://cbs.centos.org/kojifiles/packages/openvswitch2.15/2.15.0/35.el8s/x86_64/openvswitch2.15-2.15.0-35.el8s.x86_64.rpm \
-     https://cbs.centos.org/kojifiles/packages/openvswitch2.15/2.15.0/35.el8s/x86_64/openvswitch2.15-devel-2.15.0-35.el8s.x86_64.rpm
+    ./cluster/cli.sh ssh ${node} -- sudo dnf install -y centos-release-nfv-openvswitch
+    ./cluster/cli.sh ssh ${node} -- sudo dnf install -y openvswitch2.16 dpdk
     ./cluster/cli.sh ssh ${node} -- sudo systemctl daemon-reload
     ./cluster/cli.sh ssh ${node} -- sudo systemctl restart openvswitch
 done

hack/functests.sh

@@ -18,7 +18,7 @@
 
 set -e
 
-source ./cluster/kubevirtci.sh
+source ./cluster/cluster.sh
 
-KUBECONFIG=${KUBECONFIG:-$(kubevirtci::kubeconfig)}
+KUBECONFIG=${KUBECONFIG:-$(cluster::kubeconfig)}
 ${GO} test ./tests/... $E2E_TEST_ARGS --kubeconfig ${KUBECONFIG}

tests/cmd/run.go

@@ -0,0 +1,39 @@
+/*
+Copyright The Kubernetes NMState Authors.
+
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"github.com/onsi/ginkgo"
+)
+
+// Run exec a command with its arguments and return stdout and stderr
+func Run(command string, arguments ...string) (string, error) {
+	cmd := exec.Command(command, arguments...)
+	ginkgo.GinkgoWriter.Write([]byte(command + " " + strings.Join(arguments, " ") + "\n"))
+	var stdout, stderr bytes.Buffer
+	cmd.Stderr = &stderr
+	cmd.Stdout = &stdout
+	err := cmd.Run()
+	ginkgo.GinkgoWriter.Write([]byte(fmt.Sprintf("stdout: %.500s...\n, stderr %s\n", stdout.String(), stderr.String())))
+	return stdout.String(), err
+}

tests/marker_test.go

@@ -30,11 +30,11 @@ import (
 var _ = Describe("ovs-cni-marker", func() {
 	Describe("bridge resource reporting", func() {
 		It("should be reported only when available on node", func() {
-			out, err := node.RunOnNode("node01", "sudo ovs-vsctl add-br br-test")
+			out, err := node.RunAtNode("node01", "sudo ovs-vsctl add-br br-test")
 			if err != nil {
 				panic(fmt.Errorf("%v: %s", err, out))
 			}
-			defer node.RunOnNode("node01", "sudo ovs-vsctl --if-exists del-br br-test")
+			defer node.RunAtNode("node01", "sudo ovs-vsctl --if-exists del-br br-test")
 
 			Eventually(func() bool {
 				node, err := clusterApi.Clientset.CoreV1().Nodes().Get(context.TODO(), "node01", metav1.GetOptions{})
@@ -50,7 +50,7 @@ var _ = Describe("ovs-cni-marker", func() {
 				return true
 			}, 180*time.Second, 5*time.Second).Should(Equal(true))
 
-			out, err = node.RunOnNode("node01", "sudo ovs-vsctl --if-exists del-br br-test")
+			out, err = node.RunAtNode("node01", "sudo ovs-vsctl --if-exists del-br br-test")
 			if err != nil {
 				panic(fmt.Errorf("%v: %s", err, out))
 			}

tests/node/node.go

@@ -21,44 +21,51 @@ package node
 
 import (
 	"fmt"
-	"os/exec"
 	"strings"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
+	"github.com/k8snetworkplumbingwg/ovs-cni/tests/cmd"
 )
 
-// RunOnNode performs bash shell command on node
 // TODO: Use job with a node affinity instead
-func RunOnNode(node string, command string) (string, error) {
-	out, err := exec.Command("bash", "-c", fmt.Sprintf("docker ps | grep %s | awk '{ print $1}'", node)).CombinedOutput()
-	if err != nil {
-		panic(fmt.Sprintf("failed to run docker ps error output: %s", string(out)))
+func runAtNodeWithExtras(node string, command ...string) (string, error) {
+	ssh := "./cluster/ssh.sh"
+	sshCommand := []string{node, "--"}
+	sshCommand = append(sshCommand, command...)
+	output, err := cmd.Run(ssh, sshCommand...)
+	// Remove first two lines from output, ssh.sh add garbage there
+	outputLines := strings.Split(output, "\n")
+	if len(outputLines) > 2 {
+		output = strings.Join(outputLines[2:], "\n")
 	}
+	return output, err
+}
 
-	out, err = exec.Command("docker", "exec", string(out[:12]), "ssh.sh", command).CombinedOutput()
-	if err != nil {
-		panic(fmt.Sprintf("failed to run docker exec command error output: %s", string(out)))
-	}
-	outString := string(out)
-	outLines := strings.Split(outString, "\n")
-	// first two lines of output indicate that connection was successful
-	outStripped := outLines[2:]
-	outStrippedString := strings.Join(outStripped, "\n")
+func RunAtNode(node string, command ...string) (string, error) {
+	return runAtNodeWithExtras(node, command...)
+}
 
-	return outStrippedString, err
+func RunAtNodes(nodes []string, command ...string) (outputs []string, errs []error) {
+	for _, node := range nodes {
+		output, err := RunAtNode(node, command...)
+		outputs = append(outputs, output)
+		errs = append(errs, err)
+	}
+	return outputs, errs
 }
 
 // RemoveOvsBridgeOnNode removes ovs bridge on the node
 func RemoveOvsBridgeOnNode(bridgeName string) {
 	By("Removing ovs-bridge on the node")
-	out, err := RunOnNode("node01", "sudo ovs-vsctl --if-exists del-br "+bridgeName)
+	out, err := RunAtNode("node01", "sudo ovs-vsctl --if-exists del-br "+bridgeName)
 	Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to run command on node. stdout: %s", out))
 }
 
 // AddOvsBridgeOnNode add ovs bridge on the node
 func AddOvsBridgeOnNode(bridgeName string) {
 	By("Adding ovs-bridge on the node")
-	out, err := RunOnNode("node01", "sudo ovs-vsctl add-br "+bridgeName)
+	out, err := RunAtNode("node01", "sudo ovs-vsctl add-br "+bridgeName)
 	Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Failed to run command on node. stdout: %s", out))
 }

tests/tests_suite_test.go

@@ -17,6 +17,7 @@ package tests_test
 
 import (
 	"flag"
+	"os"
 	"testing"
 
 	. "github.com/onsi/ginkgo"
@@ -41,6 +42,9 @@ func TestPlugin(t *testing.T) {
 var _ = BeforeSuite(func() {
 	flag.Parse()
 
+	// Change to root directory some test expect that
+	os.Chdir("../")
+
 	clusterApi = clusterapi.NewClusterAPI(*kubeconfig)
 	clusterApi.RemoveTestNamespace()
 	clusterApi.CreateTestNamespace()