ci: refactor the ci logic

- bump version + bump go to v1.13.12 + bump kubectl to v0.18.2 + bump golangci-lint to v1.27.0 + bump controller-gen to v0.3.0 - change to only install docker client - parameterized component version - adjust the script of integration/e2e test BREAKING CHANGE: - need to change the version of go on dev laptop - need to change the version of kubectl on dev laptop - need to change the version of golangci-lint on dev laptop - need to change the version of controller-gen on dev laptop
cnrancher · Jun 22, 2020 · 3a2e4bd · 3a2e4bd
1 parent a3f8df1
commit 3a2e4bd
Show file tree

Hide file tree

Showing 40 changed files with 420 additions and 119 deletions.
diff --git a/.drone.yml b/.drone.yml
@@ -10,9 +10,20 @@ steps:
     commands:
       - commitsar -d /drone/src -v
   - name: code
-    image: golang:1.13.9-buster
+    image: maiwj/drone-dapper:v0.4.2
+    privileged: true
+    environment:
+      BY: dapper
+      DIRTY_CHECK: true
     commands:
-      - make all lint only
+      - make all lint
+    volumes:
+      - name: docker
+        path: /var/run/docker.sock
+volumes:
+  - name: docker
+    host:
+      path: /var/run/docker.sock
 trigger:
   event:
     include:

diff --git a/Dockerfile.dapper b/Dockerfile.dapper
@@ -1,17 +1,32 @@
-FROM golang:1.13.9-buster
-RUN apt-get update && \
-    apt-get install -y xz-utils unzip netcat
+FROM golang:1.13.12-buster
+RUN apt-get update -qq && apt-get install -y --no-install-recommends \
+        xz-utils \
+        unzip \
+        netcat \
+    && rm -rf /var/lib/apt/lists/*
 
 # -- for make rules
-## install docker
-RUN curl -sSfL "https://get.docker.com" | sh -s VERSION=19.03; \
+## install docker client
+RUN apt-get update -qq && apt-get install -y --no-install-recommends \
+        apt-transport-https \
+        ca-certificates \
+        curl \
+        gnupg \
+    && rm -rf /var/lib/apt/lists/*; \
+    \
+    curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - >/dev/null; \
+    echo "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/debian buster stable" > /etc/apt/sources.list.d/docker.list; \
+    \
+    apt-get update -qq && apt-get install -y --no-install-recommends \
+        docker-ce-cli=5:19.03.* \
+    && rm -rf /var/lib/apt/lists/*; \
     docker --version
 ## install kubectl
-RUN curl -fL "https://storage.googleapis.com/kubernetes-release/release/v1.17.2/bin/$(go env GOOS)/$(go env GOARCH)/kubectl" -o /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl; \
+RUN curl -fL "https://storage.googleapis.com/kubernetes-release/release/v1.18.2/bin/$(go env GOOS)/$(go env GOARCH)/kubectl" -o /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl; \
     kubectl version --short --client
 ## install golangci-lint
 RUN if [ "$(go env GOARCH)" = "amd64" ]; then \
-        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)/bin" v1.24.0; \
+        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)/bin" v1.27.0; \
         golangci-lint --version; \
     fi
 ## install protoc & generate
@@ -27,16 +42,21 @@ RUN if [ "$(go env GOARCH)" = "amd64" ]; then \
     fi
 ## install controller-gen
 RUN if [ "$(go env GOARCH)" = "amd64" ]; then \
-        GO111MODULE=on go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.5; \
+        GO111MODULE=on go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.3.0; \
         controller-gen --version; \
     fi
+## install ginkgo
+RUN if [ "$(go env GOARCH)" = "amd64" ]; then \
+        GO111MODULE=on go get github.com/onsi/ginkgo/[email protected]; \
+        ginkgo version; \
+    fi
 # -- for make rules
 
 # -- for dapper
 ENV DAPPER_RUN_ARGS --privileged --network host
 ENV GO111MODULE=off
 ENV CROSS=false
-ENV DAPPER_ENV CROSS LOCAL_CLUSTER_KIND DOCKER_USERNAME DOCKER_PASSWORD WITHOUT_MANIFEST ONLY_MANIFEST IGNORE_MISSING DRONE_TAG REPO TAG OS ARCH IMAGE_NAME
+ENV DAPPER_ENV CROSS LOCAL_CLUSTER_KIND DOCKER_USERNAME DOCKER_PASSWORD WITHOUT_MANIFEST ONLY_MANIFEST IGNORE_MISSING DRONE_TAG REPO TAG OS ARCH IMAGE_NAME DIRTY_CHECK
 ENV DAPPER_SOURCE /go/src/github.com/rancher/octopus/
 ENV DAPPER_OUTPUT ./bin ./dist ./deploy ./pkg/adaptor/api ./api
 ENV DAPPER_DOCKER_SOCKET true

diff --git a/adaptors/ble/Dockerfile.dapper b/adaptors/ble/Dockerfile.dapper
@@ -1,31 +1,51 @@
-FROM golang:1.13.9-buster
-RUN apt-get update && \
-    apt-get install -y xz-utils unzip netcat
+FROM golang:1.13.12-buster
+RUN apt-get update -qq && apt-get install -y --no-install-recommends \
+        xz-utils \
+        unzip \
+        netcat \
+    && rm -rf /var/lib/apt/lists/*
 
 # -- for make rules
-## install docker
-RUN curl -sSfL "https://get.docker.com" | sh -s VERSION=19.03; \
+## install docker client
+RUN apt-get update -qq && apt-get install -y --no-install-recommends \
+        apt-transport-https \
+        ca-certificates \
+        curl \
+        gnupg \
+    && rm -rf /var/lib/apt/lists/*; \
+    \
+    curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - >/dev/null; \
+    echo "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/debian buster stable" > /etc/apt/sources.list.d/docker.list; \
+    \
+    apt-get update -qq && apt-get install -y --no-install-recommends \
+        docker-ce-cli=5:19.03.* \
+    && rm -rf /var/lib/apt/lists/*; \
     docker --version
 ## install kubectl
-RUN curl -fL "https://storage.googleapis.com/kubernetes-release/release/v1.17.2/bin/$(go env GOOS)/$(go env GOARCH)/kubectl" -o /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl; \
+RUN curl -fL "https://storage.googleapis.com/kubernetes-release/release/v1.18.2/bin/$(go env GOOS)/$(go env GOARCH)/kubectl" -o /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl; \
     kubectl version --short --client
 ## install golangci-lint
 RUN if [ "$(go env GOARCH)" = "amd64" ]; then \
-        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)/bin" v1.24.0; \
+        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)/bin" v1.27.0; \
         golangci-lint --version; \
     fi
 ## install controller-gen
 RUN if [ "$(go env GOARCH)" = "amd64" ]; then \
-        GO111MODULE=on go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.5; \
+        GO111MODULE=on go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.3.0; \
         controller-gen --version; \
     fi
+## install ginkgo
+RUN if [ "$(go env GOARCH)" = "amd64" ]; then \
+        GO111MODULE=on go get github.com/onsi/ginkgo/[email protected]; \
+        ginkgo version; \
+    fi
 # -- for make rules
 
 # -- for dapper
 ENV DAPPER_RUN_ARGS --privileged --network host
 ENV GO111MODULE=off
 ENV CROSS=false
-ENV DAPPER_ENV CROSS LOCAL_CLUSTER_KIND DOCKER_USERNAME DOCKER_PASSWORD WITHOUT_MANIFEST ONLY_MANIFEST IGNORE_MISSING DRONE_TAG REPO TAG OS ARCH IMAGE_NAME
+ENV DAPPER_ENV CROSS LOCAL_CLUSTER_KIND DOCKER_USERNAME DOCKER_PASSWORD WITHOUT_MANIFEST ONLY_MANIFEST IGNORE_MISSING DRONE_TAG REPO TAG OS ARCH IMAGE_NAME DIRTY_CHECK
 ENV DAPPER_SOURCE /go/src/github.com/rancher/octopus/
 ENV DAPPER_OUTPUT ./adaptors/ble/bin ./adaptors/ble/dist ./adaptors/ble/deploy ./adaptors/ble/api
 ENV DAPPER_DOCKER_SOCKET true

diff --git a/adaptors/ble/deploy/e2e/all_in_one.yaml b/adaptors/ble/deploy/e2e/all_in_one.yaml
@@ -2,7 +2,6 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.5
     devices.edge.cattle.io/description: BLE stands for Bluetooth Low Energy (Bluetooth
       LE, , and marketed as Bluetooth Smart). BLE is a form of wireless communication
       designed for short-range communications. The BLE adaptor defines the device

diff --git a/adaptors/ble/deploy/manifests/crd/base/devices.edge.cattle.io_bluetoothdevices.yaml b/adaptors/ble/deploy/manifests/crd/base/devices.edge.cattle.io_bluetoothdevices.yaml
@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.5
+    {}
   creationTimestamp: null
   name: bluetoothdevices.devices.edge.cattle.io
 spec:

diff --git a/adaptors/ble/hack/make-rules/adaptor.sh b/adaptors/ble/hack/make-rules/adaptor.sh
@@ -266,6 +266,8 @@ function verify() {
 
   octopus::log::info "running integration tests for adaptor ${adaptor}..."
 
+  octopus::ginkgo::test "${CURR_DIR}/test/integration"
+
   octopus::log::info "...done"
 }
 
@@ -275,6 +277,8 @@ function e2e() {
 
   octopus::log::info "running E2E tests for adaptor ${adaptor}..."
 
+  octopus::ginkgo::test "${CURR_DIR}/test/e2e"
+
   octopus::log::info "...done"
 }
 

diff --git a/adaptors/dummy/Dockerfile.dapper b/adaptors/dummy/Dockerfile.dapper
@@ -1,31 +1,51 @@
-FROM golang:1.13.9-buster
-RUN apt-get update && \
-    apt-get install -y xz-utils unzip netcat
+FROM golang:1.13.12-buster
+RUN apt-get update -qq && apt-get install -y --no-install-recommends \
+        xz-utils \
+        unzip \
+        netcat \
+    && rm -rf /var/lib/apt/lists/*
 
 # -- for make rules
-## install docker
-RUN curl -sSfL "https://get.docker.com" | sh -s VERSION=19.03; \
+## install docker client
+RUN apt-get update -qq && apt-get install -y --no-install-recommends \
+        apt-transport-https \
+        ca-certificates \
+        curl \
+        gnupg \
+    && rm -rf /var/lib/apt/lists/*; \
+    \
+    curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - >/dev/null; \
+    echo "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/debian buster stable" > /etc/apt/sources.list.d/docker.list; \
+    \
+    apt-get update -qq && apt-get install -y --no-install-recommends \
+        docker-ce-cli=5:19.03.* \
+    && rm -rf /var/lib/apt/lists/*; \
     docker --version
 ## install kubectl
-RUN curl -fL "https://storage.googleapis.com/kubernetes-release/release/v1.17.2/bin/$(go env GOOS)/$(go env GOARCH)/kubectl" -o /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl; \
+RUN curl -fL "https://storage.googleapis.com/kubernetes-release/release/v1.18.2/bin/$(go env GOOS)/$(go env GOARCH)/kubectl" -o /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl; \
     kubectl version --short --client
 ## install golangci-lint
 RUN if [ "$(go env GOARCH)" = "amd64" ]; then \
-        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)/bin" v1.24.0; \
+        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)/bin" v1.27.0; \
         golangci-lint --version; \
     fi
 ## install controller-gen
 RUN if [ "$(go env GOARCH)" = "amd64" ]; then \
-        GO111MODULE=on go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.5; \
+        GO111MODULE=on go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.3.0; \
         controller-gen --version; \
     fi
+## install ginkgo
+RUN if [ "$(go env GOARCH)" = "amd64" ]; then \
+        GO111MODULE=on go get github.com/onsi/ginkgo/[email protected]; \
+        ginkgo version; \
+    fi
 # -- for make rules
 
 # -- for dapper
 ENV DAPPER_RUN_ARGS --privileged --network host
 ENV GO111MODULE=off
 ENV CROSS=false
-ENV DAPPER_ENV CROSS LOCAL_CLUSTER_KIND DOCKER_USERNAME DOCKER_PASSWORD WITHOUT_MANIFEST ONLY_MANIFEST IGNORE_MISSING DRONE_TAG REPO TAG OS ARCH IMAGE_NAME
+ENV DAPPER_ENV CROSS LOCAL_CLUSTER_KIND DOCKER_USERNAME DOCKER_PASSWORD WITHOUT_MANIFEST ONLY_MANIFEST IGNORE_MISSING DRONE_TAG REPO TAG OS ARCH IMAGE_NAME DIRTY_CHECK
 ENV DAPPER_SOURCE /go/src/github.com/rancher/octopus/
 ENV DAPPER_OUTPUT ./adaptors/dummy/bin ./adaptors/dummy/dist ./adaptors/dummy/deploy ./adaptors/dummy/api
 ENV DAPPER_DOCKER_SOCKET true

diff --git a/adaptors/dummy/deploy/e2e/all_in_one.yaml b/adaptors/dummy/deploy/e2e/all_in_one.yaml
@@ -2,7 +2,6 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.5
     devices.edge.cattle.io/description: dummy device description
     devices.edge.cattle.io/device-property: ""
     devices.edge.cattle.io/enable: "true"
@@ -648,7 +647,6 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.5
     devices.edge.cattle.io/description: dummy device description
     devices.edge.cattle.io/device-property: ""
     devices.edge.cattle.io/enable: "true"

diff --git a/adaptors/dummy/deploy/manifests/crd/base/devices.edge.cattle.io_dummyprotocoldevices.yaml b/adaptors/dummy/deploy/manifests/crd/base/devices.edge.cattle.io_dummyprotocoldevices.yaml
@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.5
+    {}
   creationTimestamp: null
   name: dummyprotocoldevices.devices.edge.cattle.io
 spec:

diff --git a/adaptors/dummy/deploy/manifests/crd/base/devices.edge.cattle.io_dummyspecialdevices.yaml b/adaptors/dummy/deploy/manifests/crd/base/devices.edge.cattle.io_dummyspecialdevices.yaml
@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.5
+    {}
   creationTimestamp: null
   name: dummyspecialdevices.devices.edge.cattle.io
 spec:

diff --git a/adaptors/dummy/hack/make-rules/adaptor.sh b/adaptors/dummy/hack/make-rules/adaptor.sh
@@ -268,11 +268,7 @@ function verify() {
 
   octopus::log::info "running integration tests for adaptor ${adaptor}..."
 
-  CGO_ENABLED=0 go test \
-    "${CURR_DIR}/test/integration/adaptor/..." -v -ginkgo.v -ginkgo.progress
-  CGO_ENABLED=0 go test \
-    -tags=test \
-    "${CURR_DIR}/test/integration/physical/..." -v -ginkgo.v -ginkgo.progress
+  octopus::ginkgo::test "${CURR_DIR}/test/integration"
 
   octopus::log::info "...done"
 }
@@ -283,6 +279,8 @@ function e2e() {
 
   octopus::log::info "running E2E tests for adaptor ${adaptor}..."
 
+  octopus::ginkgo::test "${CURR_DIR}/test/e2e"
+
   octopus::log::info "...done"
 }
 

diff --git a/adaptors/modbus/Dockerfile.dapper b/adaptors/modbus/Dockerfile.dapper
@@ -1,31 +1,51 @@
-FROM golang:1.13.9-buster
-RUN apt-get update && \
-    apt-get install -y xz-utils unzip netcat
+FROM golang:1.13.12-buster
+RUN apt-get update -qq && apt-get install -y --no-install-recommends \
+        xz-utils \
+        unzip \
+        netcat \
+    && rm -rf /var/lib/apt/lists/*
 
 # -- for make rules
-## install docker
-RUN curl -sSfL "https://get.docker.com" | sh -s VERSION=19.03; \
+## install docker client
+RUN apt-get update -qq && apt-get install -y --no-install-recommends \
+        apt-transport-https \
+        ca-certificates \
+        curl \
+        gnupg \
+    && rm -rf /var/lib/apt/lists/*; \
+    \
+    curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - >/dev/null; \
+    echo "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/debian buster stable" > /etc/apt/sources.list.d/docker.list; \
+    \
+    apt-get update -qq && apt-get install -y --no-install-recommends \
+        docker-ce-cli=5:19.03.* \
+    && rm -rf /var/lib/apt/lists/*; \
     docker --version
 ## install kubectl
-RUN curl -fL "https://storage.googleapis.com/kubernetes-release/release/v1.17.2/bin/$(go env GOOS)/$(go env GOARCH)/kubectl" -o /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl; \
+RUN curl -fL "https://storage.googleapis.com/kubernetes-release/release/v1.18.2/bin/$(go env GOOS)/$(go env GOARCH)/kubectl" -o /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl; \
     kubectl version --short --client
 ## install golangci-lint
 RUN if [ "$(go env GOARCH)" = "amd64" ]; then \
-        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)/bin" v1.24.0; \
+        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)/bin" v1.27.0; \
         golangci-lint --version; \
     fi
 ## install controller-gen
 RUN if [ "$(go env GOARCH)" = "amd64" ]; then \
-        GO111MODULE=on go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.5; \
+        GO111MODULE=on go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.3.0; \
         controller-gen --version; \
     fi
+## install ginkgo
+RUN if [ "$(go env GOARCH)" = "amd64" ]; then \
+        GO111MODULE=on go get github.com/onsi/ginkgo/[email protected]; \
+        ginkgo version; \
+    fi
 # -- for make rules
 
 # -- for dapper
 ENV DAPPER_RUN_ARGS --privileged --network host
 ENV GO111MODULE=off
 ENV CROSS=false
-ENV DAPPER_ENV CROSS LOCAL_CLUSTER_KIND DOCKER_USERNAME DOCKER_PASSWORD WITHOUT_MANIFEST ONLY_MANIFEST IGNORE_MISSING DRONE_TAG REPO TAG OS ARCH IMAGE_NAME
+ENV DAPPER_ENV CROSS LOCAL_CLUSTER_KIND DOCKER_USERNAME DOCKER_PASSWORD WITHOUT_MANIFEST ONLY_MANIFEST IGNORE_MISSING DRONE_TAG REPO TAG OS ARCH IMAGE_NAME DIRTY_CHECK
 ENV DAPPER_SOURCE /go/src/github.com/rancher/octopus/
 ENV DAPPER_OUTPUT ./adaptors/modbus/bin ./adaptors/modbus/dist ./adaptors/modbus/deploy ./adaptors/modbus/api
 ENV DAPPER_DOCKER_SOCKET true

diff --git a/adaptors/modbus/deploy/e2e/all_in_one.yaml b/adaptors/modbus/deploy/e2e/all_in_one.yaml
@@ -2,7 +2,6 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.5
     devices.edge.cattle.io/description: Modbus is a communication protocol developed
       by Modicon system. In short, it is a method for transmitting information between
       electronic devices through a serial line. The Modbus protocol is a master-slave

diff --git a/adaptors/modbus/deploy/manifests/crd/base/devices.edge.cattle.io_modbusdevices.yaml b/adaptors/modbus/deploy/manifests/crd/base/devices.edge.cattle.io_modbusdevices.yaml
@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.5
+    {}
   creationTimestamp: null
   name: modbusdevices.devices.edge.cattle.io
 spec:

diff --git a/adaptors/modbus/hack/make-rules/adaptor.sh b/adaptors/modbus/hack/make-rules/adaptor.sh
@@ -266,6 +266,8 @@ function verify() {
 
   octopus::log::info "running integration tests for adaptor ${adaptor}..."
 
+  octopus::ginkgo::test "${CURR_DIR}/test/integration"
+
   octopus::log::info "...done"
 }
 
@@ -275,6 +277,8 @@ function e2e() {
 
   octopus::log::info "running E2E tests for adaptor ${adaptor}..."
 
+  octopus::ginkgo::test "${CURR_DIR}/test/e2e"
+
   octopus::log::info "...done"
 }