add manifests for kustomize

minio · Oct 3, 2024 · 80e8986 · 80e8986
1 parent c657af7
commit 80e8986
Show file tree

Hide file tree

Showing 44 changed files with 1,775 additions and 122 deletions.
diff --git a/.github/workflows/functests.yml b/.github/workflows/functests.yml
@@ -69,6 +69,11 @@ jobs:
           while kubectl get pods -n kube-system --no-headers | grep -vqw Running; do echo "Waiting for Minikube pods to be ready"; sleep 3; done
           while kubectl get nodes --no-headers | grep -vqw Ready; do echo "Waiting for Minikube node to be ready"; sleep 3; done
 
+      - name: Enable kustomize
+        if: contains(fromJson('["v1.24.17", "v1.25.16", "v1.26.15", "v1.27.15", "v1.28.11", "v1.29.6", "v1.30.2"]'), matrix.kube-version)
+        run: |
+          echo "KUSTOMIZE=true" >> $GITHUB_ENV
+
       - name: Run tests
         run: |
           functests/run-tests.sh

diff --git a/functests/common.sh b/functests/common.sh
@@ -137,6 +137,41 @@ function install_directpv() {
     sleep 10
 }
 
+# install_directpv_kustomize <plugin> <pod_count>
+function install_directpv_kustomize() {
+    directpv_client="$1"
+    echo "* Installing DirectPV via kustomize"
+
+    cat > kustomization.yaml <<EOF
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../resources/base
+
+images:
+  - name: quay.io/minio/directpv
+    newTag: ${VERSION}
+EOF
+
+    kubectl kustomize | kubectl apply -f -
+
+    required_count="$2"
+    running_count=0
+    while [[ $running_count -lt $required_count ]]; do
+        echo "  ...waiting for $(( required_count - running_count )) DirectPV pods to come up"
+        sleep 1m
+        running_count=$(kubectl get pods --field-selector=status.phase=Running --no-headers --namespace=directpv | wc -l)
+    done
+
+    while ! "${directpv_client}" info --quiet; do
+        echo "  ...waiting for DirectPV to come up"
+        sleep 1m
+    done
+
+    sleep 10
+}
+
 # uninstall_directpv <plugin> <pod_count>
 function uninstall_directpv() {
     directpv_client="$1"
@@ -160,6 +195,39 @@ function uninstall_directpv() {
     return 0
 }
 
+# uninstall_directpv_kustomize  <pod_count>
+function uninstall_directpv_kustomize() {
+    echo "* Uninstalling DirectPV via kustomize"
+
+    cat > kustomization.yaml <<EOF
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../resources/base
+
+images:
+  - name: quay.io/minio/directpv
+    newTag: ${VERSION}
+EOF
+    # delete hangs sometimes even after the resources are deleted
+    kubectl kustomize | timeout 15s kubectl delete -f -
+
+    pending="$2"
+    while [[ $pending -gt 0 ]]; do
+        echo "  ...waiting for ${pending} DirectPV pods to go down"
+        sleep 5
+        pending=$(kubectl get pods --field-selector=status.phase=Running --no-headers --namespace=directpv-min-io 2>/dev/null | wc -l)
+    done
+
+    while kubectl get namespace directpv-min-io --no-headers 2>/dev/null | grep -q .; do
+        echo "  ...waiting for directpv-min-io namespace to be removed"
+        sleep 5
+    done
+
+    return 0
+}
+
 # usage: check_drives_status <plugin>
 function check_drives_status() {
     if ! is_github_workflow; then

diff --git a/functests/tests.sh b/functests/tests.sh
@@ -25,7 +25,11 @@ function run_tests() {
     setup_lvm
     setup_luks
     pod_count=$(( 3 + ACTIVE_NODES ))
-    install_directpv "${DIRECTPV_DIR}/kubectl-directpv" "${pod_count}"
+    if [ "${KUSTOMIZE}" == "true" ]; then
+        install_directpv_kustomize "${DIRECTPV_DIR}/kubectl-directpv" "${pod_count}"
+    else
+        install_directpv "${DIRECTPV_DIR}/kubectl-directpv" "${pod_count}"
+    fi
     add_drives "${DIRECTPV_DIR}/kubectl-directpv"
     deploy_minio minio.yaml
     test_force_delete
@@ -34,7 +38,11 @@ function run_tests() {
     uninstall_minio "${DIRECTPV_DIR}/kubectl-directpv" minio.yaml
     test_volume_expansion "${DIRECTPV_DIR}/kubectl-directpv" sleep.yaml
     remove_drives "${DIRECTPV_DIR}/kubectl-directpv"
-    uninstall_directpv "${DIRECTPV_DIR}/kubectl-directpv" "${pod_count}"
+    if [ "${KUSTOMIZE}" == "true" ]; then
+        uninstall_directpv_kustomize  "${pod_count}"
+    else
+        uninstall_directpv "${DIRECTPV_DIR}/kubectl-directpv" "${pod_count}"    
+    fi
     unmount_directpv
     remove_luks
     remove_lvm

diff --git a/operator/helm-charts/directpv-chart/templates/CSIDriver.yaml b/operator/helm-charts/directpv-chart/templates/CSIDriver.yaml
@@ -3,7 +3,6 @@ apiVersion: storage.k8s.io/v1
 kind: CSIDriver
 metadata:
   annotations:
-    directpv.min.io/plugin-version: v4.0.10
   creationTimestamp: null
   labels:
     application-name: directpv.min.io

diff --git a/operator/helm-charts/directpv-chart/templates/ClusterRole.yaml b/operator/helm-charts/directpv-chart/templates/ClusterRole.yaml
@@ -3,7 +3,6 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   annotations:
-    directpv.min.io/plugin-version: v4.0.10
     rbac.authorization.kubernetes.io/autoupdate: "true"
   creationTimestamp: null
   labels:
@@ -63,20 +62,6 @@ rules:
   - patch
   - update
   - watch
-- apiGroups:
-  - snapshot.storage.k8s.io
-  resources:
-  - volumesnapshots
-  verbs:
-  - get
-  - list
-- apiGroups:
-  - snapshot.storage.k8s.io
-  resources:
-  - volumesnapshotcontents
-  verbs:
-  - get
-  - list
 - apiGroups:
   - storage.k8s.io
   resources:
@@ -125,10 +110,8 @@ rules:
   - watch
 - apiGroups:
   - apiextensions.k8s.io
-  - directpv.min.io
   resources:
   - customresourcedefinitions
-  - customresourcedefinition
   verbs:
   - create
   - delete
@@ -155,7 +138,6 @@ rules:
   - ""
   resources:
   - pods
-  - pod
   verbs:
   - get
   - list
@@ -164,10 +146,7 @@ rules:
   - ""
   resources:
   - secrets
-  - secret
   verbs:
   - get
   - list
   - watch
-
-
diff --git a/operator/helm-charts/directpv-chart/templates/ClusterRoleBinding.yaml b/operator/helm-charts/directpv-chart/templates/ClusterRoleBinding.yaml
@@ -3,7 +3,6 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   annotations:
-    directpv.min.io/plugin-version: v4.0.10
     rbac.authorization.kubernetes.io/autoupdate: "true"
   creationTimestamp: null
   labels:

diff --git a/operator/helm-charts/directpv-chart/templates/DaemonSet.yaml b/operator/helm-charts/directpv-chart/templates/DaemonSet.yaml
@@ -2,9 +2,6 @@
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  annotations:
-    directpv.min.io/image-tag: v4.0.10
-    directpv.min.io/plugin-version: v4.0.10
   creationTimestamp: null
   labels:
     application-name: directpv.min.io

diff --git a/operator/helm-charts/directpv-chart/templates/Deployment.yaml b/operator/helm-charts/directpv-chart/templates/Deployment.yaml
@@ -2,9 +2,6 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  annotations:
-    directpv.min.io/image-tag: v4.0.10
-    directpv.min.io/plugin-version: v4.0.10
   creationTimestamp: null
   labels:
     application-name: directpv.min.io

diff --git a/operator/helm-charts/directpv-chart/templates/Namespace.yaml b/operator/helm-charts/directpv-chart/templates/Namespace.yaml
@@ -1,8 +1,6 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  annotations:
-    directpv.min.io/plugin-version: v4.0.10
   creationTimestamp: null
   finalizers:
   - foregroundDeletion

diff --git a/operator/helm-charts/directpv-chart/templates/Role.yaml b/operator/helm-charts/directpv-chart/templates/Role.yaml
@@ -3,7 +3,6 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
   annotations:
-    directpv.min.io/plugin-version: v4.0.10
     rbac.authorization.kubernetes.io/autoupdate: "true"
   creationTimestamp: null
   labels:

diff --git a/operator/helm-charts/directpv-chart/templates/RoleBinding.yaml b/operator/helm-charts/directpv-chart/templates/RoleBinding.yaml
@@ -3,7 +3,6 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   annotations:
-    directpv.min.io/plugin-version: v4.0.10
     rbac.authorization.kubernetes.io/autoupdate: "true"
   creationTimestamp: null
   labels:

diff --git a/operator/helm-charts/directpv-chart/templates/ServiceAccount.yaml b/operator/helm-charts/directpv-chart/templates/ServiceAccount.yaml
@@ -3,7 +3,6 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   annotations:
-    directpv.min.io/plugin-version: v4.0.10
   creationTimestamp: null
   labels:
     application-name: directpv.min.io

diff --git a/operator/helm-charts/directpv-chart/templates/StorageClass.yaml b/operator/helm-charts/directpv-chart/templates/StorageClass.yaml
@@ -8,8 +8,6 @@ allowedTopologies:
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
-  annotations:
-    directpv.min.io/plugin-version: v4.0.10
   creationTimestamp: null
   finalizers:
   - foregroundDeletion

diff --git a/pkg/admin/installer/args.go b/pkg/admin/installer/args.go
@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"io"
 	"path"
-	"regexp"
 
 	"github.com/minio/directpv/pkg/utils"
 	corev1 "k8s.io/api/core/v1"
@@ -80,18 +79,10 @@ type Args struct {
 	nodeDriverRegistrarImage string
 	livenessProbeImage       string
 	csiResizerImage          string
-	imageTag                 string
 }
 
-var imageTagRegex = regexp.MustCompile(`:([^/]+)$`)
-
 // NewArgs creates arguments for DirectPV installation.
 func NewArgs(image string) *Args {
-	imageTag := "dev"
-	matchIndex := imageTagRegex.FindStringSubmatchIndex(image)
-	if len(matchIndex) > 0 && len(image) > matchIndex[0]+1 {
-		imageTag = image[matchIndex[0]+1:]
-	}
 	return &Args{
 		image:    image,
 		Registry: "quay.io",
@@ -101,7 +92,6 @@ func NewArgs(image string) *Args {
 		nodeDriverRegistrarImage: nodeDriverRegistrarImage,
 		livenessProbeImage:       livenessProbeImage,
 		csiResizerImage:          csiResizerImage,
-		imageTag:                 imageTag,
 	}
 }
 

diff --git a/pkg/admin/installer/csidriver.go b/pkg/admin/installer/csidriver.go
@@ -21,7 +21,6 @@ import (
 	"errors"
 	"fmt"
 
-	directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types"
 	"github.com/minio/directpv/pkg/client"
 	"github.com/minio/directpv/pkg/consts"
 	legacyclient "github.com/minio/directpv/pkg/legacy/client"
@@ -95,10 +94,7 @@ func (t csiDriverTask) doCreateCSIDriver(ctx context.Context, args *Args, versio
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      name,
 				Namespace: metav1.NamespaceNone,
-				Annotations: map[string]string{
-					string(directpvtypes.PluginVersionLabelKey): args.PluginVersion,
-				},
-				Labels: defaultLabels,
+				Labels:    defaultLabels,
 			},
 			Spec: storagev1.CSIDriverSpec{
 				PodInfoOnMount: &podInfoOnMount,
@@ -128,10 +124,7 @@ func (t csiDriverTask) doCreateCSIDriver(ctx context.Context, args *Args, versio
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      name,
 				Namespace: metav1.NamespaceNone,
-				Annotations: map[string]string{
-					string(directpvtypes.PluginVersionLabelKey): args.PluginVersion,
-				},
-				Labels: defaultLabels,
+				Labels:    defaultLabels,
 			},
 			Spec: storagev1beta1.CSIDriverSpec{
 				PodInfoOnMount: &podInfoOnMount,

diff --git a/pkg/admin/installer/daemonset.go b/pkg/admin/installer/daemonset.go
@@ -20,7 +20,6 @@ import (
 	"context"
 	"fmt"
 
-	directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types"
 	"github.com/minio/directpv/pkg/client"
 	"github.com/minio/directpv/pkg/consts"
 	"github.com/minio/directpv/pkg/k8s"
@@ -215,11 +214,7 @@ func newDaemonset(podSpec corev1.PodSpec, name, selectorValue string, args *Args
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
 			Namespace: namespace,
-			Annotations: map[string]string{
-				string(directpvtypes.ImageTagLabelKey):      args.imageTag,
-				string(directpvtypes.PluginVersionLabelKey): args.PluginVersion,
-			},
-			Labels: defaultLabels,
+			Labels:    defaultLabels,
 		},
 		Spec: appsv1.DaemonSetSpec{
 			Selector: metav1.AddLabelToSelector(&metav1.LabelSelector{}, selectorKey, selectorValue),
@@ -297,7 +292,7 @@ func (t daemonsetTask) doCreateDaemonset(ctx context.Context, args *Args) (err e
 		}
 	}
 	if selectorValue == "" {
-		selectorValue = fmt.Sprintf("%v-%v", consts.Identity, getRandSuffix())
+		selectorValue = fmt.Sprintf("%v-%v", consts.Identity, consts.NodeServerName)
 	}
 
 	daemonset := newDaemonset(podSpec, consts.NodeServerName, selectorValue, args)
@@ -360,7 +355,7 @@ func (t daemonsetTask) doCreateLegacyDaemonset(ctx context.Context, args *Args)
 		}
 	}
 	if selectorValue == "" {
-		selectorValue = fmt.Sprintf("%v-%v", consts.Identity, getRandSuffix())
+		selectorValue = fmt.Sprintf("%v-%v", consts.Identity, consts.LegacyNodeServerName)
 	}
 
 	daemonset := newDaemonset(podSpec, consts.LegacyNodeServerName, selectorValue, args)

diff --git a/pkg/admin/installer/deployment.go b/pkg/admin/installer/deployment.go
@@ -20,7 +20,6 @@ import (
 	"context"
 	"fmt"
 
-	directpvtypes "github.com/minio/directpv/pkg/apis/directpv.min.io/types"
 	"github.com/minio/directpv/pkg/client"
 	"github.com/minio/directpv/pkg/consts"
 	"github.com/minio/directpv/pkg/k8s"
@@ -195,7 +194,7 @@ func (t deploymentTask) doCreateDeployment(ctx context.Context, args *Args, lega
 		}
 	}
 	if selectorValue == "" {
-		selectorValue = fmt.Sprintf("%v-%v", consts.ControllerServerName, getRandSuffix())
+		selectorValue = fmt.Sprintf("%v-%v", consts.ControllerServerName, name)
 	}
 
 	replicas := int32(3)
@@ -207,11 +206,7 @@ func (t deploymentTask) doCreateDeployment(ctx context.Context, args *Args, lega
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
 			Namespace: namespace,
-			Annotations: map[string]string{
-				string(directpvtypes.ImageTagLabelKey):      args.imageTag,
-				string(directpvtypes.PluginVersionLabelKey): args.PluginVersion,
-			},
-			Labels: defaultLabels,
+			Labels:    defaultLabels,
 		},
 		Spec: appsv1.DeploymentSpec{
 			Replicas: &replicas,