add test for vsphere driver snapshot configuration

Author: RomanBednar

go.mod

@@ -47,6 +47,7 @@ require (
 	golang.org/x/net v0.19.0
 	golang.org/x/oauth2 v0.10.0
 	google.golang.org/grpc v1.58.3
+	gopkg.in/ini.v1 v1.62.0
 	gopkg.in/src-d/go-git.v4 v4.13.1
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/api v0.29.0
@@ -266,7 +267,6 @@ require (
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/gcfg.v1 v1.2.3 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/ini.v1 v1.62.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
 	gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect

test/extended/storage/driver_configuration.go

@@ -0,0 +1,326 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	g "github.com/onsi/ginkgo/v2"
+	o "github.com/onsi/gomega"
+	operatorv1 "github.com/openshift/api/operator/v1"
+	exutil "github.com/openshift/origin/test/extended/util"
+	"gopkg.in/ini.v1"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubernetes/test/e2e/framework"
+	e2e "k8s.io/kubernetes/test/e2e/framework"
+	k8simage "k8s.io/kubernetes/test/utils/image"
+	"strings"
+	"time"
+)
+
+const (
+	projectName  = "csi-driver-configuration"
+	providerName = "csi.vsphere.vmware.com"
+)
+
+// This is [Serial] because it modifies clustercsidriver.
+var _ = g.Describe("[sig-storage][Feature:VSphereDriverConfiguration][Serial] vSphere CSI Driver Configuration", func() {
+	defer g.GinkgoRecover()
+	var oc = exutil.NewCLI(projectName)
+
+	var (
+		originalDriverConfigSpec *operatorv1.CSIDriverConfigSpec
+		snapshotOptions          = map[string]map[string]string{
+			"global": {
+				"clusterCSIDriver": "globalMaxSnapshotsPerBlockVolume",
+				"cloudConf":        "global-max-snapshots-per-block-volume",
+			},
+			"vsan": {
+				"clusterCSIDriver": "granularMaxSnapshotsPerBlockVolumeInVSAN",
+				"cloudConf":        "granular-max-snapshots-per-block-volume-vsan",
+			},
+			"vvol": {
+				"clusterCSIDriver": "granularMaxSnapshotsPerBlockVolumeInVVOL",
+				"cloudConf":        "granular-max-snapshots-per-block-volume-vvol",
+			},
+		}
+	)
+
+	g.BeforeEach(func() {
+		//TODO: remove when GA
+		if !exutil.IsTechPreviewNoUpgrade(oc) {
+			g.Skip("this test is only expected to work with TechPreviewNoUpgrade clusters")
+		}
+
+		if !framework.ProviderIs("vsphere") {
+			g.Skip("this test is only expected to work with vSphere clusters")
+		}
+
+		originalClusterCSIDriver, err := oc.AdminOperatorClient().OperatorV1().ClusterCSIDrivers().Get(context.Background(), providerName, metav1.GetOptions{})
+		o.Expect(err).NotTo(o.HaveOccurred())
+
+		originalDriverConfigSpec = originalClusterCSIDriver.Spec.DriverConfig.DeepCopy()
+		e2e.Logf("storing original driver config: %+v", originalDriverConfigSpec)
+	})
+
+	g.AfterEach(func() {
+		if originalDriverConfigSpec == nil {
+			return
+		}
+
+		clusterCSIDriver, err := oc.AdminOperatorClient().OperatorV1().ClusterCSIDrivers().Get(context.Background(), providerName, metav1.GetOptions{})
+		o.Expect(err).NotTo(o.HaveOccurred())
+
+		e2e.Logf("restoring original driver config: %+v", originalDriverConfigSpec)
+		clusterCSIDriver.Spec.DriverConfig = *originalDriverConfigSpec
+		_, err = oc.AdminOperatorClient().OperatorV1().ClusterCSIDrivers().Update(context.Background(), clusterCSIDriver, metav1.UpdateOptions{})
+		if err != nil {
+			e2e.Failf("failed to update ClusterCSIDriver: %v", err)
+		}
+		o.Expect(err).NotTo(o.HaveOccurred())
+	})
+
+	g.Context("snapshot options in clusterCSIDriver", func() {
+		var tests = []struct {
+			name                       string
+			clusterCSIDriverOptions    map[string]int
+			cloudConfigOptions         map[string]int
+			successfulSnapshotsCreated int // Number of snapshots that should be created successfully, 0 to skip.
+		}{
+			{
+				name: "global snapshot option",
+				clusterCSIDriverOptions: map[string]int{
+					snapshotOptions["global"]["clusterCSIDriver"]: 4,
+				},
+				cloudConfigOptions: map[string]int{
+					snapshotOptions["global"]["cloudConf"]: 4,
+				},
+				successfulSnapshotsCreated: 4,
+			},
+			{
+				name:                       "default snapshot limit",
+				clusterCSIDriverOptions:    map[string]int{},
+				cloudConfigOptions:         map[string]int{},
+				successfulSnapshotsCreated: 3,
+			},
+			{
+				name: "vsan snapshot option",
+				clusterCSIDriverOptions: map[string]int{
+					snapshotOptions["vsan"]["clusterCSIDriver"]: 4,
+				},
+				cloudConfigOptions: map[string]int{
+					snapshotOptions["vsan"]["cloudConf"]: 4,
+				},
+				successfulSnapshotsCreated: 0,
+			},
+			{
+				name: "vvol snapshot option",
+				clusterCSIDriverOptions: map[string]int{
+					snapshotOptions["vvol"]["clusterCSIDriver"]: 4,
+				},
+				cloudConfigOptions: map[string]int{
+					snapshotOptions["vvol"]["cloudConf"]: 4,
+				},
+				successfulSnapshotsCreated: 0,
+			},
+			{
+				name: "all snapshot options",
+				clusterCSIDriverOptions: map[string]int{
+					snapshotOptions["global"]["clusterCSIDriver"]: 5,
+					snapshotOptions["vsan"]["clusterCSIDriver"]:   10,
+					snapshotOptions["vvol"]["clusterCSIDriver"]:   15,
+				},
+				cloudConfigOptions: map[string]int{
+					snapshotOptions["global"]["cloudConf"]: 5,
+					snapshotOptions["vsan"]["cloudConf"]:   10,
+					snapshotOptions["vvol"]["cloudConf"]:   15,
+				},
+				successfulSnapshotsCreated: 0,
+			},
+		}
+
+		for _, t := range tests {
+			g.It(fmt.Sprintf("%s", t.name), func() {
+				for option, value := range t.clusterCSIDriverOptions {
+					e2e.Logf("updating %s to %d in clustercsidriver", option, value)
+					setClusterCSIDriverSnapshotOptions(oc, option, value)
+				}
+
+				for option, value := range t.cloudConfigOptions {
+					o.Eventually(func() error {
+						return loadAndCheckCloudConf(oc, "Snapshot", option, value)
+					}, time.Minute, time.Second).Should(o.Succeed())
+				}
+
+				if t.successfulSnapshotsCreated > 0 {
+					pvc, err := createTestPVC(oc, oc.Namespace(), "test-pvc", "1Gi")
+					o.Expect(err).NotTo(o.HaveOccurred())
+
+					_, err = createTestPod(oc, pvc.Name, oc.Namespace())
+					o.Expect(err).NotTo(o.HaveOccurred())
+
+					// Wait for pvc to be bound.
+					o.Eventually(func() error {
+						pvc, err := oc.AdminKubeClient().CoreV1().PersistentVolumeClaims(oc.Namespace()).Get(context.Background(), "test-pvc", metav1.GetOptions{})
+						if err != nil {
+							return err
+						}
+						if pvc.Status.Phase != v1.ClaimBound {
+							return fmt.Errorf("PVC not bound")
+						}
+						return nil
+					})
+
+					for i := 0; i < t.successfulSnapshotsCreated; i++ {
+						err := createSnapshot(oc, oc.Namespace(), fmt.Sprintf("test-snapshot-%d", i), "test-pvc")
+						o.Expect(err).NotTo(o.HaveOccurred())
+					}
+
+					// Next snapshot creation should be over the set limit and fail.
+					err = createSnapshot(oc, oc.Namespace(), "test-snapshot-failed", "test-pvc")
+					o.Expect(err).NotTo(o.HaveOccurred())
+
+					readyToUse, err := oc.Run("get").Args("volumesnapshot/test-snapshot-failed", "-o", "jsonpath={.status.readyToUse}").Output()
+					o.Expect(err).NotTo(o.HaveOccurred())
+
+					errMsg, err := oc.Run("get").Args("volumesnapshot/test-snapshot-failed", "-o", "jsonpath={.status.error.message}").Output()
+					o.Expect(err).NotTo(o.HaveOccurred())
+
+					e2e.Logf("VolumeSnapshot error message: %s readyToUse %s", errMsg, readyToUse)
+					if !strings.Contains(errMsg, "failed to take snapshot of the volume") && readyToUse != "false" {
+						e2e.Failf("VolumeSnapshot \"test-snapshot-failed\" should have failed and should not be ready to use")
+					}
+				}
+			})
+
+		}
+	})
+})
+
+func setClusterCSIDriverSnapshotOptions(oc *exutil.CLI, snapshotOptions string, value int) {
+	patch := []byte(fmt.Sprintf("{\"spec\":{\"driverConfig\":{\"vSphere\":{\"%s\": %d}}}}", snapshotOptions, value))
+	_, err := oc.AdminOperatorClient().OperatorV1().ClusterCSIDrivers().Patch(context.Background(), providerName, types.MergePatchType, patch, metav1.PatchOptions{})
+	if err != nil {
+		e2e.Failf("failed to patch ClusterCSIDriver: %v", err)
+	}
+}
+
+func loadAndCheckCloudConf(oc *exutil.CLI, sectionName string, keyName string, expectedValue int) error {
+	cm, err := oc.AdminKubeClient().CoreV1().ConfigMaps("openshift-cluster-csi-drivers").Get(context.Background(), "vsphere-csi-config", metav1.GetOptions{})
+	if err != nil {
+		e2e.Failf("failed to get ConfigMap: %v", err)
+		return err
+	}
+
+	cloudConfData, ok := cm.Data["cloud.conf"]
+	if !ok {
+		return fmt.Errorf("cloud.conf key not found in ConfigMap")
+	}
+
+	cfg, err := ini.Load([]byte(cloudConfData))
+	if err != nil {
+		e2e.Failf("failed to load cloud.conf: %v", err)
+		return err
+	}
+
+	section, err := cfg.GetSection(sectionName)
+	if err != nil {
+		return fmt.Errorf("section %s not found in cloud.conf: %v", sectionName, err)
+	}
+
+	key, err := section.GetKey(keyName)
+	if err != nil {
+		return fmt.Errorf("key %s not found in section %s: %v", keyName, sectionName, err)
+	}
+
+	o.Expect(key.String()).To(o.Equal(fmt.Sprintf("%d", expectedValue)))
+
+	return nil
+}
+
+func createTestPod(oc *exutil.CLI, pvcName string, namespace string) (*v1.Pod, error) {
+	allowPrivEsc := false
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-pod-driver-conf",
+			Namespace: namespace,
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:  "test",
+					Image: k8simage.GetE2EImage(k8simage.BusyBox),
+					VolumeMounts: []v1.VolumeMount{
+						{
+							Name:      "pvc-data",
+							MountPath: "/mnt",
+						},
+					},
+					SecurityContext: &v1.SecurityContext{
+						AllowPrivilegeEscalation: &allowPrivEsc,
+						SeccompProfile: &v1.SeccompProfile{
+							Type: v1.SeccompProfileTypeRuntimeDefault,
+						},
+						Capabilities: &v1.Capabilities{
+							Drop: []v1.Capability{"ALL"},
+						},
+					},
+				},
+			},
+			Volumes: []v1.Volume{
+				{
+					Name: "pvc-data",
+					VolumeSource: v1.VolumeSource{
+						PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
+							ClaimName: pvcName,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	return oc.AdminKubeClient().CoreV1().Pods(namespace).Create(context.Background(), pod, metav1.CreateOptions{})
+}
+
+func createTestPVC(oc *exutil.CLI, namespace string, pvcName string, volumeSize string) (*v1.PersistentVolumeClaim, error) {
+	e2e.Logf("creating PVC %s in namespace %s with size %s", pvcName, namespace, volumeSize)
+
+	pvc := &v1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      pvcName,
+			Namespace: namespace,
+		},
+		Spec: v1.PersistentVolumeClaimSpec{
+			AccessModes: []v1.PersistentVolumeAccessMode{"ReadWriteOnce"},
+			Resources: v1.VolumeResourceRequirements{
+				Requests: v1.ResourceList{
+					v1.ResourceStorage: resource.MustParse(volumeSize),
+				},
+			},
+		},
+	}
+
+	return oc.AdminKubeClient().CoreV1().PersistentVolumeClaims(namespace).Create(context.Background(), pvc, metav1.CreateOptions{})
+}
+
+func createSnapshot(oc *exutil.CLI, namespace string, snapshotName string, pvcName string) error {
+	snapshot := fmt.Sprintf(`
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshot
+metadata:
+ name: %s
+ namespace: %s
+spec:
+ source:
+   persistentVolumeClaimName: %s
+`, snapshotName, namespace, pvcName)
+
+	err := oc.AsAdmin().Run("apply").Args("-f", "-").InputString(snapshot).Execute()
+	if err != nil {
+		return fmt.Errorf("failed to create snapshot: %v", err)
+	}
+
+	return nil
+}