add test for vsphere driver snapshot configuration

go.mod

@@ -47,6 +47,7 @@ require (
  golang.org/x/net v0.19.0
  golang.org/x/oauth2 v0.10.0
  google.golang.org/grpc v1.58.3
+ gopkg.in/ini.v1 v1.62.0
  gopkg.in/src-d/go-git.v4 v4.13.1
  gopkg.in/yaml.v2 v2.4.0
  k8s.io/api v0.29.0
@@ -266,7 +267,6 @@ require (
  google.golang.org/protobuf v1.31.0 // indirect
  gopkg.in/gcfg.v1 v1.2.3 // indirect
  gopkg.in/inf.v0 v0.9.1 // indirect
- gopkg.in/ini.v1 v1.62.0 // indirect
  gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
  gopkg.in/square/go-jose.v2 v2.6.0 // indirect
  gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect

test/extended/storage/driver_configuration.go

@@ -0,0 +1,326 @@
+package storage
+
+import (
+ "context"
+ "fmt"
+ g "github.com/onsi/ginkgo/v2"
+ o "github.com/onsi/gomega"
+ operatorv1 "github.com/openshift/api/operator/v1"
+ exutil "github.com/openshift/origin/test/extended/util"
+ "gopkg.in/ini.v1"
+ v1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
+ "k8s.io/kubernetes/test/e2e/framework"
+ e2e "k8s.io/kubernetes/test/e2e/framework"
+ k8simage "k8s.io/kubernetes/test/utils/image"
+ "strings"
+ "time"
+)
+
+const (
+ projectName = "csi-driver-configuration"
+ providerName = "csi.vsphere.vmware.com"
+)
+
+// This is [Serial] because it modifies clustercsidriver.
+var _ = g.Describe("[sig-storage][Feature:VSphereDriverConfiguration][Serial] vSphere CSI Driver Configuration", func() {
+ defer g.GinkgoRecover()
+ var oc = exutil.NewCLI(projectName)
+
+ var (
+ originalDriverConfigSpec *operatorv1.CSIDriverConfigSpec
+ snapshotOptions = map[string]map[string]string{
+ "global": {
+ "clusterCSIDriver": "globalMaxSnapshotsPerBlockVolume",
+ "cloudConf": "global-max-snapshots-per-block-volume",
+ },
+ "vsan": {
+ "clusterCSIDriver": "granularMaxSnapshotsPerBlockVolumeInVSAN",
+ "cloudConf": "granular-max-snapshots-per-block-volume-vsan",
+ },
+ "vvol": {
+ "clusterCSIDriver": "granularMaxSnapshotsPerBlockVolumeInVVOL",
+ "cloudConf": "granular-max-snapshots-per-block-volume-vvol",
+ },
+ }
+ )
+
+ g.BeforeEach(func() {
+ //TODO: remove when GA
+ if !exutil.IsTechPreviewNoUpgrade(oc) {
+ g.Skip("this test is only expected to work with TechPreviewNoUpgrade clusters")
+ }
+
+ if !framework.ProviderIs("vsphere") {
+ g.Skip("this test is only expected to work with vSphere clusters")
+ }
+
+ originalClusterCSIDriver, err := oc.AdminOperatorClient().OperatorV1().ClusterCSIDrivers().Get(context.Background(), providerName, metav1.GetOptions{})
+ o.Expect(err).NotTo(o.HaveOccurred())
+
+ originalDriverConfigSpec = originalClusterCSIDriver.Spec.DriverConfig.DeepCopy()
+ e2e.Logf("storing original driver config: %+v", originalDriverConfigSpec)
+ })
+
+ g.AfterEach(func() {
+ if originalDriverConfigSpec == nil {
+ return
+ }
+
+ clusterCSIDriver, err := oc.AdminOperatorClient().OperatorV1().ClusterCSIDrivers().Get(context.Background(), providerName, metav1.GetOptions{})
+ o.Expect(err).NotTo(o.HaveOccurred())
+
+ e2e.Logf("restoring original driver config: %+v", originalDriverConfigSpec)
+ clusterCSIDriver.Spec.DriverConfig = *originalDriverConfigSpec
+ _, err = oc.AdminOperatorClient().OperatorV1().ClusterCSIDrivers().Update(context.Background(), clusterCSIDriver, metav1.UpdateOptions{})
+ if err != nil {
+ e2e.Failf("failed to update ClusterCSIDriver: %v", err)
+ }
+ o.Expect(err).NotTo(o.HaveOccurred())
+ })
+
+ g.Context("snapshot options in clusterCSIDriver", func() {
+ var tests = []struct {
+ name string
+ clusterCSIDriverOptions map[string]int
+ cloudConfigOptions map[string]int
+ successfulSnapshotsCreated int // Number of snapshots that should be created successfully, 0 to skip.
+ }{
+ {
+ name: "global snapshot option",
+ clusterCSIDriverOptions: map[string]int{
+ snapshotOptions["global"]["clusterCSIDriver"]: 4,
+ },
+ cloudConfigOptions: map[string]int{
+ snapshotOptions["global"]["cloudConf"]: 4,
+ },
+ successfulSnapshotsCreated: 4,
+ },
+ {
+ name: "default snapshot limit",
+ clusterCSIDriverOptions: map[string]int{},
+ cloudConfigOptions: map[string]int{},
+ successfulSnapshotsCreated: 3,
+ },
+ {
+ name: "vsan snapshot option",
+ clusterCSIDriverOptions: map[string]int{
+ snapshotOptions["vsan"]["clusterCSIDriver"]: 4,
+ },
+ cloudConfigOptions: map[string]int{
+ snapshotOptions["vsan"]["cloudConf"]: 4,
+ },
+ successfulSnapshotsCreated: 0,
+ },
+ {
+ name: "vvol snapshot option",
+ clusterCSIDriverOptions: map[string]int{
+ snapshotOptions["vvol"]["clusterCSIDriver"]: 4,
+ },
+ cloudConfigOptions: map[string]int{
+ snapshotOptions["vvol"]["cloudConf"]: 4,
+ },
+ successfulSnapshotsCreated: 0,
+ },
+ {
+ name: "all snapshot options",
+ clusterCSIDriverOptions: map[string]int{
+ snapshotOptions["global"]["clusterCSIDriver"]: 5,
+ snapshotOptions["vsan"]["clusterCSIDriver"]: 10,
+ snapshotOptions["vvol"]["clusterCSIDriver"]: 15,
+ },
+ cloudConfigOptions: map[string]int{
+ snapshotOptions["global"]["cloudConf"]: 5,
+ snapshotOptions["vsan"]["cloudConf"]: 10,
+ snapshotOptions["vvol"]["cloudConf"]: 15,
+ },
+ successfulSnapshotsCreated: 0,
+ },
+ }
+
+ for _, t := range tests {
+ g.It(fmt.Sprintf("%s", t.name), func() {
+ for option, value := range t.clusterCSIDriverOptions {
+ e2e.Logf("updating %s to %d in clustercsidriver", option, value)
+ setClusterCSIDriverSnapshotOptions(oc, option, value)
+ }
+
+ for option, value := range t.cloudConfigOptions {
+ o.Eventually(func() error {
+ return loadAndCheckCloudConf(oc, "Snapshot", option, value)
+ }, time.Minute, time.Second).Should(o.Succeed())
+ }
+
+ if t.successfulSnapshotsCreated > 0 {
+ pvc, err := createTestPVC(oc, oc.Namespace(), "test-pvc", "1Gi")
+ o.Expect(err).NotTo(o.HaveOccurred())
+
+ _, err = createTestPod(oc, pvc.Name, oc.Namespace())
+ o.Expect(err).NotTo(o.HaveOccurred())
+
+ // Wait for pvc to be bound.
+ o.Eventually(func() error {
+ pvc, err := oc.AdminKubeClient().CoreV1().PersistentVolumeClaims(oc.Namespace()).Get(context.Background(), "test-pvc", metav1.GetOptions{})
+ if err != nil {
+ return err
+ }
+ if pvc.Status.Phase != v1.ClaimBound {
+ return fmt.Errorf("PVC not bound")
+ }
+ return nil
+ })
+
+ for i := 0; i < t.successfulSnapshotsCreated; i++ {
+ err := createSnapshot(oc, oc.Namespace(), fmt.Sprintf("test-snapshot-%d", i), "test-pvc")
+ o.Expect(err).NotTo(o.HaveOccurred())
+ }
+
+ // Next snapshot creation should be over the set limit and fail.
+ err = createSnapshot(oc, oc.Namespace(), "test-snapshot-failed", "test-pvc")
+ o.Expect(err).NotTo(o.HaveOccurred())
+
+ readyToUse, err := oc.Run("get").Args("volumesnapshot/test-snapshot-failed", "-o", "jsonpath={.status.readyToUse}").Output()
+ o.Expect(err).NotTo(o.HaveOccurred())
+
+ errMsg, err := oc.Run("get").Args("volumesnapshot/test-snapshot-failed", "-o", "jsonpath={.status.error.message}").Output()
+ o.Expect(err).NotTo(o.HaveOccurred())
+
+ e2e.Logf("VolumeSnapshot error message: %s readyToUse %s", errMsg, readyToUse)
+ if !strings.Contains(errMsg, "failed to take snapshot of the volume") && readyToUse != "false" {
+ e2e.Failf("VolumeSnapshot \"test-snapshot-failed\" should have failed and should not be ready to use")
+ }
+ }
+ })
+
+ }
+ })
+})
+
+func setClusterCSIDriverSnapshotOptions(oc *exutil.CLI, snapshotOptions string, value int) {
+ patch := []byte(fmt.Sprintf("{\"spec\":{\"driverConfig\":{\"vSphere\":{\"%s\": %d}}}}", snapshotOptions, value))
+ _, err := oc.AdminOperatorClient().OperatorV1().ClusterCSIDrivers().Patch(context.Background(), providerName, types.MergePatchType, patch, metav1.PatchOptions{})
+ if err != nil {
+ e2e.Failf("failed to patch ClusterCSIDriver: %v", err)
+ }
+}
+
+func loadAndCheckCloudConf(oc *exutil.CLI, sectionName string, keyName string, expectedValue int) error {
+ cm, err := oc.AdminKubeClient().CoreV1().ConfigMaps("openshift-cluster-csi-drivers").Get(context.Background(), "vsphere-csi-config", metav1.GetOptions{})
+ if err != nil {
+ e2e.Failf("failed to get ConfigMap: %v", err)
+ return err
+ }
+
+ cloudConfData, ok := cm.Data["cloud.conf"]
+ if !ok {
+ return fmt.Errorf("cloud.conf key not found in ConfigMap")
+ }
+
+ cfg, err := ini.Load([]byte(cloudConfData))
+ if err != nil {
+ e2e.Failf("failed to load cloud.conf: %v", err)
+ return err
+ }
+
+ section, err := cfg.GetSection(sectionName)
+ if err != nil {
+ return fmt.Errorf("section %s not found in cloud.conf: %v", sectionName, err)
+ }
+
+ key, err := section.GetKey(keyName)
+ if err != nil {
+ return fmt.Errorf("key %s not found in section %s: %v", keyName, sectionName, err)
+ }
+
+ o.Expect(key.String()).To(o.Equal(fmt.Sprintf("%d", expectedValue)))
+
+ return nil
+}
+
+func createTestPod(oc *exutil.CLI, pvcName string, namespace string) (*v1.Pod, error) {
+ allowPrivEsc := false
+ pod := &v1.Pod{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-pod-driver-conf",
+ Namespace: namespace,
+ },
+ Spec: v1.PodSpec{
+ Containers: []v1.Container{
+ {
+ Name: "test",
+ Image: k8simage.GetE2EImage(k8simage.BusyBox),
+ VolumeMounts: []v1.VolumeMount{
+ {
+ Name: "pvc-data",
+ MountPath: "/mnt",
+ },
+ },
+ SecurityContext: &v1.SecurityContext{
+ AllowPrivilegeEscalation: &allowPrivEsc,
+ SeccompProfile: &v1.SeccompProfile{
+ Type: v1.SeccompProfileTypeRuntimeDefault,
+ },
+ Capabilities: &v1.Capabilities{
+ Drop: []v1.Capability{"ALL"},
+ },
+ },
+ },
+ },
+ Volumes: []v1.Volume{
+ {
+ Name: "pvc-data",
+ VolumeSource: v1.VolumeSource{
+ PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
+ ClaimName: pvcName,
+ },
+ },
+ },
+ },
+ },
+ }
+
+ return oc.AdminKubeClient().CoreV1().Pods(namespace).Create(context.Background(), pod, metav1.CreateOptions{})
+}
+
+func createTestPVC(oc *exutil.CLI, namespace string, pvcName string, volumeSize string) (*v1.PersistentVolumeClaim, error) {
+ e2e.Logf("creating PVC %s in namespace %s with size %s", pvcName, namespace, volumeSize)
+
+ pvc := &v1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: pvcName,
+ Namespace: namespace,
+ },
+ Spec: v1.PersistentVolumeClaimSpec{
+ AccessModes: []v1.PersistentVolumeAccessMode{"ReadWriteOnce"},
+ Resources: v1.VolumeResourceRequirements{
+ Requests: v1.ResourceList{
+ v1.ResourceStorage: resource.MustParse(volumeSize),
+ },
+ },
+ },
+ }
+
+ return oc.AdminKubeClient().CoreV1().PersistentVolumeClaims(namespace).Create(context.Background(), pvc, metav1.CreateOptions{})
+}
+
+func createSnapshot(oc *exutil.CLI, namespace string, snapshotName string, pvcName string) error {
+ snapshot := fmt.Sprintf(`
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshot
+metadata:
+ name: %s
+ namespace: %s
+spec:
+ source:
+ persistentVolumeClaimName: %s
+`, snapshotName, namespace, pvcName)
+
+ err := oc.AsAdmin().Run("apply").Args("-f", "-").InputString(snapshot).Execute()
+ if err != nil {
+ return fmt.Errorf("failed to create snapshot: %v", err)
+ }
+
+ return nil
+}