test/e2e/multik8s: refactor to parallelize the tests more aggressively

.github/workflows/e2e-multiple-k8s-clusters.yaml

@@ -22,33 +22,21 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        package:
-          - changetostandalone
-          - changetoprimary
-          - changetosecondary
         backup-transfer-part-size:
           - 3Mi # smaller than the PVC size
-        ginkgo-flags:
-          - ""
+        label-filter:
+          - full-backup
+          - incr-backup
+          - backup-failure
+          - change-to-primary
+          - change-to-secondary
+          - change-to-standalone
+          - "!//" # select unlabelled tests
         include:
-          - package: replication
-            backup-transfer-part-size: 10Mi # equal to the PVC size
-            ginkgo-flags: --label-filter="various-transfer-part-size"
-          - package: replication
-            backup-transfer-part-size: 200Gi # greater than the PVC size
-            ginkgo-flags: --label-filter="various-transfer-part-size"
-          - package: replication
-            backup-transfer-part-size: 3Mi # smaller than the PVC size
-            ginkgo-flags: --label-filter="full-backup"
-          - package: replication
-            backup-transfer-part-size: 3Mi # smaller than the PVC size
-            ginkgo-flags: --label-filter="incr-backup"
-          - package: replication
-            backup-transfer-part-size: 3Mi # smaller than the PVC size
-            ginkgo-flags: --label-filter="misc"
-          - package: replication
-            backup-transfer-part-size: 3Mi # smaller than the PVC size
-            ginkgo-flags: --label-filter="backup-failure"
+          - backup-transfer-part-size: 10Mi # equal to the PVC size
+            label-filter: various-transfer-part-size
+          - backup-transfer-part-size: 200Gi # greater than the PVC size
+            label-filter: various-transfer-part-size
     #runs-on: "ubuntu-22.04"
     runs-on: mantle_large_runner_16core
     timeout-minutes: 120
@@ -62,6 +50,5 @@ jobs:
       - run: make -C test/e2e setup
       - run: |
           make -C test/e2e test-multiple-k8s-clusters \
-            TEST_MULTIK8S_PACKAGES=${{ matrix.package }} \
             BACKUP_TRANSFER_PART_SIZE=${{ matrix.backup-transfer-part-size }} \
-            GINKGO_FLAGS=${{ matrix.ginkgo-flags }}
+            GINKGO_FLAGS=--label-filter='${{ matrix.label-filter }}'

test/e2e/Makefile

@@ -25,7 +25,6 @@ LOOP_DEV2 := /dev/loop1
 MINIKUBE_PROFILE_PRIMARY=profile1
 MINIKUBE_PROFILE_SECONDARY=profile2
 TMPDIR := tmp
-TEST_MULTIK8S_PACKAGES := replication changetostandalone changetoprimary changetosecondary
 BACKUP_TRANSFER_PART_SIZE := 3Mi # smaller than the PVC size (10Mi)
 MAKEFILE_DIR := $(shell pwd)
 
@@ -327,7 +326,7 @@ do-test-multik8s: $(GINKGO)
 	env \
 	PATH=${PATH} \
 	E2ETEST=1 \
-	$(GINKGO) --fail-fast -v $(GINKGO_FLAGS) $(addprefix multik8s/, $(TEST_MULTIK8S_PACKAGES)); \
+	$(GINKGO) --fail-fast -v $(GINKGO_FLAGS) multik8s; \
 	if [ "$$?" -ne 0 ]; then \
 		echo "Controller logs for $(MINIKUBE_PROFILE_PRIMARY):"; \
 		$${KUBECTL_PRIMARY} logs --tail 100 -n $(CEPH_CLUSTER1_NAMESPACE) -l app.kubernetes.io/name=mantle -c mantle; \

test/e2e/multik8s/replication/backup_failure_test.go → test/e2e/multik8s/backup_failure_test.go

@@ -1,4 +1,4 @@
-package replication
+package multik8s
 
 import (
 	. "github.com/cybozu-go/mantle/test/e2e/multik8s/testutil"

test/e2e/multik8s/change_to_primary_test.go

@@ -0,0 +1,133 @@
+package multik8s
+
+import (
+	"github.com/cybozu-go/mantle/internal/controller"
+	. "github.com/cybozu-go/mantle/test/e2e/multik8s/testutil"
+	"github.com/cybozu-go/mantle/test/util"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("change role from primary to standalone during full backup", Label("change-to-primary"), func() {
+	It("should cancel a full backup if the role is changed from primary to standalone", func(ctx SpecContext) {
+		namespace := util.GetUniqueName("ns-")
+		pvcName := util.GetUniqueName("pvc-")
+		backupName := util.GetUniqueName("mb-")
+		restoreName := util.GetUniqueName("mr-")
+
+		SetupEnvironment(namespace)
+
+		PauseObjectStorage(ctx)
+		defer ResumeObjectStorage(ctx)
+
+		By("should create a MantleBackup resource")
+		CreatePVC(ctx, PrimaryK8sCluster, namespace, pvcName)
+		writtenDataHash := WriteRandomDataToPV(ctx, PrimaryK8sCluster, namespace, pvcName)
+		CreateMantleBackup(PrimaryK8sCluster, namespace, pvcName, backupName)
+		WaitMantleBackupReadyToUse(PrimaryK8sCluster, namespace, backupName)
+
+		By("changing the primary mantle to standalone")
+		err := ChangeClusterRole(PrimaryK8sCluster, controller.RoleStandalone)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("checking the MantleBackup in the primary K8s cluster remains")
+		Consistently(ctx, func(g Gomega) {
+			_, err := GetMB(PrimaryK8sCluster, namespace, backupName)
+			g.Expect(err).NotTo(HaveOccurred())
+		}, "10s", "1s").Should(Succeed())
+
+		EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName, restoreName, writtenDataHash)
+
+		By("changing the standalone mantle to primary")
+		err = ChangeClusterRole(PrimaryK8sCluster, controller.RolePrimary)
+		Expect(err).NotTo(HaveOccurred())
+
+		ResumeObjectStorage(ctx)
+		WaitMantleBackupSynced(namespace, backupName)
+	})
+})
+
+var _ = Describe("change to primary", Label("change-to-primary"), func() {
+	var (
+		namespace                                                                  string
+		pvcName0, backupName00, backupName01, writtenDataHash00, writtenDataHash01 string
+		pvcName1, backupName10, writtenDataHash10                                  string
+	)
+
+	/*
+		Overview of the test:
+
+		 primary k8s cluster        | secondary k8s cluster
+		============================|==========================
+		  role=primary              | role=secondary
+		  PVC0, MB00 (created)      |
+		                            | PVC0, MB00 (synced)
+		  role=standalone (changed) |
+		  MB01, PVC1, MB10 (created)|
+		  role=primary (changed)    |
+		                            | MB01, PVC1, MB10 (synced)
+	*/
+
+	It("should replicate a MantleBackup resource", func(ctx SpecContext) {
+		namespace = util.GetUniqueName("ns-")
+		pvcName0 = util.GetUniqueName("pvc-")
+		backupName00 = util.GetUniqueName("mb-")
+
+		SetupEnvironment(namespace)
+		CreatePVC(ctx, PrimaryK8sCluster, namespace, pvcName0)
+		writtenDataHash00 = WriteRandomDataToPV(ctx, PrimaryK8sCluster, namespace, pvcName0)
+		CreateMantleBackup(PrimaryK8sCluster, namespace, pvcName0, backupName00)
+		WaitMantleBackupSynced(namespace, backupName00)
+	})
+
+	It("should change the role from primary to standalone", func() {
+		By("changing the primary mantle to standalone")
+		err := ChangeClusterRole(PrimaryK8sCluster, controller.RoleStandalone)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("should restore the synced MantleBackup in the both clusters", func(ctx SpecContext) {
+		restoreName00 := util.GetUniqueName("mr-")
+		EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName00, restoreName00, writtenDataHash00)
+		EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName00, restoreName00, writtenDataHash00)
+	})
+
+	It("should create a MantleBackup resource", func(ctx SpecContext) {
+		backupName01 = util.GetUniqueName("mb-")
+		writtenDataHash01 = WriteRandomDataToPV(ctx, PrimaryK8sCluster, namespace, pvcName0)
+
+		CreateMantleBackup(PrimaryK8sCluster, namespace, pvcName0, backupName01)
+
+		pvcName1 = util.GetUniqueName("pvc-")
+		backupName10 = util.GetUniqueName("mb-")
+
+		CreatePVC(ctx, PrimaryK8sCluster, namespace, pvcName1)
+		writtenDataHash10 = WriteRandomDataToPV(ctx, PrimaryK8sCluster, namespace, pvcName1)
+		CreateMantleBackup(PrimaryK8sCluster, namespace, pvcName1, backupName10)
+	})
+
+	It("should change the role from standalone to primary", func() {
+		By("changing the standalone mantle to primary")
+		err := ChangeClusterRole(PrimaryK8sCluster, controller.RolePrimary)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("should synchronize MantleBackups correctly", func() {
+		WaitMantleBackupSynced(namespace, backupName01)
+		WaitMantleBackupSynced(namespace, backupName10)
+	})
+
+	It("should restore MantleBackups correctly", func(ctx SpecContext) {
+		restoreName00 := util.GetUniqueName("mr-")
+		EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName00, restoreName00, writtenDataHash00)
+		EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName00, restoreName00, writtenDataHash00)
+
+		restoreName01 := util.GetUniqueName("mr-")
+		EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName01, restoreName01, writtenDataHash01)
+		EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName01, restoreName01, writtenDataHash01)
+
+		restoreName10 := util.GetUniqueName("mr-")
+		EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName10, restoreName10, writtenDataHash10)
+		EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName10, restoreName10, writtenDataHash10)
+	})
+})

test/e2e/multik8s/change_to_secondary_test.go

@@ -0,0 +1,126 @@
+package multik8s
+
+import (
+	"github.com/cybozu-go/mantle/internal/controller"
+	. "github.com/cybozu-go/mantle/test/e2e/multik8s/testutil"
+	"github.com/cybozu-go/mantle/test/util"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("change to secondary", Label("change-to-secondary"), func() {
+	var (
+		namespace                                                                  string
+		pvcName0, backupName00, writtenDataHash00                                  string
+		pvcName1, backupName10, writtenDataHash10                                  string
+		pvcName2, backupName20, backupName21, writtenDataHash20, writtenDataHash21 string
+	)
+
+	/*
+		Overview of the test:
+
+		 primary k8s cluster       | secondary k8s cluster
+		===========================|==========================
+		  role=primary             | role=secondary
+		  PVC0, MB00 (created)     |
+		                           | PVC0, MB00 (synced)
+		                           | role=standalone (changed)
+		                           | PVC1, MB10 (created)
+		  (MB10 don't exist)       |
+		                           | role=secondary (changed)
+		  PVC2, MB20 (created)     |
+		                           | PVC2, MB20 (synced)
+		  MB21 (created)           |
+		                           | MB21 (synced)
+		  (MB10 don't exist)       |
+	*/
+
+	It("should set up the environment", func(ctx SpecContext) {
+		namespace = util.GetUniqueName("ns-")
+		pvcName0 = util.GetUniqueName("pvc-")
+		backupName00 = util.GetUniqueName("mb-")
+		pvcName1 = util.GetUniqueName("pvc-")
+		backupName10 = util.GetUniqueName("mb-")
+		pvcName2 = util.GetUniqueName("pvc-")
+		backupName20 = util.GetUniqueName("mb-")
+		backupName21 = util.GetUniqueName("mb-")
+
+		SetupEnvironment(namespace)
+	})
+
+	It("should create and restore a MantleBackup resource", func(ctx SpecContext) {
+		CreatePVC(ctx, PrimaryK8sCluster, namespace, pvcName0)
+
+		writtenDataHash00 = WriteRandomDataToPV(ctx, PrimaryK8sCluster, namespace, pvcName0)
+		CreateMantleBackup(PrimaryK8sCluster, namespace, pvcName0, backupName00)
+		WaitMantleBackupSynced(namespace, backupName00)
+
+		restoreName00 := util.GetUniqueName("mr-")
+		EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName00, restoreName00, writtenDataHash00)
+		EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName00, restoreName00, writtenDataHash00)
+	})
+
+	It("should change the role from secondary to standalone", func() {
+		By("changing the secondary mantle to standalone")
+		err := ChangeClusterRole(SecondaryK8sCluster, controller.RoleStandalone)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("should restore the synced MantleBackup in the both clusters", func(ctx SpecContext) {
+		restoreName00 := util.GetUniqueName("mr-")
+		EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName00, restoreName00, writtenDataHash00)
+		EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName00, restoreName00, writtenDataHash00)
+	})
+
+	It("should create a MantleBackup resource in the secondary k8s cluster", func(ctx SpecContext) {
+		CreatePVC(ctx, SecondaryK8sCluster, namespace, pvcName1)
+		writtenDataHash10 = WriteRandomDataToPV(ctx, SecondaryK8sCluster, namespace, pvcName1)
+		CreateMantleBackup(SecondaryK8sCluster, namespace, pvcName1, backupName10)
+		WaitMantleBackupReadyToUse(SecondaryK8sCluster, namespace, backupName10)
+	})
+
+	It("should ensure the MantleBackup created by standalone mantle doesn't exist in the primary k8s cluster",
+		func(ctx SpecContext) {
+			EnsureMantleBackupNotExist(ctx, PrimaryK8sCluster, namespace, backupName10)
+		})
+
+	It("should change the role from standalone to secondary", func() {
+		By("changing the standalone mantle to secondary")
+		err := ChangeClusterRole(SecondaryK8sCluster, controller.RoleSecondary)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("should create and synchronize new MantleBackup resources", func(ctx SpecContext) {
+		CreatePVC(ctx, PrimaryK8sCluster, namespace, pvcName2)
+
+		writtenDataHash20 = WriteRandomDataToPV(ctx, PrimaryK8sCluster, namespace, pvcName2)
+		CreateMantleBackup(PrimaryK8sCluster, namespace, pvcName2, backupName20)
+		WaitMantleBackupSynced(namespace, backupName20)
+
+		writtenDataHash21 = WriteRandomDataToPV(ctx, PrimaryK8sCluster, namespace, pvcName2)
+		CreateMantleBackup(PrimaryK8sCluster, namespace, pvcName2, backupName21)
+		WaitMantleBackupSynced(namespace, backupName21)
+	})
+
+	It("should restore MantleBackups correctly", func(ctx SpecContext) {
+		restoreName00 := util.GetUniqueName("mr-")
+		EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName00, restoreName00, writtenDataHash00)
+		EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName00, restoreName00, writtenDataHash00)
+
+		restoreName10 := util.GetUniqueName("mr-")
+		EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName10, restoreName10, writtenDataHash10)
+
+		restoreName20 := util.GetUniqueName("mr-")
+		EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName20, restoreName20, writtenDataHash20)
+		EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName20, restoreName20, writtenDataHash20)
+
+		restoreName21 := util.GetUniqueName("mr-")
+		EnsureCorrectRestoration(PrimaryK8sCluster, ctx, namespace, backupName21, restoreName21, writtenDataHash21)
+		EnsureCorrectRestoration(SecondaryK8sCluster, ctx, namespace, backupName21, restoreName21, writtenDataHash21)
+	})
+
+	It("should ensure the MantleBackup created by standalone mantle doesn't exist in the primary k8s cluster",
+		func(ctx SpecContext) {
+			EnsureMantleBackupNotExist(ctx, PrimaryK8sCluster, namespace, backupName10)
+		})
+})