test: refactor integration and e2e suites

- defer the `close(done)` at first.
- remove the TLS/SSL verification case of dummy adaptor as
  it is easy to block the CI flow.
- change the MQTT verification cases of dummy adaptor to access a local
  mosquitto container server.
- refactor some e2e suite's package name.
- replace a shellscript-based local cluster launcher with a code-based one.

adaptors/ble/test/integration/adaptor/suite_test.go

@@ -6,7 +6,8 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
+
+	"github.com/rancher/octopus/test/framework/envtest/printer"
 )
 
 var (
@@ -25,12 +26,14 @@ func TestAPIs(t *testing.T) {
 }
 
 var _ = BeforeSuite(func(done Done) {
-	testCtx, testCtxCancel = context.WithCancel(context.Background())
+	defer close(done)
 
-	close(done)
+	testCtx, testCtxCancel = context.WithCancel(context.Background())
 }, 600)
 
-var _ = AfterSuite(func() {
+var _ = AfterSuite(func(done Done) {
+	defer close(done)
+
 	if testCtxCancel != nil {
 		testCtxCancel()
 	}

adaptors/dummy/test/e2e/usability/suite_test.go

@@ -1,4 +1,4 @@
-package usability_test
+package usability
 
 import (
 	"context"
@@ -14,16 +14,14 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	clientsetscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
-	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/envtest"
-	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
 
 	dummyv1alpha1 "github.com/rancher/octopus/adaptors/dummy/api/v1alpha1"
 	"github.com/rancher/octopus/pkg/brain"
 	"github.com/rancher/octopus/pkg/limb"
 	"github.com/rancher/octopus/pkg/util/object"
-	"github.com/rancher/octopus/test/framework"
+	"github.com/rancher/octopus/test/framework/envtest"
+	"github.com/rancher/octopus/test/framework/envtest/printer"
 	"github.com/rancher/octopus/test/util/exec"
 )
 
@@ -49,14 +47,14 @@ func TestDummyAdaptor(t *testing.T) {
 }
 
 var _ = BeforeSuite(func(done Done) {
+	defer close(done)
+
 	testCtx, testCtxCancel = context.WithCancel(context.Background())
 
 	var err error
 
 	By("bootstrapping test environment")
-	testEnv = &envtest.Environment{
-		UseExistingCluster: pointer.BoolPtr(true),
-	}
+	testEnv = &envtest.Environment{}
 
 	By("creating kubernetes client")
 	var k8sSchema = clientsetscheme.Scheme
@@ -68,7 +66,7 @@ var _ = BeforeSuite(func(done Done) {
 	err = registerScheme(k8sSchema)
 	Expect(err).NotTo(HaveOccurred())
 
-	k8sCfg, err = framework.StartEnv(testRootDir, testEnv, GinkgoWriter)
+	k8sCfg, err = testEnv.Start()
 	Expect(err).ToNot(HaveOccurred())
 	Expect(k8sCfg).ToNot(BeNil())
 
@@ -77,22 +75,20 @@ var _ = BeforeSuite(func(done Done) {
 	Expect(k8sCli).ToNot(BeNil())
 
 	installOctopus()
-
-	close(done)
 }, 600)
 
 var _ = AfterSuite(func(done Done) {
+	defer close(done)
+
 	uninstallOctopus()
 
 	By("tearing down test environment")
-	var err = framework.StopEnv(testRootDir, testEnv, GinkgoWriter)
+	var err = testEnv.Stop()
 	Expect(err).ToNot(HaveOccurred())
 
 	if testCtxCancel != nil {
 		testCtxCancel()
 	}
-
-	close(done)
 }, 600)
 
 func init() {

adaptors/dummy/test/e2e/usability/usability_test.go

@@ -1,4 +1,4 @@
-package usability_test
+package usability
 
 import (
 	"fmt"
@@ -14,8 +14,7 @@ import (
 
 	apiv1alpha1 "github.com/rancher/octopus/adaptors/dummy/api/v1alpha1"
 	edgev1alpha1 "github.com/rancher/octopus/api/v1alpha1"
-	"github.com/rancher/octopus/test/framework"
-	. "github.com/rancher/octopus/test/framework/ginkgo"
+	. "github.com/rancher/octopus/test/framework/envtest/dsl"
 	"github.com/rancher/octopus/test/util/content"
 	"github.com/rancher/octopus/test/util/exec"
 	"github.com/rancher/octopus/test/util/node"
@@ -492,7 +491,7 @@ func deleteCorrespondingNode() {
 }
 
 func redeployCorrespondingNode() {
-	Expect(framework.GetCluster().AddWorker(testRootDir, GinkgoWriter, testDeviceLink.Spec.Adaptor.Node)).Should(Succeed())
+	Expect(testEnv.AddWorker(testDeviceLink.Spec.Adaptor.Node)).Should(Succeed())
 }
 
 func deleteDummyDeviceModel() {

adaptors/dummy/test/integration/adaptor/suite_test.go

@@ -6,7 +6,8 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
+
+	"github.com/rancher/octopus/test/framework/envtest/printer"
 )
 
 var (
@@ -25,12 +26,14 @@ func TestAdaptor(t *testing.T) {
 }
 
 var _ = BeforeSuite(func(done Done) {
-	testCtx, testCtxCancel = context.WithCancel(context.Background())
+	defer close(done)
 
-	close(done)
+	testCtx, testCtxCancel = context.WithCancel(context.Background())
 }, 600)
 
-var _ = AfterSuite(func() {
+var _ = AfterSuite(func(done Done) {
+	defer close(done)
+
 	if testCtxCancel != nil {
 		testCtxCancel()
 	}

adaptors/dummy/test/integration/physical/mqtt_test.go

@@ -19,14 +19,10 @@ import (
 	"github.com/rancher/octopus/pkg/util/converter"
 	"github.com/rancher/octopus/pkg/util/log/zap"
 	"github.com/rancher/octopus/pkg/util/object"
-	"github.com/rancher/octopus/test/util/testdata"
 )
 
 var _ = Describe("verify MQTT extension", func() {
 	var (
-		testUnencryptedBrokerAddress              = "tcp://test.mosquitto.org:1883"
-		testEncryptedAndClientAuthedBrokerAddress = "tcps://test.mosquitto.org:8884"
-
 		testNamespace = "default"
 
 		err error
@@ -70,7 +66,7 @@ var _ = Describe("verify MQTT extension", func() {
 			*/
 
 			var testSubscriptionStream *mqtttest.SubscriptionStream
-			testSubscriptionStream, err = mqtttest.NewSubscriptionStream(testUnencryptedBrokerAddress, fmt.Sprintf("cattle.io/octopus/%s", testInstanceNamespacedName), 0)
+			testSubscriptionStream, err = mqtttest.NewSubscriptionStream(testMQTTBrokerAddress, fmt.Sprintf("cattle.io/octopus/%s", testInstanceNamespacedName), 0)
 			Expect(err).ToNot(HaveOccurred())
 			defer testSubscriptionStream.Close()
 
@@ -85,7 +81,7 @@ var _ = Describe("verify MQTT extension", func() {
 				Extension: &v1alpha1.DummyDeviceExtension{
 					MQTT: &mqttapi.MQTTOptions{
 						Client: mqttapi.MQTTClientOptions{
-							Server: testUnencryptedBrokerAddress,
+							Server: testMQTTBrokerAddress,
 						},
 						Message: mqttapi.MQTTMessageOptions{
 							// dynamic topic with namespaced name
@@ -115,61 +111,6 @@ var _ = Describe("verify MQTT extension", func() {
 			Expect(err).ToNot(HaveOccurred())
 		})
 
-		It("should publish the status changes via SSL/TLS", func() {
-
-			/*
-				since test.mosquitto.org provides a public CA, we can download it and generate the client cert key pair
-				via https://test.mosquitto.org/ssl/index.php.
-			*/
-
-			var testSubscriptionStream *mqtttest.SubscriptionStream
-			// subscribe via unencrypted endpoint
-			testSubscriptionStream, err = mqtttest.NewSubscriptionStream(testUnencryptedBrokerAddress, fmt.Sprintf("cattle.io/octopus/%s", testInstance.OwnerReferences[0].UID), 0)
-			Expect(err).ToNot(HaveOccurred())
-			defer testSubscriptionStream.Close()
-
-			var testDevice = physical.NewSpecialDevice(
-				log.WithValues("device", testInstanceNamespacedName),
-				testInstance.ObjectMeta,
-				nil,
-			)
-			defer testDevice.Shutdown()
-
-			testInstance.Spec = v1alpha1.DummySpecialDeviceSpec{
-				Extension: &v1alpha1.DummyDeviceExtension{
-					MQTT: &mqttapi.MQTTOptions{
-						Client: mqttapi.MQTTClientOptions{
-							// publish via encrypted endpoint
-							Server: testEncryptedAndClientAuthedBrokerAddress,
-							TLSConfig: &mqttapi.MQTTClientTLS{
-								CAFilePEM:   testdata.MustLoadString("mosquitto.org.crt", GinkgoT()),
-								CertFilePEM: testdata.MustLoadString("client.crt", GinkgoT()),
-								KeyFilePEM:  testdata.MustLoadString("client-key.pem", GinkgoT()),
-								ServerName:  "test.mosquitto.org",
-							},
-						},
-						Message: mqttapi.MQTTMessageOptions{
-							// dynamic topic with uid
-							Topic: "cattle.io/octopus/:uid",
-						},
-					},
-				},
-				Protocol: v1alpha1.DummySpecialDeviceProtocol{
-					Location: "living-room",
-				},
-				On:   true,
-				Gear: v1alpha1.DummySpecialDeviceGearFast,
-			}
-			err = testDevice.Configure(nil, testInstance)
-			Expect(err).ToNot(HaveOccurred())
-
-			err = testSubscriptionStream.Intercept(15*time.Second, func(actual *packet.Message) bool {
-				GinkgoT().Logf("topic: %s, qos: %d, retain: %v, payload: %s", actual.Topic, actual.QOS, actual.Retain, converter.UnsafeBytesToString(actual.Payload))
-				return true
-			})
-			Expect(err).ToNot(HaveOccurred())
-		})
-
 		It("should work if modified extension settings", func() {
 
 			/*
@@ -181,7 +122,7 @@ var _ = Describe("verify MQTT extension", func() {
 			*/
 
 			var testSubscriptionStream *mqtttest.SubscriptionStream
-			testSubscriptionStream, err = mqtttest.NewSubscriptionStream(testUnencryptedBrokerAddress, fmt.Sprintf("cattle.io/octopus/%s", testInstanceNamespacedName), 0)
+			testSubscriptionStream, err = mqtttest.NewSubscriptionStream(testMQTTBrokerAddress, fmt.Sprintf("cattle.io/octopus/%s", testInstanceNamespacedName), 0)
 			Expect(err).ToNot(HaveOccurred())
 
 			var testDevice = physical.NewSpecialDevice(
@@ -195,7 +136,7 @@ var _ = Describe("verify MQTT extension", func() {
 				Extension: &v1alpha1.DummyDeviceExtension{
 					MQTT: &mqttapi.MQTTOptions{
 						Client: mqttapi.MQTTClientOptions{
-							Server: testUnencryptedBrokerAddress,
+							Server: testMQTTBrokerAddress,
 						},
 						Message: mqttapi.MQTTMessageOptions{
 							Topic: "cattle.io/octopus/:namespace/:name",
@@ -222,14 +163,14 @@ var _ = Describe("verify MQTT extension", func() {
 				change to static topic
 			*/
 
-			testSubscriptionStream, err = mqtttest.NewSubscriptionStream(testUnencryptedBrokerAddress, "cattle.io/octopus/default/test3/static", 0)
+			testSubscriptionStream, err = mqtttest.NewSubscriptionStream(testMQTTBrokerAddress, "cattle.io/octopus/default/test3/static", 0)
 			Expect(err).ToNot(HaveOccurred())
 
 			testInstance.Spec = v1alpha1.DummySpecialDeviceSpec{
 				Extension: &v1alpha1.DummyDeviceExtension{
 					MQTT: &mqttapi.MQTTOptions{
 						Client: mqttapi.MQTTClientOptions{
-							Server: testUnencryptedBrokerAddress,
+							Server: testMQTTBrokerAddress,
 						},
 						Message: mqttapi.MQTTMessageOptions{
 							Topic: "cattle.io/octopus/default/test3/static",
@@ -287,7 +228,7 @@ var _ = Describe("verify MQTT extension", func() {
 			*/
 
 			var testSubscriptionStream *mqtttest.SubscriptionStream
-			testSubscriptionStream, err = mqtttest.NewSubscriptionStream(testUnencryptedBrokerAddress, fmt.Sprintf("cattle.io/octopus/%s", testInstanceNamespacedName), 0)
+			testSubscriptionStream, err = mqtttest.NewSubscriptionStream(testMQTTBrokerAddress, fmt.Sprintf("cattle.io/octopus/%s", testInstanceNamespacedName), 0)
 			Expect(err).ToNot(HaveOccurred())
 			defer testSubscriptionStream.Close()
 
@@ -302,7 +243,7 @@ var _ = Describe("verify MQTT extension", func() {
 				Extension: &v1alpha1.DummyDeviceExtension{
 					MQTT: &mqttapi.MQTTOptions{
 						Client: mqttapi.MQTTClientOptions{
-							Server: testUnencryptedBrokerAddress,
+							Server: testMQTTBrokerAddress,
 						},
 						Message: mqttapi.MQTTMessageOptions{
 							// dynamic topic