[WIP] Add initial maintenance mode test

go.mod

@@ -38,7 +38,7 @@ require (
 	github.com/lf-edge/adam v0.0.0-20201209042112-7d1fa049e66b
 	github.com/lf-edge/eden/eserver v0.0.0-20201210161141-8551a3b0751b
 	github.com/lf-edge/edge-containers v0.0.0-20201111200732-5491ea93dbe4
-	github.com/lf-edge/eve/api/go v0.0.0-20210211140602-4ce4d3187bb9
+	github.com/lf-edge/eve/api/go v0.0.0-20210219053903-a52e3a1e37f9
 	github.com/lunixbochs/struc v0.0.0-20200707160740-784aaebc1d40 // indirect
 	github.com/mcuadros/go-lookup v0.0.0-20200831155250-80f87a4fa5ee
 	github.com/moby/term v0.0.0-20201110203204-bea5bbe245bf

go.sum

@@ -497,6 +497,8 @@ github.com/lf-edge/eve/api/go v0.0.0-20210128112727-417721637d65 h1:9qDT/QDJU/wO
 github.com/lf-edge/eve/api/go v0.0.0-20210128112727-417721637d65/go.mod h1:DuAv0PyTTzmd3n25iHJ/Hx2SrbON4hf3I0oXfnUH4d8=
 github.com/lf-edge/eve/api/go v0.0.0-20210211140602-4ce4d3187bb9 h1:3hGI2gaC9W24E8y37j6Cj7fKKHDxJ1SqL690+MsEC+A=
 github.com/lf-edge/eve/api/go v0.0.0-20210211140602-4ce4d3187bb9/go.mod h1:DuAv0PyTTzmd3n25iHJ/Hx2SrbON4hf3I0oXfnUH4d8=
+github.com/lf-edge/eve/api/go v0.0.0-20210219053903-a52e3a1e37f9 h1:81+YS5FLSbmXG3KpoTa835HDN40Qqa/90dcnYmB44j0=
+github.com/lf-edge/eve/api/go v0.0.0-20210219053903-a52e3a1e37f9/go.mod h1:DuAv0PyTTzmd3n25iHJ/Hx2SrbON4hf3I0oXfnUH4d8=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
 github.com/lunixbochs/struc v0.0.0-20200707160740-784aaebc1d40 h1:EnfXoSqDfSNJv0VBNqY/88RNnhSGYkrHaO0mmFGbVsc=

tests/maintenance_mode/Makefile

@@ -0,0 +1,77 @@
+DEBUG ?= "debug"
+
+# HOSTARCH is the host architecture
+# ARCH is the target architecture
+# we need to keep track of them separately
+HOSTARCH ?= $(shell uname -m)
+HOSTOS ?= $(shell uname -s | tr A-Z a-z)
+
+# canonicalized names for host architecture
+override HOSTARCH := $(subst aarch64,arm64,$(subst x86_64,amd64,$(HOSTARCH)))
+
+# unless otherwise set, I am building for my own architecture, i.e. not cross-compiling
+# and for my OS
+ARCH ?= $(HOSTARCH)
+OS ?= $(HOSTOS)
+
+# canonicalized names for target architecture
+override ARCH := $(subst aarch64,arm64,$(subst x86_64,amd64,$(ARCH)))
+
+WORKDIR ?= $(CURDIR)/../../dist
+TESTDIR := tests/$(shell basename $(CURDIR))
+BINDIR := $(WORKDIR)/bin
+DATADIR := $(WORKDIR)/$(TESTDIR)/
+BIN := eden
+LOCALBIN := $(BINDIR)/$(BIN)-$(OS)-$(ARCH)
+TESTNAME := eden.maintenance
+TESTBIN := $(TESTNAME).test
+TESTSCN := $(TESTNAME).tests.txt
+LOCALTESTBIN := $(TESTBIN)-$(OS)-$(ARCH)
+LINKDIR := ../../tests/maintenance
+
+.DEFAULT_GOAL := help
+
+clean:
+	rm -rf $(LOCALTESTBIN) $(BINDIR)/$(TESTBIN) $(WORKDIR)/$(TESTSCN) $(CURDIR)/$(TESTBIN) $(BINDIR)/$(TESTBIN)
+
+$(BINDIR):
+	mkdir -p $@
+$(DATADIR):
+	mkdir -p $@
+
+test_maintenance:
+	go test maintenance_test.go common.go -v -count=1 -timeout 3000s
+
+test:
+	$(LOCALBIN) test $(CURDIR) -v $(DEBUG)
+
+build: setup
+
+testbin: $(TESTBIN)
+$(LOCALTESTBIN): $(BINDIR) *.go
+	CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go test -c -ldflags "-s -w" -o $@ *.go
+
+$(TESTBIN): $(LOCALTESTBIN)
+	ln -sf $(LOCALTESTBIN) $(CURDIR)/$(TESTBIN)
+
+setup: testbin $(BINDIR) $(DATADIR)
+	cp -a $(LOCALTESTBIN) $(CURDIR)/$(TESTBIN) $(DATADIR)
+	cp -a *.yml $(TESTSCN) $(DATADIR)
+	ln -sf ../$(TESTDIR)/$(TESTBIN) $(BINDIR)
+
+.PHONY: test build setup clean all testbin
+
+help:
+	@echo "EDEN is the harness for testing EVE and ADAM"
+	@echo
+	@echo "This Makefile automates commons tasks of EDEN testing"
+	@echo
+	@echo "Commonly used maintenance and development targets:"
+	@echo "   build         build test-binary (OS and ARCH options supported, for ex. OS=linux ARCH=arm64)"
+	@echo "   setup         setup of test environment"
+	@echo "   test          run tests"
+	@echo "   clean         cleanup of test harness"
+	@echo
+	@echo "You need install requirements for EVE (look at https://github.com/lf-edge/eve#install-dependencies)."
+	@echo "You need access to docker socket and installed qemu packages."
+

tests/maintenance_mode/README.md

@@ -0,0 +1,11 @@
+# Test manually setting device in maintenance mode
+
+The syntax for calling this detector is:
+
+```console
+eden.mainenance.test [options]
+```
+
+Where specific "options":
+
+* -timewait -- Timewait for waiting (1 min by default).

tests/maintenance_mode/eden-config.yml

@@ -0,0 +1,7 @@
+---
+eden:
+    # test binary
+    test-bin: "eden.maintenance.test"
+
+    # test scenario
+    test-scenario: "eden.maintenance.tests.txt"

tests/maintenance_mode/eden.maintenance.test

@@ -0,0 +1 @@
+eden.maintenance.test-linux-amd64

tests/maintenance_mode/eden.maintenance.tests.txt

@@ -0,0 +1 @@
+eden.maintenance.test

tests/maintenance_mode/eden.reboot.test

@@ -0,0 +1 @@
+eden.reboot.test-linux-amd64

tests/maintenance_mode/maintenance_test.go

@@ -0,0 +1,237 @@
+package maintenance
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/lf-edge/eve/api/go/info"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/lf-edge/eden/pkg/device"
+	"github.com/lf-edge/eden/pkg/projects"
+	"github.com/lf-edge/eden/pkg/tests"
+	"github.com/lf-edge/eden/pkg/utils"
+)
+
+// This context holds all the configuration items in the same
+// way that Eden context works: the commands line options override
+// YAML settings. In addition to that, context is polymorphic in
+// a sense that it abstracts away a particular controller (currently
+// Adam and Zedcloud are supported)
+/*
+tc *TestContext // TestContext is at least {
+                //    controller *Controller
+                //    project *Project
+                //    nodes []EdgeNode
+                //    ...
+                // }
+*/
+
+var (
+	// XXX lower default?
+	timewait = flag.Duration("timewait", 3*time.Minute, "Timewait for waiting")
+
+	tc *projects.TestContext
+
+	configRebootCounter uint32
+)
+
+func checkRebootCounterMatch() projects.ProcInfoFunc {
+	return func(im *info.ZInfoMsg) error {
+		if im.GetZtype() != info.ZInfoTypes_ZiDevice {
+			return nil
+		}
+		statusRebootCounter := im.GetDinfo().RebootConfigCounter
+		if statusRebootCounter == configRebootCounter {
+			return fmt.Errorf("reached RebootCounter: %d",
+				configRebootCounter)
+		}
+		return nil
+	}
+}
+
+func checkMaintenanceMode() projects.ProcInfoFunc {
+	return func(im *info.ZInfoMsg) error {
+		if im.GetZtype() != info.ZInfoTypes_ZiDevice {
+			return nil
+		}
+		log.Infof("checkMaintenceMode: got %+v", im.GetDinfo())
+		maintenanceMode := im.GetDinfo().MaintenanceMode
+		maintenanceModeReason := im.GetDinfo().MaintenanceModeReason
+		log.Infof("checkMaintenanceMode: maintenanceMode %t maintenanceModeReason %s",
+			maintenanceMode, maintenanceModeReason)
+		if maintenanceMode {
+			return fmt.Errorf("entered mainteance mode with reason %s",
+				maintenanceModeReason)
+		}
+		return nil
+	}
+}
+
+func checkNoMaintenanceMode() projects.ProcInfoFunc {
+	return func(im *info.ZInfoMsg) error {
+		if im.GetZtype() != info.ZInfoTypes_ZiDevice {
+			return nil
+		}
+		log.Infof("checkNoMaintenceMode: got %+v", im.GetDinfo())
+		maintenanceMode := im.GetDinfo().MaintenanceMode
+		maintenanceModeReason := im.GetDinfo().MaintenanceModeReason
+		log.Infof("checkNoMaintenanceMode: maintenanceMode %t maintenanceModeReason %s",
+			maintenanceMode, maintenanceModeReason)
+		if !maintenanceMode {
+			return fmt.Errorf("done with mainteance mode (reason %s)",
+				maintenanceModeReason)
+		}
+		return nil
+	}
+}
+
+// TestMain is used to provide setup and teardown for the rest of the
+// tests. As part of setup we make sure that context has a slice of
+// EVE instances that we can operate on. For any action, if the instance
+// is not specified explicitly it is assumed to be the first one in the slice
+func TestMain(m *testing.M) {
+	fmt.Println("Maintenance Test")
+
+	tests.TestArgsParse()
+
+	tc = projects.NewTestContext()
+
+	projectName := fmt.Sprintf("%s_%s", "TestMaintenance", time.Now())
+
+	// Registering our own project namespace with controller for easy cleanup
+	tc.InitProject(projectName)
+
+	// Create representation of EVE instances (based on the names
+	// or UUIDs that were passed in) in the context. This is the first place
+	// where we're using zcli-like API:
+	for _, node := range tc.GetNodeDescriptions() {
+		edgeNode := node.GetEdgeNode(tc)
+		if edgeNode == nil {
+			// Couldn't find existing edgeNode record in the controller.
+			// Need to create it from scratch now:
+			// this is modeled after: zcli edge-node create <name>
+			// --project=<project> --model=<model> [--title=<title>]
+			// ([--edge-node-certificate=<certificate>] |
+			// [--onboarding-certificate=<certificate>] |
+			// [(--onboarding-key=<key> --serial=<serial-number>)])
+			// [--network=<network>...]
+			//
+			// XXX: not sure if struct (giving us optional fields) would be better
+			edgeNode = tc.NewEdgeNode(tc.WithNodeDescription(node), tc.WithCurrentProject())
+		} else {
+			// make sure to move EdgeNode to the project we created, again
+			// this is modeled after zcli edge-node update <name> [--title=<title>]
+			// [--lisp-mode=experimental|default] [--project=<project>]
+			// [--clear-onboarding-certs] [--config=<key:value>...] [--network=<network>...]
+			edgeNode.SetProject(projectName)
+		}
+
+		tc.ConfigSync(edgeNode)
+
+		// finally we need to make sure that the edgeNode is in a state that we need
+		// it to be, before the test can run -- this could be multiple checks on its
+		// status, but for example:
+		if edgeNode.GetState() == device.NotOnboarded {
+			log.Fatal("Node is not onboarded now")
+		}
+
+		// this is a good node -- lets add it to the test context
+		tc.AddNode(edgeNode)
+	}
+
+	tc.StartTrackingState(false)
+
+	// we now have a situation where TestContext has enough EVE nodes known
+	// for the rest of the tests to run. So run them:
+	res := m.Run()
+
+	// Finally, we need to cleanup whatever objects may be in in the project we created
+	// and then we can exit
+	os.Exit(res)
+}
+
+func TestMaintenance(t *testing.T) {
+	// note that GetEdgeNode() without any argument is
+	// equivalent to the default (first one). Otherwise
+	// one can specify a name GetEdgeNode("foo")
+	edgeNode := tc.GetEdgeNode(tc.WithTest(t))
+
+	t.Log(utils.AddTimestamp(fmt.Sprintf("Wait for state of %s", edgeNode.GetID())))
+	tc.WaitForState(edgeNode, int(timewait.Seconds()))
+
+	t.Log(utils.AddTimestamp(fmt.Sprintf("timewait: %s", timewait)))
+
+	statusRebootCounter := tc.GetState(edgeNode).GetDinfo().RebootConfigCounter
+	configRebootCounter, _ = edgeNode.GetRebootCounter()
+	if statusRebootCounter != configRebootCounter {
+		t.Logf(utils.AddTimestamp(fmt.Sprintf("Wait for match: statusRebootCounter: %d, configRebootCounter %d",
+			statusRebootCounter, configRebootCounter)))
+
+		// Wait for match
+		tc.AddProcInfo(edgeNode, checkRebootCounterMatch())
+		tc.WaitForProc(int(timewait.Seconds()))
+	}
+	statusRebootCounter = tc.GetState(edgeNode).GetDinfo().RebootConfigCounter
+	configRebootCounter, _ = edgeNode.GetRebootCounter()
+	t.Logf(utils.AddTimestamp(fmt.Sprintf("Wait done: statusRebootCounter: %d, configRebootCounter %d",
+		statusRebootCounter, configRebootCounter)))
+	t.Logf(utils.AddTimestamp(fmt.Sprintf("lastRestartCounter: %d",
+		tc.GetState(edgeNode).GetDinfo().RestartCounter)))
+
+	// XXX part of sanitize? Otherwise we need to clear from the config as well.
+	maintenanceMode := tc.GetState(edgeNode).GetDinfo().MaintenanceMode
+	maintenanceModeReason := tc.GetState(edgeNode).GetDinfo().MaintenanceModeReason
+	t.Logf(utils.AddTimestamp(fmt.Sprintf("maintenanceMode %t maintenanceModeReason %s",
+		maintenanceMode, maintenanceModeReason)))
+	if maintenanceMode {
+		t.Logf(utils.AddTimestamp(fmt.Sprintf("Wait for maintenance mode clear for %s",
+			edgeNode.GetID())))
+
+		tc.AddProcInfo(edgeNode, checkNoMaintenanceMode())
+
+		tc.WaitForProc(int(timewait.Seconds()))
+	}
+	maintenanceMode = tc.GetState(edgeNode).GetDinfo().MaintenanceMode
+	maintenanceModeReason = tc.GetState(edgeNode).GetDinfo().MaintenanceModeReason
+	t.Logf(utils.AddTimestamp(fmt.Sprintf("Wait clear done: maintenanceMode %t maintenanceModeReason %s",
+		maintenanceMode, maintenanceModeReason)))
+
+	// send command
+	edgeNode.SetConfigItem("maintenance.mode", "enabled")
+	tc.ConfigSync(edgeNode)
+	t.Logf(utils.AddTimestamp(fmt.Sprintf("Wait for maintenance mode for %s", edgeNode.GetID())))
+
+	tc.AddProcInfo(edgeNode, checkMaintenanceMode())
+
+	tc.WaitForProc(int(timewait.Seconds()))
+
+	maintenanceMode = tc.GetState(edgeNode).GetDinfo().MaintenanceMode
+	maintenanceModeReason = tc.GetState(edgeNode).GetDinfo().MaintenanceModeReason
+	t.Logf(utils.AddTimestamp(fmt.Sprintf("Wait done: maintenanceMode %t maintenanceModeReason %s",
+		maintenanceMode, maintenanceModeReason)))
+
+	// XXX check that changes do not take effect while in maintenance mode
+
+	// XXX check that reboot does take effect while in maintenance mode
+
+	// XXX restore or reboot? Manual means restore is only choice.
+	// XXX for testability maybe we should add a triggered "local" maintenance mode?
+	// send command
+	edgeNode.SetConfigItem("maintenance.mode", "disabled")
+	tc.ConfigSync(edgeNode)
+	t.Logf(utils.AddTimestamp(fmt.Sprintf("Wait for maintenance mode clear for %s",
+		edgeNode.GetID())))
+
+	tc.AddProcInfo(edgeNode, checkNoMaintenanceMode())
+
+	tc.WaitForProc(int(timewait.Seconds()))
+
+	maintenanceMode = tc.GetState(edgeNode).GetDinfo().MaintenanceMode
+	maintenanceModeReason = tc.GetState(edgeNode).GetDinfo().MaintenanceModeReason
+	t.Logf(utils.AddTimestamp(fmt.Sprintf("Wait clear done: maintenanceMode %t maintenanceModeReason %s",
+		maintenanceMode, maintenanceModeReason)))
+}