test(e2e): improve universal app run (#13033)

## Motivation

It was very hard to have a complete and readable state of things in the
test report because we were executing one of ssh cmds in different
places.
We now keep a persistent ssh session for the whole time of the app which
makes things easier to track and the output easier to consume.

## Implementation information

the ssh package was completely redesigned to keep an ssh session rather
than forking to execute `ssh` commands. This makes things easier to
follow, easier to write lengthy scripts without caring about escaping
things correctly for bash.
Once this was done we could start a CP or run a DP with a single cmdLine
instead of a list of complex calls.

This makes the report a lot easier as it avoids a lot of noisy stuff.

Other minor changes:
- Stop doing retries inside the framework. Either we should retry
because it's setup or simply in an eventually. Also stop using the retry
from terratest as it's very verbose.
- ignore for dockerfiles were including things that were not necessary,
this made the building of images longer.
- Once this change was done we could run into clock skews with tokens.
So moved the token issuance date a few seconds in the past

## Supporting documentation

Fix #13123

---------

Signed-off-by: Charly Molter <[email protected]>

Author: lahabana

go.mod

@@ -214,7 +214,7 @@ require (
 	github.com/moby/patternmatcher v0.6.0 // indirect
 	github.com/moby/spdystream v0.5.0 // indirect
 	github.com/moby/sys/sequential v0.5.0 // indirect
-	github.com/moby/sys/user v0.1.0 // indirect
+	github.com/moby/sys/user v0.3.0 // indirect
 	github.com/moby/sys/userns v0.1.0 // indirect
 	github.com/moby/term v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -252,10 +252,9 @@ require (
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 // indirect
 	go.opentelemetry.io/otel/metric v1.35.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
-	golang.org/x/crypto v0.36.0 // indirect
+	golang.org/x/crypto v0.36.0
 	golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c
 	golang.org/x/mod v0.24.0 // indirect
 	golang.org/x/oauth2 v0.27.0 // indirect
@@ -277,6 +276,7 @@ require (
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/google/cel-go v0.22.0 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 // indirect
 	k8s.io/apiserver v0.32.3 // indirect
 	k8s.io/component-base v0.32.3 // indirect
 )

go.sum

@@ -368,8 +368,8 @@ github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9Kou
 github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4=
 github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=
 github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo=
-github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg=
-github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU=
+github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo=
+github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
 github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
 github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
 github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=

pkg/core/tokens/issuer.go

@@ -37,12 +37,12 @@ func (j *jwtTokenIssuer) Generate(ctx context.Context, claims Claims, validFor t
 		return "", err
 	}
 
-	now := core.Now()
+	now := core.Now().Add(-5 * time.Second) // todo(jakubdyszkiewicz) parametrize via config and go through all clock skews in the project
 	claims.SetRegisteredClaims(jwt.RegisteredClaims{
 		ID:        core.NewUUID(),
 		IssuedAt:  jwt.NewNumericDate(now),
-		NotBefore: jwt.NewNumericDate(now.Add(time.Minute * -5)), // todo(jakubdyszkiewicz) parametrize via config and go through all clock skews in the project
-		ExpiresAt: jwt.NewNumericDate(now.Add(validFor)),
+		NotBefore: jwt.NewNumericDate(now),
+		ExpiresAt: jwt.NewNumericDate(now.Add(validFor).Add(5 * time.Second)),
 	})
 
 	token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims)

pkg/util/files/files.go

@@ -1,6 +1,8 @@
 package files
 
 import (
+	"fmt"
+	"io"
 	"io/fs"
 	"os"
 	"path/filepath"
@@ -49,3 +51,18 @@ func ToValidUnixFilename(input ...string) string {
 
 	return sanitized
 }
+
+// CopyFile copies the file content to a new path.
+// This should be used with caution as it was first written for tests
+func CopyFile(src, dst string) error {
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return fmt.Errorf("failed to open src file:%s %w", src, err)
+	}
+	dstFile, err := os.Create(dst)
+	if err != nil {
+		return fmt.Errorf("failed to create dest file:%s %w", dst, err)
+	}
+	_, err = io.Copy(dstFile, srcFile)
+	return err
+}

test/dockerfiles/universal.dockerignore

@@ -11,12 +11,6 @@
 !build/artifacts-linux-arm64/kumactl/kumactl
 !build/artifacts-linux-arm64/coredns/coredns
 !build/artifacts-linux-arm64/test-server/test-server
-!build/artifacts-darwin-arm64/kuma-cp/kuma-cp
-!build/artifacts-darwin-arm64/kuma-dp/kuma-dp
-!build/artifacts-darwin-arm64/envoy/*
-!build/artifacts-darwin-arm64/kumactl/kumactl
-!build/artifacts-darwin-arm64/coredns/coredns
-!build/artifacts-darwin-arm64/test-server/test-server
 !pkg/config/app/kuma-cp/kuma-cp.defaults.yaml
 !tools/releases/templates/LICENSE
 !tools/releases/templates/NOTICE

test/e2e_env/multizone/externalservices/externalservices_multizone_universal.go

@@ -155,42 +155,42 @@ routing:
 		Expect(err).ToNot(HaveOccurred())
 
 		Eventually(func(g Gomega) {
-			stdout, _, err := client.CollectResponse(
+			stdout, stderr, err := client.CollectResponse(
 				zone1, "demo-client", "external-service-1.mesh",
 				client.WithVerbose(),
 			)
 			g.Expect(err).ToNot(HaveOccurred())
-			g.Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK"))
+			g.Expect(stderr).To(ContainSubstring("HTTP/1.1 200 OK"))
 			g.Expect(stdout).ToNot(ContainSubstring("HTTPS"))
 		}, "1m", "3s").Should(Succeed())
 
 		Eventually(func(g Gomega) {
-			stdout, _, err := client.CollectResponse(
+			stdout, stderr, err := client.CollectResponse(
 				zone1, "demo-client", "kuma-es-4_es-http:80",
 				client.WithVerbose(),
 			)
 			g.Expect(err).ToNot(HaveOccurred())
-			g.Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK"))
+			g.Expect(stderr).To(ContainSubstring("HTTP/1.1 200 OK"))
 			g.Expect(stdout).ToNot(ContainSubstring("HTTPS"))
 		}, "1m", "3s").Should(Succeed())
 
 		Eventually(func(g Gomega) {
-			stdout, _, err := client.CollectResponse(
+			stdout, stderr, err := client.CollectResponse(
 				zone2, "demo-client", "external-service-1.mesh",
 				client.WithVerbose(),
 			)
 			g.Expect(err).ToNot(HaveOccurred())
-			g.Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK"))
+			g.Expect(stderr).To(ContainSubstring("HTTP/1.1 200 OK"))
 			g.Expect(stdout).ToNot(ContainSubstring("HTTPS"))
 		}, "1m", "3s").Should(Succeed())
 
 		Eventually(func(g Gomega) {
-			stdout, _, err := client.CollectResponse(
+			stdout, stderr, err := client.CollectResponse(
 				zone2, "demo-client", "kuma-es-4_es-http:80",
 				client.WithVerbose(),
 			)
 			g.Expect(err).ToNot(HaveOccurred())
-			g.Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK"))
+			g.Expect(stderr).To(ContainSubstring("HTTP/1.1 200 OK"))
 			g.Expect(stdout).ToNot(ContainSubstring("HTTPS"))
 		}, "1m", "3s").Should(Succeed())
 	})
@@ -219,22 +219,22 @@ routing:
 
 		// then accessing the secured external service succeeds
 		Eventually(func(g Gomega) {
-			stdout, _, err := client.CollectResponse(
+			stdout, stderr, err := client.CollectResponse(
 				zone1, "demo-client", "http://kuma-es-4_es-https:443",
 				client.WithVerbose(),
 			)
 			g.Expect(err).ToNot(HaveOccurred())
-			g.Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK"))
+			g.Expect(stderr).To(ContainSubstring("HTTP/1.1 200 OK"))
 			g.Expect(stdout).To(ContainSubstring("es-https"))
 		}, "1m", "1s").Should(Succeed())
 
 		Eventually(func(g Gomega) {
-			stdout, _, err := client.CollectResponse(
+			stdout, stderr, err := client.CollectResponse(
 				zone2, "demo-client", "http://kuma-es-4_es-https:443",
 				client.WithVerbose(),
 			)
 			g.Expect(err).ToNot(HaveOccurred())
-			g.Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK"))
+			g.Expect(stderr).To(ContainSubstring("HTTP/1.1 200 OK"))
 			g.Expect(stdout).To(ContainSubstring("es-https"))
 		}, "1m", "3s").Should(Succeed())
 	})

test/e2e_env/multizone/ownership/ownership.go

@@ -62,13 +62,11 @@ func MultizoneUniversal() {
 	}
 
 	killKumaDP := func(appname string) {
-		_, _, err := zoneUniversal.Exec("", "", appname, "pkill", "-9", "envoy")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(zoneUniversal.(*UniversalCluster).Kill(appname, "kuma-dp")).To(Succeed())
 	}
 
 	killZone := func() {
-		_, _, err := zoneUniversal.Exec("", "", AppModeCP, "pkill", "-9", "kuma-cp")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(zoneUniversal.(*UniversalCluster).Kill(AppModeCP, "kuma-cp run")).To(Succeed())
 		Eventually(func() (string, error) {
 			return global.GetKumactlOptions().RunKumactlAndGetOutput("inspect", "zones")
 		}, "30s", "1s").Should(ContainSubstring("Offline"))

test/e2e_env/multizone/resilience/resilience_multizone_universal.go

@@ -65,8 +65,7 @@ func ResilienceMultizoneUniversal() {
 			return nil
 		}, "30s", "1s").ShouldNot(HaveOccurred())
 
-		_, _, err := zone1.Exec("", "", AppModeCP, "pkill", "-9", "kuma-cp")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(zone1.(*UniversalCluster).Kill(AppModeCP, "kuma-cp run")).To(Succeed())
 
 		Eventually(func() error {
 			output, err := global.GetKumactlOptions().RunKumactlAndGetOutput("inspect", "zones")

test/e2e_env/multizone/resilience/resilience_multizone_universal_postgres.go

@@ -91,12 +91,10 @@ func ResilienceMultizoneUniversalPostgres() {
 		Expect(kumaCP).ToNot(BeNil())
 
 		// when global is killed
-		_, _, err := global.Exec("", "", AppModeCP, "pkill", "-9", "kuma-cp")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(global.(*UniversalCluster).Kill(AppModeCP, "kuma-cp run")).To(Succeed())
 
 		// and zone is killed while global is down
-		_, _, err = zoneUniversal.Exec("", "", AppModeCP, "pkill", "-9", "kuma-cp")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(zoneUniversal.(*UniversalCluster).Kill(AppModeCP, "kuma-cp run")).To(Succeed())
 
 		// and global is restarted
 		Expect(kumaCP.ReStart()).Should(Succeed())
@@ -122,8 +120,7 @@ func ResilienceMultizoneUniversalPostgres() {
 		Expect(kumaCP).ToNot(BeNil())
 
 		// when global is killed
-		_, _, err := global.Exec("", "", AppModeCP, "pkill", "-9", "kuma-cp")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(global.(*UniversalCluster).Kill(AppModeCP, "kuma-cp run")).To(Succeed())
 
 		// and global is restarted
 		Expect(kumaCP.ReStart()).Should(Succeed())
@@ -133,8 +130,7 @@ func ResilienceMultizoneUniversalPostgres() {
 		time.Sleep(10 * time.Second) // ZoneInsightFlushInterval
 
 		// and zone is killed
-		_, _, err = zoneUniversal.Exec("", "", AppModeCP, "pkill", "-9", "kuma-cp")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(zoneUniversal.(*UniversalCluster).Kill(AppModeCP, "kuma-cp run")).To(Succeed())
 
 		// then zone is offline
 		Eventually(func() (string, error) {
@@ -156,12 +152,10 @@ func ResilienceMultizoneUniversalPostgres() {
 		Expect(kumaCP).ToNot(BeNil())
 
 		// when Zone CP is killed
-		_, _, err := zoneUniversal.Exec("", "", AppModeCP, "pkill", "-9", "kuma-cp")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(zoneUniversal.(*UniversalCluster).Kill(AppModeCP, "kuma-cp run")).To(Succeed())
 
 		// and zone-ingress is killed while Zone CP is down
-		_, _, err = zoneUniversal.Exec("", "", AppIngress, "pkill", "-9", "envoy")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(zoneUniversal.(*UniversalCluster).Kill(AppIngress, "kuma-dp")).To(Succeed())
 
 		// and Zone CP is restarted
 		Expect(kumaCP.ReStart()).Should(Succeed())
@@ -181,8 +175,7 @@ func ResilienceMultizoneUniversalPostgres() {
 		}, "30s", "1s").Should(ContainSubstring("Online"))
 
 		// when Zone CP is killed
-		_, _, err := zoneUniversal.Exec("", "", AppModeCP, "pkill", "-9", "kuma-cp")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(zoneUniversal.(*UniversalCluster).Kill(AppModeCP, "kuma-cp run")).To(Succeed())
 
 		// then zone is offline immediately
 		Eventually(func() (string, error) {

test/e2e_env/universal/auth/offline_auth.go

@@ -120,9 +120,9 @@ dpServer:
 			token,
 			[]string{},
 		)
+		Expect(err).ToNot(HaveOccurred())
 
 		// then the new admin can access secrets
-		Expect(err).ToNot(HaveOccurred())
 		Expect(kumactl.RunKumactl("get", "secrets")).To(Succeed())
 	})