test(e2e): improve universal app run

go.mod

@@ -214,7 +214,7 @@ require (
 	github.com/moby/patternmatcher v0.6.0 // indirect
 	github.com/moby/spdystream v0.5.0 // indirect
 	github.com/moby/sys/sequential v0.5.0 // indirect
-	github.com/moby/sys/user v0.1.0 // indirect
+	github.com/moby/sys/user v0.3.0 // indirect
 	github.com/moby/sys/userns v0.1.0 // indirect
 	github.com/moby/term v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -252,10 +252,9 @@ require (
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.35.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 // indirect
 	go.opentelemetry.io/otel/metric v1.35.0 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
-	golang.org/x/crypto v0.36.0 // indirect
+	golang.org/x/crypto v0.36.0
 	golang.org/x/exp v0.0.0-20250128182459-e0ece0dbea4c
 	golang.org/x/mod v0.24.0 // indirect
 	golang.org/x/oauth2 v0.27.0 // indirect
@@ -277,6 +276,7 @@ require (
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/google/cel-go v0.22.0 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 // indirect
 	k8s.io/apiserver v0.32.3 // indirect
 	k8s.io/component-base v0.32.3 // indirect
 )

go.sum

@@ -368,8 +368,8 @@ github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9Kou
 github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4=
 github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=
 github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo=
-github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg=
-github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU=
+github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo=
+github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
 github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
 github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
 github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=

pkg/core/tokens/issuer.go

@@ -37,12 +37,12 @@ func (j *jwtTokenIssuer) Generate(ctx context.Context, claims Claims, validFor t
 		return "", err
 	}
 
-	now := core.Now()
+	now := core.Now().Add(-5 * time.Second) // todo(jakubdyszkiewicz) parametrize via config and go through all clock skews in the project
 	claims.SetRegisteredClaims(jwt.RegisteredClaims{
 		ID:        core.NewUUID(),
 		IssuedAt:  jwt.NewNumericDate(now),
-		NotBefore: jwt.NewNumericDate(now.Add(time.Minute * -5)), // todo(jakubdyszkiewicz) parametrize via config and go through all clock skews in the project
-		ExpiresAt: jwt.NewNumericDate(now.Add(validFor)),
+		NotBefore: jwt.NewNumericDate(now),
+		ExpiresAt: jwt.NewNumericDate(now.Add(validFor).Add(5 * time.Second)),
 	})
 
 	token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims)

pkg/util/files/files.go

@@ -1,6 +1,8 @@
 package files
 
 import (
+	"fmt"
+	"io"
 	"io/fs"
 	"os"
 	"path/filepath"
@@ -49,3 +51,18 @@ func ToValidUnixFilename(input ...string) string {
 
 	return sanitized
 }
+
+// CopyFile copies the file content to a new path.
+// This should be used with caution as it was first written for tests
+func CopyFile(src, dst string) error {
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return fmt.Errorf("failed to open src file:%s %w", src, err)
+	}
+	dstFile, err := os.Create(dst)
+	if err != nil {
+		return fmt.Errorf("failed to create dest file:%s %w", dst, err)
+	}
+	_, err = io.Copy(dstFile, srcFile)
+	return err
+}

test/dockerfiles/universal.dockerignore

@@ -11,12 +11,6 @@
 !build/artifacts-linux-arm64/kumactl/kumactl
 !build/artifacts-linux-arm64/coredns/coredns
 !build/artifacts-linux-arm64/test-server/test-server
-!build/artifacts-darwin-arm64/kuma-cp/kuma-cp
-!build/artifacts-darwin-arm64/kuma-dp/kuma-dp
-!build/artifacts-darwin-arm64/envoy/*
-!build/artifacts-darwin-arm64/kumactl/kumactl
-!build/artifacts-darwin-arm64/coredns/coredns
-!build/artifacts-darwin-arm64/test-server/test-server
 !pkg/config/app/kuma-cp/kuma-cp.defaults.yaml
 !tools/releases/templates/LICENSE
 !tools/releases/templates/NOTICE

test/e2e_env/multizone/externalservices/externalservices_multizone_universal.go

@@ -155,42 +155,42 @@ routing:
 		Expect(err).ToNot(HaveOccurred())
 
 		Eventually(func(g Gomega) {
-			stdout, _, err := client.CollectResponse(
+			stdout, stderr, err := client.CollectResponse(
 				zone1, "demo-client", "external-service-1.mesh",
 				client.WithVerbose(),
 			)
 			g.Expect(err).ToNot(HaveOccurred())
-			g.Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK"))
+			g.Expect(stderr).To(ContainSubstring("HTTP/1.1 200 OK"))
 			g.Expect(stdout).ToNot(ContainSubstring("HTTPS"))
 		}, "1m", "3s").Should(Succeed())
 
 		Eventually(func(g Gomega) {
-			stdout, _, err := client.CollectResponse(
+			stdout, stderr, err := client.CollectResponse(
 				zone1, "demo-client", "kuma-es-4_es-http:80",
 				client.WithVerbose(),
 			)
 			g.Expect(err).ToNot(HaveOccurred())
-			g.Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK"))
+			g.Expect(stderr).To(ContainSubstring("HTTP/1.1 200 OK"))
 			g.Expect(stdout).ToNot(ContainSubstring("HTTPS"))
 		}, "1m", "3s").Should(Succeed())
 
 		Eventually(func(g Gomega) {
-			stdout, _, err := client.CollectResponse(
+			stdout, stderr, err := client.CollectResponse(
 				zone2, "demo-client", "external-service-1.mesh",
 				client.WithVerbose(),
 			)
 			g.Expect(err).ToNot(HaveOccurred())
-			g.Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK"))
+			g.Expect(stderr).To(ContainSubstring("HTTP/1.1 200 OK"))
 			g.Expect(stdout).ToNot(ContainSubstring("HTTPS"))
 		}, "1m", "3s").Should(Succeed())
 
 		Eventually(func(g Gomega) {
-			stdout, _, err := client.CollectResponse(
+			stdout, stderr, err := client.CollectResponse(
 				zone2, "demo-client", "kuma-es-4_es-http:80",
 				client.WithVerbose(),
 			)
 			g.Expect(err).ToNot(HaveOccurred())
-			g.Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK"))
+			g.Expect(stderr).To(ContainSubstring("HTTP/1.1 200 OK"))
 			g.Expect(stdout).ToNot(ContainSubstring("HTTPS"))
 		}, "1m", "3s").Should(Succeed())
 	})
@@ -219,22 +219,22 @@ routing:
 
 		// then accessing the secured external service succeeds
 		Eventually(func(g Gomega) {
-			stdout, _, err := client.CollectResponse(
+			stdout, stderr, err := client.CollectResponse(
 				zone1, "demo-client", "http://kuma-es-4_es-https:443",
 				client.WithVerbose(),
 			)
 			g.Expect(err).ToNot(HaveOccurred())
-			g.Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK"))
+			g.Expect(stderr).To(ContainSubstring("HTTP/1.1 200 OK"))
 			g.Expect(stdout).To(ContainSubstring("es-https"))
 		}, "1m", "1s").Should(Succeed())
 
 		Eventually(func(g Gomega) {
-			stdout, _, err := client.CollectResponse(
+			stdout, stderr, err := client.CollectResponse(
 				zone2, "demo-client", "http://kuma-es-4_es-https:443",
 				client.WithVerbose(),
 			)
 			g.Expect(err).ToNot(HaveOccurred())
-			g.Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK"))
+			g.Expect(stderr).To(ContainSubstring("HTTP/1.1 200 OK"))
 			g.Expect(stdout).To(ContainSubstring("es-https"))
 		}, "1m", "3s").Should(Succeed())
 	})

test/e2e_env/multizone/ownership/ownership.go

@@ -62,13 +62,11 @@ func MultizoneUniversal() {
 	}
 
 	killKumaDP := func(appname string) {
-		_, _, err := zoneUniversal.Exec("", "", appname, "pkill", "-9", "envoy")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(zoneUniversal.(*UniversalCluster).Kill(appname, "kuma-dp")).To(Succeed())
 	}
 
 	killZone := func() {
-		_, _, err := zoneUniversal.Exec("", "", AppModeCP, "pkill", "-9", "kuma-cp")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(zoneUniversal.(*UniversalCluster).Kill(AppModeCP, "kuma-cp run")).To(Succeed())
 		Eventually(func() (string, error) {
 			return global.GetKumactlOptions().RunKumactlAndGetOutput("inspect", "zones")
 		}, "30s", "1s").Should(ContainSubstring("Offline"))

test/e2e_env/multizone/resilience/resilience_multizone_universal.go

@@ -65,8 +65,7 @@ func ResilienceMultizoneUniversal() {
 			return nil
 		}, "30s", "1s").ShouldNot(HaveOccurred())
 
-		_, _, err := zone1.Exec("", "", AppModeCP, "pkill", "-9", "kuma-cp")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(zone1.(*UniversalCluster).Kill(AppModeCP, "kuma-cp run")).To(Succeed())
 
 		Eventually(func() error {
 			output, err := global.GetKumactlOptions().RunKumactlAndGetOutput("inspect", "zones")

test/e2e_env/multizone/resilience/resilience_multizone_universal_postgres.go

@@ -91,12 +91,10 @@ func ResilienceMultizoneUniversalPostgres() {
 		Expect(kumaCP).ToNot(BeNil())
 
 		// when global is killed
-		_, _, err := global.Exec("", "", AppModeCP, "pkill", "-9", "kuma-cp")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(global.(*UniversalCluster).Kill(AppModeCP, "kuma-cp run")).To(Succeed())
 
 		// and zone is killed while global is down
-		_, _, err = zoneUniversal.Exec("", "", AppModeCP, "pkill", "-9", "kuma-cp")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(zoneUniversal.(*UniversalCluster).Kill(AppModeCP, "kuma-cp run")).To(Succeed())
 
 		// and global is restarted
 		Expect(kumaCP.ReStart()).Should(Succeed())
@@ -122,8 +120,7 @@ func ResilienceMultizoneUniversalPostgres() {
 		Expect(kumaCP).ToNot(BeNil())
 
 		// when global is killed
-		_, _, err := global.Exec("", "", AppModeCP, "pkill", "-9", "kuma-cp")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(global.(*UniversalCluster).Kill(AppModeCP, "kuma-cp run")).To(Succeed())
 
 		// and global is restarted
 		Expect(kumaCP.ReStart()).Should(Succeed())
@@ -133,8 +130,7 @@ func ResilienceMultizoneUniversalPostgres() {
 		time.Sleep(10 * time.Second) // ZoneInsightFlushInterval
 
 		// and zone is killed
-		_, _, err = zoneUniversal.Exec("", "", AppModeCP, "pkill", "-9", "kuma-cp")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(zoneUniversal.(*UniversalCluster).Kill(AppModeCP, "kuma-cp run")).To(Succeed())
 
 		// then zone is offline
 		Eventually(func() (string, error) {
@@ -156,12 +152,10 @@ func ResilienceMultizoneUniversalPostgres() {
 		Expect(kumaCP).ToNot(BeNil())
 
 		// when Zone CP is killed
-		_, _, err := zoneUniversal.Exec("", "", AppModeCP, "pkill", "-9", "kuma-cp")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(zoneUniversal.(*UniversalCluster).Kill(AppModeCP, "kuma-cp run")).To(Succeed())
 
 		// and zone-ingress is killed while Zone CP is down
-		_, _, err = zoneUniversal.Exec("", "", AppIngress, "pkill", "-9", "envoy")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(zoneUniversal.(*UniversalCluster).Kill(AppIngress, "kuma-dp")).To(Succeed())
 
 		// and Zone CP is restarted
 		Expect(kumaCP.ReStart()).Should(Succeed())
@@ -181,8 +175,7 @@ func ResilienceMultizoneUniversalPostgres() {
 		}, "30s", "1s").Should(ContainSubstring("Online"))
 
 		// when Zone CP is killed
-		_, _, err := zoneUniversal.Exec("", "", AppModeCP, "pkill", "-9", "kuma-cp")
-		Expect(err).ToNot(HaveOccurred())
+		Expect(zoneUniversal.(*UniversalCluster).Kill(AppModeCP, "kuma-cp run")).To(Succeed())
 
 		// then zone is offline immediately
 		Eventually(func() (string, error) {

test/e2e_env/universal/auth/offline_auth.go

@@ -120,9 +120,9 @@ dpServer:
 			token,
 			[]string{},
 		)
+		Expect(err).ToNot(HaveOccurred())
 
 		// then the new admin can access secrets
-		Expect(err).ToNot(HaveOccurred())
 		Expect(kumactl.RunKumactl("get", "secrets")).To(Succeed())
 	})
 

test/e2e_env/universal/externalservices/externalservices.go

@@ -116,12 +116,12 @@ networking:
 
 	checkSuccessfulRequest := func(url, clientName string, matcher types.GomegaMatcher) {
 		Eventually(func(g Gomega) {
-			stdout, _, err := client.CollectResponse(
+			stdout, stderr, err := client.CollectResponse(
 				universal.Cluster, clientName, url,
 				client.WithVerbose(),
 			)
 			g.Expect(err).ToNot(HaveOccurred())
-			g.Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK"))
+			g.Expect(stderr).To(ContainSubstring("HTTP/1.1 200 OK"))
 			g.Expect(stdout).To(matcher)
 		}, "30s", "500ms").WithOffset(1).Should(Succeed())
 	}

test/e2e_env/universal/externalservices/zoneegress_externalservices.go

@@ -88,12 +88,12 @@ networking:
 
 		// then should reach external service
 		Eventually(func(g Gomega) {
-			stdout, _, err := client.CollectResponse(
+			_, stderr, err := client.CollectResponse(
 				cluster, "demo-client", "external-service-1.mesh",
 				client.WithVerbose(),
 			)
 			g.Expect(err).ToNot(HaveOccurred())
-			g.Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK"))
+			g.Expect(stderr).To(ContainSubstring("HTTP/1.1 200 OK"))
 		}, "30s", "1s").Should(Succeed())
 
 		// and increase stats at egress

test/e2e_env/universal/meshexternalservice/meshexternalservice.go

@@ -130,12 +130,12 @@ networking:
 
 	checkSuccessfulRequest := func(url, clientName string, matcher types.GomegaMatcher) {
 		Eventually(func(g Gomega) {
-			stdout, _, err := client.CollectResponse(
+			stdout, stderr, err := client.CollectResponse(
 				universal.Cluster, clientName, url,
 				client.WithVerbose(),
 			)
 			g.Expect(err).ToNot(HaveOccurred())
-			g.Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK"))
+			g.Expect(stderr).To(ContainSubstring("HTTP/1.1 200 OK"))
 			g.Expect(stdout).To(matcher)
 		}, "30s", "500ms").WithOffset(1).Should(Succeed())
 	}

test/e2e_env/universal/meshratelimit/meshratelimit.go

@@ -105,8 +105,7 @@ spec:
 	})
 
 	It("should limit tcp connections", func() {
-		admin, err := universal.Cluster.GetApp("test-server-tcp").GetEnvoyAdminTunnel()
-		Expect(err).ToNot(HaveOccurred())
+		admin := universal.Cluster.GetApp("test-server-tcp").GetEnvoyAdminTunnel()
 		// should have no ratelimited connections
 		Expect(tcpRateLimitStats(admin)).To(stats.BeEqualZero())
 

test/e2e_env/universal/meshretry/grpc.go

@@ -81,8 +81,7 @@ spec:
         grpc:
           numRetries: 5
 `, meshName)
-		admin, err := universal.Cluster.GetApp("test-client").GetEnvoyAdminTunnel()
-		Expect(err).ToNot(HaveOccurred())
+		admin := universal.Cluster.GetApp("test-client").GetEnvoyAdminTunnel()
 
 		lastFailureStats := stats.StatItem{Name: "", Value: float64(0)}
 		grpcFailureStats := func(g Gomega) *stats.Stats {

test/e2e_env/universal/meshtls/policy.go

@@ -266,8 +266,7 @@ spec:
 
 	It("should tls version for 1.3", func() {
 		// given
-		admin, err := universal.Cluster.GetApp(testServerName).GetEnvoyAdminTunnel()
-		Expect(err).ToNot(HaveOccurred())
+		admin := universal.Cluster.GetApp(testServerName).GetEnvoyAdminTunnel()
 
 		policy := fmt.Sprintf(`
 type: MeshTLS
@@ -332,8 +331,7 @@ spec:
 
 	It("should set cypher and version", func() {
 		// given
-		admin, err := universal.Cluster.GetApp(testServerName).GetEnvoyAdminTunnel()
-		Expect(err).ToNot(HaveOccurred())
+		admin := universal.Cluster.GetApp(testServerName).GetEnvoyAdminTunnel()
 
 		policy := fmt.Sprintf(`
 type: MeshTLS

test/e2e_env/universal/mtls/mtls.go

@@ -33,15 +33,17 @@ func Policy() {
 		)
 	}
 	trafficAllowed := func(addr string, fn ...client.CollectResponsesOptsFn) {
+		GinkgoHelper()
 		expectation := func(g Gomega) {
-			stdout, _, err := curlAddr(addr, fn...)
+			_, stderr, err := curlAddr(addr, fn...)
 			g.Expect(err).ToNot(HaveOccurred())
-			g.Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK"))
+			g.Expect(stderr).To(ContainSubstring("HTTP/1.1 200 OK"))
 		}
 		Eventually(expectation).WithOffset(1).WithPolling(time.Millisecond * 250).WithTimeout(time.Second * 20).Should(Succeed())
 		Consistently(expectation).WithOffset(1).WithPolling(time.Millisecond * 250).WithTimeout(time.Second * 2).Should(Succeed())
 	}
 	trafficBlocked := func(addr string, fn ...client.CollectResponsesOptsFn) {
+		GinkgoHelper()
 		expectation := func(g Gomega) {
 			_, _, err := curlAddr(addr, fn...)
 			g.Expect(err).To(HaveOccurred())
@@ -149,9 +151,9 @@ mtls:
 
 		By("inside-mesh communication never fails")
 		Consistently(func(g Gomega) {
-			stdout, _, err := curlAddr("test-server.mesh")
+			_, stderr, err := curlAddr("test-server.mesh")
 			g.Expect(err).ToNot(HaveOccurred())
-			g.Expect(stdout).To(ContainSubstring("HTTP/1.1 200 OK"))
+			g.Expect(stderr).To(ContainSubstring("HTTP/1.1 200 OK"))
 		}).Should(Succeed())
 	})
 	// Added Flake because: https://github.com/kumahq/kuma/issues/4700