MockWebhook: track shutdown status more gracefully (#4761)

* MockWebhook: track shutdown status more gracefully

When we close the server sometimes the messages are interrupted and
can't be decoded by the webhook. This change accepts any decoding
failure after server shutdown.

Signed-off-by: Guido Trotter <[email protected]>

* Call NewWebhook passing 't', and protect against nil cmd in Terminate

Signed-off-by: Guido Trotter <[email protected]>

---------

Signed-off-by: Guido Trotter <[email protected]>
Co-authored-by: Guido Trotter <[email protected]>

Author: ultrotter

test/cli/acceptance/cli_test.go

@@ -422,7 +422,7 @@ receivers:
 		Tolerance: 1 * time.Second,
 	})
 	co := at.Collector("webhook")
-	wh := NewWebhook(co)
+	wh := NewWebhook(t, co)
 
 	amc := at.AlertmanagerCluster(fmt.Sprintf(conf, wh.Address()), 1)
 	require.NoError(t, amc.Start())
@@ -475,7 +475,7 @@ receivers:
 		Tolerance: 1 * time.Second,
 	})
 	co := at.Collector("webhook")
-	wh := NewWebhook(co)
+	wh := NewWebhook(t, co)
 
 	amc := at.AlertmanagerCluster(fmt.Sprintf(conf, wh.Address()), 1)
 	require.NoError(t, amc.Start())

test/testutils/acceptance.go

@@ -383,7 +383,7 @@ func (amc *AlertmanagerCluster) Terminate() {
 // data.
 func (am *Alertmanager) Terminate() {
 	am.T.Helper()
-	if am.cmd.Process != nil {
+	if am.cmd != nil && am.cmd.Process != nil {
 		if err := syscall.Kill(am.cmd.Process.Pid, syscall.SIGTERM); err != nil {
 			am.T.Logf("Error sending SIGTERM to Alertmanager process: %v", err)
 		}

test/testutils/mock.go

@@ -16,10 +16,12 @@ package testutils
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"maps"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -182,6 +184,7 @@ type MockWebhook struct {
 	opts      *AcceptanceOpts
 	collector *Collector
 	addr      string
+	closing   atomic.Bool
 
 	// Func is called early on when retrieving a notification by an
 	// Alertmanager. If Func returns true, the given notification is dropped.
@@ -202,6 +205,7 @@ func NewWebhook(t *testing.T, c *Collector) *MockWebhook {
 	wh.addr = server.Listener.Addr().String()
 
 	t.Cleanup(func() {
+		wh.closing.Store(true)
 		server.Close()
 	})
 
@@ -222,6 +226,10 @@ func (ws *MockWebhook) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 
 	var v webhook.Message
 	if err := dec.Decode(&v); err != nil {
+		// During shutdown, ignore EOF errors from interrupted connections
+		if ws.closing.Load() && (err == io.EOF || err.Error() == "EOF") {
+			return
+		}
 		panic(err)
 	}