[Protocol][Observability] Track RelayMiner non-2xx HTTP status codes (#436)

## Summary

Track non-2xx HTTP status codes from RelayMiner via metrics.

Author: adshmh

observation/protocol/shannon.pb.go

@@ -124,6 +124,12 @@ enum ShannonEndpointErrorType {
 
   // The relay request sent to the endpoint via WebSocket failed to validate the relay response.
   SHANNON_ENDPOINT_ERROR_WEBSOCKET_RELAY_RESPONSE_VALIDATION_FAILED = 41;
+
+  // RelayMiner returned a 4XX HTTP status code
+  SHANNON_ENDPOINT_ERROR_RELAY_MINER_HTTP_4XX = 42;
+
+  // RelayMiner returned a 5XX HTTP status code
+  SHANNON_ENDPOINT_ERROR_RELAY_MINER_HTTP_5XX = 43;
 }
 
 // ShannonSanctionType specifies the duration type for endpoint sanctions

proto/path/protocol/shannon.proto

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"maps"
 	"math/rand"
+	"net/http"
 	"strconv"
 	"sync"
 	"time"
@@ -337,7 +338,9 @@ func (rc *requestContext) executeRelayRequestStrategy(payload protocol.Payload)
 	// Session rollover periods
 	// - Protocol relay with fallback protection during session rollover periods
 	// - Sends requests in parallel to ensure reliability during network transitions
-	case rc.fullNode.IsInSessionRollover():
+	//
+	// TODO_DELETE(@adshmh): No session rollover fallback for hey service.
+	case rc.fullNode.IsInSessionRollover() && rc.serviceID != "hey":
 		rc.logger.Debug().Msg("Executing protocol relay with fallback protection during session rollover periods")
 		// TODO_TECHDEBT(@adshmh): Separate error handling for fallback and Shannon endpoints.
 		return rc.sendRelayWithFallback(payload)
@@ -509,12 +512,23 @@ func (rc *requestContext) sendProtocolRelay(payload protocol.Payload) (protocol.
 		return defaultResponse, fmt.Errorf("SHOULD NEVER HAPPEN: failed to marshal relay request: %w", err)
 	}
 
+	// TODO_TECHDEBT(@adshmh): Add a new struct to track details about the HTTP call.
+	// It should contain at-least:
+	// - endpoint payload
+	// - HTTP status code
+	// Use the new struct to pass data around for logging/metrics/etc.
+	//
 	// Send the HTTP request to the protocol endpoint.
-	httpRelayResponseBz, _, err := rc.sendHTTPRequest(payload, selectedEndpoint.PublicURL(), relayRequestBz)
+	httpRelayResponseBz, httpStatusCode, err := rc.sendHTTPRequest(payload, selectedEndpoint.PublicURL(), relayRequestBz)
 	if err != nil {
 		return defaultResponse, err
 	}
 
+	// Non-2xx HTTP status code received from the endpoint: build and return an error
+	if httpStatusCode != http.StatusOK {
+		return defaultResponse, fmt.Errorf("%w %w: %d", errSendHTTPRelay, errEndpointNon2XXHTTPStatusCode, httpStatusCode)
+	}
+
 	// Validate and process the response
 	response, err := rc.validateAndProcessResponse(httpRelayResponseBz)
 	if err != nil {
@@ -709,12 +723,23 @@ func (rc *requestContext) sendFallbackRelay(
 		fallbackURL,
 		[]byte(payload.Data),
 	)
+
 	if err != nil {
 		return protocol.Response{
 			EndpointAddr: fallbackEndpoint.Addr(),
 		}, err
 	}
 
+	// TODO_CONSIDERATION(@adshmh): Are there any scenarios where a fallback endpoint should return a non-2xx HTTP status code?
+	// Examples: a fallback endpoint for a RESTful service.
+	//
+	// Non-2xx HTTP status code: build and return an error.
+	if httpStatusCode != http.StatusOK {
+		return protocol.Response{
+			EndpointAddr: fallbackEndpoint.Addr(),
+		}, fmt.Errorf("%w %w: %d", errSendHTTPRelay, errEndpointNon2XXHTTPStatusCode, httpStatusCode)
+	}
+
 	// Build and return the fallback response
 	return protocol.Response{
 		Bytes:          httpResponseBz,

protocol/shannon/context.go

@@ -67,6 +67,9 @@ var (
 	// Helps track more fine-grained metrics on endpoint errors.
 	errMalformedEndpointPayload = errors.New("endpoint returned malformed payload")
 
+	// The endpoint returned a non-2XX response.
+	errEndpointNon2XXHTTPStatusCode = errors.New("endpoint returned non-2xx HTTP status code")
+
 	// ** WebSocket errors **
 
 	// Error creating a WebSocket connection.

protocol/shannon/errors.go

@@ -3,6 +3,7 @@ package shannon
 import (
 	"errors"
 	"regexp"
+	"strconv"
 	"strings"
 
 	"github.com/pokt-network/poktroll/pkg/polylog"
@@ -149,6 +150,13 @@ func classifyHttpError(logger polylog.Logger, err error) (protocolobservations.S
 			protocolobservations.ShannonSanctionType_SHANNON_SANCTION_SESSION
 	}
 
+	// RelayMiner returned non-2xx HTTP status code.
+	if errors.Is(err, errEndpointNon2XXHTTPStatusCode) {
+		return getNon2XXHTTPStatusCodeObservation(err),
+			// TODO_UPNEXT(@adshmh): Make this a sanction that lasts a few blocks.
+			protocolobservations.ShannonSanctionType_SHANNON_SANCTION_DO_NOT_SANCTION
+	}
+
 	errStr := err.Error()
 
 	// Connection establishment failures
@@ -294,3 +302,49 @@ func classifyMalformedEndpointPayload(logger polylog.Logger, payloadContent stri
 	//
 	return protocolobservations.ShannonEndpointErrorType_SHANNON_ENDPOINT_ERROR_RAW_PAYLOAD_UNKNOWN, protocolobservations.ShannonSanctionType_SHANNON_SANCTION_DO_NOT_SANCTION
 }
+
+// getNon2XXHTTPStatusCodeObservation returns ShannonEndpointErrorType based on HTTP status code:
+//   - 4xx: SHANNON_ENDPOINT_ERROR_RELAY_MINER_HTTP_4XX
+//   - 5xx: SHANNON_ENDPOINT_ERROR_RELAY_MINER_HTTP_5XX
+//   - other/parse error: SHANNON_ENDPOINT_ERROR_HTTP_UNKNOWN
+func getNon2XXHTTPStatusCodeObservation(non2XXHTTPStatusCodeErr error) protocolobservations.ShannonEndpointErrorType {
+	statusCode, ok := extractHTTPStatusCode(non2XXHTTPStatusCodeErr)
+	if !ok {
+		return protocolobservations.ShannonEndpointErrorType_SHANNON_ENDPOINT_ERROR_HTTP_UNKNOWN
+	}
+
+	switch {
+	case statusCode >= 400 && statusCode < 500:
+		return protocolobservations.ShannonEndpointErrorType_SHANNON_ENDPOINT_ERROR_RELAY_MINER_HTTP_4XX
+	case statusCode >= 500 && statusCode < 600:
+		return protocolobservations.ShannonEndpointErrorType_SHANNON_ENDPOINT_ERROR_RELAY_MINER_HTTP_5XX
+	default:
+		return protocolobservations.ShannonEndpointErrorType_SHANNON_ENDPOINT_ERROR_HTTP_UNKNOWN
+	}
+}
+
+// extractHTTPStatusCode extracts the HTTP status code from the error message.
+// Expects the status code to be at the end of the error string after ": ".
+func extractHTTPStatusCode(err error) (int, bool) {
+	errStr := err.Error()
+
+	// Look for ": " followed by 3 digits at the end of the string
+	re := regexp.MustCompile(`: (\d{3})$`)
+	matches := re.FindStringSubmatch(errStr)
+
+	if len(matches) < 2 {
+		return 0, false
+	}
+
+	statusCode, parseErr := strconv.Atoi(matches[1])
+	if parseErr != nil {
+		return 0, false
+	}
+
+	// Basic validation that it's a valid HTTP status code
+	if statusCode < 100 || statusCode > 599 {
+		return 0, false
+	}
+
+	return statusCode, true
+}