[QoS][EVM] Add new error type for JSONRPC backend service unmarshaling failures (#451)

## Summary

Add new error type for JSONRPC backend service unmarshaling failures and
improve error handling

### Primary Changes:

- Add REQUEST_ERROR_INTERNAL_JSONRPC_BACKEND_SERVICE_UNMARSHAL_ERROR
enum value to track when backend service payloads fail to parse as valid
JSONRPC responses
- Implement detection logic in requestContext.GetObservations() to set
this error when no valid JSONRPC responses are received from endpoints
- Add GetRequestErrorForJSONRPCBackendServiceUnmarshalError() helper
function to create the new error type

### Secondary Changes:

- Enhanced logging in unmarshalResponse() with request context and
clearer error messages for debugging
- Add helper function getSingleJSONRPCRequestPayload() for single
JSONRPC request handling
- Update protobuf generated code to include the new error enum value

## Issue
Distinguish backend service JSONRPC error responses from error parsing
endpoint payload as JSONRPC response.

- Issue or PR: #{ISSUE_OR_PR_NUMBER}

## Type of change

Select one or more from the following:

- [x] New feature, functionality or library
- [ ] Bug fix
- [ ] Code health or cleanup
- [ ] Documentation
- [ ] Other (specify)

## QoS Checklist

### E2E Validation & Tests

- [ ] `make path_up`
- [ ] `make test_e2e_evm_shannon`

### Observability

- [ ] 1. `make path_up`
- [ ] 2. Run the following E2E test: `make
test_request__shannon_relay_util_100`
- [ ] 3. View results in LocalNet's [PATH Relay Grafana
Dashboard](http://localhost:3003/d/relays/path-service-requests)

## Sanity Checklist

- [x] I have updated the GitHub Issue `assignees`, `reviewers`,
`labels`, `project`, `iteration` and `milestone`
- [ ] For docs, I have run `make docusaurus_start`
- [ ] For code, I have run `make test_all`
- [ ] For configurations, I have updated the documentation
- [x] I added `TODO`s where applicable

Author: adshmh

observation/qos/request_error.pb.go

@@ -14,6 +14,7 @@ enum RequestErrorKind {
   REQUEST_ERROR_INTERNAL_JSONRPC_PAYLOAD_BUILD_ERROR = 6; // Internal error: Failed to build service payload from JSONRPC request.
   REQUEST_ERROR_USER_ERROR_REST_SERVICE_DETECTION_ERROR = 7; // User error: Failed to detect service type from REST request.
   REQUEST_ERROR_USER_ERROR_REST_UNSUPPORTED_RPC_TYPE = 8; // User error: unsupported service type in REST request.
+  REQUEST_ERROR_INTERNAL_JSONRPC_BACKEND_SERVICE_UNMARSHAL_ERROR = 9; // Internal error: JSONRPC backend service payload failed to unmarshal as valid JSONRPC response.
 }
 
 // RequestError tracks the details of a request error.

proto/path/qos/request_error.proto

@@ -225,6 +225,24 @@ func (rc requestContext) GetObservations() qosobservations.Observations {
 		requestError = qos.GetRequestErrorForProtocolError()
 	}
 
+	// Check for any successfully parsedresponses.
+	var foundParsedJSONRPCResponse bool
+	for _, endpointResponse := range rc.endpointResponses {
+		// Endpoint payload was successfully parsed as JSONRPC response.
+		// Mark the request as having no errors.
+		if endpointResponse.unmarshalErr == nil {
+			foundParsedJSONRPCResponse = true
+			break
+		}
+	}
+
+	// No valid JSONRPC responses received from any endpoints.
+	// Set request error as backend service malformed payload error.
+	// i.e. the payload from the the endpoint/backend service failed to parse as a valid JSONRPC response.
+	if !foundParsedJSONRPCResponse {
+		requestError = qos.GetRequestErrorForJSONRPCBackendServiceUnmarshalError()
+	}
+
 	return qosobservations.Observations{
 		ServiceObservations: &qosobservations.Observations_Evm{
 			Evm: &qosobservations.EVMRequestObservations{

qos/evm/context.go

@@ -61,10 +61,14 @@ func unmarshalResponse(
 	var jsonrpcResponse jsonrpc.Response
 	err := json.Unmarshal(data, &jsonrpcResponse)
 	if err != nil {
+		// Get the request payload, for single JSONRPC requests.
+		requestPayload := getSingleJSONRPCRequestPayload(servicePayloads)
+
 		// The response raw payload (e.g. as received from an endpoint) could not be unmarshalled as a JSONRC response.
 		// Return a generic response to the user.
 		payloadStr := string(data)
 		logger.With(
+			"jsonrpc_request", requestPayload,
 			"unmarshal_err", err,
 			"raw_payload", log.Preview(payloadStr),
 			"endpoint_addr", endpointAddr,
@@ -85,22 +89,35 @@ func unmarshalResponse(
 	// Get the JSON-RPC request from the service payload.
 	jsonrpcReq, err := jsonrpc.GetJsonRpcReqFromServicePayload(servicePayload)
 	if err != nil {
-		logger.Error().Err(err).Msg("Failed to get JSONRPC request from service payload")
+		logger.Error().Err(err).Msg("SHOULD NEVER HAPPEN: Failed to get JSONRPC request from service payload")
 		return getGenericJSONRPCErrResponse(logger, getJsonRpcIDForErrorResponse(servicePayloads), data, err), err
 	}
 
 	// Validate the JSONRPC response.
 	if err := jsonrpcResponse.Validate(jsonrpcReq.ID); err != nil {
 		payloadStr := string(data)
 		logger.With(
-			"request", jsonrpcReq,
+			"jsonrpc_request", jsonrpcReq,
 			"validation_err", err,
 			"raw_payload", log.Preview(payloadStr),
 			"endpoint_addr", endpointAddr,
-		).Debug().Msg("JSON-RPC response validation failed")
+		).Error().Msg("❌ Failed to unmarshal response payload as valid JSON-RPC")
+
 		return getGenericJSONRPCErrResponse(logger, jsonrpcReq.ID, data, err), err
 	}
 
+	// The parsed JSONRPC response contains an error:
+	// - Log the request and the response
+	// - Used to track potential endpoint quality issues.
+	if jsonrpcResponseErr := jsonrpcResponse.Error; jsonrpcResponseErr != nil {
+		logger.With(
+			"jsonrpc_request", jsonrpcReq,
+			"jsonrpc_response_error_code", jsonrpcResponseErr.Code,
+			"jsonrpc_response_error_message", jsonrpcResponseErr.Message,
+			"endpoint_addr", endpointAddr,
+		).Error().Msg("❌ Endpoint returned JSONRPC response with error")
+	}
+
 	// Unmarshal the JSONRPC response into a method-specific response.
 	unmarshaller, found := methodResponseMappings[jsonrpcReq.Method]
 	if found {
@@ -147,3 +164,22 @@ func findServicePayloadByID(servicePayloads map[jsonrpc.ID]protocol.Payload, tar
 	}
 	return protocol.Payload{}, false
 }
+
+// TODO_HACK(@adshmh): Drop this once single and batch JSONRPC request handling logic is fully separated.
+// - There should be no need to rely on endpoint's payload to match the request from a batch.
+// - Single JSONRPC requests do not need the complexity of batch request logic.
+//
+// This only targets single JSONRPC requests.
+func getSingleJSONRPCRequestPayload(servicePayloads map[jsonrpc.ID]protocol.Payload) string {
+	if len(servicePayloads) != 1 {
+		return ""
+	}
+
+	for _, payload := range servicePayloads {
+		if len(payload.Data) > 0 {
+			return payload.Data
+		}
+	}
+
+	return ""
+}

qos/evm/response.go

@@ -24,3 +24,17 @@ func GetRequestErrorForProtocolError() *qosobservations.RequestError {
 		HttpStatusCode: int32(jsonrpcErrorResponse.GetRecommendedHTTPStatusCode()),
 	}
 }
+
+// GetRequestErrorForJSONRPCBackendServiceUnmarshalError returns a request error for a JSONRPC backend service unmarshaling error.
+// i.e. the payload returned by the endpoint/backend service failed to parse as a valid JSONRPC response.
+func GetRequestErrorForJSONRPCBackendServiceUnmarshalError() *qosobservations.RequestError {
+	err := errors.New("internal error: JSONRPC backend service error: payload failed to parse as valid JSONRPC response")
+	// initialize a JSONRPC error response to derive the HTTP status code.
+	jsonrpcErrorResponse := jsonrpc.NewErrResponseInternalErr(jsonrpc.ID{}, err)
+
+	return &qosobservations.RequestError{
+		ErrorKind:      qosobservations.RequestErrorKind_REQUEST_ERROR_INTERNAL_JSONRPC_BACKEND_SERVICE_UNMARSHAL_ERROR,
+		ErrorDetails:   err.Error(),
+		HttpStatusCode: int32(jsonrpcErrorResponse.GetRecommendedHTTPStatusCode()),
+	}
+}