[QoS][CometBFT] Add CometBFT QoS metrics and observation infrastructure (#349)

## Summary

Add CometBFT QoS metrics collection and observation infrastructure with
request tracking capabilities

### Primary Changes:

- Add CometBFTRequestObservations protobuf with chain_id, service_id,
request_origin, and request_error fields for comprehensive request
tracking
- Implement CometBFTObservationInterpreter to extract metrics-relevant
data from observations for Prometheus reporting
- Create metrics/qos/cometbft/metrics.go with requestsTotal counter
tracking requests by chain, service, method, success status, and error
types
- Update CometBFT QoS context to generate complete observations with
proper error handling and JSONRPC response mapping

### Secondary Changes:

- Separate logger contexts in cmd/qos.go - use hydratedLogger for
function-level logging and qosLogger for QoS instances
- Refactor CometBFT response interface to return GetJSONRPCResponse()
instead of separate payload/status methods
- Add serviceID field to ServiceState for observation generation
consistency

## Issue

No CometBFT metrics, log entries missing fields required for analysis of
errors.

- Issue or PR: #{ISSUE_OR_PR_NUMBER}

## Type of change

Select one or more from the following:

- [x] New feature, functionality or library
- [ ] Bug fix
- [ ] Code health or cleanup
- [ ] Documentation
- [ ] Other (specify)

## QoS Checklist

### E2E Validation & Tests

- [ ] `make path_up`
- [ ] `make test_e2e_evm_shannon`
- [ ] `make test_e2e_evm_morse`

### Observability

- [ ] 1. `make path_up`
- [ ] 2. Run one of the following:
- For `Shannon` with `anvil`: `make
test_request__shannon_relay_util_100`
  - For `Morse` with `F00C`: `make test_request__morse_relay_util_100`
- [ ] 3. Visit [PATH Relay Grafana
Dashboard](http://localhost:3003/d/relays/path-service-requests) to view
results

## Sanity Checklist

- [ ] I have updated the GitHub Issue `assignees`, `reviewers`,
`labels`, `project`, `iteration` and `milestone`
- [ ] For docs, I have run `make docusaurus_start`
- [ ] For code, I have run `make test_all`
- [ ] For configurations, I have updated the documentation
- [ ] I added `TODO`s where applicable

Author: adshmh

cmd/qos.go

@@ -24,41 +24,45 @@ func getServiceQoSInstances(
 	// need to manually add entries for every new QoS implementation.
 	qosServices := make(map[protocol.ServiceID]gateway.QoSService)
 
-	logger = logger.With("module", "qos").With("method", "getServiceQoSInstances").With("protocol", protocolInstance.Name())
+	// Create a logger for this function's own messages with method-specific context
+	hydratedLogger := logger.With("module", "qos").With("method", "getServiceQoSInstances").With("protocol", protocolInstance.Name())
+
+	// Create a separate logger for QoS instances without method-specific context
+	qosLogger := logger.With("module", "qos").With("protocol", protocolInstance.Name())
 
 	// Wait for the protocol to become healthy BEFORE configuring and starting the hydrator.
 	// - Ensures the protocol instance's configured service IDs are available before hydrator startup.
-	err := waitForProtocolHealth(logger, protocolInstance, defaultProtocolHealthTimeout)
+	err := waitForProtocolHealth(hydratedLogger, protocolInstance, defaultProtocolHealthTimeout)
 	if err != nil {
 		return nil, err
 	}
 
 	// Get configured service IDs from the protocol instance.
 	// - Used to run hydrator checks on all configured service IDs (except those manually disabled by the user).
 	gatewayServiceIDs := protocolInstance.ConfiguredServiceIDs()
-	logGatewayServiceIDs(logger, gatewayServiceIDs)
+	logGatewayServiceIDs(hydratedLogger, gatewayServiceIDs)
 
 	// Remove any service IDs that are manually disabled by the user.
 	for _, disabledQoSServiceIDForGateway := range gatewayConfig.HydratorConfig.QoSDisabledServiceIDs {
 		// Throw error if any manually disabled service IDs are not found in the protocol's configured service IDs.
 		if _, found := gatewayServiceIDs[disabledQoSServiceIDForGateway]; !found {
 			return nil, fmt.Errorf("[INVALID CONFIGURATION] QoS manually disabled for service ID: %s BUT NOT not found in protocol's configured service IDs", disabledQoSServiceIDForGateway)
 		}
-		logger.Info().Msgf("Gateway manually disabled QoS for service ID: %s", disabledQoSServiceIDForGateway)
+		hydratedLogger.Info().Msgf("Gateway manually disabled QoS for service ID: %s", disabledQoSServiceIDForGateway)
 		delete(gatewayServiceIDs, disabledQoSServiceIDForGateway)
 	}
 
 	// Get the service configs for the current protocol
 	qosServiceConfigs := config.QoSServiceConfigs.GetServiceConfigs(gatewayConfig)
-	logQoSServiceConfigs(logger, qosServiceConfigs)
+	logQoSServiceConfigs(hydratedLogger, qosServiceConfigs)
 
 	// Initialize QoS services for all service IDs with a corresponding QoS
 	// implementation, as defined in the `config/service_qos.go` file.
 	for _, qosServiceConfig := range qosServiceConfigs {
 		serviceID := qosServiceConfig.GetServiceID()
 		// Skip service IDs that are not configured for the PATH instance.
 		if _, found := gatewayServiceIDs[serviceID]; !found {
-			logger.Warn().Msgf("Service ID %s has an available QoS configuration but is not configured for the gateway. Skipping...", serviceID)
+			hydratedLogger.Warn().Msgf("Service ID %s has an available QoS configuration but is not configured for the gateway. Skipping...", serviceID)
 			continue
 		}
 
@@ -69,18 +73,18 @@ func getServiceQoSInstances(
 				return nil, fmt.Errorf("SHOULD NEVER HAPPEN: error building QoS instances: service ID %q is not an EVM service", serviceID)
 			}
 
-			evmQoS := evm.NewQoSInstance(logger, evmServiceQoSConfig)
+			evmQoS := evm.NewQoSInstance(qosLogger, evmServiceQoSConfig)
 			qosServices[serviceID] = evmQoS
 
-			logger.With("service_id", serviceID).Debug().Msg("Added EVM QoS instance for the service ID.")
+			hydratedLogger.With("service_id", serviceID).Debug().Msg("Added EVM QoS instance for the service ID.")
 
 		case cometbft.QoSType:
 			cometBFTServiceQoSConfig, ok := qosServiceConfig.(cometbft.CometBFTServiceQoSConfig)
 			if !ok {
 				return nil, fmt.Errorf("SHOULD NEVER HAPPEN: error building QoS instances: service ID %q is not a CometBFT service", serviceID)
 			}
 
-			cometBFTQoS := cometbft.NewQoSInstance(logger, cometBFTServiceQoSConfig)
+			cometBFTQoS := cometbft.NewQoSInstance(qosLogger, cometBFTServiceQoSConfig)
 			qosServices[serviceID] = cometBFTQoS
 
 		case solana.QoSType:
@@ -89,10 +93,10 @@ func getServiceQoSInstances(
 				return nil, fmt.Errorf("SHOULD NEVER HAPPEN: error building QoS instances: service ID %q is not a Solana service", serviceID)
 			}
 
-			solanaQoS := solana.NewQoSInstance(logger, solanaServiceQoSConfig)
+			solanaQoS := solana.NewQoSInstance(qosLogger, solanaServiceQoSConfig)
 			qosServices[serviceID] = solanaQoS
 
-			logger.With("service_id", serviceID).Debug().Msg("Added Solana QoS instance for the service ID.")
+			hydratedLogger.With("service_id", serviceID).Debug().Msg("Added Solana QoS instance for the service ID.")
 		default:
 			return nil, fmt.Errorf("SHOULD NEVER HAPPEN: error building QoS instances: service ID %q not supported by PATH", serviceID)
 		}

metrics/qos/cometbft/metrics.go

@@ -0,0 +1,95 @@
+package cometbft
+
+import (
+	"fmt"
+
+	"github.com/pokt-network/poktroll/pkg/polylog"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/buildwithgrove/path/observation/qos"
+)
+
+const (
+	// The POSIX process that emits metrics
+	pathProcess = "path"
+
+	// The list of metrics being tracked for CometBFT QoS
+	requestsTotalMetric = "cometbft_requests_total"
+)
+
+func init() {
+	prometheus.MustRegister(requestsTotal)
+}
+
+var (
+	// TODO_MVP(@adshmh):
+	// - Add 'errorSubType' label for more granular error categorization
+	// - Use 'errorType' for broad error categories (e.g., request validation, protocol error)
+	// - Use 'errorSubType' for specifics (e.g., endpoint maxed out, timed out)
+	// - Remove 'success' label (success = absence of errorType)
+	// - Update EVM observations proto files and add interpreter support
+	//
+	// TODO_MVP(@adshmh):
+	// - Track endpoint responses separately from requests if/when retries are implemented
+	//   (A single request may generate multiple responses due to retries)
+	//
+	// requestsTotal tracks total CometBFT requests processed
+	//
+	// - Labels:
+	//   - chain_id: Target CometBFT chain identifier
+	//   - service_id: Service ID of the CometBFT QoS instance
+	//   - request_origin: origin of the request: User or Hydrator.
+	//   - request_method: CometBFT RPC method name (e.g., health, status)
+	//   - success: Whether a valid response was received
+	//   - error_type: Type of error if request failed (empty for success)
+	//   - http_status_code: HTTP status code returned to user
+	//
+	// - Use cases:
+	//   - Analyze request volume by chain and method
+	//   - Track success rates across PATH deployment regions
+	//   - Identify method usage patterns per chain
+	//   - Measure end-to-end request success rates
+	//   - Review error types by method and chain
+	//   - Examine HTTP status code distribution
+	requestsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem: pathProcess,
+			Name:      requestsTotalMetric,
+			Help:      "Total number of requests processed by CometBFT QoS instance(s)",
+		},
+		[]string{"chain_id", "service_id", "request_origin", "request_method", "success", "error_type", "http_status_code"},
+	)
+)
+
+// PublishMetrics:
+// - Exports all CometBFT-related Prometheus metrics using observations from CometBFT QoS service
+// - Logs errors for unexpected (should-never-happen) conditions
+func PublishMetrics(logger polylog.Logger, observations *qos.CometBFTRequestObservations) {
+	logger = logger.With("method", "PublishMetricsCometBFT")
+
+	// Skip if observations is nil.
+	// This should never happen as PublishQoSMetrics uses nil checks to identify which QoS service produced the observations.
+	if observations == nil {
+		logger.ProbabilisticDebugInfo(polylog.ProbabilisticDebugInfoProb).Msg("SHOULD RARELY HAPPEN: Unable to publish CometBFT metrics: received nil observations.")
+		return
+	}
+
+	// Create an interpreter for the observations
+	interpreter := &qos.CometBFTObservationInterpreter{
+		Logger:       logger,
+		Observations: observations,
+	}
+
+	// Increment request counters with all corresponding labels
+	requestsTotal.With(
+		prometheus.Labels{
+			"chain_id":         interpreter.GetChainID(),
+			"service_id":       interpreter.GetServiceID(),
+			"request_origin":   observations.GetRequestOrigin().String(),
+			"request_method":   interpreter.GetRequestMethod(),
+			"success":          fmt.Sprintf("%t", interpreter.IsRequestSuccessful()),
+			"error_type":       interpreter.GetRequestErrorType(),
+			"http_status_code": fmt.Sprintf("%d", interpreter.GetRequestHTTPStatus()),
+		},
+	).Inc()
+}