feat: enhanced retry system with endpoint rotation and latency budget

Implemented comprehensive retry system improvements:

- Retry endpoint rotation: never reuse same endpoint on retry, select new endpoint following QoS/reputation rules
- Latency budget: configurable max_retry_latency (default 500ms) to prevent retrying slow requests
- Concurrency configuration: made max_parallel_endpoints, max_concurrent_relays, and max_batch_payloads configurable via YAML
- Endpoint exhaustion handling: exponential backoff (100ms, 200ms, 400ms) when all endpoints tried

Bug fixes:
- Fixed empty response handling in retry loops
- Fixed attempt number in retry metrics (removed incorrect +1)
- Fixed break/return in select statements
- Fixed empty endpoint domain in metrics recording
- Fixed Redis test to use testcontainer address
- Added missing error storage before loop breaks
- Enhanced context cancellation logging

Metrics improvements:
- Added endpoint_domain label to retry_latency metric
- Added retry_reason label to retry_budget_skipped metric
- New endpoint rotation metrics: shannon_retry_endpoint_switches_total, shannon_retry_endpoint_exhaustion_total
- All metrics follow lowercase_underscore naming convention

Configuration:
- Updated config schema with max_retry_latency and concurrency_config
- Added comprehensive examples in config.shannon_example.yaml
- Per-service override support for all retry and concurrency settings

Testing:
- Fixed Redis test conflicts by using testcontainer addresses
- Updated test mocks with GetConcurrencyConfig()
- E2E tests passing at 99.33% success rate (298/300 requests)

Author: jorgecuesta

cmd/extractor_factory.go

@@ -39,7 +39,7 @@ func buildExtractorRegistry(unifiedConfig *gateway.UnifiedServicesConfig) *qosty
 			registry.Register(serviceID, cosmosExtractor)
 		case gateway.ServiceTypeSolana:
 			registry.Register(serviceID, solanaExtractor)
-		// Default: falls back to NoOpDataExtractor via registry.Get()
+			// Default: falls back to NoOpDataExtractor via registry.Get()
 		}
 	}
 

cmd/qos.go

@@ -110,4 +110,3 @@ func logGatewayServiceIDs(logger polylog.Logger, serviceConfigs map[protocol.Ser
 	}
 	logger.Info().Msgf("Service IDs configured by the gateway: %s.", strings.Join(serviceIDs, ", "))
 }
-

config/config.schema.yaml

@@ -312,6 +312,11 @@ properties:
             description: "Retry on connection errors."
             type: boolean
             default: true
+          max_retry_latency:
+            description: "Maximum latency budget for retries. Only retry if failed request took less than this duration. Prevents retrying requests that already consumed significant time."
+            type: string
+            pattern: "^[0-9]+m?s$"
+            default: "500ms"
 
       # Observation Pipeline Configuration
       observation_pipeline:
@@ -497,45 +502,18 @@ properties:
               minimum: 0.0
               maximum: 1.0
 
-      # Service Defaults - Settings inherited by all services
-      defaults:
-        description: "Default settings inherited by all services. Per-service overrides only need to specify differences."
-        type: object
-        additionalProperties: false
-        properties:
-          type:
-            description: "Default QoS type. Options: evm, solana, cosmos, generic, passthrough"
-            type: string
-            enum: ["evm", "solana", "cosmos", "generic", "passthrough"]
-            default: "passthrough"
-          rpc_types:
-            description: "Default supported RPC types."
-            type: array
-            items:
-              type: string
-              enum: ["json_rpc", "rest", "websocket", "comet_bft", "grpc"]
-          latency_profile:
-            description: "Default latency profile name (references latency_profiles or built-in)."
-            type: string
-            default: "standard"
-          reputation_config:
-            $ref: "#/definitions/service_reputation_config"
-          latency:
-            $ref: "#/definitions/service_latency_config"
-          tiered_selection:
-            $ref: "#/definitions/service_tiered_selection_config"
-          probation:
-            $ref: "#/definitions/service_probation_config"
-          retry_config:
-            $ref: "#/definitions/service_retry_config"
-          observation_pipeline:
-            $ref: "#/definitions/service_observation_config"
-          active_health_checks:
-            $ref: "#/definitions/service_health_check_override"
+      # Note: Service defaults are inherited from gateway_config top-level settings:
+      # - reputation_config.tiered_selection -> default tiered selection
+      # - reputation_config.tiered_selection.probation -> default probation
+      # - retry_config -> default retry settings
+      # - observation_pipeline -> default observation settings
+      # - active_health_checks -> default health check settings
+      #
+      # Services only need to specify overrides in the services[] array.
 
       # Services - Array of per-service configurations
       services:
-        description: "List of configured services. Each service inherits from defaults unless explicitly overridden."
+        description: "List of configured services. Each service inherits from gateway_config settings unless explicitly overridden."
         type: array
         uniqueItems: true
         items:
@@ -572,6 +550,8 @@ properties:
               $ref: "#/definitions/service_retry_config"
             observation_pipeline:
               $ref: "#/definitions/service_observation_config"
+            concurrency_config:
+              $ref: "#/definitions/service_concurrency_config"
             fallback:
               description: "Fallback endpoint configuration (no defaults - must be explicitly set per-service)."
               type: object
@@ -635,6 +615,31 @@ properties:
         description: "Buffer size for websocket messages."
         type: integer
 
+  # Concurrency Configuration (optional)
+  concurrency_config:
+    description: "Optional configuration for controlling concurrency limits in request processing. These limits protect against resource exhaustion from batch requests and parallel relays."
+    type: object
+    additionalProperties: false
+    properties:
+      max_parallel_endpoints:
+        description: "Maximum number of endpoints to query in parallel per request. Higher values reduce latency but increase load. Range: 1-10."
+        type: integer
+        minimum: 1
+        maximum: 10
+        default: 1
+      max_concurrent_relays:
+        description: "Global limit on concurrent relay goroutines across all requests. Prevents resource exhaustion from too many simultaneous relays. Range: 100-10000."
+        type: integer
+        minimum: 100
+        maximum: 10000
+        default: 5500
+      max_batch_payloads:
+        description: "Maximum number of payloads allowed in a batch request. Must be less than or equal to max_concurrent_relays. Range: 1-10000."
+        type: integer
+        minimum: 1
+        maximum: 10000
+        default: 5500
+
   # Hydrator Configuration (optional)
   hydrator_config:
     description: "Configuration for the hydrator, which is used to run QoS checks against endpoints of a service."
@@ -832,6 +837,10 @@ definitions:
       retry_on_connection:
         description: "Retry on connection errors."
         type: boolean
+      max_retry_latency:
+        description: "Maximum latency budget for retries."
+        type: string
+        pattern: "^[0-9]+m?s$"
 
   # Per-service observation pipeline configuration
   # Note: worker_count and queue_size are GLOBAL only (gateway_config.observation_pipeline)
@@ -849,6 +858,24 @@ definitions:
         minimum: 0.0
         maximum: 1.0
 
+  # Per-service concurrency configuration
+  # Note: max_concurrent_relays is GLOBAL only (cannot be overridden per-service)
+  service_concurrency_config:
+    description: "Per-service concurrency configuration. Allows fine-tuning parallel execution and batch limits per service."
+    type: object
+    additionalProperties: false
+    properties:
+      max_parallel_endpoints:
+        description: "Maximum endpoints to query in parallel for this service. Use >1 for unreliable services to reduce latency. Range: 1-10."
+        type: integer
+        minimum: 1
+        maximum: 10
+      max_batch_payloads:
+        description: "Maximum payloads in a batch request for this service. Lower for heavy services, higher for light services. Range: 1-10000."
+        type: integer
+        minimum: 1
+        maximum: 10000
+
   # Per-service health check configuration
   service_health_check_override:
     description: "Per-service health check configuration."

config/config_test.go

@@ -39,7 +39,7 @@ func Test_LoadGatewayConfigFromYAML(t *testing.T) {
 			name:        "should load valid config from example file",
 			filePath:    "./examples/config.shannon_example.yaml",
 			skipCompare: true, // Example config is a reference doc, not a test fixture
-			want:        GatewayConfig{
+			want: GatewayConfig{
 				FullNodeConfig: shannonprotocol.FullNodeConfig{
 					RpcURL:                "https://shannon-grove-rpc.mainnet.poktroll.com",
 					SessionRolloverBlocks: 10,

config/examples/config.shannon_example.yaml

@@ -9,7 +9,7 @@
 # 2. Full node connection settings
 # 3. Gateway settings with unified service configuration
 #
-# Services inherit from `defaults` and can override any setting.
+# Services inherit from gateway_config settings and can override any setting.
 # Only specify what differs from defaults to keep configs clean.
 
 # =============================================================================
@@ -47,6 +47,13 @@ data_reporter_config:
 logger_config:
   level: "info"
 
+# Concurrency Configuration (optional)
+# Controls parallel request processing and batch limits
+concurrency_config:
+  max_parallel_endpoints: 1      # How many endpoints to query in parallel per request (1-10)
+  max_concurrent_relays: 5500    # Global limit on concurrent relay goroutines (100-10000)
+  max_batch_payloads: 5500       # Max payloads in batch request (1-10000, ≤ max_concurrent_relays)
+
 # =============================================================================
 # FULL NODE CONFIGURATION
 # =============================================================================
@@ -96,7 +103,8 @@ gateway_config:
   # ===========================================================================
   # GLOBAL REPUTATION CONFIGURATION
   # ===========================================================================
-  # These settings apply globally and CANNOT be overridden per-service
+  # These settings apply globally and serve as defaults for all services.
+  # Per-service overrides can be specified in the services[] array.
 
   reputation_config:
     # Enable/disable the entire reputation system
@@ -109,7 +117,7 @@ gateway_config:
     # NOTE: This is GLOBAL ONLY - cannot be overridden per-service
     storage_type: "memory"
 
-    # Global initial score (can be overridden per-service in `defaults` or `services`)
+    # Global initial score (can be overridden per-service)
     initial_score: 80
 
     # Global minimum threshold (can be overridden per-service)
@@ -118,14 +126,30 @@ gateway_config:
     # Time before inactive low-scoring endpoints can recover
     recovery_timeout: 5m
 
+    # Tiered endpoint selection
+    # Cascade: tier1 first, then tier2, then tier3
+    tiered_selection:
+      enabled: true
+      tier1_threshold: 70    # Premium tier (highest priority)
+      tier2_threshold: 50    # Good tier
+
+      # Probation system for recovering endpoints
+      # Low-scoring endpoints get limited traffic to prove reliability
+      probation:
+        enabled: true
+        threshold: 10          # Score below which endpoint enters probation
+        traffic_percent: 10    # % of traffic routed to probation endpoints
+        recovery_multiplier: 2.0  # Boost for successful probation requests
+
   # ===========================================================================
   # GLOBAL RETRY CONFIGURATION (optional)
   # ===========================================================================
-  # Global retry settings - use `defaults` or `services` for per-service control
+  # Global retry settings - per-service overrides in services[] array
 
   retry_config:
     enabled: true
     max_retries: 1
+    max_retry_latency: 500ms  # Only retry if failed request took < 500ms
     retry_on_5xx: true
     retry_on_timeout: true
     retry_on_connection: true
@@ -145,7 +169,7 @@ gateway_config:
   # GLOBAL ACTIVE HEALTH CHECKS
   # ===========================================================================
   # Proactive endpoint monitoring - runs health checks on all endpoints
-  # Use `defaults` or `services` for per-service health check rules
+  # Per-service health check rules can be defined in services[].health_checks
 
   active_health_checks:
     enabled: true
@@ -254,87 +278,22 @@ gateway_config:
       slow_penalty: 0.7
       very_slow_penalty: 0.3
 
-  # ===========================================================================
-  # SERVICE DEFAULTS
-  # ===========================================================================
-  # Settings inherited by ALL services unless overridden per-service
-  # Only specify what you want as default behavior
-
-  defaults:
-    # QoS type determines how requests are validated and processed
-    # Options: evm, solana, cosmos, generic, passthrough
-    type: passthrough
-
-    # Supported RPC types for endpoints
-    # Options: json_rpc, rest, websocket, comet_bft, grpc
-    rpc_types:
-      - json_rpc
-
-    # Reference to a latency profile (from latency_profiles or built-in)
-    latency_profile: "standard"
-
-    # Per-service reputation overrides
-    # NOTE: storage_type is GLOBAL ONLY (set in gateway_config.reputation_config)
-    reputation_config:
-      enabled: true
-      initial_score: 70      # Default starting score
-      min_threshold: 40      # Default minimum for selection
-      recovery_timeout: 5m   # Time before recovery attempt
-      # key_granularity: "per-endpoint"  # per-endpoint, per-domain, per-supplier
-
-    # Inline latency config (alternative to latency_profile)
-    # If target_ms > 0, this OVERRIDES the latency_profile
-    latency:
-      enabled: true
-      target_ms: 0           # 0 means use latency_profile instead
-      penalty_weight: 0.3    # How much latency affects scoring (0.0-1.0)
-
-    # Tiered endpoint selection
-    # Cascade: tier1 first, then tier2, then tier3
-    tiered_selection:
-      enabled: true
-      tier1_threshold: 70    # Premium tier (highest priority)
-      tier2_threshold: 50    # Good tier
-
-    # Probation system for recovering endpoints
-    # Low-scoring endpoints get limited traffic to prove reliability
-    probation:
-      enabled: true
-      threshold: 10          # Score below which endpoint enters probation
-      traffic_percent: 10    # % of traffic routed to probation endpoints
-      recovery_multiplier: 2.0  # Boost for successful probation requests
-
-    # Retry configuration
-    retry_config:
-      enabled: true
-      max_retries: 1
-      retry_on_5xx: true
-      retry_on_timeout: true
-      retry_on_connection: true
-
-    # Observation pipeline per-service settings
-    # NOTE: worker_count and queue_size are GLOBAL ONLY
-    observation_pipeline:
-      enabled: true
-      sample_rate: 0.1       # Per-service sample rate (this WORKS)
-      # worker_count: 4      # GLOBAL ONLY - per-service value ignored
-      # queue_size: 1000     # GLOBAL ONLY - per-service value ignored
-
-    # Health check defaults
-    active_health_checks:
-      enabled: true
-      interval: 30s
-      sync_allowance: 5      # Blocks behind latest before considered out-of-sync
-      external:              # Per-service external URL (overrides global)
-        url: ""
-        refresh_interval: "1h"
-        timeout: "30s"
-      local: []              # Per-service local rules (override external by name)
-
   # ===========================================================================
   # SERVICE CONFIGURATIONS
   # ===========================================================================
-  # Define services and their overrides. Only specify what differs from defaults.
+  # Define services with their per-service overrides.
+  # Each service inherits from gateway_config settings and can override:
+  # - type: QoS type (evm, solana, cosmos, generic, passthrough)
+  # - rpc_types: Supported RPC types
+  # - latency_profile: Reference to a named profile
+  # - reputation_config: Per-service reputation overrides
+  # - tiered_selection: Per-service tier thresholds
+  # - probation: Per-service probation settings
+  # - retry_config: Per-service retry settings
+  # - concurrency_config: Per-service concurrency overrides (max_parallel_endpoints, max_batch_payloads)
+  # - observation_pipeline: Per-service sample rate
+  # - fallback: Fallback endpoints (no defaults - must be explicitly configured)
+  # - health_checks: Per-service health check rules
 
   services:
     # -------------------------------------------------------------------------
@@ -505,3 +464,15 @@ gateway_config:
       latency_profile: "slow"
       retry_config:
         max_retries: 2
+
+      # Per-service concurrency overrides (optional)
+      # Use these to override global concurrency_config for specific services
+      # concurrency_config:
+      #   # max_parallel_endpoints: Race multiple endpoints in parallel (1-10)
+      #   # Use >1 for unreliable services to get faster responses
+      #   # ⚠️ WARNING: Values >1 multiply token burn (e.g., 3 endpoints = 3x cost)
+      #   max_parallel_endpoints: 3
+      #
+      #   # max_batch_payloads: Limit batch size for this service (1-10000)
+      #   # Useful for heavy services that process large batches
+      #   max_batch_payloads: 100