Skip to content

Commit 8b9f7f8

Browse files
OlshanskOlshansk
authored
[Session] Improve relay response metrics and timeout propogation (#341)
- Main process: better management of kill signal - Config fix: s/MaxRequestBodySize/MaxRequestHeaderBytes - Full Node: Extract the full node interface into its own file - Historgram: New historgram to track response sizes - Backup timeout for relay requests - Code health: A bit of misc cleanup across the board
1 parent e6891a5 commit 8b9f7f8

File tree

18 files changed

+305
-142
lines changed

18 files changed

+305
-142
lines changed

‎cmd/main.go‎

Lines changed: 38 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,11 @@ import (
66
"fmt"
77
"log"
88
"os"
9+
"os/signal"
910
"path/filepath"
1011
"strings"
12+
"syscall"
13+
"time"
1114

1215
"github.com/pokt-network/poktroll/pkg/polylog"
1316
"github.com/pokt-network/poktroll/pkg/polylog/polyzero"
@@ -39,45 +42,50 @@ func main() {
3942
// Initialize version metrics for Prometheus monitoring
4043
metrics.SetVersionInfo(Version, Commit, BuildDate)
4144

45+
// Get the config path
4246
configPath, err := getConfigPath(defaultConfigPath)
4347
if err != nil {
4448
log.Fatalf("failed to get config path: %v", err)
4549
}
4650

51+
// Load the config
4752
config, err := configpkg.LoadGatewayConfigFromYAML(configPath)
4853
if err != nil {
4954
log.Fatalf("failed to load config: %v", err)
5055
}
5156

57+
// Initialize the logger
5258
log.Printf("Initializing PATH logger with level: %s", config.Logger.Level)
53-
5459
loggerOpts := []polylog.LoggerOption{
5560
polyzero.WithLevel(polyzero.ParseLevel(config.Logger.Level)),
5661
}
57-
5862
logger := polyzero.NewLogger(loggerOpts...)
5963

64+
// Log the config path
6065
logger.Info().Msgf("Starting PATH using config file: %s", configPath)
6166

67+
// Create the protocol
6268
protocol, err := getProtocol(logger, config)
6369
if err != nil {
6470
log.Fatalf("failed to create protocol: %v", err)
6571
}
6672

73+
// Prepare the QoS instances
6774
qosInstances, err := getServiceQoSInstances(logger, config, protocol)
6875
if err != nil {
6976
log.Fatalf("failed to setup QoS instances: %v", err)
7077
}
7178

72-
// setup metrics reporter, to be used by Gateway and Hydrator.
79+
// Setup metrics reporter, to be used by Gateway and Hydrator
7380
metricsReporter, err := setupMetricsServer(logger, prometheusMetricsServerAddr)
7481
if err != nil {
7582
log.Fatalf("failed to start metrics server: %v", err)
7683
}
7784

85+
// Setup the pprof server
7886
setupPprofServer(context.TODO(), logger, pprofAddr)
7987

80-
// setup data reporter, to be used by Gateway and Hydrator.
88+
// Setup the data reporter
8189
dataReporter, err := setupHTTPDataReporter(logger, config.DataReporterConfig)
8290
if err != nil {
8391
log.Fatalf("failed to start the configured HTTP data reporter: %v", err)
@@ -98,10 +106,9 @@ func main() {
98106
log.Fatalf("failed to setup endpoint hydrator: %v", err)
99107
}
100108

101-
// setup the request parser which maps requests to the correct QoS instance.
109+
// Setup the request parser which maps requests to the correct QoS instance.
102110
requestParser := &request.Parser{
103-
Logger: logger,
104-
111+
Logger: logger,
105112
QoSServices: qosInstances,
106113
}
107114

@@ -151,22 +158,39 @@ func main() {
151158
config.GetRouterConfig(),
152159
)
153160

154-
// -------------------- Start PATH API Router -------------------- */
161+
// -------------------- Log PATH Startup Info --------------------
155162

156-
// Log out some basic info about the running PATH instance.
163+
// Log out some basic info about the running PATH instance
157164
configuredServiceIDs := make([]string, 0, len(protocol.ConfiguredServiceIDs()))
158165
for serviceID := range protocol.ConfiguredServiceIDs() {
159166
configuredServiceIDs = append(configuredServiceIDs, string(serviceID))
160167
}
161-
// log.Printf is used here to ensure this info is printed to the console regardless of the log level.
162-
log.Printf("🌿 PATH gateway started.\n Port: %d\n Protocol: %s\n Configured Service IDs: %s",
168+
logger.Info().Msgf("🌿 PATH gateway starting on port %d for Protocol: %s with Configured Service IDs: %s",
163169
config.GetRouterConfig().Port, protocol.Name(), strings.Join(configuredServiceIDs, ", "))
164170

165-
// Start the API router.
171+
// -------------------- Start PATH API Router --------------------
172+
166173
// This will block until the router is stopped.
167-
if err := apiRouter.Start(); err != nil {
168-
log.Fatalf("failed to start API router: %v", err)
174+
server, err := apiRouter.Start()
175+
if err != nil {
176+
logger.Error().Err(err).Msg("failed to start PATH API router")
169177
}
178+
179+
// -------------------- PATH Shutdown --------------------
180+
stop := make(chan os.Signal, 1)
181+
signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)
182+
<-stop
183+
184+
logger.Info().Msg("Shutting down PATH...")
185+
186+
// TODO_IMPROVE: Make shutdown timeout configurable and add graceful shutdown of dependencies
187+
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
188+
defer cancel()
189+
if err := server.Shutdown(ctx); err != nil {
190+
logger.Error().Err(err).Msg("PATH forced to shutdown")
191+
}
192+
193+
logger.Info().Msg("PATH exited properly")
170194
}
171195

172196
/* -------------------- Gateway Init Helpers -------------------- */

‎config/config.schema.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -212,8 +212,8 @@ properties:
212212
port:
213213
description: "Port number for the router."
214214
type: integer
215-
max_request_body_size:
216-
description: "Maximum size of the request body."
215+
max_request_header_bytes:
216+
description: "Maximum size of the request header."
217217
type: integer
218218
read_timeout:
219219
description: "Read timeout duration for the router."

‎config/config_test.go‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,11 @@ func Test_LoadGatewayConfigFromYAML(t *testing.T) {
4545
},
4646
},
4747
Router: RouterConfig{
48-
Port: defaultPort,
49-
MaxRequestBodySize: defaultMaxRequestBodySize,
50-
ReadTimeout: defaultHTTPServerReadTimeout,
51-
WriteTimeout: defaultHTTPServerWriteTimeout,
52-
IdleTimeout: defaultHTTPServerIdleTimeout,
48+
Port: defaultPort,
49+
MaxRequestHeaderBytes: defaultMaxRequestHeaderBytes,
50+
ReadTimeout: defaultHTTPServerReadTimeout,
51+
WriteTimeout: defaultHTTPServerWriteTimeout,
52+
IdleTimeout: defaultHTTPServerIdleTimeout,
5353
},
5454
Logger: LoggerConfig{
5555
Level: defaultLogLevel,
@@ -82,11 +82,11 @@ func Test_LoadGatewayConfigFromYAML(t *testing.T) {
8282
},
8383
},
8484
Router: RouterConfig{
85-
Port: defaultPort,
86-
MaxRequestBodySize: defaultMaxRequestBodySize,
87-
ReadTimeout: defaultHTTPServerReadTimeout,
88-
WriteTimeout: defaultHTTPServerWriteTimeout,
89-
IdleTimeout: defaultHTTPServerIdleTimeout,
85+
Port: defaultPort,
86+
MaxRequestHeaderBytes: defaultMaxRequestHeaderBytes,
87+
ReadTimeout: defaultHTTPServerReadTimeout,
88+
WriteTimeout: defaultHTTPServerWriteTimeout,
89+
IdleTimeout: defaultHTTPServerIdleTimeout,
9090
},
9191
Logger: LoggerConfig{
9292
Level: defaultLogLevel,
@@ -214,11 +214,11 @@ logger_config:
214214
},
215215
},
216216
Router: RouterConfig{
217-
Port: defaultPort,
218-
MaxRequestBodySize: defaultMaxRequestBodySize,
219-
ReadTimeout: defaultHTTPServerReadTimeout,
220-
WriteTimeout: defaultHTTPServerWriteTimeout,
221-
IdleTimeout: defaultHTTPServerIdleTimeout,
217+
Port: defaultPort,
218+
MaxRequestHeaderBytes: defaultMaxRequestHeaderBytes,
219+
ReadTimeout: defaultHTTPServerReadTimeout,
220+
WriteTimeout: defaultHTTPServerWriteTimeout,
221+
IdleTimeout: defaultHTTPServerIdleTimeout,
222222
},
223223
Logger: LoggerConfig{
224224
Level: "debug",

‎config/router.go‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,21 +6,33 @@ import (
66

77
/* --------------------------------- Router Config Defaults -------------------------------- */
88

9+
// TODO_IMPROVE: Make all of these configurable for PATH users
910
const (
10-
defaultPort = 3069
11-
defaultMaxRequestBodySize = 1 << 20 // 1 MB
11+
// default PATH port
12+
defaultPort = 3069
13+
14+
// defaultMaxRequestHeaderBytes is the default maximum size of the HTTP request header.
15+
defaultMaxRequestHeaderBytes = 2 * 1e6 // 2 MB
16+
17+
// HTTP server's default timeout values.
18+
defaultHTTPServerReadTimeout = 10 * time.Second
19+
defaultHTTPServerIdleTimeout = 120 * time.Second
20+
21+
// HTTP request handler's WriteTimeout.
22+
// https://pkg.go.dev/net/http#Server
23+
defaultHTTPServerWriteTimeout = 20 * time.Second
1224
)
1325

1426
/* --------------------------------- Router Config Struct -------------------------------- */
1527

1628
// RouterConfig contains server configuration settings.
1729
// See default values above.
1830
type RouterConfig struct {
19-
Port int `yaml:"port"`
20-
MaxRequestBodySize int `yaml:"max_request_body_size"`
21-
ReadTimeout time.Duration `yaml:"read_timeout"`
22-
WriteTimeout time.Duration `yaml:"write_timeout"`
23-
IdleTimeout time.Duration `yaml:"idle_timeout"`
31+
Port int `yaml:"port"`
32+
MaxRequestHeaderBytes int `yaml:"max_request_header_bytes"`
33+
ReadTimeout time.Duration `yaml:"read_timeout"`
34+
WriteTimeout time.Duration `yaml:"write_timeout"`
35+
IdleTimeout time.Duration `yaml:"idle_timeout"`
2436
}
2537

2638
/* --------------------------------- Router Config Private Helpers -------------------------------- */
@@ -30,8 +42,8 @@ func (c *RouterConfig) hydrateRouterDefaults() {
3042
if c.Port == 0 {
3143
c.Port = defaultPort
3244
}
33-
if c.MaxRequestBodySize == 0 {
34-
c.MaxRequestBodySize = defaultMaxRequestBodySize
45+
if c.MaxRequestHeaderBytes == 0 {
46+
c.MaxRequestHeaderBytes = defaultMaxRequestHeaderBytes
3547
}
3648
if c.ReadTimeout == 0 {
3749
c.ReadTimeout = defaultHTTPServerReadTimeout

‎config/router_test.go‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -59,11 +59,11 @@ func TestRouterConfig_hydrateRouterDefaults(t *testing.T) {
5959
name: "should set all defaults",
6060
cfg: RouterConfig{},
6161
want: RouterConfig{
62-
Port: defaultPort,
63-
MaxRequestBodySize: defaultMaxRequestBodySize,
64-
ReadTimeout: defaultHTTPServerReadTimeout,
65-
WriteTimeout: defaultHTTPServerWriteTimeout,
66-
IdleTimeout: defaultHTTPServerIdleTimeout,
62+
Port: defaultPort,
63+
MaxRequestHeaderBytes: defaultMaxRequestHeaderBytes,
64+
ReadTimeout: defaultHTTPServerReadTimeout,
65+
WriteTimeout: defaultHTTPServerWriteTimeout,
66+
IdleTimeout: defaultHTTPServerIdleTimeout,
6767
},
6868
},
6969
{
@@ -72,11 +72,11 @@ func TestRouterConfig_hydrateRouterDefaults(t *testing.T) {
7272
Port: 8080,
7373
},
7474
want: RouterConfig{
75-
Port: 8080,
76-
MaxRequestBodySize: defaultMaxRequestBodySize,
77-
ReadTimeout: defaultHTTPServerReadTimeout,
78-
WriteTimeout: defaultHTTPServerWriteTimeout,
79-
IdleTimeout: defaultHTTPServerIdleTimeout,
75+
Port: 8080,
76+
MaxRequestHeaderBytes: defaultMaxRequestHeaderBytes,
77+
ReadTimeout: defaultHTTPServerReadTimeout,
78+
WriteTimeout: defaultHTTPServerWriteTimeout,
79+
IdleTimeout: defaultHTTPServerIdleTimeout,
8080
},
8181
},
8282
}

‎config/timeout.go‎

Lines changed: 0 additions & 15 deletions
This file was deleted.

‎docusaurus/docs/develop/path/5_configurations_path.md‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -264,13 +264,13 @@ hydrator_config:
264264

265265
In particular, allows specifying server parameters for how the gateway handles incoming requests.
266266

267-
| Field | Type | Required | Default | Description |
268-
| ----------------------- | ------- | -------- | ----------------- | ----------------------------------------------- |
269-
| `port` | integer | No | 3070 | Port number on which the gateway server listens |
270-
| `max_request_body_size` | integer | No | 1MB | Maximum request size in bytes |
271-
| `read_timeout` | string | No | "5000ms" (5s) | Time limit for reading request data |
272-
| `write_timeout` | string | No | "10000ms" (10s) | Time limit for writing response data |
273-
| `idle_timeout` | string | No | "120000ms" (120s) | Time limit for closing idle connections |
267+
| Field | Type | Required | Default | Description |
268+
| -------------------------- | ------- | -------- | ----------------- | ----------------------------------------------- |
269+
| `port` | integer | No | 3069 | Port number on which the gateway server listens |
270+
| `max_request_header_bytes` | integer | No | 1MB | Maximum request header size in bytes |
271+
| `read_timeout` | string | No | "5000ms" (5s) | Time limit for reading request data |
272+
| `write_timeout` | string | No | "10000ms" (10s) | Time limit for writing response data |
273+
| `idle_timeout` | string | No | "120000ms" (120s) | Time limit for closing idle connections |
274274

275275
---
276276

@@ -299,10 +299,10 @@ data_reporter_config:
299299
post_timeout_ms: 5000
300300
```
301301

302-
| Field | Type | Required | Default | Description |
303-
| ---------------- | ------- | -------- | ------- | --------------------------------------------------------------------------------------------------- |
304-
| `target_url` | string | Yes | - | HTTP endpoint URL where data will be reported (must start with http:// or https://) |
305-
| `post_timeout_ms`| integer | No | 10000 | Timeout in milliseconds for HTTP POST operations. If zero or negative, default of 10000ms is used |
302+
| Field | Type | Required | Default | Description |
303+
| ----------------- | ------- | -------- | ------- | ------------------------------------------------------------------------------------------------- |
304+
| `target_url` | string | Yes | - | HTTP endpoint URL where data will be reported (must start with http:// or https://) |
305+
| `post_timeout_ms` | integer | No | 10000 | Timeout in milliseconds for HTTP POST operations. If zero or negative, default of 10000ms is used |
306306

307307
:::info
308308
Currently, only JSON-accepting data pipelines are supported as of PR #215.

‎gateway/hydrator.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,8 @@ func (eph *EndpointHydrator) performChecks(serviceID protocol.ServiceID, service
186186

187187
// Prepare a request context to submit a synthetic relay request to the endpoint on behalf of the gateway for QoS purposes.
188188
gatewayRequestCtx := requestContext{
189-
logger: endpointLogger,
189+
logger: endpointLogger,
190+
context: context.TODO(),
190191
// TODO_MVP(@adshmh): populate the fields of gatewayObservations struct.
191192
// Mark the request as Synthetic using the following steps:
192193
// 1. Define a `gatewayObserver` function as a field in the `requestContext` struct.
@@ -202,7 +203,6 @@ func (eph *EndpointHydrator) performChecks(serviceID protocol.ServiceID, service
202203
metricsReporter: eph.MetricsReporter,
203204
// data reporter for exporting data on hydrator service requests to the data pipeline.
204205
dataReporter: eph.DataReporter,
205-
context: context.TODO(),
206206
}
207207

208208
err = gatewayRequestCtx.HandleRelayRequest()

0 commit comments

Comments
 (0)