Skip to content

Commit 1d311ad

Browse files
peterarguepeterargue
authored
Merge pull request #591 from onflow/petera/add-profiler
Add profiler server for pprof
2 parents e7a47eb + 4487599 commit 1d311ad

File tree

5 files changed

+173
-35
lines changed

5 files changed

+173
-35
lines changed

‎Makefile‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,4 +45,14 @@ start:
4545
start-local:
4646
rm -rf db/
4747
rm -rf metrics/data/
48-
go run cmd/main/main.go --flow-network-id=flow-emulator --coinbase=FACF71692421039876a5BB4F10EF7A439D8ef61E --coa-address=f8d6e0586b0a20c7 --coa-key=2619878f0e2ff438d17835c2a4561cb87b4d24d72d12ec34569acd0dd4af7c21 --wallet-api-key=2619878f0e2ff438d17835c2a4561cb87b4d24d72d12ec34569acd0dd4af7c21 --coa-resource-create=true --gas-price=0 --log-writer=console
48+
go run cmd/main/main.go \
49+
--flow-network-id=flow-emulator \
50+
--coinbase=FACF71692421039876a5BB4F10EF7A439D8ef61E \
51+
--coa-address=f8d6e0586b0a20c7 \
52+
--coa-key=2619878f0e2ff438d17835c2a4561cb87b4d24d72d12ec34569acd0dd4af7c21 \
53+
--wallet-api-key=2619878f0e2ff438d17835c2a4561cb87b4d24d72d12ec34569acd0dd4af7c21 \
54+
--coa-resource-create=true \
55+
--gas-price=0 \
56+
--log-writer=console \
57+
--profiler-enabled=true \
58+
--profiler-port=6060

‎README.md‎

Lines changed: 56 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -172,40 +172,43 @@ Running the EVM gateway for mainnet requires additional security and stability m
172172

173173
The application can be configured using the following flags at runtime:
174174

175-
| Flag | Default Value | Description |
176-
|------------------------------|---------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|
177-
| `database-dir` | `./db` | Path to the directory for the database |
178-
| `rpc-host` | `""` | Host for the RPC API server |
179-
| `rpc-port` | `8545` | Port for the RPC API server |
180-
| `ws-enabled` | `false` | Enable websocket connections |
181-
| `access-node-grpc-host` | `localhost:3569` | Host to the flow access node gRPC API |
182-
| `access-node-spork-hosts` | `""` | Previous spork AN hosts, defined as a comma-separated list (e.g. `"host-1.com,host2.com"`) |
183-
| `evm-network-id` | `testnet ` | EVM network ID (options: `testnet`, `mainnet`) |
184-
| `flow-network-id` | `flow-emulator` | Flow network ID (options: `flow-emulator`, `flow-testnet`, `flow-mainnet`) |
185-
| `coinbase` | `""` | Coinbase address to use for fee collection |
186-
| `init-cadence-height` | `0` | Cadence block height to start indexing; avoid using on a new network |
187-
| `gas-price` | `1` | Static gas price for EVM transactions |
188-
| `coa-address` | `""` | Flow address holding COA account for submitting transactions |
189-
| `coa-key` | `""` | Private key for the COA address used for transactions |
190-
| `coa-key-file` | `""` | Path to a JSON file of COA keys for key-rotation (exclusive with `coa-key` flag) |
191-
| `coa-resource-create` | `false` | Auto-create the COA resource if it doesn't exist in the Flow COA account |
192-
| `coa-cloud-kms-project-id` | `""` | Project ID for KMS keys (e.g. `flow-evm-gateway`) |
193-
| `coa-cloud-kms-location-id` | `""` | Location ID for KMS key ring (e.g. 'global') |
194-
| `coa-cloud-kms-key-ring-id` | `""` | Key ring ID for KMS keys (e.g. 'tx-signing') |
195-
| `coa-cloud-kms-keys` | `""` | KMS keys and versions, comma-separated (e.g. `"gw-key-6@1,gw-key-7@1"`) |
196-
| `log-level` | `debug` | Log verbosity level (`debug`, `info`, `warn`, `error`, `fatal`, `panic`) |
197-
| `log-writer` | `stderr` | Output method for logs (`stderr`, `console`) |
198-
| `stream-limit` | `10` | Rate-limit for client events sent per second |
199-
| `rate-limit` | `50` | Requests per second limit for clients over any protocol (ws/http) |
200-
| `address-header` | `""` | Header for client IP when server is behind a proxy |
201-
| `heartbeat-interval` | `100` | Interval for AN event subscription heartbeats |
202-
| `stream-timeout` | `3` | Timeout in seconds for sending events to clients |
203-
| `force-start-height` | `0` | Force-set starting Cadence height (local/testing use only) |
204-
| `wallet-api-key` | `""` | ECDSA private key for wallet APIs (local/testing use only) |
205-
| `filter-expiry` | `5m` | Expiry time for idle filters |
206-
| `traces-gcp-bucket` | `""` | GCP bucket name for transaction traces |
207-
| `prometheus-config-file-path`| `./metrics/prometheus.yml` | Path to the Prometheus configuration file |
208-
| `index-only` | `false` | Run in index-only mode, allowing state queries and indexing but no transaction sending |
175+
| Flag | Default Value | Description |
176+
|------------------------------|---------------------------------|------------------------------------------------------------------------------------------|
177+
| `database-dir` | `./db` | Path to the directory for the database |
178+
| `rpc-host` | `""` | Host for the RPC API server |
179+
| `rpc-port` | `8545` | Port for the RPC API server |
180+
| `ws-enabled` | `false` | Enable websocket connections |
181+
| `access-node-grpc-host` | `localhost:3569` | Host to the flow access node gRPC API |
182+
| `access-node-spork-hosts` | `""` | Previous spork AN hosts, defined as a comma-separated list (e.g. `"host-1.com,host2.com"`) |
183+
| `evm-network-id` | `testnet ` | EVM network ID (options: `testnet`, `mainnet`) |
184+
| `flow-network-id` | `flow-emulator` | Flow network ID (options: `flow-emulator`, `flow-testnet`, `flow-mainnet`) |
185+
| `coinbase` | `""` | Coinbase address to use for fee collection |
186+
| `init-cadence-height` | `0` | Cadence block height to start indexing; avoid using on a new network |
187+
| `gas-price` | `1` | Static gas price for EVM transactions |
188+
| `coa-address` | `""` | Flow address holding COA account for submitting transactions |
189+
| `coa-key` | `""` | Private key for the COA address used for transactions |
190+
| `coa-key-file` | `""` | Path to a JSON file of COA keys for key-rotation (exclusive with `coa-key` flag) |
191+
| `coa-resource-create` | `false` | Auto-create the COA resource if it doesn't exist in the Flow COA account |
192+
| `coa-cloud-kms-project-id` | `""` | Project ID for KMS keys (e.g. `flow-evm-gateway`) |
193+
| `coa-cloud-kms-location-id` | `""` | Location ID for KMS key ring (e.g. 'global') |
194+
| `coa-cloud-kms-key-ring-id` | `""` | Key ring ID for KMS keys (e.g. 'tx-signing') |
195+
| `coa-cloud-kms-keys` | `""` | KMS keys and versions, comma-separated (e.g. `"gw-key-6@1,gw-key-7@1"`) |
196+
| `log-level` | `debug` | Log verbosity level (`debug`, `info`, `warn`, `error`, `fatal`, `panic`) |
197+
| `log-writer` | `stderr` | Output method for logs (`stderr`, `console`) |
198+
| `stream-limit` | `10` | Rate-limit for client events sent per second |
199+
| `rate-limit` | `50` | Requests per second limit for clients over any protocol (ws/http) |
200+
| `address-header` | `""` | Header for client IP when server is behind a proxy |
201+
| `heartbeat-interval` | `100` | Interval for AN event subscription heartbeats |
202+
| `stream-timeout` | `3` | Timeout in seconds for sending events to clients |
203+
| `force-start-height` | `0` | Force-set starting Cadence height (local/testing use only) |
204+
| `wallet-api-key` | `""` | ECDSA private key for wallet APIs (local/testing use only) |
205+
| `filter-expiry` | `5m` | Expiry time for idle filters |
206+
| `traces-gcp-bucket` | `""` | GCP bucket name for transaction traces |
207+
| `prometheus-config-file-path`| `./metrics/prometheus.yml` | Path to the Prometheus configuration file |
208+
| `index-only` | `false` | Run in index-only mode, allowing state queries and indexing but no transaction sending |
209+
| `profiler-enabled` | `false` | Enable the pprof profiler server |
210+
| `profiler-host` | `localhost` | Host for the pprof profiler |
211+
| `profiler-port` | `6060` | Port for the pprof profiler |
209212

210213
# Deploying
211214
Deploying the EVM Gateway node comes with some prerequisites as well as expectations and they are best explained in the WIP document: https://flowfoundation.notion.site/EVM-Gateway-Deployment-3c41da6710af40acbaf971e22ce0a9fd
@@ -252,6 +255,25 @@ The EVM Gateway implements APIs according to the Ethereum specification: https:/
252255
- Access Lists: we don't yet support creating access lists as they don't affect the fees we charge. We might support this in the future
253256
to optimize fees, but it currently is not part of our priorities.
254257

258+
# Debugging
259+
260+
## Profiler
261+
262+
The EVM Gateway supports profiling via the `pprof` package. To enable profiling, add the following flags to the command line:
263+
```
264+
--profiler-enabled=true
265+
--profiler-host=localhost
266+
--profiler-port=6060
267+
```
268+
269+
This will start a pprof server on the provided `host` and `port`. You can generate profiles using the following `go tool` commands
270+
```
271+
go tool pprof -http :2000 http://localhost:6060/debug/pprof/profile
272+
```
273+
```
274+
curl --output trace.out http://localhost:6060/debug/pprof/trace
275+
go tool trace -http :2001 trace.out
276+
```
255277

256278
# Contributing
257279
We welcome contributions from the community! Please read our [Contributing Guide](./CONTRIBUTING.md) for information on how to get involved.

‎api/profiler.go‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package api
2+
3+
import (
4+
"context"
5+
"errors"
6+
"net"
7+
"net/http"
8+
_ "net/http/pprof"
9+
"strconv"
10+
11+
"github.com/rs/zerolog"
12+
)
13+
14+
type ProfileServer struct {
15+
logger zerolog.Logger
16+
server *http.Server
17+
endpoint string
18+
}
19+
20+
func NewProfileServer(
21+
logger zerolog.Logger,
22+
host string,
23+
port int,
24+
) *ProfileServer {
25+
endpoint := net.JoinHostPort(host, strconv.Itoa(port))
26+
return &ProfileServer{
27+
logger: logger,
28+
server: &http.Server{Addr: endpoint},
29+
endpoint: endpoint,
30+
}
31+
}
32+
33+
func (h *ProfileServer) ListenAddr() string {
34+
return h.endpoint
35+
}
36+
37+
func (s *ProfileServer) Start() {
38+
go func() {
39+
err := s.server.ListenAndServe()
40+
if err != nil {
41+
if errors.Is(err, http.ErrServerClosed) {
42+
s.logger.Warn().Msg("Profiler server shutdown")
43+
return
44+
}
45+
s.logger.Err(err).Msg("failed to start Profiler server")
46+
panic(err)
47+
}
48+
}()
49+
}
50+
51+
func (s *ProfileServer) Stop() error {
52+
ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
53+
defer cancel()
54+
55+
return s.server.Shutdown(ctx)
56+
}
57+
58+
func (s *ProfileServer) Close() error {
59+
return s.server.Close()
60+
}

‎bootstrap/bootstrap.go‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ type Bootstrap struct {
5252
metrics *metrics.Server
5353
events *ingestion.Engine
5454
traces *traces.Engine
55+
profiler *api.ProfileServer
5556
}
5657

5758
func New(config *config.Config) (*Bootstrap, error) {
@@ -340,6 +341,38 @@ func (b *Bootstrap) StopMetricsServer() {
340341
b.metrics.Stop()
341342
}
342343

344+
func (b *Bootstrap) StartProfilerServer(_ context.Context) error {
345+
if !b.config.ProfilerEnabled {
346+
return nil
347+
}
348+
b.logger.Info().Msg("bootstrap starting profiler server")
349+
350+
b.profiler = api.NewProfileServer(b.logger, b.config.ProfilerHost, b.config.ProfilerPort)
351+
352+
b.profiler.Start()
353+
b.logger.Info().Msgf("Profiler server started: %s", b.profiler.ListenAddr())
354+
355+
return nil
356+
}
357+
358+
func (b *Bootstrap) StopProfilerServer() {
359+
if b.profiler == nil {
360+
return
361+
}
362+
363+
b.logger.Warn().Msg("shutting down profiler server")
364+
365+
err := b.profiler.Stop()
366+
if err != nil {
367+
if errors.Is(err, context.DeadlineExceeded) {
368+
b.logger.Warn().Msg("Profiler server graceful shutdown timed out")
369+
b.profiler.Close()
370+
} else {
371+
b.logger.Err(err).Msg("Profiler server graceful shutdown failed")
372+
}
373+
}
374+
}
375+
343376
// startEngine starts provided engine and panics if there are startup errors.
344377
func startEngine(
345378
ctx context.Context,
@@ -478,6 +511,10 @@ func Run(ctx context.Context, cfg *config.Config, ready chan struct{}) error {
478511
return fmt.Errorf("failed to start metrics server: %w", err)
479512
}
480513

514+
if err := boot.StartProfilerServer(ctx); err != nil {
515+
return fmt.Errorf("failed to start profiler server: %w", err)
516+
}
517+
481518
// mark ready
482519
close(ready)
483520

‎config/config.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,12 @@ type Config struct {
9797
IndexOnly bool
9898
// Cache size in units of items in cache, one unit in cache takes approximately 64 bytes
9999
CacheSize uint
100+
// ProfilerEnabled sets whether the profiler server is enabled
101+
ProfilerEnabled bool
102+
// ProfilerHost is the host for the profiler server will listen to (e.g. localhost, 0.0.0.0)
103+
ProfilerHost string
104+
// ProfilerPort is the port for the profiler server
105+
ProfilerPort int
100106
}
101107

102108
func FromFlags() (*Config, error) {
@@ -159,6 +165,9 @@ func FromFlags() (*Config, error) {
159165
flag.StringVar(&walletKey, "wallet-api-key", "", "ECDSA private key used for wallet APIs. WARNING: This should only be used locally or for testing, never in production.")
160166
flag.IntVar(&cfg.MetricsPort, "metrics-port", 9091, "Port for the metrics server")
161167
flag.BoolVar(&cfg.IndexOnly, "index-only", false, "Run the gateway in index-only mode which only allows querying the state and indexing, but disallows sending transactions.")
168+
flag.BoolVar(&cfg.ProfilerEnabled, "profiler-enabled", false, "Run the profiler server to capture pprof data.")
169+
flag.StringVar(&cfg.ProfilerHost, "profiler-host", "localhost", "Host for the Profiler server")
170+
flag.IntVar(&cfg.ProfilerPort, "profiler-port", 6060, "Port for the Profiler server")
162171
flag.Parse()
163172

164173
if coinbase == "" {

0 commit comments

Comments
 (0)