Skip to content

Commit 5b8d549

Browse files
authored
Merge pull request #760 from onflow/mpeter/make-ingestion-idempotent
Make ingestion of EVM events idempotent
2 parents 7274b59 + 55713b8 commit 5b8d549

File tree

9 files changed

+315
-144
lines changed

9 files changed

+315
-144
lines changed

api/debug.go

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -419,18 +419,11 @@ func (d *DebugAPI) traceBlockByNumber(
419419
}
420420

421421
func (d *DebugAPI) executorAtBlock(block *models.Block) (*evm.BlockExecutor, error) {
422-
previousBlock, err := d.blocks.GetByHeight(block.Height - 1)
423-
if err != nil {
424-
return nil, err
425-
}
426-
427-
// We need to re-execute all the transactions from the given block,
428-
// on top of the previous block state, to generate the correct traces.
429-
snapshot, err := d.registerStore.GetSnapshotAt(previousBlock.Height)
422+
snapshot, err := d.registerStore.GetSnapshotAt(block.Height)
430423
if err != nil {
431424
return nil, fmt.Errorf(
432425
"failed to get register snapshot at block height %d: %w",
433-
previousBlock.Height,
426+
block.Height,
434427
err,
435428
)
436429
}

bootstrap/bootstrap.go

Lines changed: 37 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ func (b *Bootstrap) StopEventIngestion() {
207207
}
208208

209209
func (b *Bootstrap) StartAPIServer(ctx context.Context) error {
210-
b.logger.Info().Msg("bootstrap starting metrics server")
210+
b.logger.Info().Msg("bootstrap starting API server")
211211

212212
b.server = api.NewServer(b.logger, b.collector, b.config)
213213

@@ -372,7 +372,6 @@ func (b *Bootstrap) StopAPIServer() {
372372
}
373373

374374
func (b *Bootstrap) StartMetricsServer(ctx context.Context) error {
375-
376375
b.metrics = newMetricsWrapper(b.logger, b.config.MetricsPort)
377376
return b.metrics.Start(ctx)
378377
}
@@ -639,45 +638,62 @@ func setupStorage(
639638
}, nil
640639
}
641640

642-
// Run will run complete bootstrap of the EVM gateway with all the engines.
643-
// Run is a blocking call, but it does signal readiness of the service
644-
// through a channel provided as an argument.
645-
func Run(ctx context.Context, cfg config.Config, ready component.ReadyFunc) error {
646-
boot, err := New(cfg)
647-
if err != nil {
648-
return err
649-
}
650-
641+
func (b *Bootstrap) Run(
642+
ctx context.Context,
643+
cfg config.Config,
644+
ready component.ReadyFunc,
645+
) error {
651646
// Start the API Server first, to avoid any races with incoming
652647
// EVM events, that might affect the starting state.
653-
if err := boot.StartAPIServer(ctx); err != nil {
648+
if err := b.StartAPIServer(ctx); err != nil {
654649
return fmt.Errorf("failed to start API server: %w", err)
655650
}
656651

657-
if err := boot.StartEventIngestion(ctx); err != nil {
652+
if err := b.StartEventIngestion(ctx); err != nil {
658653
return fmt.Errorf("failed to start event ingestion engine: %w", err)
659654
}
660655

661-
if err := boot.StartMetricsServer(ctx); err != nil {
656+
if err := b.StartMetricsServer(ctx); err != nil {
662657
return fmt.Errorf("failed to start metrics server: %w", err)
663658
}
664659

665-
if err := boot.StartProfilerServer(ctx); err != nil {
660+
if err := b.StartProfilerServer(ctx); err != nil {
666661
return fmt.Errorf("failed to start profiler server: %w", err)
667662
}
668663

669664
// mark ready
670665
ready()
671666

667+
return nil
668+
}
669+
670+
func (b *Bootstrap) Stop() {
671+
b.logger.Info().Msg("bootstrap received context cancellation, stopping services")
672+
673+
b.StopEventIngestion()
674+
b.StopMetricsServer()
675+
b.StopAPIServer()
676+
b.StopClient()
677+
b.StopDB()
678+
}
679+
680+
// Run will run complete bootstrap of the EVM gateway with all the engines.
681+
// Run is a blocking call, but it does signal readiness of the service
682+
// through a channel provided as an argument.
683+
func Run(ctx context.Context, cfg config.Config, ready component.ReadyFunc) error {
684+
boot, err := New(cfg)
685+
if err != nil {
686+
return err
687+
}
688+
689+
if err := boot.Run(ctx, cfg, ready); err != nil {
690+
return err
691+
}
692+
672693
// if context is canceled start shutdown
673694
<-ctx.Done()
674-
boot.logger.Warn().Msg("bootstrap received context cancellation, stopping services")
675695

676-
boot.StopEventIngestion()
677-
boot.StopMetricsServer()
678-
boot.StopAPIServer()
679-
boot.StopClient()
680-
boot.StopDB()
696+
boot.Stop()
681697

682698
return nil
683699
}

go.mod

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ require (
88
github.com/hashicorp/go-multierror v1.1.1
99
github.com/onflow/atree v0.9.0
1010
github.com/onflow/cadence v1.3.1
11-
github.com/onflow/flow-go v0.38.1-0.20250213171922-77f4db56bb54
11+
github.com/onflow/flow-go v0.38.1-0.20250218174738-2181389f9f7d
1212
github.com/onflow/flow-go-sdk v1.3.1
1313
github.com/onflow/go-ethereum v1.14.7
1414
github.com/prometheus/client_golang v1.20.5
@@ -135,13 +135,13 @@ require (
135135
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
136136
github.com/olekukonko/tablewriter v0.0.5 // indirect
137137
github.com/onflow/crypto v0.25.2 // indirect
138-
github.com/onflow/flow-core-contracts/lib/go/contracts v1.4.0 // indirect
139-
github.com/onflow/flow-core-contracts/lib/go/templates v1.4.0 // indirect
138+
github.com/onflow/flow-core-contracts/lib/go/contracts v1.5.1-preview // indirect
139+
github.com/onflow/flow-core-contracts/lib/go/templates v1.5.1-preview // indirect
140140
github.com/onflow/flow-ft/lib/go/contracts v1.0.1 // indirect
141141
github.com/onflow/flow-ft/lib/go/templates v1.0.1 // indirect
142-
github.com/onflow/flow-nft/lib/go/contracts v1.2.2 // indirect
142+
github.com/onflow/flow-nft/lib/go/contracts v1.2.3 // indirect
143143
github.com/onflow/flow-nft/lib/go/templates v1.2.1 // indirect
144-
github.com/onflow/flow/protobuf/go/flow v0.4.7 // indirect
144+
github.com/onflow/flow/protobuf/go/flow v0.4.9 // indirect
145145
github.com/onflow/sdks v0.6.0-preview.1 // indirect
146146
github.com/onsi/ginkgo v1.16.4 // indirect
147147
github.com/onsi/gomega v1.18.1 // indirect

go.sum

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -520,24 +520,24 @@ github.com/onflow/cadence v1.3.1 h1:bs9TFHQy8HHbwTtCtg5cLdyndWhmwq55RSwID1cb220=
520520
github.com/onflow/cadence v1.3.1/go.mod h1:6/47FljVAdl3/31tShI8JOJW0sXYZHK1PwXkE+yk0qA=
521521
github.com/onflow/crypto v0.25.2 h1:GjHunqVt+vPcdqhxxhAXiMIF3YiLX7gTuTR5O+VG2ns=
522522
github.com/onflow/crypto v0.25.2/go.mod h1:fY7eLqUdMKV8EGOw301unP8h7PvLVy8/6gVR++/g0BY=
523-
github.com/onflow/flow-core-contracts/lib/go/contracts v1.4.0 h1:R86HaOuk6vpuECZnriEUE7bw9inC2AtdSn8lL/iwQLQ=
524-
github.com/onflow/flow-core-contracts/lib/go/contracts v1.4.0/go.mod h1:9asTBnB6Tw2UlVVtQKyS/egYv3xr4zVlJnJ75z1dfac=
525-
github.com/onflow/flow-core-contracts/lib/go/templates v1.4.0 h1:u2DAG8pk0xFH7TwS70t1gSZ/FtIIZWMSNyiu4SeXBYg=
526-
github.com/onflow/flow-core-contracts/lib/go/templates v1.4.0/go.mod h1:pN768Al/wLRlf3bwugv9TyxniqJxMu4sxnX9eQJam64=
523+
github.com/onflow/flow-core-contracts/lib/go/contracts v1.5.1-preview h1:W+QkNQcIbhtR+zXVROKq0bdDEnvzUfUrQrCmegmwzvc=
524+
github.com/onflow/flow-core-contracts/lib/go/contracts v1.5.1-preview/go.mod h1:LyCICUK6sK1jtEyb+3GuRw5tYfHT1uxACLwLTLxw/0I=
525+
github.com/onflow/flow-core-contracts/lib/go/templates v1.5.1-preview h1:C0PraQFfwpav4nJAf/RPE9BJyYD6lUMvt+cJyiMDeis=
526+
github.com/onflow/flow-core-contracts/lib/go/templates v1.5.1-preview/go.mod h1:pN768Al/wLRlf3bwugv9TyxniqJxMu4sxnX9eQJam64=
527527
github.com/onflow/flow-ft/lib/go/contracts v1.0.1 h1:Ts5ob+CoCY2EjEd0W6vdLJ7hLL3SsEftzXG2JlmSe24=
528528
github.com/onflow/flow-ft/lib/go/contracts v1.0.1/go.mod h1:PwsL8fC81cjnUnTfmyL/HOIyHnyaw/JA474Wfj2tl6A=
529529
github.com/onflow/flow-ft/lib/go/templates v1.0.1 h1:FDYKAiGowABtoMNusLuRCILIZDtVqJ/5tYI4VkF5zfM=
530530
github.com/onflow/flow-ft/lib/go/templates v1.0.1/go.mod h1:uQ8XFqmMK2jxyBSVrmyuwdWjTEb+6zGjRYotfDJ5pAE=
531-
github.com/onflow/flow-go v0.38.1-0.20250213171922-77f4db56bb54 h1:JizTOEQ6ME++LOEqOmKUxtM0Q8tpHUvVJBhqBrgDeE0=
532-
github.com/onflow/flow-go v0.38.1-0.20250213171922-77f4db56bb54/go.mod h1:p88l1DhHObIUr+8b0zqkwH0bklZmQksb6IFH8ZaL1Bg=
531+
github.com/onflow/flow-go v0.38.1-0.20250218174738-2181389f9f7d h1:XRefc4rcBjGDEqsj3OB6XjSjSeYwMtysl7jmaLYpg+s=
532+
github.com/onflow/flow-go v0.38.1-0.20250218174738-2181389f9f7d/go.mod h1:VS7MlNHZeDrGm9/jkuMCSvAQTLFXpzQD0BIMB8/QYB8=
533533
github.com/onflow/flow-go-sdk v1.3.1 h1:2YdTL/R1/DjMYYmyKgArTeQ93GKvLlfCeCpMVH7b8q4=
534534
github.com/onflow/flow-go-sdk v1.3.1/go.mod h1:0rMuCLShdX9F4pLBCPhlMGCFu8gu9SfiXT/Lc9qAi24=
535-
github.com/onflow/flow-nft/lib/go/contracts v1.2.2 h1:XFERNVUDGbZ4ViZjt7P1cGD80mO1PzUJYPfdhXFsGbQ=
536-
github.com/onflow/flow-nft/lib/go/contracts v1.2.2/go.mod h1:eZ9VMMNfCq0ho6kV25xJn1kXeCfxnkhj3MwF3ed08gY=
535+
github.com/onflow/flow-nft/lib/go/contracts v1.2.3 h1:4ju20g1xgDKWBT63rOj5f/Sa4Lc+naCSWT4p31x9yQk=
536+
github.com/onflow/flow-nft/lib/go/contracts v1.2.3/go.mod h1:eZ9VMMNfCq0ho6kV25xJn1kXeCfxnkhj3MwF3ed08gY=
537537
github.com/onflow/flow-nft/lib/go/templates v1.2.1 h1:SAALMZPDw9Eb9p5kSLnmnFxjyig1MLiT4JUlLp0/bSE=
538538
github.com/onflow/flow-nft/lib/go/templates v1.2.1/go.mod h1:W6hOWU0xltPqNpv9gQX8Pj8Jtf0OmRxc1XX2V0kzJaI=
539-
github.com/onflow/flow/protobuf/go/flow v0.4.7 h1:iP6DFx4wZ3ETORsyeqzHu7neFT3d1CXF6wdK+AOOjmc=
540-
github.com/onflow/flow/protobuf/go/flow v0.4.7/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk=
539+
github.com/onflow/flow/protobuf/go/flow v0.4.9 h1:UfsWWqj6VQbEHvaw8kSGvIawCpEfz3gOGZfcdugNxVE=
540+
github.com/onflow/flow/protobuf/go/flow v0.4.9/go.mod h1:NA2pX2nw8zuaxfKphhKsk00kWLwfd+tv8mS23YXO4Sk=
541541
github.com/onflow/go-ethereum v1.14.7 h1:gg3awYqI02e3AypRdpJKEvNTJ6kz/OhAqRti0h54Wlc=
542542
github.com/onflow/go-ethereum v1.14.7/go.mod h1:zV14QLrXyYu5ucvcwHUA0r6UaqveqbXaehAVQJlSW+I=
543543
github.com/onflow/nft-storefront/lib/go/contracts v1.0.0 h1:sxyWLqGm/p4EKT6DUlQESDG1ZNMN9GjPCm1gTq7NGfc=

metrics/collector.go

Lines changed: 82 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,84 @@ import (
99
"github.com/rs/zerolog"
1010
)
1111

12+
var apiErrors = prometheus.NewCounter(prometheus.CounterOpts{
13+
Name: prefixedName("api_errors_total"),
14+
Help: "Total number of API errors",
15+
})
16+
17+
var serverPanicsCounters = prometheus.NewCounterVec(prometheus.CounterOpts{
18+
Name: prefixedName("api_server_panics_total"),
19+
Help: "Total number of panics in the API server",
20+
}, []string{"reason"})
21+
22+
var operatorBalance = prometheus.NewGauge(prometheus.GaugeOpts{
23+
Name: prefixedName("operator_balance"),
24+
Help: "Flow balance of the EVM gateway operator wallet",
25+
})
26+
27+
var cadenceBlockHeight = prometheus.NewGauge(prometheus.GaugeOpts{
28+
Name: prefixedName("cadence_block_height"),
29+
Help: "Current Cadence block height",
30+
})
31+
32+
var evmBlockHeight = prometheus.NewGauge(prometheus.GaugeOpts{
33+
Name: prefixedName("evm_block_height"),
34+
Help: "Current EVM block height",
35+
})
36+
37+
var evmBlockIndexedCounter = prometheus.NewCounter(prometheus.CounterOpts{
38+
Name: prefixedName("blocks_indexed_total"),
39+
Help: "Total number of blocks indexed",
40+
})
41+
42+
var evmTxIndexedCounter = prometheus.NewCounter(prometheus.CounterOpts{
43+
Name: prefixedName("txs_indexed_total"),
44+
Help: "Total number transactions indexed",
45+
})
46+
47+
var evmAccountCallCounters = prometheus.NewCounterVec(prometheus.CounterOpts{
48+
Name: prefixedName("evm_account_interactions_total"),
49+
Help: "Total number of account interactions",
50+
}, []string{"address"})
51+
52+
// TODO: Think of adding 'status_code'
53+
var requestDurations = prometheus.NewHistogramVec(prometheus.HistogramOpts{
54+
Name: prefixedName("api_request_duration_seconds"),
55+
Help: "Duration of the request made a specific API endpoint",
56+
Buckets: prometheus.DefBuckets,
57+
}, []string{"method"})
58+
59+
var availableSigningKeys = prometheus.NewGauge(prometheus.GaugeOpts{
60+
Name: prefixedName("available_signing_keys"),
61+
Help: "Number of keys available for transaction signing",
62+
})
63+
64+
var gasEstimationIterations = prometheus.NewGauge(prometheus.GaugeOpts{
65+
Name: prefixedName("gas_estimation_iterations"),
66+
Help: "Number of iterations taken to estimate the gas of a EVM call/tx",
67+
})
68+
69+
var blockIngestionTime = prometheus.NewHistogram(prometheus.HistogramOpts{
70+
Name: prefixedName("block_ingestion_time_seconds"),
71+
Help: "Time taken to fully ingest an EVM block in the local state index",
72+
Buckets: prometheus.DefBuckets,
73+
})
74+
75+
var metrics = []prometheus.Collector{
76+
apiErrors,
77+
serverPanicsCounters,
78+
cadenceBlockHeight,
79+
evmBlockHeight,
80+
evmBlockIndexedCounter,
81+
evmTxIndexedCounter,
82+
operatorBalance,
83+
evmAccountCallCounters,
84+
requestDurations,
85+
availableSigningKeys,
86+
gasEstimationIterations,
87+
blockIngestionTime,
88+
}
89+
1290
type Collector interface {
1391
ApiErrorOccurred()
1492
ServerPanicked(reason string)
@@ -42,83 +120,6 @@ type DefaultCollector struct {
42120
}
43121

44122
func NewCollector(logger zerolog.Logger) Collector {
45-
apiErrors := prometheus.NewCounter(prometheus.CounterOpts{
46-
Name: prefixedName("api_errors_total"),
47-
Help: "Total number of API errors",
48-
})
49-
50-
serverPanicsCounters := prometheus.NewCounterVec(prometheus.CounterOpts{
51-
Name: prefixedName("api_server_panics_total"),
52-
Help: "Total number of panics in the API server",
53-
}, []string{"reason"})
54-
55-
operatorBalance := prometheus.NewGauge(prometheus.GaugeOpts{
56-
Name: prefixedName("operator_balance"),
57-
Help: "Flow balance of the EVM gateway operator wallet",
58-
})
59-
60-
cadenceBlockHeight := prometheus.NewGauge(prometheus.GaugeOpts{
61-
Name: prefixedName("cadence_block_height"),
62-
Help: "Current Cadence block height",
63-
})
64-
65-
evmBlockHeight := prometheus.NewGauge(prometheus.GaugeOpts{
66-
Name: prefixedName("evm_block_height"),
67-
Help: "Current EVM block height",
68-
})
69-
70-
evmBlockIndexedCounter := prometheus.NewCounter(prometheus.CounterOpts{
71-
Name: prefixedName("blocks_indexed_total"),
72-
Help: "Total number of blocks indexed",
73-
})
74-
75-
evmTxIndexedCounter := prometheus.NewCounter(prometheus.CounterOpts{
76-
Name: prefixedName("txs_indexed_total"),
77-
Help: "Total number transactions indexed",
78-
})
79-
80-
evmAccountCallCounters := prometheus.NewCounterVec(prometheus.CounterOpts{
81-
Name: prefixedName("evm_account_interactions_total"),
82-
Help: "Total number of account interactions",
83-
}, []string{"address"})
84-
85-
// TODO: Think of adding 'status_code'
86-
requestDurations := prometheus.NewHistogramVec(prometheus.HistogramOpts{
87-
Name: prefixedName("api_request_duration_seconds"),
88-
Help: "Duration of the request made a specific API endpoint",
89-
Buckets: prometheus.DefBuckets,
90-
}, []string{"method"})
91-
92-
availableSigningKeys := prometheus.NewGauge(prometheus.GaugeOpts{
93-
Name: prefixedName("available_signing_keys"),
94-
Help: "Number of keys available for transaction signing",
95-
})
96-
97-
gasEstimationIterations := prometheus.NewGauge(prometheus.GaugeOpts{
98-
Name: prefixedName("gas_estimation_iterations"),
99-
Help: "Number of iterations taken to estimate the gas of a EVM call/tx",
100-
})
101-
102-
blockIngestionTime := prometheus.NewHistogram(prometheus.HistogramOpts{
103-
Name: prefixedName("block_ingestion_time_seconds"),
104-
Help: "Time taken to fully ingest an EVM block in the local state index",
105-
Buckets: prometheus.DefBuckets,
106-
})
107-
108-
metrics := []prometheus.Collector{
109-
apiErrors,
110-
serverPanicsCounters,
111-
cadenceBlockHeight,
112-
evmBlockHeight,
113-
evmBlockIndexedCounter,
114-
evmTxIndexedCounter,
115-
operatorBalance,
116-
evmAccountCallCounters,
117-
requestDurations,
118-
availableSigningKeys,
119-
gasEstimationIterations,
120-
blockIngestionTime,
121-
}
122123
if err := registerMetrics(logger, metrics...); err != nil {
123124
logger.Info().Msg("using noop collector as metric register failed")
124125
return NopCollector
@@ -142,6 +143,10 @@ func NewCollector(logger zerolog.Logger) Collector {
142143

143144
func registerMetrics(logger zerolog.Logger, metrics ...prometheus.Collector) error {
144145
for _, m := range metrics {
146+
// During E2E tests, the EVM GW might be bootstrapped again
147+
// and again, so we make sure to register the metrics on a
148+
// clean state.
149+
prometheus.Unregister(m)
145150
if err := prometheus.Register(m); err != nil {
146151
logger.Err(err).Msg("failed to register metric")
147152
return err

storage/pebble/register_storage.go

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -178,10 +178,23 @@ func newLookupKey(height uint64, key []byte) *lookupKey {
178178
return &lookupKey
179179
}
180180

181-
// GetSnapshotAt returns a snapshot of the register index at the given block height.
182-
// the snapshot has a cache. Nil values are cached.
183-
func (r *RegisterStorage) GetSnapshotAt(evmBlockHeight uint64) (types.BackendStorageSnapshot, error) {
184-
return NewStorageSnapshot(r.Get, evmBlockHeight), nil
181+
// GetSnapshotAt returns a snapshot of the register index at the start of the
182+
// given block height (which is the end of the previous block).
183+
// The snapshot has a cache. Nil values are cached.
184+
func (r *RegisterStorage) GetSnapshotAt(
185+
evmBlockHeightOfStartStateToQuery uint64,
186+
) (types.BackendStorageSnapshot, error) {
187+
var snapshotHeightOfEndState uint64
188+
if evmBlockHeightOfStartStateToQuery > 0 {
189+
// `evmBlockHeightOfStartStateToQuery-1` to get the end state of the previous block.
190+
snapshotHeightOfEndState = evmBlockHeightOfStartStateToQuery - 1
191+
} else {
192+
// Avoid a possible underflow
193+
snapshotHeightOfEndState = uint64(0)
194+
}
195+
196+
// NewStorageSnapshot return the end state of a given height.
197+
return NewStorageSnapshot(r.Get, snapshotHeightOfEndState), nil
185198
}
186199

187200
func registerOwnerMismatch(expected flow.Address, owner flow.Address) error {

0 commit comments

Comments
 (0)