Implement the RPCBlockHeaderSubscriber for indexing finalized results

Makefile

@@ -227,3 +227,36 @@ endif
 
 	docker run $(MODE) -p $(HOST_PORT):8545 -p $(HOST_METRICS_PORT):8080 $(MOUNT) "$(CONTAINER_REGISTRY)/flow-evm-gateway:$(IMAGE_TAG)" $(CMD_ARGS)
 
+.PHONY: start-testnet
+start-testnet:
+	rm -rf testnet-db-block-headers/
+	rm -rf metrics/data/
+	go run cmd/main.go run \
+		--database-dir=testnet-db-block-headers \
+		--access-node-grpc-host=access.devnet.nodes.onflow.org:9000 \
+		--access-node-spork-hosts=access-001.devnet51.nodes.onflow.org:9000 \
+		--flow-network-id=flow-testnet \
+		--init-cadence-height=211176670 \
+		--ws-enabled=true \
+		--coinbase=FACF71692421039876a5BB4F10EF7A439D8ef61E \
+		--coa-address=0x62631c28c9fc5a91 \
+		--coa-key=2892fba444f1d5787739708874e3b01160671924610411ac787ac1379d420f49 \
+		--gas-price=100 \
+		--log-level=info
+
+.PHONY: start-mainnet
+start-mainnet:
+	rm -rf mainnet-db-block-headers/
+	rm -rf metrics/data/
+	go run cmd/main.go run \
+		--database-dir=mainnet-db-block-headers \
+		--access-node-grpc-host=access.mainnet.nodes.onflow.org:9000 \
+		--access-node-spork-hosts=access-001.mainnet25.nodes.onflow.org:9000 \
+		--flow-network-id=flow-mainnet \
+		--init-cadence-height=85981135 \
+		--ws-enabled=true \
+		--coinbase=FACF71692421039876a5BB4F10EF7A439D8ef61E \
+		--coa-address=0xf1ab99c82dee3526 \
+		--coa-key=2892fba444f1d5787739708874e3b01160671924610411ac787ac1379d420f49 \
+		--gas-price=100 \
+		--log-level=info

bootstrap/bootstrap.go

@@ -142,13 +142,24 @@ func (b *Bootstrap) StartEventIngestion(ctx context.Context) error {
 	chainID := b.config.FlowNetworkID
 
 	// create event subscriber
-	subscriber := ingestion.NewRPCEventSubscriber(
-		b.logger,
-		b.client,
-		chainID,
-		b.keystore,
-		latestCadenceHeight,
-	)
+	var subscriber ingestion.EventSubscriber
+	if b.config.ExperimentalSoftFinalityEnabled {
+		subscriber = ingestion.NewRPCBlockTrackingSubscriber(
+			b.logger,
+			b.client,
+			chainID,
+			b.keystore,
+			latestCadenceHeight,
+		)
+	} else {
+		subscriber = ingestion.NewRPCEventSubscriber(
+			b.logger,
+			b.client,
+			chainID,
+			b.keystore,
+			latestCadenceHeight,
+		)
+	}
 
 	callTracerCollector, err := replayer.NewCallTracerCollector(b.logger)
 	if err != nil {

cmd/run/cmd.go

@@ -222,6 +222,8 @@ func parseConfigFromFlags() error {
 		return fmt.Errorf("unknown tx state validation: %s", txStateValidation)
 	}
 
+	cfg.ExperimentalSoftFinalityEnabled = experimentalSoftFinalityEnabled
+
 	return nil
 }
 
@@ -246,6 +248,8 @@ var (
 
 	initHeight,
 	forceStartHeight uint64
+
+	experimentalSoftFinalityEnabled bool
 )
 
 func init() {
@@ -280,4 +284,5 @@ func init() {
 	Cmd.Flags().StringVar(&cfg.ProfilerHost, "profiler-host", "localhost", "Host for the Profiler server")
 	Cmd.Flags().IntVar(&cfg.ProfilerPort, "profiler-port", 6060, "Port for the Profiler server")
 	Cmd.Flags().StringVar(&txStateValidation, "tx-state-validation", "tx-seal", "Sets the transaction validation mechanism. It can validate using the local state index, or wait for the outer Flow transaction to seal. Available values ('local-index' / 'tx-seal'), defaults to 'tx-seal'.")
+	Cmd.Flags().BoolVar(&experimentalSoftFinalityEnabled, "experimental-soft-finality-enabled", false, "Sets whether the gateway should use the experimental soft finality feature. WARNING: This may result in incorrect results being returned in certain circumstances. Use only if you know what you are doing.")
 }

config/config.go

@@ -89,4 +89,8 @@ type Config struct {
 	// TxStateValidation sets the transaction validation mechanism. It can validate
 	// using the local state index, or wait for the outer Flow transaction to seal.
 	TxStateValidation string
+	// ExperimentalSoftFinalityEnabled enables the experimental soft finality feature which syncs
+	// EVM block and transaction data from the upstream Access node before the block is sealed.
+	// CAUTION: This feature is experimental and may return incorrect data in certain circumstances.
+	ExperimentalSoftFinalityEnabled bool
 }

go.mod

@@ -6,10 +6,10 @@ require (
 	github.com/cockroachdb/pebble v1.1.1
 	github.com/goccy/go-json v0.10.2
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/onflow/atree v0.8.0
-	github.com/onflow/cadence v1.2.2
+	github.com/onflow/atree v0.8.1
+	github.com/onflow/cadence v1.3.0
 	github.com/onflow/flow-go v0.38.0-preview.0.4
-	github.com/onflow/flow-go-sdk v1.2.3
+	github.com/onflow/flow-go-sdk v1.3.0
 	github.com/onflow/go-ethereum v1.14.7
 	github.com/prometheus/client_golang v1.18.0
 	github.com/rs/cors v1.8.0
@@ -18,20 +18,19 @@ require (
 	github.com/sethvargo/go-limiter v1.0.0
 	github.com/sethvargo/go-retry v0.2.3
 	github.com/spf13/cobra v1.8.1
-	github.com/stretchr/testify v1.9.0
+	github.com/stretchr/testify v1.10.0
 	go.uber.org/ratelimit v0.3.1
 	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
 	golang.org/x/sync v0.8.0
-	google.golang.org/grpc v1.63.2
+	google.golang.org/grpc v1.64.1
 )
 
 require (
-	cloud.google.com/go v0.112.0 // indirect
-	cloud.google.com/go/compute v1.24.0 // indirect
-	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	cloud.google.com/go v0.112.1 // indirect
+	cloud.google.com/go/compute/metadata v0.5.0 // indirect
 	cloud.google.com/go/iam v1.1.6 // indirect
 	cloud.google.com/go/kms v1.15.7 // indirect
-	cloud.google.com/go/storage v1.36.0 // indirect
+	cloud.google.com/go/storage v1.38.0 // indirect
 	github.com/DataDog/zstd v1.5.2 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/SaveTheRbtz/mph v0.1.1-0.20240117162131-4166ec7869bc // indirect
@@ -88,7 +87,7 @@ require (
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
-	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.12.2 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -181,10 +180,10 @@ require (
 	github.com/vmihailenco/msgpack/v4 v4.3.11 // indirect
 	github.com/vmihailenco/tagparser v0.1.1 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
-	github.com/zeebo/blake3 v0.2.3 // indirect
+	github.com/zeebo/blake3 v0.2.4 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
 	go.opentelemetry.io/otel v1.24.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0 // indirect
@@ -196,19 +195,19 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/crypto v0.28.0 // indirect
-	golang.org/x/net v0.25.0 // indirect
-	golang.org/x/oauth2 v0.17.0 // indirect
+	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/oauth2 v0.18.0 // indirect
 	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/text v0.19.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	gonum.org/v1/gonum v0.14.0 // indirect
-	google.golang.org/api v0.162.0 // indirect
+	google.golang.org/api v0.169.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect
-	google.golang.org/protobuf v1.33.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/blake3 v1.3.0 // indirect