Allow adding back-up AN hosts to be used for client-side request load balancing

Author: m-Peter

bootstrap/bootstrap.go

@@ -20,8 +20,11 @@ import (
 	"github.com/onflow/flow-go/module/util"
 	"github.com/rs/zerolog"
 	"github.com/sethvargo/go-limiter/memorystore"
+	_ "google.golang.org/grpc"
 	grpcOpts "google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/resolver"
+	"google.golang.org/grpc/resolver/manual"
 	"google.golang.org/grpc/status"
 
 	"github.com/onflow/flow-evm-gateway/api"
@@ -482,16 +485,53 @@ func StartEngine(
 // setupCrossSporkClient sets up a cross-spork AN client.
 func setupCrossSporkClient(config config.Config, logger zerolog.Logger) (*requester.CrossSporkClient, error) {
 	// create access client with cross-spork capabilities
-	currentSporkClient, err := grpc.NewClient(
-		config.AccessNodeHost,
-		grpc.WithGRPCDialOptions(
-			grpcOpts.WithDefaultCallOptions(grpcOpts.MaxCallRecvMsgSize(DefaultMaxMessageSize)),
-			grpcOpts.WithUnaryInterceptor(retryInterceptor(
-				DefaultResourceExhaustedMaxRetryDelay,
-				DefaultResourceExhaustedRetryDelay,
-			)),
-		),
-	)
+	var currentSporkClient *grpc.Client
+	var err error
+
+	if len(config.AccessNodeBackupHosts) > 0 {
+		mr := manual.NewBuilderWithScheme("dns")
+		defer mr.Close()
+
+		json := `{"loadBalancingConfig": [{"pick_first":{}}]}`
+		endpoints := []resolver.Endpoint{
+			{Addresses: []resolver.Address{{Addr: config.AccessNodeHost}}},
+		}
+
+		for _, accessNodeBackupAddr := range config.AccessNodeBackupHosts {
+			endpoints = append(endpoints, resolver.Endpoint{
+				Addresses: []resolver.Address{{Addr: accessNodeBackupAddr}},
+			})
+		}
+
+		mr.InitialState(resolver.State{
+			Endpoints: endpoints,
+		})
+
+		currentSporkClient, err = grpc.NewClient(
+			mr.Scheme(),
+			grpc.WithGRPCDialOptions(
+				grpcOpts.WithDefaultCallOptions(grpcOpts.MaxCallRecvMsgSize(DefaultMaxMessageSize)),
+				grpcOpts.WithResolvers(mr),
+				grpcOpts.WithDefaultServiceConfig(json),
+				grpcOpts.WithUnaryInterceptor(retryInterceptor(
+					DefaultResourceExhaustedMaxRetryDelay,
+					DefaultResourceExhaustedRetryDelay,
+				)),
+			),
+		)
+	} else {
+		currentSporkClient, err = grpc.NewClient(
+			config.AccessNodeHost,
+			grpc.WithGRPCDialOptions(
+				grpcOpts.WithDefaultCallOptions(grpcOpts.MaxCallRecvMsgSize(DefaultMaxMessageSize)),
+				grpcOpts.WithUnaryInterceptor(retryInterceptor(
+					DefaultResourceExhaustedMaxRetryDelay,
+					DefaultResourceExhaustedRetryDelay,
+				)),
+			),
+		)
+	}
+
 	if err != nil {
 		return nil, fmt.Errorf(
 			"failed to create client connection for host: %s, with error: %w",

cmd/run/cmd.go

@@ -201,6 +201,10 @@ func parseConfigFromFlags() error {
 	}
 	cfg.FilterExpiry = exp
 
+	if accessBackupHosts != "" {
+		cfg.AccessNodeBackupHosts = strings.Split(accessBackupHosts, ",")
+	}
+
 	if accessSporkHosts != "" {
 		heightHosts := strings.Split(accessSporkHosts, ",")
 		cfg.AccessNodePreviousSporkHosts = append(cfg.AccessNodePreviousSporkHosts, heightHosts...)
@@ -242,6 +246,7 @@ var (
 	logWriter,
 	filterExpiry,
 	accessSporkHosts,
+	accessBackupHosts,
 	cloudKMSKey,
 	cloudKMSProjectID,
 	cloudKMSLocationID,
@@ -259,6 +264,7 @@ func init() {
 	Cmd.Flags().IntVar(&cfg.RPCPort, "rpc-port", 8545, "Port for the RPC API server")
 	Cmd.Flags().BoolVar(&cfg.WSEnabled, "ws-enabled", false, "Enable websocket connections")
 	Cmd.Flags().StringVar(&cfg.AccessNodeHost, "access-node-grpc-host", "localhost:3569", "Host to the flow access node gRPC API")
+	Cmd.Flags().StringVar(&accessBackupHosts, "access-node-backup-hosts", "", `Backup AN hosts to use in case of connectivity issues, defined following the schema: {host1},{host2} as a comma separated list (e.g. "host-1.com,host2.com")`)
 	Cmd.Flags().StringVar(&accessSporkHosts, "access-node-spork-hosts", "", `Previous spork AN hosts, defined following the schema: {host1},{host2} as a comma separated list (e.g. "host-1.com,host2.com")`)
 	Cmd.Flags().StringVar(&flowNetwork, "flow-network-id", "flow-emulator", "Flow network ID (flow-emulator, flow-previewnet, flow-testnet, flow-mainnet)")
 	Cmd.Flags().StringVar(&coinbase, "coinbase", "", "Coinbase address to use for fee collection")

config/config.go

@@ -45,6 +45,9 @@ type Config struct {
 	DatabaseDir string
 	// AccessNodeHost defines the current spork Flow network AN host.
 	AccessNodeHost string
+	// AccessNodeBackupHosts contains a list of ANs hosts to use as backup, in
+	// case of connectivity issues with `AccessNodeHost`.
+	AccessNodeBackupHosts []string
 	// AccessNodePreviousSporkHosts contains a list of the ANs hosts for each spork
 	AccessNodePreviousSporkHosts []string
 	// GRPCPort for the RPC API server