From d7621fc6374e416ab5f61763688b48b5d071807b Mon Sep 17 00:00:00 2001 From: Eric Nielson Date: Wed, 21 Feb 2024 11:06:31 -0500 Subject: [PATCH 01/28] Support infinite connection attempt retries --- pkg/client/events/replay_client.go | 2 +- pkg/retry/retry.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/client/events/replay_client.go b/pkg/client/events/replay_client.go index 112fa44c3..8224a2a89 100644 --- a/pkg/client/events/replay_client.go +++ b/pkg/client/events/replay_client.go @@ -21,7 +21,7 @@ const ( // eventsBytesRetryLimit is the maximum number of times to attempt to // re-establish the events query bytes subscription when the events bytes // observable returns an error or closes. - eventsBytesRetryLimit = 10 + eventsBytesRetryLimit = 0 eventsBytesRetryResetTimeout = 10 * time.Second // replayObsCacheBufferSize is the replay buffer size of the // replayObsCache replay observable which is used to cache the replay diff --git a/pkg/retry/retry.go b/pkg/retry/retry.go index bd06af941..b73af9f3e 100644 --- a/pkg/retry/retry.go +++ b/pkg/retry/retry.go @@ -58,7 +58,7 @@ func OnError( return nil } - if retryCount >= retryLimit { + if retryLimit > 0 && retryCount >= retryLimit { return err } From 4cb108c24534bdd63d2ec445fdc53544ed853fbf Mon Sep 17 00:00:00 2001 From: Eric Nielson Date: Wed, 20 Mar 2024 11:11:28 -0400 Subject: [PATCH 02/28] Added TODO --- pkg/client/events/replay_client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/client/events/replay_client.go b/pkg/client/events/replay_client.go index 8224a2a89..5175a78e0 100644 --- a/pkg/client/events/replay_client.go +++ b/pkg/client/events/replay_client.go @@ -21,6 +21,7 @@ const ( // eventsBytesRetryLimit is the maximum number of times to attempt to // re-establish the events query bytes subscription when the events bytes // observable returns an error or closes. + // TODO to make this a customisable parameter in the appgateserver and relayminer config files. eventsBytesRetryLimit = 0 eventsBytesRetryResetTimeout = 10 * time.Second // replayObsCacheBufferSize is the replay buffer size of the From efca1f2c5ecb2b4a2efd8ac868db554a3e009c4b Mon Sep 17 00:00:00 2001 From: Eric Nielson Date: Tue, 26 Mar 2024 20:06:56 -0400 Subject: [PATCH 03/28] Removed panics in replay_client --- pkg/client/events/replay_client.go | 38 +++++++++++++++++------------- pkg/retry/retry.go | 2 +- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/pkg/client/events/replay_client.go b/pkg/client/events/replay_client.go index 5175a78e0..4da7cc032 100644 --- a/pkg/client/events/replay_client.go +++ b/pkg/client/events/replay_client.go @@ -2,7 +2,6 @@ package events import ( "context" - "fmt" "time" "cosmossdk.io/depinject" @@ -11,6 +10,7 @@ import ( "github.com/pokt-network/poktroll/pkg/either" "github.com/pokt-network/poktroll/pkg/observable" "github.com/pokt-network/poktroll/pkg/observable/channel" + "github.com/pokt-network/poktroll/pkg/polylog" "github.com/pokt-network/poktroll/pkg/retry" ) @@ -22,7 +22,7 @@ const ( // re-establish the events query bytes subscription when the events bytes // observable returns an error or closes. // TODO to make this a customisable parameter in the appgateserver and relayminer config files. - eventsBytesRetryLimit = 0 + eventsBytesRetryLimit = 10 eventsBytesRetryResetTimeout = 10 * time.Second // replayObsCacheBufferSize is the replay buffer size of the // replayObsCache replay observable which is used to cache the replay @@ -118,7 +118,16 @@ func NewEventsReplayClient[T any]( } // Concurrently publish events to the observable emitted by replayObsCache. - go rClient.goPublishEvents(ctx) + go func() { + for { + select { + case <-ctx.Done(): + return + default: + rClient.publishEvents(ctx) + } + } + }() return rClient, nil } @@ -176,11 +185,12 @@ func (rClient *replayClient[T]) Close() { close(rClient.replayObsCachePublishCh) } -// goPublishEvents runs the work function returned by retryPublishEventsFactory, +// publishEvents runs the work function returned by retryPublishEventsFactory, // re-invoking it according to the arguments to retry.OnError when the events bytes // observable returns an asynchronous error. -// This function is intended to be called in a goroutine. -func (rClient *replayClient[T]) goPublishEvents(ctx context.Context) { +func (rClient *replayClient[T]) publishEvents(ctx context.Context) { + logger := polylog.Ctx(ctx) + // React to errors by getting a new events bytes observable, re-mapping it, // and send it to replayObsCachePublishCh such that // replayObsCache.Last(ctx, 1) will return it. @@ -194,11 +204,8 @@ func (rClient *replayClient[T]) goPublishEvents(ctx context.Context) { ) // If we get here, the retry limit was reached and the retry loop exited. - // Since this function runs in a goroutine, we can't return the error to the - // caller. Instead, we panic. - if publishError != nil { - panic(fmt.Errorf("EventsReplayClient[%T].goPublishEvents should never reach this spot: %w", *new(T), publishError)) - } + logger.Error().Err(publishError).Msgf("EventsReplayClient[%T].goPublishEvents error") + return } // retryPublishEventsFactory returns a function which is intended to be passed @@ -264,9 +271,10 @@ func (rClient *replayClient[T]) newMapEventsBytesToTFn(errCh chan<- error) func( either.Bytes, ) (T, bool) { return func( - _ context.Context, + ctx context.Context, eitherEventBz either.Bytes, ) (_ T, skip bool) { + logger := polylog.Ctx(ctx) eventBz, err := eitherEventBz.ValueOrError() if err != nil { errCh <- err @@ -286,10 +294,8 @@ func (rClient *replayClient[T]) newMapEventsBytesToTFn(errCh chan<- error) func( return *new(T), true } - panic(fmt.Sprintf( - "unexpected error deserialising event: %v; eventBz: %s", - err, string(eventBz), - )) + logger.Error().Err(err).Msgf("unexpected error deserialising event") + return *new(T), true } return event, false } diff --git a/pkg/retry/retry.go b/pkg/retry/retry.go index b73af9f3e..bd06af941 100644 --- a/pkg/retry/retry.go +++ b/pkg/retry/retry.go @@ -58,7 +58,7 @@ func OnError( return nil } - if retryLimit > 0 && retryCount >= retryLimit { + if retryCount >= retryLimit { return err } From bc721daaabff20e2a13d32e57ac7ea3b1974fd78 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 2 Apr 2024 20:34:16 +0200 Subject: [PATCH 04/28] refactor: factor out connection retry limit --- e2e/tests/session_steps_test.go | 3 +- pkg/client/block/client.go | 22 +++++- pkg/client/block/options.go | 13 ++++ pkg/client/delegation/client.go | 22 +++++- pkg/client/delegation/options.go | 13 ++++ pkg/client/events/options.go | 10 +++ pkg/client/events/replay_client.go | 68 ++++++++++++------- pkg/client/interface.go | 9 +++ pkg/client/tx/client.go | 7 ++ pkg/client/tx/options.go | 10 +++ pkg/retry/retry.go | 5 ++ pkg/sdk/deps_builder.go | 10 ++- pkg/sdk/sdk.go | 1 + testutil/testclient/testeventsquery/client.go | 2 +- 14 files changed, 159 insertions(+), 36 deletions(-) create mode 100644 pkg/client/block/options.go create mode 100644 pkg/client/delegation/options.go diff --git a/e2e/tests/session_steps_test.go b/e2e/tests/session_steps_test.go index 83a78e2f1..cc2acd89d 100644 --- a/e2e/tests/session_steps_test.go +++ b/e2e/tests/session_steps_test.go @@ -122,7 +122,8 @@ func (s *suite) TheSupplierHasServicedASessionWithRelaysForServiceForApplication deps, msgSenderQuery, tx.UnmarshalTxResult, - eventsReplayClientBufferSize, + testEventsReplayClientBufferSize, + events.DefaultConnRetryLimit, ) require.NoError(s, err) s.scenarioState[txResultEventsReplayClientKey] = txSendEventsReplayClient diff --git a/pkg/client/block/client.go b/pkg/client/block/client.go index ba809194d..239f5464c 100644 --- a/pkg/client/block/client.go +++ b/pkg/client/block/client.go @@ -35,18 +35,29 @@ const ( func NewBlockClient( ctx context.Context, deps depinject.Config, -) (client.BlockClient, error) { - client, err := events.NewEventsReplayClient[client.Block]( + opts ...client.BlockClientOption, +) (_ client.BlockClient, err error) { + bClient := &blockClient{ + connRetryLimit: events.DefaultConnRetryLimit, + } + + for _, opt := range opts { + opt(bClient) + } + + bClient.eventsReplayClient, err = events.NewEventsReplayClient[client.Block]( ctx, deps, committedBlocksQuery, UnmarshalNewBlock, defaultBlocksReplayLimit, + events.WithConnRetryLimit[client.Block](bClient.connRetryLimit), ) if err != nil { return nil, err } - return &blockClient{eventsReplayClient: client}, nil + + return bClient, nil } // blockClient is a wrapper around an EventsReplayClient that implements the @@ -58,6 +69,11 @@ type blockClient struct { // These enable the EventsReplayClient to correctly map the raw event bytes // to Block objects and to correctly return a BlockReplayObservable eventsReplayClient client.EventsReplayClient[client.Block] + + // connRetryLimit is the number of times the underlying replay client + // should retry in the event that it encounters an error or its connection is interrupted. + // If connRetryLimit is < 0, it will retry indefinitely. + connRetryLimit int } // CommittedBlocksSequence returns a replay observable of new block events. diff --git a/pkg/client/block/options.go b/pkg/client/block/options.go new file mode 100644 index 000000000..b793306fa --- /dev/null +++ b/pkg/client/block/options.go @@ -0,0 +1,13 @@ +package block + +import "github.com/pokt-network/poktroll/pkg/client" + +// WithConnRetryLimit returns an option function which sets the number +// of times the underlying replay client should retry in the event that it encounters +// an error or its connection is interrupted. +// If connRetryLimit is < 0, it will retry indefinitely. +func WithConnRetryLimit(limit int) client.BlockClientOption { + return func(client client.BlockClient) { + client.(*blockClient).connRetryLimit = limit + } +} diff --git a/pkg/client/delegation/client.go b/pkg/client/delegation/client.go index 013b094df..20173c620 100644 --- a/pkg/client/delegation/client.go +++ b/pkg/client/delegation/client.go @@ -46,18 +46,29 @@ const ( func NewDelegationClient( ctx context.Context, deps depinject.Config, -) (client.DelegationClient, error) { - client, err := events.NewEventsReplayClient[client.Redelegation]( + opts ...client.DelegationClientOption, +) (_ client.DelegationClient, err error) { + dClient := &delegationClient{ + connRetryLimit: events.DefaultConnRetryLimit, + } + + for _, opt := range opts { + opt(dClient) + } + + dClient.eventsReplayClient, err = events.NewEventsReplayClient[client.Redelegation]( ctx, deps, delegationEventQuery, newRedelegationEventFactoryFn(), defaultRedelegationsReplayLimit, + events.WithConnRetryLimit[client.Redelegation](dClient.connRetryLimit), ) if err != nil { return nil, err } - return &delegationClient{eventsReplayClient: client}, nil + + return dClient, nil } // delegationClient is a wrapper around an EventsReplayClient that implements @@ -69,6 +80,11 @@ type delegationClient struct { // These enable the EventsReplayClient to correctly map the raw event bytes // to Redelegation objects and to correctly return a RedelegationReplayObservable eventsReplayClient client.EventsReplayClient[client.Redelegation] + + // connRetryLimit is the number of times the underlying replay client + // should retry in the event that it encounters an error or its connection is interrupted. + // If connRetryLimit is < 0, it will retry indefinitely. + connRetryLimit int } // RedelegationsSequence returns a replay observable of Redelgation events diff --git a/pkg/client/delegation/options.go b/pkg/client/delegation/options.go new file mode 100644 index 000000000..b618a0b3b --- /dev/null +++ b/pkg/client/delegation/options.go @@ -0,0 +1,13 @@ +package delegation + +import "github.com/pokt-network/poktroll/pkg/client" + +// WithConnRetryLimit returns an option function which sets the number +// of times the underlying replay client should retry in the event that it encounters +// an error or its connection is interrupted. +// If connRetryLimit is < 0, it will retry indefinitely. +func WithConnRetryLimit(limit int) client.DelegationClientOption { + return func(client client.DelegationClient) { + client.(*delegationClient).connRetryLimit = limit + } +} diff --git a/pkg/client/events/options.go b/pkg/client/events/options.go index 41fca7548..caf0a8841 100644 --- a/pkg/client/events/options.go +++ b/pkg/client/events/options.go @@ -9,3 +9,13 @@ func WithDialer(dialer client.Dialer) client.EventsQueryClientOption { evtClient.(*eventsQueryClient).dialer = dialer } } + +// WithConnRetryLimit returns an option function which sets the number +// of times the replay client should retry in the event that it encounters +// an error or its connection is interrupted. +// If connRetryLimit is < 0, it will retry indefinitely. +func WithConnRetryLimit[T any](limit int) client.EventsReplayClientOption[T] { + return func(client client.EventsReplayClient[T]) { + client.(*replayClient[T]).connRetryLimit = limit + } +} diff --git a/pkg/client/events/replay_client.go b/pkg/client/events/replay_client.go index 4da7cc032..7df1a0997 100644 --- a/pkg/client/events/replay_client.go +++ b/pkg/client/events/replay_client.go @@ -2,6 +2,7 @@ package events import ( "context" + "fmt" "time" "cosmossdk.io/depinject" @@ -10,11 +11,18 @@ import ( "github.com/pokt-network/poktroll/pkg/either" "github.com/pokt-network/poktroll/pkg/observable" "github.com/pokt-network/poktroll/pkg/observable/channel" - "github.com/pokt-network/poktroll/pkg/polylog" "github.com/pokt-network/poktroll/pkg/retry" ) const ( + // DefaultConnRetryLimit is used to indicate how many times the + // underlying replay client should attempt to retry if it encounters an error + // or its connection is interrupted. + // + // TODO_IMPROVE: this should be configurable but can be overridden at compile-time: + // go build -ldflags "-X github.com/pokt-network/poktroll/DefaultConnRetryLimit=value". + DefaultConnRetryLimit = 10 + // eventsBytesRetryDelay is the delay between retry attempts when the events // bytes observable returns an error. eventsBytesRetryDelay = time.Second @@ -77,6 +85,11 @@ type replayClient[T any] struct { // observable; // For example when the connection is re-established after erroring. replayObsCachePublishCh chan<- observable.ReplayObservable[T] + + // connRetryLimit is the number of times the replay client should retry + // in the event that it encounters an error or its connection is interrupted. + // If connRetryLimit is < 0, it will retry indefinitely. + connRetryLimit int } // NewEventsReplayClient creates a new EventsReplayClient from the given @@ -94,6 +107,7 @@ func NewEventsReplayClient[T any]( queryString string, newEventFn NewEventsFn[T], replayObsBufferSize int, + opts ...client.EventsReplayClientOption[T], ) (client.EventsReplayClient[T], error) { // Initialize the replay client rClient := &replayClient[T]{ @@ -101,6 +115,11 @@ func NewEventsReplayClient[T any]( eventDecoder: newEventFn, replayObsBufferSize: replayObsBufferSize, } + + for _, opt := range opts { + opt(rClient) + } + // TODO_REFACTOR(@h5law): Look into making this a regular observable as // we may no longer depend on it being replayable. replayObsCache, replayObsCachePublishCh := channel.NewReplayObservable[observable.ReplayObservable[T]]( @@ -118,16 +137,7 @@ func NewEventsReplayClient[T any]( } // Concurrently publish events to the observable emitted by replayObsCache. - go func() { - for { - select { - case <-ctx.Done(): - return - default: - rClient.publishEvents(ctx) - } - } - }() + go rClient.goPublishEvents(ctx, rClient.connRetryLimit) return rClient, nil } @@ -185,26 +195,28 @@ func (rClient *replayClient[T]) Close() { close(rClient.replayObsCachePublishCh) } -// publishEvents runs the work function returned by retryPublishEventsFactory, +// goPublishEvents runs the work function returned by retryPublishEventsFactory, // re-invoking it according to the arguments to retry.OnError when the events bytes // observable returns an asynchronous error. -func (rClient *replayClient[T]) publishEvents(ctx context.Context) { - logger := polylog.Ctx(ctx) - +func (rClient *replayClient[T]) goPublishEvents(ctx context.Context, retryLimit int) { // React to errors by getting a new events bytes observable, re-mapping it, // and send it to replayObsCachePublishCh such that // replayObsCache.Last(ctx, 1) will return it. - publishError := retry.OnError( + publishErr := retry.OnError( ctx, - eventsBytesRetryLimit, + retryLimit, eventsBytesRetryDelay, eventsBytesRetryResetTimeout, "goPublishEvents", rClient.retryPublishEventsFactory(ctx), ) - // If we get here, the retry limit was reached and the retry loop exited. - logger.Error().Err(publishError).Msgf("EventsReplayClient[%T].goPublishEvents error") + // Since this function runs in a goroutine, we can't return the error to the + // caller. Instead, we panic. + if publishErr != nil { + panic(fmt.Errorf("EventsReplayClient[%T].goPublishEvents should never reach this spot: %w", *new(T), publishErr)) + } + return } @@ -214,20 +226,24 @@ func (rClient *replayClient[T]) publishEvents(ctx context.Context) { // replayObsCache replay observable. func (rClient *replayClient[T]) retryPublishEventsFactory(ctx context.Context) func() chan error { return func() chan error { + eventsBzCtx, cancelEventsBzObs := context.WithCancel(ctx) errCh := make(chan error, 1) - eventsBytesObs, err := rClient.eventsClient.EventsBytes(ctx, rClient.queryString) + + eventsBytesObs, err := rClient.eventsClient.EventsBytes(eventsBzCtx, rClient.queryString) if err != nil { + // No need to cancel eventsBytesObs in the case of a synchronous error. errCh <- err return errCh } // NB: must cast back to generic observable type to use with Map. eventsBzObs := observable.Observable[either.Either[[]byte]](eventsBytesObs) + typedObs := channel.MapReplay( - ctx, + eventsBzCtx, replayObsCacheBufferSize, eventsBzObs, - rClient.newMapEventsBytesToTFn(errCh), + rClient.newMapEventsBytesToTFn(errCh, cancelEventsBzObs), ) // Subscribe to the eventBzObs and block until the channel closes. @@ -266,7 +282,7 @@ func (rClient *replayClient[T]) retryPublishEventsFactory(ctx context.Context) f // If deserialisation failed because the event bytes were for a different event // type, this value is also skipped. If deserialisation failed for some other // reason, this function panics. -func (rClient *replayClient[T]) newMapEventsBytesToTFn(errCh chan<- error) func( +func (rClient *replayClient[T]) newMapEventsBytesToTFn(errCh chan<- error, cancel context.CancelFunc) func( context.Context, either.Bytes, ) (T, bool) { @@ -274,7 +290,6 @@ func (rClient *replayClient[T]) newMapEventsBytesToTFn(errCh chan<- error) func( ctx context.Context, eitherEventBz either.Bytes, ) (_ T, skip bool) { - logger := polylog.Ctx(ctx) eventBz, err := eitherEventBz.ValueOrError() if err != nil { errCh <- err @@ -294,7 +309,10 @@ func (rClient *replayClient[T]) newMapEventsBytesToTFn(errCh chan<- error) func( return *new(T), true } - logger.Error().Err(err).Msgf("unexpected error deserialising event") + // Don't publish (skip) if there was some other kind of error, + // and send that error on the errCh. + cancel() + errCh <- err return *new(T), true } return event, false diff --git a/pkg/client/interface.go b/pkg/client/interface.go index 14c8b6a6c..e02eb7bec 100644 --- a/pkg/client/interface.go +++ b/pkg/client/interface.go @@ -229,6 +229,15 @@ type TxClientOption func(TxClient) // SupplierClientOption defines a function type that modifies the SupplierClient. type SupplierClientOption func(SupplierClient) +// DelegationClientOption defines a function type that modifies the BlockClient. +type DelegationClientOption func(DelegationClient) + +// BlockClientOption defines a function type that modifies the BlockClient. +type BlockClientOption func(BlockClient) + +// EventsReplayClientOption defines a function type that modifies the ReplayClient. +type EventsReplayClientOption[T any] func(EventsReplayClient[T]) + // AccountQueryClient defines an interface that enables the querying of the // on-chain account information type AccountQueryClient interface { diff --git a/pkg/client/tx/client.go b/pkg/client/tx/client.go index 95e1a2599..3d46b2527 100644 --- a/pkg/client/tx/client.go +++ b/pkg/client/tx/client.go @@ -100,6 +100,11 @@ type txClient struct { // is used to ensure that transactions error channels receive and close in the event // that they have not already by the given timeout height. txTimeoutPool txTimeoutPool + + // connRetryLimit is the number of times the underlying replay client + // should retry in the event that it encounters an error or its connection is interrupted. + // If connRetryLimit is < 0, it will retry indefinitely. + connRetryLimit int } type ( @@ -139,6 +144,7 @@ func NewTxClient( commitTimeoutHeightOffset: DefaultCommitTimeoutHeightOffset, txErrorChans: make(txErrorChansByHash), txTimeoutPool: make(txTimeoutPool), + connRetryLimit: events.DefaultConnRetryLimit, } if err = depinject.Inject( @@ -167,6 +173,7 @@ func NewTxClient( eventQuery, UnmarshalTxResult, defaultTxReplayLimit, + events.WithConnRetryLimit[*abci.TxResult](txnClient.connRetryLimit), ) if err != nil { return nil, err diff --git a/pkg/client/tx/options.go b/pkg/client/tx/options.go index 5364008a5..46e3dbcd2 100644 --- a/pkg/client/tx/options.go +++ b/pkg/client/tx/options.go @@ -18,3 +18,13 @@ func WithSigningKeyName(keyName string) client.TxClientOption { client.(*txClient).signingKeyName = keyName } } + +// WithConnRetryLimit returns an option function which sets the number +// of times the underlying replay client should retry in the event that it encounters +// an error or its connection is interrupted. +// If connRetryLimit is < 0, it will retry indefinitely. +func WithConnRetryLimit(limit int) client.TxClientOption { + return func(client client.TxClient) { + client.(*txClient).connRetryLimit = limit + } +} diff --git a/pkg/retry/retry.go b/pkg/retry/retry.go index bd06af941..c864278a3 100644 --- a/pkg/retry/retry.go +++ b/pkg/retry/retry.go @@ -58,6 +58,11 @@ func OnError( return nil } + if retryLimit < 0 { + time.Sleep(retryDelay) + continue + } + if retryCount >= retryLimit { return err } diff --git a/pkg/sdk/deps_builder.go b/pkg/sdk/deps_builder.go index 57c9ab745..c4f56045d 100644 --- a/pkg/sdk/deps_builder.go +++ b/pkg/sdk/deps_builder.go @@ -11,7 +11,7 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" - block "github.com/pokt-network/poktroll/pkg/client/block" + "github.com/pokt-network/poktroll/pkg/client/block" "github.com/pokt-network/poktroll/pkg/client/delegation" eventsquery "github.com/pokt-network/poktroll/pkg/client/events" "github.com/pokt-network/poktroll/pkg/client/query" @@ -38,8 +38,10 @@ func (sdk *poktrollSDK) buildDeps( eventsQueryClient := eventsquery.NewEventsQueryClient(pocketNodeWebsocketURL) deps = depinject.Configs(deps, depinject.Supply(eventsQueryClient)) + blockClientConnRetryLimitOpt := block.WithConnRetryLimit(config.ConnRetryLimit) + // Create and supply the block client that depends on the events query client - blockClient, err := block.NewBlockClient(ctx, deps) + blockClient, err := block.NewBlockClient(ctx, deps, blockClientConnRetryLimitOpt) if err != nil { return nil, err } @@ -83,8 +85,10 @@ func (sdk *poktrollSDK) buildDeps( } deps = depinject.Configs(deps, depinject.Supply(sessionQuerier)) + delegationClientConnRetryLimitOpt := delegation.WithConnRetryLimit(config.ConnRetryLimit) + // Create and supply the delegation client - delegationClient, err := delegation.NewDelegationClient(ctx, deps) + delegationClient, err := delegation.NewDelegationClient(ctx, deps, delegationClientConnRetryLimitOpt) if err != nil { return nil, err } diff --git a/pkg/sdk/sdk.go b/pkg/sdk/sdk.go index 5cc4d1458..790173abb 100644 --- a/pkg/sdk/sdk.go +++ b/pkg/sdk/sdk.go @@ -26,6 +26,7 @@ var _ POKTRollSDK = (*poktrollSDK)(nil) type POKTRollSDKConfig struct { QueryNodeGRPCUrl *url.URL QueryNodeUrl *url.URL + ConnRetryLimit int PrivateKey cryptotypes.PrivKey Deps depinject.Config } diff --git a/testutil/testclient/testeventsquery/client.go b/testutil/testclient/testeventsquery/client.go index aec74609f..b16461ab6 100644 --- a/testutil/testclient/testeventsquery/client.go +++ b/testutil/testclient/testeventsquery/client.go @@ -103,7 +103,7 @@ func NewAnyTimesEventsBytesEventsQueryClient( eventsQueryClient := mockclient.NewMockEventsQueryClient(ctrl) eventsQueryClient.EXPECT().Close().Times(1) eventsQueryClient.EXPECT(). - EventsBytes(gomock.AssignableToTypeOf(ctx), gomock.Eq(expectedQuery)). + EventsBytes(gomock.Any(), gomock.Eq(expectedQuery)). DoAndReturn( func(ctx context.Context, query string) (client.EventsBytesObservable, error) { bytesObsvbl, bytesPublishCh := channel.NewReplayObservable[either.Bytes](ctx, 1) From 6f8153b4688a54c41c13c8526fd573822dddb3a9 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 3 Apr 2024 10:20:56 +0200 Subject: [PATCH 05/28] fix: default conn retry limit & simplify config usage --- e2e/tests/session_steps_test.go | 1 - pkg/client/block/client.go | 4 +--- pkg/client/delegation/client.go | 4 +--- pkg/client/events/options.go | 7 ++++++- pkg/client/events/replay_client.go | 21 ++++++++++++------- pkg/client/tx/client.go | 1 - pkg/sdk/deps_builder.go | 4 ++-- pkg/sdk/sdk.go | 2 +- testutil/testclient/testeventsquery/client.go | 2 +- 9 files changed, 25 insertions(+), 21 deletions(-) diff --git a/e2e/tests/session_steps_test.go b/e2e/tests/session_steps_test.go index cc2acd89d..e60362da0 100644 --- a/e2e/tests/session_steps_test.go +++ b/e2e/tests/session_steps_test.go @@ -123,7 +123,6 @@ func (s *suite) TheSupplierHasServicedASessionWithRelaysForServiceForApplication msgSenderQuery, tx.UnmarshalTxResult, testEventsReplayClientBufferSize, - events.DefaultConnRetryLimit, ) require.NoError(s, err) s.scenarioState[txResultEventsReplayClientKey] = txSendEventsReplayClient diff --git a/pkg/client/block/client.go b/pkg/client/block/client.go index 239f5464c..4411d9b0b 100644 --- a/pkg/client/block/client.go +++ b/pkg/client/block/client.go @@ -37,9 +37,7 @@ func NewBlockClient( deps depinject.Config, opts ...client.BlockClientOption, ) (_ client.BlockClient, err error) { - bClient := &blockClient{ - connRetryLimit: events.DefaultConnRetryLimit, - } + bClient := &blockClient{} for _, opt := range opts { opt(bClient) diff --git a/pkg/client/delegation/client.go b/pkg/client/delegation/client.go index 20173c620..00f45e2fa 100644 --- a/pkg/client/delegation/client.go +++ b/pkg/client/delegation/client.go @@ -48,9 +48,7 @@ func NewDelegationClient( deps depinject.Config, opts ...client.DelegationClientOption, ) (_ client.DelegationClient, err error) { - dClient := &delegationClient{ - connRetryLimit: events.DefaultConnRetryLimit, - } + dClient := &delegationClient{} for _, opt := range opts { opt(dClient) diff --git a/pkg/client/events/options.go b/pkg/client/events/options.go index caf0a8841..b3a959fb0 100644 --- a/pkg/client/events/options.go +++ b/pkg/client/events/options.go @@ -16,6 +16,11 @@ func WithDialer(dialer client.Dialer) client.EventsQueryClientOption { // If connRetryLimit is < 0, it will retry indefinitely. func WithConnRetryLimit[T any](limit int) client.EventsReplayClientOption[T] { return func(client client.EventsReplayClient[T]) { - client.(*replayClient[T]).connRetryLimit = limit + // Ignore the zero value because limit may be provided via a partially + // configured config struct (i.e. no retry limit set). + // The default will be used instead. + if limit != 0 { + client.(*replayClient[T]).connRetryLimit = limit + } } } diff --git a/pkg/client/events/replay_client.go b/pkg/client/events/replay_client.go index 7df1a0997..14c246a7f 100644 --- a/pkg/client/events/replay_client.go +++ b/pkg/client/events/replay_client.go @@ -114,6 +114,7 @@ func NewEventsReplayClient[T any]( queryString: queryString, eventDecoder: newEventFn, replayObsBufferSize: replayObsBufferSize, + connRetryLimit: DefaultConnRetryLimit, } for _, opt := range opts { @@ -137,7 +138,7 @@ func NewEventsReplayClient[T any]( } // Concurrently publish events to the observable emitted by replayObsCache. - go rClient.goPublishEvents(ctx, rClient.connRetryLimit) + go rClient.goPublishEvents(ctx) return rClient, nil } @@ -198,13 +199,13 @@ func (rClient *replayClient[T]) Close() { // goPublishEvents runs the work function returned by retryPublishEventsFactory, // re-invoking it according to the arguments to retry.OnError when the events bytes // observable returns an asynchronous error. -func (rClient *replayClient[T]) goPublishEvents(ctx context.Context, retryLimit int) { +func (rClient *replayClient[T]) goPublishEvents(ctx context.Context) { // React to errors by getting a new events bytes observable, re-mapping it, // and send it to replayObsCachePublishCh such that // replayObsCache.Last(ctx, 1) will return it. publishErr := retry.OnError( ctx, - retryLimit, + rClient.connRetryLimit, eventsBytesRetryDelay, eventsBytesRetryResetTimeout, "goPublishEvents", @@ -282,10 +283,10 @@ func (rClient *replayClient[T]) retryPublishEventsFactory(ctx context.Context) f // If deserialisation failed because the event bytes were for a different event // type, this value is also skipped. If deserialisation failed for some other // reason, this function panics. -func (rClient *replayClient[T]) newMapEventsBytesToTFn(errCh chan<- error, cancel context.CancelFunc) func( - context.Context, - either.Bytes, -) (T, bool) { +func (rClient *replayClient[T]) newMapEventsBytesToTFn( + errCh chan<- error, + cancelEventsBzObs context.CancelFunc, +) func(context.Context, either.Bytes) (T, bool) { return func( ctx context.Context, eitherEventBz either.Bytes, @@ -311,8 +312,12 @@ func (rClient *replayClient[T]) newMapEventsBytesToTFn(errCh chan<- error, cance // Don't publish (skip) if there was some other kind of error, // and send that error on the errCh. - cancel() errCh <- err + + // The source observable may not necessarily close automatically in this case, + // cancel its context to ensure its closure and prevent a memory/goroutine leak. + cancelEventsBzObs() + return *new(T), true } return event, false diff --git a/pkg/client/tx/client.go b/pkg/client/tx/client.go index 3d46b2527..e1ff933d4 100644 --- a/pkg/client/tx/client.go +++ b/pkg/client/tx/client.go @@ -144,7 +144,6 @@ func NewTxClient( commitTimeoutHeightOffset: DefaultCommitTimeoutHeightOffset, txErrorChans: make(txErrorChansByHash), txTimeoutPool: make(txTimeoutPool), - connRetryLimit: events.DefaultConnRetryLimit, } if err = depinject.Inject( diff --git a/pkg/sdk/deps_builder.go b/pkg/sdk/deps_builder.go index c4f56045d..7f2574319 100644 --- a/pkg/sdk/deps_builder.go +++ b/pkg/sdk/deps_builder.go @@ -38,7 +38,7 @@ func (sdk *poktrollSDK) buildDeps( eventsQueryClient := eventsquery.NewEventsQueryClient(pocketNodeWebsocketURL) deps = depinject.Configs(deps, depinject.Supply(eventsQueryClient)) - blockClientConnRetryLimitOpt := block.WithConnRetryLimit(config.ConnRetryLimit) + blockClientConnRetryLimitOpt := block.WithConnRetryLimit(sdk.config.ConnRetryLimit) // Create and supply the block client that depends on the events query client blockClient, err := block.NewBlockClient(ctx, deps, blockClientConnRetryLimitOpt) @@ -85,7 +85,7 @@ func (sdk *poktrollSDK) buildDeps( } deps = depinject.Configs(deps, depinject.Supply(sessionQuerier)) - delegationClientConnRetryLimitOpt := delegation.WithConnRetryLimit(config.ConnRetryLimit) + delegationClientConnRetryLimitOpt := delegation.WithConnRetryLimit(sdk.config.ConnRetryLimit) // Create and supply the delegation client delegationClient, err := delegation.NewDelegationClient(ctx, deps, delegationClientConnRetryLimitOpt) diff --git a/pkg/sdk/sdk.go b/pkg/sdk/sdk.go index 790173abb..93b7bb3cd 100644 --- a/pkg/sdk/sdk.go +++ b/pkg/sdk/sdk.go @@ -80,7 +80,7 @@ func NewPOKTRollSDK(ctx context.Context, config *POKTRollSDKConfig) (POKTRollSDK // Build the dependencies if they are not provided in the config. if config.Deps != nil { deps = config.Deps - } else if deps, err = sdk.buildDeps(ctx, config); err != nil { + } else if deps, err = sdk.buildDeps(ctx); err != nil { return nil, err } diff --git a/testutil/testclient/testeventsquery/client.go b/testutil/testclient/testeventsquery/client.go index b16461ab6..462764af7 100644 --- a/testutil/testclient/testeventsquery/client.go +++ b/testutil/testclient/testeventsquery/client.go @@ -45,7 +45,7 @@ func NewOneTimeEventsQuery( ctrl := gomock.NewController(t) eventsQueryClient := mockclient.NewMockEventsQueryClient(ctrl) - eventsQueryClient.EXPECT().EventsBytes(gomock.Eq(ctx), gomock.Eq(query)). + eventsQueryClient.EXPECT().EventsBytes(gomock.Any(), gomock.Eq(query)). DoAndReturn(func( ctx context.Context, query string, From 5c8f542246e5fbe35003873d63b0fa1573b0edd1 Mon Sep 17 00:00:00 2001 From: Eric Nielson Date: Fri, 12 Apr 2024 10:19:19 -0400 Subject: [PATCH 06/28] Fix in OnError and test update --- pkg/retry/retry.go | 9 ++-- pkg/retry/retry_test.go | 97 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 6 deletions(-) diff --git a/pkg/retry/retry.go b/pkg/retry/retry.go index c864278a3..3f8f244aa 100644 --- a/pkg/retry/retry.go +++ b/pkg/retry/retry.go @@ -58,12 +58,9 @@ func OnError( return nil } - if retryLimit < 0 { - time.Sleep(retryDelay) - continue - } - - if retryCount >= retryLimit { + // Return error if retry limit reached + // A negative retryLimit allows limitless retries + if retryLimit >= 0 && retryCount >= retryLimit { return err } diff --git a/pkg/retry/retry_test.go b/pkg/retry/retry_test.go index c9dab961c..e1adb3142 100644 --- a/pkg/retry/retry_test.go +++ b/pkg/retry/retry_test.go @@ -335,3 +335,100 @@ func TestOnError_RetryCountResetTimeout(t *testing.T) { require.Contains(t, line, expectedPrefix) } } + +// assert that a negative retry limit continually calls workFn +func TestOnError_NegativeRetryLimit(t *testing.T) { + t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") + + // Setup test variables and log capture + var ( + logOutput bytes.Buffer + testFnCallCount int32 + minimumCallCount = 100 + expectedRetryDelay = time.Millisecond + retryLimit = -1 + retryResetTimeout = 3 * time.Millisecond + testFnCallTimeCh = make(chan time.Time, minimumCallCount) + ctx = context.Background() + ) + + // Redirect the log output for verification later + log.SetOutput(&logOutput) + + // Define the test function that simulates an error and counts its invocations + testFn := func() chan error { + // Track the invocation time + testFnCallTimeCh <- time.Now() + + errCh := make(chan error, 1) + + count := atomic.LoadInt32(&testFnCallCount) + if count == int32(retryLimit) { + go func() { + time.Sleep(retryResetTimeout) + errCh <- testErr + }() + } else { + errCh <- testErr + } + + // Increment the invocation count atomically + atomic.AddInt32(&testFnCallCount, 1) + return errCh + } + + retryOnErrorErrCh := make(chan error, 1) + // Spawn a goroutine to test the OnError function + go func() { + retryOnErrorErrCh <- retry.OnError( + ctx, + retryLimit, + expectedRetryDelay, + retryResetTimeout, + "TestOnError", + testFn, + ) + }() + + // Wait for the OnError function to execute and retry the expected number of times + totalExpectedDelay := expectedRetryDelay * time.Duration(minimumCallCount) + time.Sleep(totalExpectedDelay + 100*time.Millisecond) + + // Assert that the test function was called the expected number of times + require.GreaterOrEqual(t, minimumCallCount, int(testFnCallCount)) + + // Assert that the retry delay between function calls matches the expected delay + var prevCallTime = new(time.Time) + for i := 0; i < minimumCallCount; i++ { + select { + case nextCallTime := <-testFnCallTimeCh: + if i != 0 { + actualRetryDelay := nextCallTime.Sub(*prevCallTime) + require.GreaterOrEqual(t, actualRetryDelay, expectedRetryDelay) + } + + *prevCallTime = nextCallTime + default: + t.Fatalf( + "expected %d calls to testFn, but only received %d", + minimumCallCount, i+1, + ) + } + } + + // Verify the logged error messages + var ( + logOutputLines = strings.Split(strings.Trim(logOutput.String(), "\n"), "\n") + expectedPrefix = "ERROR: retrying TestOnError after error: test error" + ) + + require.Lenf( + t, logOutputLines, + minimumCallCount-1, + "expected %d log lines, got %d", + minimumCallCount-1, len(logOutputLines), + ) + for _, line := range logOutputLines { + require.Contains(t, line, expectedPrefix) + } +} From 3b3b7cb803d6a8e324518256041bf1f1b3b2f834 Mon Sep 17 00:00:00 2001 From: Eric Nielson Date: Mon, 15 Apr 2024 11:22:00 -0400 Subject: [PATCH 07/28] retry test test fixes --- pkg/retry/retry_test.go | 53 ++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/pkg/retry/retry_test.go b/pkg/retry/retry_test.go index e1adb3142..f289165da 100644 --- a/pkg/retry/retry_test.go +++ b/pkg/retry/retry_test.go @@ -11,7 +11,6 @@ import ( "bytes" "context" "fmt" - "log" "strings" "sync/atomic" "testing" @@ -19,6 +18,8 @@ import ( "github.com/stretchr/testify/require" + "github.com/pokt-network/poktroll/pkg/polylog/polyzero" + _ "github.com/pokt-network/poktroll/pkg/polylog/polyzero" "github.com/pokt-network/poktroll/pkg/retry" ) @@ -28,12 +29,12 @@ var testErr = fmt.Errorf("test error") // It ensures that the function correctly retries a failing operation for a specified // number of times with the expected delay between retries. func TestOnError(t *testing.T) { - t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") + // t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") // Setting up the test variables. var ( // logOutput captures the log output for verification of logged messages. - logOutput bytes.Buffer + logOutput = new(bytes.Buffer) // expectedRetryDelay is the duration we expect between retries. expectedRetryDelay = time.Millisecond // expectedRetryLimit is the maximum number of retries the test expects. @@ -48,8 +49,10 @@ func TestOnError(t *testing.T) { ctx = context.Background() ) - // Redirect the standard logger's output to our custom buffer for later verification. - log.SetOutput(&logOutput) + // Redirect the log output for verification later + logOpt := polyzero.WithOutput(logOutput) + // Construct a new polylog logger & attach it to the context. + ctx = polyzero.NewLogger(logOpt).WithContext(ctx) // Define testFn, a function that simulates a failing operation and logs its invocation times. testFn := func() chan error { @@ -118,7 +121,7 @@ func TestOnError(t *testing.T) { } // Verify the error messages logged during the retries. - expectedErrLine := "ERROR: retrying TestOnError after error: test error" + expectedErrLine := `"error":"test error"` trimmedLogOutput := strings.Trim(logOutput.String(), "\n") logOutputLines := strings.Split(trimmedLogOutput, "\n") require.Lenf(t, logOutputLines, expectedRetryLimit, "unexpected number of log lines") @@ -133,11 +136,11 @@ func TestOnError_ExitsWhenCtxCloses(t *testing.T) { } func TestOnError_ExitsWhenErrChCloses(t *testing.T) { - t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") + // t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") // Setup test variables and log capture var ( - logOutput bytes.Buffer + logOutput = new(bytes.Buffer) testFnCallCount int32 expectedRetryDelay = time.Millisecond expectedRetryLimit = 3 @@ -148,7 +151,9 @@ func TestOnError_ExitsWhenErrChCloses(t *testing.T) { ) // Redirect the log output for verification later - log.SetOutput(&logOutput) + logOpt := polyzero.WithOutput(logOutput) + // Construct a new polylog logger & attach it to the context. + ctx = polyzero.NewLogger(logOpt).WithContext(ctx) // Define the test function that simulates an error and counts its invocations testFn := func() chan error { @@ -216,8 +221,8 @@ func TestOnError_ExitsWhenErrChCloses(t *testing.T) { logOutputLines = strings.Split(strings.Trim(logOutput.String(), "\n"), "\n") errorLines = logOutputLines[:len(logOutputLines)-1] warnLine = logOutputLines[len(logOutputLines)-1] - expectedWarnMsg = "WARN: error channel for TestOnError_ExitsWhenErrChCloses closed, will no longer retry on error" - expectedErrMsg = "ERROR: retrying TestOnError_ExitsWhenErrChCloses after error: test error" + expectedWarnMsg = "error channel closed, will no longer retry on error" + expectedErrMsg = `"error":"test error"` ) require.Lenf( @@ -234,11 +239,11 @@ func TestOnError_ExitsWhenErrChCloses(t *testing.T) { // assert that retryCount resets on success func TestOnError_RetryCountResetTimeout(t *testing.T) { - t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") + // t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") // Setup test variables and log capture var ( - logOutput bytes.Buffer + logOutput = new(bytes.Buffer) testFnCallCount int32 expectedRetryDelay = time.Millisecond expectedRetryLimit = 9 @@ -249,7 +254,9 @@ func TestOnError_RetryCountResetTimeout(t *testing.T) { ) // Redirect the log output for verification later - log.SetOutput(&logOutput) + logOpt := polyzero.WithOutput(logOutput) + // Construct a new polylog logger & attach it to the context. + ctx = polyzero.NewLogger(logOpt).WithContext(ctx) // Define the test function that simulates an error and counts its invocations testFn := func() chan error { @@ -315,7 +322,7 @@ func TestOnError_RetryCountResetTimeout(t *testing.T) { // Verify the logged error messages var ( logOutputLines = strings.Split(strings.Trim(logOutput.String(), "\n"), "\n") - expectedPrefix = "ERROR: retrying TestOnError after error: test error" + expectedErrMsg = `"error":"test error"` ) select { @@ -332,19 +339,19 @@ func TestOnError_RetryCountResetTimeout(t *testing.T) { expectedRetryLimit-1, len(logOutputLines), ) for _, line := range logOutputLines { - require.Contains(t, line, expectedPrefix) + require.Contains(t, line, expectedErrMsg) } } // assert that a negative retry limit continually calls workFn func TestOnError_NegativeRetryLimit(t *testing.T) { - t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") + // t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") // Setup test variables and log capture var ( - logOutput bytes.Buffer + logOutput = new(bytes.Buffer) testFnCallCount int32 - minimumCallCount = 100 + minimumCallCount = 99 expectedRetryDelay = time.Millisecond retryLimit = -1 retryResetTimeout = 3 * time.Millisecond @@ -353,7 +360,9 @@ func TestOnError_NegativeRetryLimit(t *testing.T) { ) // Redirect the log output for verification later - log.SetOutput(&logOutput) + logOpt := polyzero.WithOutput(logOutput) + // Construct a new polylog logger & attach it to the context. + ctx = polyzero.NewLogger(logOpt).WithContext(ctx) // Define the test function that simulates an error and counts its invocations testFn := func() chan error { @@ -419,7 +428,7 @@ func TestOnError_NegativeRetryLimit(t *testing.T) { // Verify the logged error messages var ( logOutputLines = strings.Split(strings.Trim(logOutput.String(), "\n"), "\n") - expectedPrefix = "ERROR: retrying TestOnError after error: test error" + expectedErrMsg = `"error":"test error"` ) require.Lenf( @@ -429,6 +438,6 @@ func TestOnError_NegativeRetryLimit(t *testing.T) { minimumCallCount-1, len(logOutputLines), ) for _, line := range logOutputLines { - require.Contains(t, line, expectedPrefix) + require.Contains(t, line, expectedErrMsg) } } From 983f58dcb0aa4791789843db7a97abccac4f5382 Mon Sep 17 00:00:00 2001 From: Eric Nielson Date: Tue, 26 Mar 2024 20:06:56 -0400 Subject: [PATCH 08/28] Removed panics in replay_client --- pkg/client/events/replay_client.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/pkg/client/events/replay_client.go b/pkg/client/events/replay_client.go index 14c246a7f..8ccd9d314 100644 --- a/pkg/client/events/replay_client.go +++ b/pkg/client/events/replay_client.go @@ -2,7 +2,6 @@ package events import ( "context" - "fmt" "time" "cosmossdk.io/depinject" @@ -11,6 +10,7 @@ import ( "github.com/pokt-network/poktroll/pkg/either" "github.com/pokt-network/poktroll/pkg/observable" "github.com/pokt-network/poktroll/pkg/observable/channel" + "github.com/pokt-network/poktroll/pkg/polylog" "github.com/pokt-network/poktroll/pkg/retry" ) @@ -138,7 +138,16 @@ func NewEventsReplayClient[T any]( } // Concurrently publish events to the observable emitted by replayObsCache. - go rClient.goPublishEvents(ctx) + go func() { + for { + select { + case <-ctx.Done(): + return + default: + rClient.publishEvents(ctx) + } + } + }() return rClient, nil } @@ -196,7 +205,7 @@ func (rClient *replayClient[T]) Close() { close(rClient.replayObsCachePublishCh) } -// goPublishEvents runs the work function returned by retryPublishEventsFactory, +// publishEvents runs the work function returned by retryPublishEventsFactory, // re-invoking it according to the arguments to retry.OnError when the events bytes // observable returns an asynchronous error. func (rClient *replayClient[T]) goPublishEvents(ctx context.Context) { @@ -291,6 +300,7 @@ func (rClient *replayClient[T]) newMapEventsBytesToTFn( ctx context.Context, eitherEventBz either.Bytes, ) (_ T, skip bool) { + logger := polylog.Ctx(ctx) eventBz, err := eitherEventBz.ValueOrError() if err != nil { errCh <- err From def63d9838a9d510d9263485f357ec03841d4d1c Mon Sep 17 00:00:00 2001 From: Bryan White Date: Tue, 2 Apr 2024 20:34:16 +0200 Subject: [PATCH 09/28] refactor: factor out connection retry limit --- pkg/client/events/replay_client.go | 16 +++------------- pkg/client/tx/client.go | 1 + 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/pkg/client/events/replay_client.go b/pkg/client/events/replay_client.go index 8ccd9d314..9540672bb 100644 --- a/pkg/client/events/replay_client.go +++ b/pkg/client/events/replay_client.go @@ -2,6 +2,7 @@ package events import ( "context" + "fmt" "time" "cosmossdk.io/depinject" @@ -10,7 +11,6 @@ import ( "github.com/pokt-network/poktroll/pkg/either" "github.com/pokt-network/poktroll/pkg/observable" "github.com/pokt-network/poktroll/pkg/observable/channel" - "github.com/pokt-network/poktroll/pkg/polylog" "github.com/pokt-network/poktroll/pkg/retry" ) @@ -138,16 +138,7 @@ func NewEventsReplayClient[T any]( } // Concurrently publish events to the observable emitted by replayObsCache. - go func() { - for { - select { - case <-ctx.Done(): - return - default: - rClient.publishEvents(ctx) - } - } - }() + go rClient.goPublishEvents(ctx, rClient.connRetryLimit) return rClient, nil } @@ -205,7 +196,7 @@ func (rClient *replayClient[T]) Close() { close(rClient.replayObsCachePublishCh) } -// publishEvents runs the work function returned by retryPublishEventsFactory, +// goPublishEvents runs the work function returned by retryPublishEventsFactory, // re-invoking it according to the arguments to retry.OnError when the events bytes // observable returns an asynchronous error. func (rClient *replayClient[T]) goPublishEvents(ctx context.Context) { @@ -300,7 +291,6 @@ func (rClient *replayClient[T]) newMapEventsBytesToTFn( ctx context.Context, eitherEventBz either.Bytes, ) (_ T, skip bool) { - logger := polylog.Ctx(ctx) eventBz, err := eitherEventBz.ValueOrError() if err != nil { errCh <- err diff --git a/pkg/client/tx/client.go b/pkg/client/tx/client.go index e1ff933d4..3d46b2527 100644 --- a/pkg/client/tx/client.go +++ b/pkg/client/tx/client.go @@ -144,6 +144,7 @@ func NewTxClient( commitTimeoutHeightOffset: DefaultCommitTimeoutHeightOffset, txErrorChans: make(txErrorChansByHash), txTimeoutPool: make(txTimeoutPool), + connRetryLimit: events.DefaultConnRetryLimit, } if err = depinject.Inject( From c397674af70648502f66fca4626a8ae8feaa851a Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 3 Apr 2024 10:20:56 +0200 Subject: [PATCH 10/28] fix: default conn retry limit & simplify config usage --- pkg/client/events/replay_client.go | 2 +- pkg/client/tx/client.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/client/events/replay_client.go b/pkg/client/events/replay_client.go index 9540672bb..14c246a7f 100644 --- a/pkg/client/events/replay_client.go +++ b/pkg/client/events/replay_client.go @@ -138,7 +138,7 @@ func NewEventsReplayClient[T any]( } // Concurrently publish events to the observable emitted by replayObsCache. - go rClient.goPublishEvents(ctx, rClient.connRetryLimit) + go rClient.goPublishEvents(ctx) return rClient, nil } diff --git a/pkg/client/tx/client.go b/pkg/client/tx/client.go index 3d46b2527..e1ff933d4 100644 --- a/pkg/client/tx/client.go +++ b/pkg/client/tx/client.go @@ -144,7 +144,6 @@ func NewTxClient( commitTimeoutHeightOffset: DefaultCommitTimeoutHeightOffset, txErrorChans: make(txErrorChansByHash), txTimeoutPool: make(txTimeoutPool), - connRetryLimit: events.DefaultConnRetryLimit, } if err = depinject.Inject( From 9fe81eb0c2a39c54a5fa61cbce9a818e56ef0dd6 Mon Sep 17 00:00:00 2001 From: Eric Nielson Date: Fri, 12 Apr 2024 10:19:19 -0400 Subject: [PATCH 11/28] Fix in OnError and test update --- pkg/retry/retry_test.go | 97 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/pkg/retry/retry_test.go b/pkg/retry/retry_test.go index f289165da..afead046a 100644 --- a/pkg/retry/retry_test.go +++ b/pkg/retry/retry_test.go @@ -441,3 +441,100 @@ func TestOnError_NegativeRetryLimit(t *testing.T) { require.Contains(t, line, expectedErrMsg) } } + +// assert that a negative retry limit continually calls workFn +func TestOnError_NegativeRetryLimit(t *testing.T) { + t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") + + // Setup test variables and log capture + var ( + logOutput bytes.Buffer + testFnCallCount int32 + minimumCallCount = 100 + expectedRetryDelay = time.Millisecond + retryLimit = -1 + retryResetTimeout = 3 * time.Millisecond + testFnCallTimeCh = make(chan time.Time, minimumCallCount) + ctx = context.Background() + ) + + // Redirect the log output for verification later + log.SetOutput(&logOutput) + + // Define the test function that simulates an error and counts its invocations + testFn := func() chan error { + // Track the invocation time + testFnCallTimeCh <- time.Now() + + errCh := make(chan error, 1) + + count := atomic.LoadInt32(&testFnCallCount) + if count == int32(retryLimit) { + go func() { + time.Sleep(retryResetTimeout) + errCh <- testErr + }() + } else { + errCh <- testErr + } + + // Increment the invocation count atomically + atomic.AddInt32(&testFnCallCount, 1) + return errCh + } + + retryOnErrorErrCh := make(chan error, 1) + // Spawn a goroutine to test the OnError function + go func() { + retryOnErrorErrCh <- retry.OnError( + ctx, + retryLimit, + expectedRetryDelay, + retryResetTimeout, + "TestOnError", + testFn, + ) + }() + + // Wait for the OnError function to execute and retry the expected number of times + totalExpectedDelay := expectedRetryDelay * time.Duration(minimumCallCount) + time.Sleep(totalExpectedDelay + 100*time.Millisecond) + + // Assert that the test function was called the expected number of times + require.GreaterOrEqual(t, minimumCallCount, int(testFnCallCount)) + + // Assert that the retry delay between function calls matches the expected delay + var prevCallTime = new(time.Time) + for i := 0; i < minimumCallCount; i++ { + select { + case nextCallTime := <-testFnCallTimeCh: + if i != 0 { + actualRetryDelay := nextCallTime.Sub(*prevCallTime) + require.GreaterOrEqual(t, actualRetryDelay, expectedRetryDelay) + } + + *prevCallTime = nextCallTime + default: + t.Fatalf( + "expected %d calls to testFn, but only received %d", + minimumCallCount, i+1, + ) + } + } + + // Verify the logged error messages + var ( + logOutputLines = strings.Split(strings.Trim(logOutput.String(), "\n"), "\n") + expectedPrefix = "ERROR: retrying TestOnError after error: test error" + ) + + require.Lenf( + t, logOutputLines, + minimumCallCount-1, + "expected %d log lines, got %d", + minimumCallCount-1, len(logOutputLines), + ) + for _, line := range logOutputLines { + require.Contains(t, line, expectedPrefix) + } +} From 0efe6697840096d2ae251473c9e3d45e6a4b4879 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Fri, 12 Apr 2024 15:21:43 -0700 Subject: [PATCH 12/28] Update pkg/client/events/replay_client.go --- pkg/client/events/replay_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/client/events/replay_client.go b/pkg/client/events/replay_client.go index 14c246a7f..d645cc758 100644 --- a/pkg/client/events/replay_client.go +++ b/pkg/client/events/replay_client.go @@ -29,7 +29,7 @@ const ( // eventsBytesRetryLimit is the maximum number of times to attempt to // re-establish the events query bytes subscription when the events bytes // observable returns an error or closes. - // TODO to make this a customisable parameter in the appgateserver and relayminer config files. + // TODO_TECHDEBT: to make this a customizable parameter in the appgateserver and relayminer config files. eventsBytesRetryLimit = 10 eventsBytesRetryResetTimeout = 10 * time.Second // replayObsCacheBufferSize is the replay buffer size of the From 5b2b59ba125cb70048043ff096d273798d8bf1a7 Mon Sep 17 00:00:00 2001 From: Eric Nielson Date: Mon, 15 Apr 2024 11:26:42 -0400 Subject: [PATCH 13/28] Variable rename --- e2e/tests/session_steps_test.go | 2 +- pkg/retry/retry_test.go | 107 ++------------------------------ pkg/sdk/sdk.go | 2 +- 3 files changed, 7 insertions(+), 104 deletions(-) diff --git a/e2e/tests/session_steps_test.go b/e2e/tests/session_steps_test.go index e60362da0..83a78e2f1 100644 --- a/e2e/tests/session_steps_test.go +++ b/e2e/tests/session_steps_test.go @@ -122,7 +122,7 @@ func (s *suite) TheSupplierHasServicedASessionWithRelaysForServiceForApplication deps, msgSenderQuery, tx.UnmarshalTxResult, - testEventsReplayClientBufferSize, + eventsReplayClientBufferSize, ) require.NoError(s, err) s.scenarioState[txResultEventsReplayClientKey] = txSendEventsReplayClient diff --git a/pkg/retry/retry_test.go b/pkg/retry/retry_test.go index afead046a..1f0cf6844 100644 --- a/pkg/retry/retry_test.go +++ b/pkg/retry/retry_test.go @@ -29,7 +29,7 @@ var testErr = fmt.Errorf("test error") // It ensures that the function correctly retries a failing operation for a specified // number of times with the expected delay between retries. func TestOnError(t *testing.T) { - // t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") + t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") // Setting up the test variables. var ( @@ -136,7 +136,7 @@ func TestOnError_ExitsWhenCtxCloses(t *testing.T) { } func TestOnError_ExitsWhenErrChCloses(t *testing.T) { - // t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") + t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") // Setup test variables and log capture var ( @@ -239,7 +239,7 @@ func TestOnError_ExitsWhenErrChCloses(t *testing.T) { // assert that retryCount resets on success func TestOnError_RetryCountResetTimeout(t *testing.T) { - // t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") + t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") // Setup test variables and log capture var ( @@ -345,13 +345,13 @@ func TestOnError_RetryCountResetTimeout(t *testing.T) { // assert that a negative retry limit continually calls workFn func TestOnError_NegativeRetryLimit(t *testing.T) { - // t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") + t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") // Setup test variables and log capture var ( logOutput = new(bytes.Buffer) testFnCallCount int32 - minimumCallCount = 99 + minimumCallCount = 90 expectedRetryDelay = time.Millisecond retryLimit = -1 retryResetTimeout = 3 * time.Millisecond @@ -441,100 +441,3 @@ func TestOnError_NegativeRetryLimit(t *testing.T) { require.Contains(t, line, expectedErrMsg) } } - -// assert that a negative retry limit continually calls workFn -func TestOnError_NegativeRetryLimit(t *testing.T) { - t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") - - // Setup test variables and log capture - var ( - logOutput bytes.Buffer - testFnCallCount int32 - minimumCallCount = 100 - expectedRetryDelay = time.Millisecond - retryLimit = -1 - retryResetTimeout = 3 * time.Millisecond - testFnCallTimeCh = make(chan time.Time, minimumCallCount) - ctx = context.Background() - ) - - // Redirect the log output for verification later - log.SetOutput(&logOutput) - - // Define the test function that simulates an error and counts its invocations - testFn := func() chan error { - // Track the invocation time - testFnCallTimeCh <- time.Now() - - errCh := make(chan error, 1) - - count := atomic.LoadInt32(&testFnCallCount) - if count == int32(retryLimit) { - go func() { - time.Sleep(retryResetTimeout) - errCh <- testErr - }() - } else { - errCh <- testErr - } - - // Increment the invocation count atomically - atomic.AddInt32(&testFnCallCount, 1) - return errCh - } - - retryOnErrorErrCh := make(chan error, 1) - // Spawn a goroutine to test the OnError function - go func() { - retryOnErrorErrCh <- retry.OnError( - ctx, - retryLimit, - expectedRetryDelay, - retryResetTimeout, - "TestOnError", - testFn, - ) - }() - - // Wait for the OnError function to execute and retry the expected number of times - totalExpectedDelay := expectedRetryDelay * time.Duration(minimumCallCount) - time.Sleep(totalExpectedDelay + 100*time.Millisecond) - - // Assert that the test function was called the expected number of times - require.GreaterOrEqual(t, minimumCallCount, int(testFnCallCount)) - - // Assert that the retry delay between function calls matches the expected delay - var prevCallTime = new(time.Time) - for i := 0; i < minimumCallCount; i++ { - select { - case nextCallTime := <-testFnCallTimeCh: - if i != 0 { - actualRetryDelay := nextCallTime.Sub(*prevCallTime) - require.GreaterOrEqual(t, actualRetryDelay, expectedRetryDelay) - } - - *prevCallTime = nextCallTime - default: - t.Fatalf( - "expected %d calls to testFn, but only received %d", - minimumCallCount, i+1, - ) - } - } - - // Verify the logged error messages - var ( - logOutputLines = strings.Split(strings.Trim(logOutput.String(), "\n"), "\n") - expectedPrefix = "ERROR: retrying TestOnError after error: test error" - ) - - require.Lenf( - t, logOutputLines, - minimumCallCount-1, - "expected %d log lines, got %d", - minimumCallCount-1, len(logOutputLines), - ) - for _, line := range logOutputLines { - require.Contains(t, line, expectedPrefix) - } -} diff --git a/pkg/sdk/sdk.go b/pkg/sdk/sdk.go index 93b7bb3cd..790173abb 100644 --- a/pkg/sdk/sdk.go +++ b/pkg/sdk/sdk.go @@ -80,7 +80,7 @@ func NewPOKTRollSDK(ctx context.Context, config *POKTRollSDKConfig) (POKTRollSDK // Build the dependencies if they are not provided in the config. if config.Deps != nil { deps = config.Deps - } else if deps, err = sdk.buildDeps(ctx); err != nil { + } else if deps, err = sdk.buildDeps(ctx, config); err != nil { return nil, err } From 094f8bf4b59d371a7bb7d066d536deae36722bb0 Mon Sep 17 00:00:00 2001 From: Eric Nielson Date: Fri, 19 Apr 2024 12:27:01 -0400 Subject: [PATCH 14/28] Empty commit From 579c3441d5820c9d5f992331d711c4c132537ce8 Mon Sep 17 00:00:00 2001 From: Eric Nielson Date: Fri, 19 Apr 2024 13:13:52 -0400 Subject: [PATCH 15/28] Empty commit From e382fe573cb5192e759a7cf1a3afc2fb511fa093 Mon Sep 17 00:00:00 2001 From: ezeike Date: Tue, 23 Apr 2024 11:27:48 -0400 Subject: [PATCH 16/28] Update pkg/client/interface.go Co-authored-by: Bryan White --- pkg/client/interface.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/client/interface.go b/pkg/client/interface.go index e02eb7bec..e4cf03d00 100644 --- a/pkg/client/interface.go +++ b/pkg/client/interface.go @@ -229,7 +229,7 @@ type TxClientOption func(TxClient) // SupplierClientOption defines a function type that modifies the SupplierClient. type SupplierClientOption func(SupplierClient) -// DelegationClientOption defines a function type that modifies the BlockClient. +// DelegationClientOption defines a function type that modifies the DelegationClient. type DelegationClientOption func(DelegationClient) // BlockClientOption defines a function type that modifies the BlockClient. From c37e36c86713114a3d89a2209e66f0d7a7c2bf93 Mon Sep 17 00:00:00 2001 From: ezeike Date: Tue, 23 Apr 2024 13:02:55 -0400 Subject: [PATCH 17/28] Update pkg/retry/retry_test.go Co-authored-by: Bryan White --- pkg/retry/retry_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/retry/retry_test.go b/pkg/retry/retry_test.go index 1f0cf6844..ca52ac5b3 100644 --- a/pkg/retry/retry_test.go +++ b/pkg/retry/retry_test.go @@ -371,7 +371,8 @@ func TestOnError_NegativeRetryLimit(t *testing.T) { errCh := make(chan error, 1) - count := atomic.LoadInt32(&testFnCallCount) + // Increment the invocation count atomically + count := atomic.AddInt32(&testFnCallCount, 1) - 1 if count == int32(retryLimit) { go func() { time.Sleep(retryResetTimeout) @@ -380,9 +381,6 @@ func TestOnError_NegativeRetryLimit(t *testing.T) { } else { errCh <- testErr } - - // Increment the invocation count atomically - atomic.AddInt32(&testFnCallCount, 1) return errCh } From f91721c665daa32686a22eb10b4b4d1e1fce6424 Mon Sep 17 00:00:00 2001 From: ezeike Date: Wed, 24 Apr 2024 14:32:39 -0400 Subject: [PATCH 18/28] Update pkg/retry/retry_test.go Co-authored-by: Daniel Olshansky --- pkg/retry/retry_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/retry/retry_test.go b/pkg/retry/retry_test.go index ca52ac5b3..a99761a68 100644 --- a/pkg/retry/retry_test.go +++ b/pkg/retry/retry_test.go @@ -345,7 +345,7 @@ func TestOnError_RetryCountResetTimeout(t *testing.T) { // assert that a negative retry limit continually calls workFn func TestOnError_NegativeRetryLimit(t *testing.T) { - t.Skip("TODO_TECHDEBT: this test should pass but contains a race condition around the logOutput buffer") + t.Skip("TODO_TECHDEBT(@bryanchriswhite): this test should pass but contains a race condition around the logOutput buffer") // Setup test variables and log capture var ( From 108303a25b71bbb64447955715093eae2d6a4ba5 Mon Sep 17 00:00:00 2001 From: Redouane Lakrache Date: Tue, 23 Apr 2024 23:06:38 +0200 Subject: [PATCH 19/28] [Config] feat: Simplify relay miner config (#477) Co-authored-by: Daniel Olshansky Co-authored-by: ezeike --- .../operate/configs/app_staking_config.md | 2 +- .../docs/operate/configs/relayminer_config.md | 301 +++--- .../configs/supplier_staking_config.md | 54 +- docusaurus/yarn.lock | 16 +- localnet/kubernetes/values-relayminer-1.yaml | 10 +- localnet/kubernetes/values-relayminer-2.yaml | 10 +- localnet/kubernetes/values-relayminer-3.yaml | 10 +- .../kubernetes/values-relayminer-common.yaml | 4 - .../poktrolld/config/relayminer_config.yaml | 20 +- .../relayminer_config_full_example.yaml | 154 +-- .../config/supplier1_stake_config.yaml | 2 +- .../config/supplier2_stake_config.yaml | 2 +- .../config/supplier3_stake_config.yaml | 2 +- pkg/relayer/cmd/cmd.go | 16 +- pkg/relayer/config/errors.go | 2 +- pkg/relayer/config/proxies_config_hydrator.go | 63 -- .../config/proxy_http_config_parser.go | 50 +- .../config/relayminer_configs_reader.go | 32 +- .../config/relayminer_configs_reader_test.go | 880 ++++-------------- pkg/relayer/config/servers_config_hydrator.go | 60 ++ pkg/relayer/config/supplier_hydrator.go | 72 +- .../config/suppliers_config_hydrator.go | 27 +- pkg/relayer/config/types.go | 96 +- pkg/relayer/proxy/error_reply.go | 17 +- pkg/relayer/proxy/errors.go | 20 +- pkg/relayer/proxy/metrics.go | 19 +- pkg/relayer/proxy/options.go | 9 +- pkg/relayer/proxy/proxy.go | 35 +- pkg/relayer/proxy/proxy_test.go | 172 ++-- pkg/relayer/proxy/server_builder.go | 38 +- pkg/relayer/proxy/synchronous.go | 112 ++- pkg/relayer/session/proof.go | 5 + testutil/testproxy/relayerproxy.go | 48 +- x/proof/keeper/msg_server_submit_proof.go | 4 + x/supplier/config/supplier_configs_reader.go | 42 +- .../config/supplier_configs_reader_test.go | 371 +++++++- x/supplier/module/tx_stake_supplier_test.go | 30 +- 37 files changed, 1224 insertions(+), 1583 deletions(-) delete mode 100644 pkg/relayer/config/proxies_config_hydrator.go create mode 100644 pkg/relayer/config/servers_config_hydrator.go diff --git a/docusaurus/docs/operate/configs/app_staking_config.md b/docusaurus/docs/operate/configs/app_staking_config.md index c52ab6afd..0443b9839 100644 --- a/docusaurus/docs/operate/configs/app_staking_config.md +++ b/docusaurus/docs/operate/configs/app_staking_config.md @@ -67,7 +67,7 @@ service_ids: Defines the list of services the `Application` is willing to consume on the Pocket network. Each entry in the list is a `service_id` that identifies a service -that is available on the Pocket network. +that is available on Pocket network. It MUST be a string of 8 or fewer alphanumeric characters, underscores, and dashes (i.e. matching the regex `^[a-zA-Z0-9_-]{1,8}$`). diff --git a/docusaurus/docs/operate/configs/relayminer_config.md b/docusaurus/docs/operate/configs/relayminer_config.md index 8b410c0fe..5128edb82 100644 --- a/docusaurus/docs/operate/configs/relayminer_config.md +++ b/docusaurus/docs/operate/configs/relayminer_config.md @@ -3,16 +3,18 @@ title: RelayMiner config sidebar_position: 1 --- -# `relayer/config/relayminer_configs_reader` +# RelayMiner config _This document describes the configuration options available through the -`relayminer_config.yaml` file. It drives how the `RelayMiner` is setup in terms -of Pocket network connectivity, the servers it starts, which domains it accepts -queries from and which services it forwards requests to._ +`relayminer_config.yaml` file. It configures how the `RelayMiner` is setup in terms +of Pocket network connectivity, starting up backend servers, querying requests +and which domains to accept queries from._ +- [Full config reference example](#full-config-reference-example) +- [RelayMiner (off-chain) config -\> Supplier (on-chain) configs](#relayminer-off-chain-config---supplier-on-chain-configs) - [Usage](#usage) - [Structure](#structure) -- [Top level options](#top-level-options) +- [Global options](#global-options) - [`signing_key_name`](#signing_key_name) - [`smt_store_path`](#smt_store_path) - [`metrics`](#metrics) @@ -20,23 +22,69 @@ queries from and which services it forwards requests to._ - [`query_node_rpc_url`](#query_node_rpc_url) - [`query_node_grpc_url`](#query_node_grpc_url) - [`tx_node_rpc_url`](#tx_node_rpc_url) -- [RelayMiner proxies](#relayminer-proxies) - - [`proxy_name`](#proxy_name) - - [`type`](#type) - - [`host`](#host) - [Suppliers](#suppliers) - [`service_id`](#service_id) - - [`type`](#type-1) + - [`listen_url`](#listen_url) - [`service_config`](#service_config) - - [`url`](#url) + - [`backend_url`](#backend_url) - [`authentication`](#authentication) - [`headers`](#headers) - - [`hosts`](#hosts) - - [`proxy_names`](#proxy_names) -- [Proxy to Supplier referencing](#proxy-to-supplier-referencing) -- [RelayMiner config -\> On-chain service relationship](#relayminer-config---on-chain-service-relationship) -- [Full config example](#full-config-example) -- [Supported proxy types](#supported-proxy-types) + - [`publicly_exposed_endpoints`](#publicly_exposed_endpoints) + - [Why should one supplier have multiple `publicly_exposed_endpoints`?](#why-should-one-supplier-have-multiple-publicly_exposed_endpoints) +- [Supported server types](#supported-server-types) + +## Full config reference example + +A full and commented example of a `RelayMiner` configuration file can be found +at [localnet/poktrolld/config/relayminer_config_full_example.yaml](https://github.com/pokt-network/poktroll/tree/main/localnet/poktrolld/config/relayminer_config_full_example.yaml) + +## RelayMiner (off-chain) config -> Supplier (on-chain) configs + +The following diagram illustrates how the _off-chain_ `RelayMiner` operator +config (yaml) MUST match the _on-chain_ `Supplier` actor service endpoints +for correct and deterministic behavior. + +If these do not match, the behavior is non-deterministic and could result in +a variety of errors such as bad QoS, incorrect proxying, burning of the actor, etc... + +_Assuming that the on-chain endpoints 1 and 2 have different hosts_ + +```mermaid +flowchart LR + +subgraph "Supplier Actor (On-Chain)" + subgraph "SupplierServiceConfig (protobuf)" + subgraph svc1["Service1 (protobuf)"] + svc1Id[Service1.Id] + subgraph SupplierEndpoint + EP1[Endpoint1] + EP2[Endpoint2] + end + end + subgraph svc2 ["Service2 (protobuf)"] + svc2Id[Service2.Id] + end + end +end + +subgraph "RelayMiner Operator (Off-Chain)" + subgraph "DevOps Operator Configs (yaml)" + subgraph svc1Config ["Service1 Config (yaml)"] + svc1IdConfig[service_id=Service1.Id]-->svc1Id + subgraph Hosts + H1[Endpoint1.Host]-->EP1 + H2[Endpoint2.Host]-->EP2 + H3[Internal Host] + end + end + subgraph svc2Config ["Service2 Config (yaml)"] + svc2IdConfig[Service2.Id] + end + end +end + +svc2Config-->svc2 +``` ## Usage @@ -49,10 +97,10 @@ poktrolld relayminer --config ./relayminer_config.yaml --keyring-backend test ## Structure -The `RelayMiner` configuration file is a `yaml` file that contains `top level options`, -`proxies` and `suppliers` sections. +The `RelayMiner` configuration file is a `yaml` file that contains `global options` +and `supplier` specific sections and configurations. -## Top level options +## Global options ```yaml signing_key_name: @@ -64,7 +112,7 @@ smt_store_path: _`Required`_ The name of the key that will be used to sign transactions, derive the public key -and the corresponding address. This key name must be present in the keyring that is used +and the corresponding address. This key name MUST be present in the keyring that is used to start the `RelayMiner` instance. ### `smt_store_path` @@ -130,72 +178,32 @@ The RPC URL of the Pocket node that allows the `RelayMiner` to broadcast transac It may have a different host than the `query_node_rpc_url` but the same value is acceptable too. -## RelayMiner proxies - -The `proxies` section of the configuration file is a list of proxies that the -`RelayMiner` will use to start servers by listening on the configured host. - -At least one proxy is required for the `RelayMiner` to start. - -```yaml -proxies: - - proxy_name: - type: - host: -``` - -### `proxy_name` - -_`Required`_, _`Unique`_ - -Is the name of the proxy which will be used as a unique identifier to reference -proxies in the [Suppliers](#suppliers) section of the configuration file. -It corresponds to a server that will be started by the `RelayMiner` instance -and must be unique across all proxies. - -### `type` - -_`Required`_ - -The type of the proxy server to be started. Must be one of the [supported types](#supported-proxy-types). -When other types are supported, the `type` field could determine if additional -configuration options are allowed be them optional or required. - -### `host` - -_`Required`_, _`Unique`_ - -The host to which the proxy server will be started and listening to. It must be -a valid host according to the `type` filed and must be unique across all proxies. - ## Suppliers -The `suppliers` section of the configuration file is a list of suppliers that -represent the services that the `RelayMiner` will offer to the Pocket network -through the configured proxies and their corresponding services to where the -requests will be forwarded to. +The `suppliers` section configures the services that the `RelayMiner` will offer +to Pocket Network. It specifies exactly where those requests will be forwarded +to by the Supplier's infrastructure. -Each suppliers entry's `service_id` must reflect the on-chain `Service.Id` the supplier -staked for and the `hosts` list must contain the same endpoints hosts that the -supplier advertised when staking for that service. +Each suppliers entry's `service_id` MUST reflect the on-chain `Service.Id` the +supplier staked for. In addition, the `publicly_exposed_endpoints` list MUST +contain the same endpoints that the Supplier advertised on-chain when staking for +that service. At least one supplier is required for the `RelayMiner` to be functional. ```yaml suppliers: - service_id: - type: + listen_url: :// service_config: - url: + backend_url: authentication: username: password: headers: : - hosts: - - - proxy_names: - - + publicly_exposed_endpoints: + - ``` ### `service_id` @@ -205,16 +213,20 @@ _`Required`_, _`Unique`_ The Id of the service which will be used as a unique identifier to reference a service provided by the `Supplier` and served by the `RelayMiner` instance. -It must match the `Service.Id` specified by the supplier when staking for the +It MUST match the `Service.Id` specified by the supplier when staking for the service. -### `type` +### `listen_url` _`Required`_ -The transport type that the service will be offered on. It must match the `type` field -of the proxy that the supplier is referencing through `proxy_names`. -Must be one of the [supported types](#supported-proxy-types). +The address on which the `RelayMiner` will start a server to listen for incoming +requests. The server type is inferred from the URL scheme (http, https, etc...). + +The same `listen_url` can be used for multiple suppliers and/or different +`publicly_exposed_endpoints`, the `RelayMiner` takes care of routing the requests +to the correct `backend_url` based on the `service_id` and the `publicly_exposed_endpoints` +it received a request form. ### `service_config` @@ -224,13 +236,13 @@ The `service_config` section of the supplier configuration is a set of options that are specific to the service that the `RelayMiner` will be offering to the Pocket network. -#### `url` +#### `backend_url` _`Required`_ The URL of the service that the `RelayMiner` will forward the requests to when a relay is received, also known as **data node** or **service node**. -It must be a valid URL (not just a host) and be reachable from the `RelayMiner` instance. +It MUST be a valid URL (not just a host) and be reachable from the `RelayMiner` instance. #### `authentication` @@ -249,135 +261,42 @@ that will be added to the request headers when the `RelayMiner` forwards the requests to the service. It can be used to add additional headers like `Authorization: Bearer ` for example. -### `hosts` +#### `publicly_exposed_endpoints` -_`Required`_, _`Unique` for each referenced proxy_, _`Unique` within the supplier's `hosts` list_ +_`Required`_, _`Unique` within the supplier's `publicly_exposed_endpoints` list_ -The `hosts` section of the supplier configuration is a list of hosts that the -`RelayMiner` will accept requests from. It must be a valid host that reflects -the on-chain supplier staking service endpoints. +The `publicly_exposed_endpoints` section of the supplier configuration is a list +of hosts that the `RelayMiner` will accept requests from. It MUST be a valid host +that reflects the on-chain supplier staking service endpoints. It is used to determine if the incoming request is allowed to be processed by -the referenced proxy server as well as to check if the request's RPC-Type matches -the on-chain endpoint's RPC-Type. +the server listening on `listen_url` host address as well as to check if the +request's RPC-Type matches the on-chain endpoint's RPC-Type. + +##### Why should one supplier have multiple `publicly_exposed_endpoints`? -There are various reasons to having multiple hosts for the same supplier services. +There are various reasons to having multiple `publicly_exposed_endpoints` +for the same supplier service. - The on-chain Supplier may provide the same Service on multiple domains (e.g. for different regions). - The operator may want to route requests of different RPC types to - the same proxy + the same server - Migrating from one domain to another. Where the operator could still accept requests on the old domain while the new domain is being propagated. - The operator may want to have a different domain for internal requests. - The on-chain Service configuration accepts multiple endpoints. -It must be unique across all the `hosts` lined to a given proxy. - -_Note: The `service_id` of the supplier is automatically added to the `hosts` list as -it may help troubleshooting the `RelayMiner` and/or send requests internally -from a k8s cluster for example._ - -### `proxy_names` - -_`Required`_, _`Unique` within the `proxy_names` list_ - -The `proxy_names` section of the supplier configuration is the list of proxies -that the `RelayMiner` will use to serve the requests for the given supplier entry. - -It must be a valid proxy name that is defined in the `proxies` section of the -configuration file, must be unique across the supplier's `proxy_names` and the -`supplier` `type` must match the `type` of the referenced `proxy`. - -## Proxy to Supplier referencing - -To illustrate how the `suppliers.proxy_names` and `proxies.proxy_name` fields are used -to reference proxies and suppliers, let's consider the following configuration file: - -```yaml -proxies: - - proxy_name: http-example - ... - - proxy_name: http-example-2 -suppliers: - - service_id: ethereum - ... - proxy_names: - - http-example - - http-example-2 - - name: 7b-llm-model - ... - proxy_names: - - http-example -``` - -In this example, the `ethereum` supplier is referencing two proxies, `http-example` -and `http-example-2` and the `7b-llm-model` supplier is referencing only the -`http-example` proxy. This would result in the following setup: - -```yaml -- http-example - - ethereum - - 7b-llm-model -- http-example-2 - - ethereum -``` - -## RelayMiner config -> On-chain service relationship - -The following diagram illustrates how the _off-chain_ `RelayMiner` operator -config (yaml) must match the _on-chain_ `Supplier` actor service endpoints -for correct and deterministic behavior. - -If these do not match, the behavior is non-deterministic and could result in -a variety of errors such as bad QoS, incorrect proxying, burning of the actor, etc... - -_Assuming that the on-chain endpoints 1 and 2 have different hosts_ - -```mermaid -flowchart LR - -subgraph "Supplier Actor (On-Chain)" - subgraph "SupplierServiceConfig (protobuf)" - subgraph svc1["Service1 (protobuf)"] - svc1Id[Service1.Id] - subgraph SupplierEndpoint - EP1[Endpoint1] - EP2[Endpoint2] - end - end - subgraph svc2 ["Service2 (protobuf)"] - svc2Id[Service2.Id] - end - end -end - -subgraph "RelayMiner Operator (Off-Chain)" - subgraph "DevOps Operator Configs (yaml)" - subgraph svc1Config ["Service1 Config (yaml)"] - svc1IdConfig[service_id=Service1.Id]-->svc1Id - subgraph Hosts - H1[Endpoint1.Host]-->EP1 - H2[Endpoint2.Host]-->EP2 - H3[Internal Host] - end - end - subgraph svc2Config ["Service2 Config (yaml)"] - svc2IdConfig[Service2.Id] - end - end -end - -svc2Config-->svc2 -``` +:::note -## Full config example +The `service_id` of the supplier is automatically added to the +`publicly_exposed_endpoints` list as it may help troubleshooting the `RelayMiner` +and/or send requests internally from a k8s cluster for example. -A full and commented example of a `RelayMiner` configuration file can be found -at [localnet/poktrolld/config/relayminer_config_full_example.yaml](https://github.com/pokt-network/poktroll/tree/main/localnet/poktrolld/config/relayminer_config_full_example.yaml) +::: --- -## Supported proxy types +## Supported server types -The list of supported proxy types can be found at [pkg/relayer/config/types.go](https://github.com/pokt-network/poktroll/tree/main/pkg/relayer/config/types.go#L8) +The list of supported server types can be found at [pkg/relayer/config/types.go](https://github.com/pokt-network/poktroll/tree/main/pkg/relayer/config/types.go#L8) diff --git a/docusaurus/docs/operate/configs/supplier_staking_config.md b/docusaurus/docs/operate/configs/supplier_staking_config.md index 8ec8fe3cf..bd46622d4 100644 --- a/docusaurus/docs/operate/configs/supplier_staking_config.md +++ b/docusaurus/docs/operate/configs/supplier_staking_config.md @@ -6,34 +6,32 @@ sidebar_position: 3 # Supplier staking config _This document describes the configuration file used by the `Supplier` to submit -a stake transaction required to provide RPC services on the Pocket Network._ +a stake transaction required to provide RPC services on Pocket Network._ +- [Reference Example](#reference-example) - [Usage](#usage) - [Configuration](#configuration) - [`stake_amount`](#stake_amount) - [`services`](#services) - [`service_id`](#service_id) - [`endpoints`](#endpoints) - - [`url`](#url) + - [`publicly_exposed_url`](#publicly_exposed_url) - [`rpc_type`](#rpc_type) -- [Example](#example) + +## Reference Example + +A full example of the configuration file could be found at [supplier_staking_config.yaml](https://github.com/pokt-network/poktroll/tree/main/localnet/poktrolld/config/supplier1_stake_config.yaml). ## Usage The `stake-supplier` transaction submission command accepts a `--config` flag that points to a `yaml` configuration file that defines their staking -configuration. This includes, but is not limited to things like `stake_amount`, the `service`s, their respective advertised `endpoints`, etc. +configuration. This includes, but is not limited to, things like `stake_amount`, +provided `service`s, their respective advertised `endpoints`, etc. The following is an example command of how to stake a supplier in a LocalNet environment. -:::warning - -TestNet is not ready as of writing this documentation, so you may -need to adjust the command below appropriately. - -::: - ```bash poktrolld tx supplier stake-supplier \ --home=./poktroll \ @@ -59,10 +57,13 @@ This amount covers all the `service`s defined in the `services` section. :::note If the `Supplier` account already has a stake and wishes to change or add -to the `service`s that it provides, then it MUST to increase the current +to the `service`s that it provides, then it MUST increase the current `stake_amount` by at least `1upokt`. -For example, if the current stake is `1000upokt` and the `Supplier` wants to add a new `service` then `stake_amount: 1001upokt` should be specified in the configuration file. This will increase the stake by `1upokt` and deduct `1upokt` from the `Supplier`'s account balance. +For example, if the current stake is `1000upokt` and the `Supplier` wants to add +a new `service`, then `stake_amount: 1001upokt` should be specified in the +configuration file. This will increase the stake by `1upokt` and deduct `1upokt` +from the `Supplier`'s account balance. ::: @@ -74,14 +75,14 @@ _`Required`_, _`Non-empty`_ services: - service_id: endpoints: - - url: ://: + - publicly_exposed_url: ://: rpc_type: ``` `services` define the list of services that the `Supplier` wants to provide. It takes the form of a list of `service` objects. Each `service` object consists of a `service_id` and a list of `endpoints` that the `Supplier` will -advertise on the Pocket Network. +advertise on Pocket Network. #### `service_id` @@ -98,24 +99,27 @@ For example, it must match the regex `^[a-zA-Z0-9_-]{1,8}$`, and spaces are disa _`Required`_, _`Non-empty`_ `endpoints` is a list of `endpoint` objects that the `Supplier` will advertise -to the Pocket Network. Each `endpoint` object consists of an `url` and a `rpc_type`. +to the Pocket Network. Each `endpoint` object consists of a `publicly_exposed_url` +and a `rpc_type`. -##### `url` +##### `publicly_exposed_url` _`Required`_ -The `url` defines the endpoint for sending `RelayRequests` from the Pocket Network's `Gateways` and `Applications`. Typically, this endpoint is provided by a RelayMiner, which acts as a proxy. This setup means that the specified URL directs to the RelayMiner's endpoint, which in turn, routes the requests to the designated service node. - -- **Example**: `https://etherium-relayminer1.relayminers.com:443` demonstrates how a RelayMiner endpoint, typically offered by the RelayMiner, proxies requests to an Etherium backend data node. +The `publicly_exposed_url` defines the endpoint for sending `RelayRequests` from +the Pocket Network's `Gateways` and `Applications`. This endpoint is provided by +the `Supplier` when staking, and is meant to point to (or route requests to) +the `Supplier`'s `RelayMiner` which in turn forwards these requests to the service node. +- **Example**: When a `Supplier` stakes with a config file that contains + `https://ethereum-relayminer1.relayminers.com:443` as a `publicly_exposed_url`, + this endpoint will be discoverable on the Pocket Network by `Gateways` and + `Applications`, which can send it Ethereum `RelayRequests` to be processed by the + `Supplier`'s `RelayMiner`. ##### `rpc_type` _`Required`_ `rpc_type` is a string that defines the type of RPC service that the `Supplier` -is providing. The `rpc_type` MUST be one of the [supported types found here](https://github.com/pokt-network/poktroll/tree/main/pkg/relayer/config/types.go#L8) - -## Example - -A full example of the configuration file could be found at [supplier_staking_config.yaml](https://github.com/pokt-network/poktroll/tree/main/localnet/poktrolld/config/supplier1_stake_config.yaml) +is providing. The `rpc_type` MUST be one of the [supported types found here](https://github.com/pokt-network/poktroll/tree/main/pkg/relayer/config/types.go#L8). diff --git a/docusaurus/yarn.lock b/docusaurus/yarn.lock index 67e91df77..ef13ae772 100644 --- a/docusaurus/yarn.lock +++ b/docusaurus/yarn.lock @@ -1810,15 +1810,10 @@ dependencies: "@types/mdx" "^2.0.0" -"@node-rs/jieba-linux-x64-gnu@1.10.0": +"@node-rs/jieba-darwin-arm64@1.10.0": version "1.10.0" - resolved "https://registry.npmjs.org/@node-rs/jieba-linux-x64-gnu/-/jieba-linux-x64-gnu-1.10.0.tgz" - integrity sha512-rS5Shs8JITxJjFIjoIZ5a9O+GO21TJgKu03g2qwFE3QaN5ZOvXtz+/AqqyfT4GmmMhCujD83AGqfOGXDmItF9w== - -"@node-rs/jieba-linux-x64-musl@1.10.0": - version "1.10.0" - resolved "https://registry.npmjs.org/@node-rs/jieba-linux-x64-musl/-/jieba-linux-x64-musl-1.10.0.tgz" - integrity sha512-BvSiF2rR8Birh2oEVHcYwq0WGC1cegkEdddWsPrrSmpKmukJE2zyjcxaOOggq2apb8fIRsjyeeUh6X3R5AgjvA== + resolved "https://registry.npmjs.org/@node-rs/jieba-darwin-arm64/-/jieba-darwin-arm64-1.10.0.tgz" + integrity sha512-IhR5r+XxFcfhVsF93zQ3uCJy8ndotRntXzoW/JCyKqOahUo/ITQRT6vTKHKMyD9xNmjl222OZonBSo2+mlI2fQ== "@node-rs/jieba@^1.6.0": version "1.10.0" @@ -4621,6 +4616,11 @@ fs.realpath@^1.0.0: resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== +fsevents@~2.3.2: + version "2.3.3" + resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz" + integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== + function-bind@^1.1.2: version "1.1.2" resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz" diff --git a/localnet/kubernetes/values-relayminer-1.yaml b/localnet/kubernetes/values-relayminer-1.yaml index 6054bf02a..956f99cd3 100644 --- a/localnet/kubernetes/values-relayminer-1.yaml +++ b/localnet/kubernetes/values-relayminer-1.yaml @@ -2,10 +2,8 @@ config: signing_key_name: supplier1 suppliers: - service_id: anvil - type: http + listen_url: http://0.0.0.0:8545 service_config: - url: http://anvil:8547/ - proxy_names: - - http-proxy - hosts: - - tcp://relayminer1:8545 + backend_url: http://anvil:8547/ + publicly_exposed_endpoints: + - relayminer1 diff --git a/localnet/kubernetes/values-relayminer-2.yaml b/localnet/kubernetes/values-relayminer-2.yaml index 64fb6af3d..78e9ea4f9 100644 --- a/localnet/kubernetes/values-relayminer-2.yaml +++ b/localnet/kubernetes/values-relayminer-2.yaml @@ -2,10 +2,8 @@ config: signing_key_name: supplier2 suppliers: - service_id: anvil - type: http + listen_url: http://0.0.0.0:8545 service_config: - url: http://anvil:8547/ - proxy_names: - - http-proxy - hosts: - - tcp://relayminer2:8545 + backend_url: http://anvil:8547/ + publicly_exposed_endpoints: + - relayminer2 diff --git a/localnet/kubernetes/values-relayminer-3.yaml b/localnet/kubernetes/values-relayminer-3.yaml index 8e6a2098f..506d0ab7f 100644 --- a/localnet/kubernetes/values-relayminer-3.yaml +++ b/localnet/kubernetes/values-relayminer-3.yaml @@ -2,10 +2,8 @@ config: signing_key_name: supplier3 suppliers: - service_id: anvil - type: http + listen_url: http://0.0.0.0:8545 service_config: - url: http://anvil:8547/ - proxy_names: - - http-proxy - hosts: - - tcp://relayminer3:8545 + backend_url: http://anvil:8547/ + publicly_exposed_endpoints: + - relayminer3 diff --git a/localnet/kubernetes/values-relayminer-common.yaml b/localnet/kubernetes/values-relayminer-common.yaml index 6192fb9be..f222f7a39 100644 --- a/localnet/kubernetes/values-relayminer-common.yaml +++ b/localnet/kubernetes/values-relayminer-common.yaml @@ -7,8 +7,4 @@ config: query_node_rpc_url: tcp://validator-poktroll-validator:36657 query_node_grpc_url: tcp://validator-poktroll-validator:36658 tx_node_rpc_url: tcp://validator-poktroll-validator:36657 - proxies: - - proxy_name: http-proxy - type: http - host: 0.0.0.0:8545 suppliers: [] diff --git a/localnet/poktrolld/config/relayminer_config.yaml b/localnet/poktrolld/config/relayminer_config.yaml index 18833d93b..a44d59e6a 100644 --- a/localnet/poktrolld/config/relayminer_config.yaml +++ b/localnet/poktrolld/config/relayminer_config.yaml @@ -1,22 +1,16 @@ signing_key_name: supplier1 smt_store_path: smt_stores +metrics: + enabled: true + addr: :9090 pocket_node: query_node_rpc_url: tcp://poktroll-validator:36657 query_node_grpc_url: tcp://poktroll-validator:36658 tx_node_rpc_url: tcp://poktroll-validator:36657 -proxies: - - proxy_name: http-proxy - type: http - host: 0.0.0.0:8545 suppliers: - service_id: anvil - type: http + listen_url: http://0.0.0.0:8545 service_config: - url: http://anvil:8547/ - proxy_names: - - http-proxy - hosts: - - tcp://relayminers:8545 -metrics: - enabled: true - addr: :9090 + backend_url: http://anvil:8547/ + publicly_exposed_endpoints: + - relayminers diff --git a/localnet/poktrolld/config/relayminer_config_full_example.yaml b/localnet/poktrolld/config/relayminer_config_full_example.yaml index 286c42c65..4dab186d1 100644 --- a/localnet/poktrolld/config/relayminer_config_full_example.yaml +++ b/localnet/poktrolld/config/relayminer_config_full_example.yaml @@ -1,133 +1,81 @@ -# TODO_CONSIDERATION: We don't need this now, but it would be beneficial if the -# logic handling this config file could be designed in such a way that it allows for -# "hot" config changes in the future, meaning changes without restarting a process. -# This would be useful for adding a proxy or a supplier without interrupting the service. - -# Name of the key (in the keyring) to sign transactions. +# Name of the key (in the keyring) used to sign transactions. signing_key_name: supplier1 -# Relative path (on the relayminer's machine) to where the data backing -# SMT KV store exists on disk. -smt_store_path: smt_stores + +# Relative path (on the relayminer's machine) to the SMT KV store data on disk. +smt_store_path: ./path/to/smt_stores # Prometheus exporter configuration metrics: - # Turn the metrics exporter on/off + # Enable or disable the metrics exporter enabled: true - # The address that the metrics exporter will listen on. Can be just a port, or host:port + # The address (host:port or just port) for the metrics exporter to listen on. addr: :9090 pocket_node: - # Pocket node URL that exposes CometBFT JSON-RPC API. - # This can be used by the Cosmos client SDK, event subscriptions, etc... + # Pocket node URL exposing the CometBFT JSON-RPC API. + # Used by the Cosmos client SDK, event subscriptions, etc. # If unspecified, defaults to `tx_node_rpc_url`. query_node_rpc_url: tcp://poktroll-validator:36657 - # Pocket node URL that exposes the Cosmos gRPC service, dedicated to querying purposes. + # Pocket node URL exposing the Cosmos gRPC service for querying purposes. query_node_grpc_url: tcp://poktroll-validator:36658 - # Pocket node URL that exposes the TendermintRPC service. + # Pocket node URL exposing the CometRPC service. tx_node_rpc_url: tcp://poktroll-validator:36658 -# Proxies are endpoints that expose different suppliers to the internet. -proxies: - # Name of the proxy. It will be used to reference in a supplier. - # Must be unique. - # Required. - # TODO_CONSIDERATION: if we enforce DNS compliant names, it can potentially - # become handy in the future. - # More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names - - proxy_name: http-example - # Type of proxy: currently only http is supported but will support more - # (https, tcp, quic ...) in the future. - # MUST match the type of the supplier. - # Required. - type: http - # Hostname to open port on. - # Use 0.0.0.0 in containerized environments. - # 127.0.0.1 with a reverse proxy when there's another process on localhost - # that can be used as a reverse proxy (nginx, apache, traefik, etc.). - # Required. - host: 127.0.0.1:8080 - - # TODO_IMPROVE: https is not currently supported, but this is how it could potentially look. - # - name: example-how-we-can-support-https - # type: https - # host: 0.0.0.0:8443 - # tls: - # enabled: true - # certificate: /path/to/crt - # key: /path/to/key - -# Suppliers are different services that can be offered through RelayMiner. -# When a supplier is configured to use a proxy and staked appropriately, -# the relays will start flowing through RelayMiner. +# Suppliers are different services offered on Pocket Network, +# proxied through Relay Miner. suppliers: - # The serviceId the supplier is offering on the network . - # It must match the Service.Id of the service that has been staked for. - # Must be unique. + # The `service_id` of the service offered on the network. + # Must match the Service.Id of the staked service. + # Must be unique in the `suppliers` array. # Required. - service_id: ethereum - # Type of how the supplier offers service through the network. - # Must match the type of the proxy the supplier is connected to. - # Required. - type: http + listen_url: http://0.0.0.0:80 # Configuration of the service offered through RelayMiner. service_config: - # URL RelayMiner proxies the requests to. - # Also known as the data node, or service node in some cases. + # Backend URL is the endpoint that the RelayMiner proxies the requests to. + # Also known as the data node or service node. # Required. - url: http://anvil.servicer:8545 - # Authentication for the service. - # HTTP Basic Auth: https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication + backend_url: http://anvil.servicer:8545 + + # Authentication for the service (HTTP Basic Auth). # Optional. authentication: username: user password: pwd - # TODO_IMPROVE: This is not supported in code yet, - # but some services authenticate via a header. - # Example, if the service requires a header like `Authorization: Bearer ` - # Authorization: Bearer + # TODO_IMPROVE: This is not implemented in code yet. + # Authentication via a static header might be desirable for high-traffic services. + # Example: If the service requires a header like `Authorization: Bearer ` # Optional. headers: {} - # A list of hosts the proxy is accepting requests from. - # When linked to the proxy, each host is going to be used to lookup the - # the Supplier.Service in Pocket Network. - # Each host in the list must match a Supplier.Service.Endpoint that the Supplier - # has advertised on-chain when staking for that Service. - # There are various reasons to having multiple hosts for the same supplier services, - # - The on-chain Supplier may provide the same Service on multiple domains - # (e.g. for different regions). - # - The operator may want to route requests of different RPC types to - # the same proxy. - # - Migrating from one domain to another. Where the operator could still - # accept requests on the old domain while the new domain is being propagated. - # - The operator may want to have a different domain for internal requests. - # - The on-chain Service configuration accepts multiple endpoints. - # Must be unique within the proxy it is referenced in. - # Required. - hosts: - - ethereum.devnet1.poktroll.com - # The `service_id` of the supplier is automatically added to the hosts section - # for potential troubleshooting/debugging purposes such as: - # Having internal requests coming from non-FQDNs because of complex routing. - # Sending requests from k8s pods. - # Specify the `host` in curl requests when testing `curl -H "Host: ethereum" ...` - # and make the proxy server process the request without the need for an - # on-chain Endpoint entry. - # - ethereum # <- this part is added automatically. + # A list of addresses that the RelayMiner accepts requests from. + # + # 1. Multiple hosts can be configured per supplier. + # 2. Must be unique per listen_address to avoid conflicts. + # 3. Host must be reachable from the internet if staked on the network. + # 4. A supplier must be staked on this endpoint for traffic to be routed. + # + # In most scenarios, only one host is specified. Multiple hosts might be useful for: + # - Migrating from one domain to another (accepting requests on both old and new domains). + # - Having a different domain for internal requests. + # + # Required. + publicly_exposed_endpoints: + # Note: No schema or port is specified. + # - HTTPS/TLS termination can be handled by a separate layer (e.g., nginx or HAproxy). + # - Port can differ from the staked port (e.g., Relay Miner on port 80, TLS on port 443). + - ethereum.devnet1.relayminer.com - # Names of proxies that this supplier is connected to. - # This MUST correspond to the `proxy_name` entry in the `proxies` section - # in order for the supplier to be available to the external network. + # Listen url, usually `http://0.0.0.0:80` (all network interfaces, port `80`). + # The scheme in the URL is required in order to infer the server type. + # Multiple suppliers can share one listen address. # Required. - proxy_names: - - http-example # when the RelayMiner server builder runs. - - service_id: 7b-llm-model - type: http + + # Example of exposing an ollama LLM endpoint. + - service_id: ollama:mistral:7b + listen_url: http://0.0.0.0:80 service_config: - url: http://llama-endpoint - hosts: - - 7b-llm-model.devnet1.poktroll.com - # - 7b-llm-model # <- this part is added automatically. - proxy_names: - - http-example + backend_url: http://localhost:11434 + publicly_exposed_endpoints: + - mistral-7b.devnet1.poktroll.com diff --git a/localnet/poktrolld/config/supplier1_stake_config.yaml b/localnet/poktrolld/config/supplier1_stake_config.yaml index 9384657b1..fa1abaff1 100644 --- a/localnet/poktrolld/config/supplier1_stake_config.yaml +++ b/localnet/poktrolld/config/supplier1_stake_config.yaml @@ -11,5 +11,5 @@ services: # This setup allows for flexible and dynamic service provisioning within the network. - service_id: anvil endpoints: - - url: http://relayminer1:8545 + - publicly_exposed_url: http://relayminer1:8545 rpc_type: json_rpc diff --git a/localnet/poktrolld/config/supplier2_stake_config.yaml b/localnet/poktrolld/config/supplier2_stake_config.yaml index 831e56091..e03f730bc 100644 --- a/localnet/poktrolld/config/supplier2_stake_config.yaml +++ b/localnet/poktrolld/config/supplier2_stake_config.yaml @@ -5,5 +5,5 @@ services: # The endpoint URL for the Anvil service is provided via the RelayMiner. # The RelayMiner acts as a proxy, forwarding requests to the actual Anvil data node behind it. # This setup allows for flexible and dynamic service provisioning within the network. - - url: http://relayminer2:8545 + - publicly_exposed_url: http://relayminer2:8545 rpc_type: json_rpc diff --git a/localnet/poktrolld/config/supplier3_stake_config.yaml b/localnet/poktrolld/config/supplier3_stake_config.yaml index 78b6402a2..ee0d95b25 100644 --- a/localnet/poktrolld/config/supplier3_stake_config.yaml +++ b/localnet/poktrolld/config/supplier3_stake_config.yaml @@ -5,5 +5,5 @@ services: # The endpoint URL for the Anvil service is provided via the RelayMiner. # The RelayMiner acts as a proxy, forwarding requests to the actual Anvil data node behind it. # This setup allows for flexible and dynamic service provisioning within the network. - - url: http://relayminer3:8545 + - publicly_exposed_url: http://relayminer3:8545 rpc_type: json_rpc diff --git a/pkg/relayer/cmd/cmd.go b/pkg/relayer/cmd/cmd.go index 1d96f83cf..1dba8122b 100644 --- a/pkg/relayer/cmd/cmd.go +++ b/pkg/relayer/cmd/cmd.go @@ -47,10 +47,10 @@ var ( func RelayerCmd() *cobra.Command { cmd := &cobra.Command{ Use: "relayminer", - Short: "Run a relay miner", - Long: `Run a relay miner. The relay miner process configures and starts -relay servers for each service the supplier actor identified by --signing-key is -staked for (configured on-chain). + Short: "Start a RelayMiner", + Long: `Run a RelayMiner. A RelayMiner is the off-chain complementary +middleware that handles incoming requests for all the services a Supplier staked +for on-chain. Relay requests received by the relay servers are validated and proxied to their respective service endpoints, maintained by the relayer off-chain. The responses @@ -177,7 +177,7 @@ func setupRelayerDependencies( } signingKeyName := relayMinerConfig.SigningKeyName - proxiedServiceEndpoints := relayMinerConfig.Proxies + servicesConfigMap := relayMinerConfig.Servers smtStorePath := relayMinerConfig.SmtStorePath supplierFuncs := []config.SupplierFn{ @@ -197,7 +197,7 @@ func setupRelayerDependencies( supplyTxContext, newSupplyTxClientFn(signingKeyName), newSupplySupplierClientFn(signingKeyName), - newSupplyRelayerProxyFn(signingKeyName, proxiedServiceEndpoints), + newSupplyRelayerProxyFn(signingKeyName, servicesConfigMap), newSupplyRelayerSessionsManagerFn(smtStorePath), } @@ -304,7 +304,7 @@ func newSupplySupplierClientFn(signingKeyName string) config.SupplierFn { // is supplied with the given deps and the new RelayerProxy. func newSupplyRelayerProxyFn( signingKeyName string, - proxiedServiceEndpoints map[string]*relayerconfig.RelayMinerProxyConfig, + servicesConfigMap map[string]*relayerconfig.RelayMinerServerConfig, ) config.SupplierFn { return func( _ context.Context, @@ -314,7 +314,7 @@ func newSupplyRelayerProxyFn( relayerProxy, err := proxy.NewRelayerProxy( deps, proxy.WithSigningKeyName(signingKeyName), - proxy.WithProxiedServicesEndpoints(proxiedServiceEndpoints), + proxy.WithServicesConfigMap(servicesConfigMap), ) if err != nil { return nil, err diff --git a/pkg/relayer/config/errors.go b/pkg/relayer/config/errors.go index 16544e209..db9fe2395 100644 --- a/pkg/relayer/config/errors.go +++ b/pkg/relayer/config/errors.go @@ -10,5 +10,5 @@ var ( ErrRelayMinerConfigInvalidSmtStorePath = sdkerrors.Register(codespace, 2103, "invalid smt store path in RelayMiner config") ErrRelayMinerConfigEmpty = sdkerrors.Register(codespace, 2104, "empty RelayMiner config") ErrRelayMinerConfigInvalidSupplier = sdkerrors.Register(codespace, 2105, "invalid supplier in RelayMiner config") - ErrRelayMinerConfigInvalidProxy = sdkerrors.Register(codespace, 2106, "invalid proxy in RelayMiner config") + ErrRelayMinerConfigInvalidServer = sdkerrors.Register(codespace, 2106, "invalid server in RelayMiner config") ) diff --git a/pkg/relayer/config/proxies_config_hydrator.go b/pkg/relayer/config/proxies_config_hydrator.go deleted file mode 100644 index f3b7628ed..000000000 --- a/pkg/relayer/config/proxies_config_hydrator.go +++ /dev/null @@ -1,63 +0,0 @@ -package config - -// HydrateProxies populates the proxies fields of the RelayMinerConfig that -// are relevant to the "proxies" section in the config file. -func (relayMinerConfig *RelayMinerConfig) HydrateProxies( - yamlProxyConfigs []YAMLRelayMinerProxyConfig, -) error { - // At least one proxy is required - if len(yamlProxyConfigs) == 0 { - return ErrRelayMinerConfigInvalidProxy.Wrap("no proxies provided") - } - - relayMinerConfig.Proxies = make(map[string]*RelayMinerProxyConfig) - - for _, yamlProxyConfig := range yamlProxyConfigs { - // Proxy name is required - if len(yamlProxyConfig.ProxyName) == 0 { - return ErrRelayMinerConfigInvalidProxy.Wrap("proxy name is required") - } - - // Proxy name should not be unique - if _, ok := relayMinerConfig.Proxies[yamlProxyConfig.ProxyName]; ok { - return ErrRelayMinerConfigInvalidProxy.Wrapf( - "duplicate porxy name %s", - yamlProxyConfig.ProxyName, - ) - } - - proxyConfig := &RelayMinerProxyConfig{ - ProxyName: yamlProxyConfig.ProxyName, - XForwardedHostLookup: yamlProxyConfig.XForwardedHostLookup, - Suppliers: make(map[string]*RelayMinerSupplierConfig), - } - - // Populate the proxy fields that are relevant to each supported proxy type - switch yamlProxyConfig.Type { - case "http": - if err := proxyConfig.parseHTTPProxyConfig(yamlProxyConfig); err != nil { - return err - } - default: - // Fail if the proxy type is not supported - return ErrRelayMinerConfigInvalidProxy.Wrapf( - "invalid proxy type %s", - yamlProxyConfig.Type, - ) - } - - switch yamlProxyConfig.Type { - case "http": - proxyConfig.Type = ProxyTypeHTTP - default: - ErrRelayMinerConfigInvalidProxy.Wrapf( - "invalid proxy type %s", - yamlProxyConfig.Type, - ) - } - - relayMinerConfig.Proxies[proxyConfig.ProxyName] = proxyConfig - } - - return nil -} diff --git a/pkg/relayer/config/proxy_http_config_parser.go b/pkg/relayer/config/proxy_http_config_parser.go index a8f655f14..ad704d306 100644 --- a/pkg/relayer/config/proxy_http_config_parser.go +++ b/pkg/relayer/config/proxy_http_config_parser.go @@ -1,59 +1,53 @@ package config -import ( - "fmt" - "net/url" -) +import "net/url" -// parseHTTPProxyConfig populates the proxy fields of the target structure that -// are relevant to the "http" type in the proxy section of the config file. -// This function alters the target RelayMinerProxyConfig structure as a side effect. -func (proxyConfig *RelayMinerProxyConfig) parseHTTPProxyConfig( - yamlProxyConfig YAMLRelayMinerProxyConfig, +// parseHTTPServerConfig populates the server fields of the target structure that +// are relevant to the "http" type. +// This function alters the target RelayMinerServerConfig structure as a side effect. +func (serverConfig *RelayMinerServerConfig) parseHTTPServerConfig( + yamlSupplierConfig YAMLRelayMinerSupplierConfig, ) error { - // Check if the proxy host is a valid URL. - // Since `yamlProxyConfig.Host` is a string representing the host, we need to - // prepend it with the "http://" scheme to make it a valid URL; we end up - // using the `Host` field of the resulting `url.URL` struct, so the prepended - // scheme is irrelevant. - proxyUrl, err := url.Parse(fmt.Sprintf("http://%s", yamlProxyConfig.Host)) + // Validate yamlSupplierConfig.ListenUrl. + listenUrl, err := url.Parse(yamlSupplierConfig.ListenUrl) if err != nil { - return ErrRelayMinerConfigInvalidProxy.Wrapf( - "invalid proxy host %s", + return ErrRelayMinerConfigInvalidServer.Wrapf( + "invalid relay miner server listen address %s", err.Error(), ) } - if proxyUrl.Host == "" { - return ErrRelayMinerConfigInvalidProxy.Wrap("empty proxy host") + // Ensure that the host is not empty and use it as the server listen address. + if listenUrl.Host == "" { + return ErrRelayMinerConfigInvalidServer.Wrap("empty relay miner server listen address") } - proxyConfig.Host = proxyUrl.Host + serverConfig.ListenAddress = listenUrl.Host return nil } // parseHTTPSupplierConfig populates the supplier fields of the target structure -// that are relevant to the "http" type in the supplier section of the config file. +// that are relevant to "http" specific service configurations. // This function alters the target RelayMinerSupplierServiceConfig structure // as a side effect. func (supplierServiceConfig *RelayMinerSupplierServiceConfig) parseHTTPSupplierConfig( yamlSupplierServiceConfig YAMLRelayMinerSupplierServiceConfig, ) error { - // Check if the supplier url is not empty - if len(yamlSupplierServiceConfig.Url) == 0 { - return ErrRelayMinerConfigInvalidSupplier.Wrap("empty supplier url") + // Check if the supplier backend url is empty + if len(yamlSupplierServiceConfig.BackendUrl) == 0 { + return ErrRelayMinerConfigInvalidSupplier.Wrap("empty supplier backend url") } - // Check if the supplier url is a valid URL - supplierServiceUrl, err := url.Parse(yamlSupplierServiceConfig.Url) + // Check if the supplier backend url is a valid URL + supplierServiceBackendUrl, err := url.Parse(yamlSupplierServiceConfig.BackendUrl) if err != nil { return ErrRelayMinerConfigInvalidSupplier.Wrapf( - "invalid supplier url %s", + "invalid supplier backend url %s", err.Error(), ) } - supplierServiceConfig.Url = supplierServiceUrl + supplierServiceConfig.BackendUrl = supplierServiceBackendUrl // If the Authentication section is not empty, populate the supplier service // authentication fields diff --git a/pkg/relayer/config/relayminer_configs_reader.go b/pkg/relayer/config/relayminer_configs_reader.go index b546d451d..e3ae3a52b 100644 --- a/pkg/relayer/config/relayminer_configs_reader.go +++ b/pkg/relayer/config/relayminer_configs_reader.go @@ -19,7 +19,7 @@ func ParseRelayMinerConfigs(configContent []byte) (*RelayMinerConfig, error) { return nil, ErrRelayMinerConfigUnmarshalYAML.Wrap(err.Error()) } - // Top level section + // Global section // SigningKeyName is required if len(yamlRelayMinerConfig.SigningKeyName) == 0 { return nil, ErrRelayMinerConfigInvalidSigningKeyName @@ -44,8 +44,8 @@ func ParseRelayMinerConfigs(configContent []byte) (*RelayMinerConfig, error) { return nil, err } - // Hydrate the proxies - if err := relayMinerConfig.HydrateProxies(yamlRelayMinerConfig.Proxies); err != nil { + // Hydrate the relay miner servers config + if err := relayMinerConfig.HydrateServers(yamlRelayMinerConfig.Suppliers); err != nil { return nil, err } @@ -54,31 +54,5 @@ func ParseRelayMinerConfigs(configContent []byte) (*RelayMinerConfig, error) { return nil, err } - // Check if proxies are referencing hosts more than once - if err := relayMinerConfig.EnsureUniqueHosts(); err != nil { - return nil, err - } - return relayMinerConfig, nil } - -// EnsureUniqueHosts checks if each proxy is referencing a host more than once -func (relayMinerConfig *RelayMinerConfig) EnsureUniqueHosts() error { - for _, proxyConfig := range relayMinerConfig.Proxies { - existingHosts := make(map[string]bool) - for _, supplierConfig := range proxyConfig.Suppliers { - for _, host := range supplierConfig.Hosts { - if _, ok := existingHosts[host]; ok { - return ErrRelayMinerConfigInvalidProxy.Wrapf( - "duplicate host %s in proxy %s", - host, - proxyConfig.ProxyName, - ) - } - existingHosts[host] = true - } - } - } - - return nil -} diff --git a/pkg/relayer/config/relayminer_configs_reader_test.go b/pkg/relayer/config/relayminer_configs_reader_test.go index e82f39fa1..472a893f0 100644 --- a/pkg/relayer/config/relayminer_configs_reader_test.go +++ b/pkg/relayer/config/relayminer_configs_reader_test.go @@ -2,6 +2,7 @@ package config_test import ( "net/url" + "os" "testing" sdkerrors "cosmossdk.io/errors" @@ -12,10 +13,17 @@ import ( "github.com/pokt-network/poktroll/testutil/yaml" ) +func Test_ParseRelayMinerConfig_ReferenceExample(t *testing.T) { + configContent, err := os.ReadFile("../../../localnet/poktrolld/config/relayminer_config_full_example.yaml") + require.NoError(t, err) + + _, err = config.ParseRelayMinerConfigs(configContent) + require.NoError(t, err) +} + func Test_ParseRelayMinerConfigs(t *testing.T) { tests := []struct { - desc string - + desc string inputConfigYAML string expectedErr *sdkerrors.Error @@ -32,24 +40,18 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { tx_node_rpc_url: tcp://127.0.0.1:36659 signing_key_name: supplier1 smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http suppliers: - service_id: ethereum - type: http + listen_url: http://127.0.0.1:8080 service_config: - url: http://anvil.servicer:8545 + backend_url: http://anvil.servicer:8545 authentication: username: user password: pwd headers: {} - hosts: - - tcp://ethereum.devnet1.poktroll.com - - tcp://ethereum - proxy_names: - - http-example + publicly_exposed_endpoints: + - ethereum.devnet1.poktroll.com + - ethereum `, expectedErr: nil, @@ -61,25 +63,24 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { }, SigningKeyName: "supplier1", SmtStorePath: "smt_stores", - Proxies: map[string]*config.RelayMinerProxyConfig{ - "http-example": { - ProxyName: "http-example", - Host: "127.0.0.1:8080", - Type: config.ProxyTypeHTTP, + Servers: map[string]*config.RelayMinerServerConfig{ + "http://127.0.0.1:8080": { + ListenAddress: "127.0.0.1:8080", + ServerType: config.RelayMinerServerTypeHTTP, XForwardedHostLookup: false, - Suppliers: map[string]*config.RelayMinerSupplierConfig{ + SupplierConfigsMap: map[string]*config.RelayMinerSupplierConfig{ "ethereum": { - ServiceId: "ethereum", - Type: config.ProxyTypeHTTP, + ServiceId: "ethereum", + ServerType: config.RelayMinerServerTypeHTTP, ServiceConfig: &config.RelayMinerSupplierServiceConfig{ - Url: &url.URL{Scheme: "http", Host: "anvil.servicer:8545"}, + BackendUrl: &url.URL{Scheme: "http", Host: "anvil.servicer:8545"}, Authentication: &config.RelayMinerSupplierServiceAuthentication{ Username: "user", Password: "pwd", }, Headers: map[string]string{}, }, - Hosts: []string{ + PubliclyExposedEndpoints: []string{ "ethereum.devnet1.poktroll.com", "ethereum", }, @@ -90,7 +91,7 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { }, }, { - desc: "valid: multiple suppliers, single proxy", + desc: "valid: multiple suppliers, single server", inputConfigYAML: ` pocket_node: @@ -99,32 +100,25 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { tx_node_rpc_url: tcp://127.0.0.1:36659 signing_key_name: supplier1 smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http suppliers: - service_id: ethereum - type: http + listen_url: http://127.0.0.1:8080 service_config: - url: http://anvil.servicer:8545 + backend_url: http://anvil.servicer:8545 authentication: username: user password: pwd headers: {} - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example + publicly_exposed_endpoints: + - ethereum.devnet1.poktroll.com - service_id: 7b-llm-model - type: http + listen_url: http://127.0.0.1:8080 service_config: - url: http://llama-endpoint - hosts: - - tcp://7b-llm-model.devnet1.poktroll.com - - tcp://7b-llm-model - proxy_names: - - http-example + backend_url: http://llama-endpoint + publicly_exposed_endpoints: + - 7b-llm-model.devnet1.poktroll.com + - 7b-llm-model + `, expectedErr: nil, @@ -136,36 +130,35 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { }, SigningKeyName: "supplier1", SmtStorePath: "smt_stores", - Proxies: map[string]*config.RelayMinerProxyConfig{ - "http-example": { - ProxyName: "http-example", - Host: "127.0.0.1:8080", - Type: config.ProxyTypeHTTP, + Servers: map[string]*config.RelayMinerServerConfig{ + "http://127.0.0.1:8080": { + ListenAddress: "127.0.0.1:8080", + ServerType: config.RelayMinerServerTypeHTTP, XForwardedHostLookup: false, - Suppliers: map[string]*config.RelayMinerSupplierConfig{ + SupplierConfigsMap: map[string]*config.RelayMinerSupplierConfig{ "ethereum": { - ServiceId: "ethereum", - Type: config.ProxyTypeHTTP, + ServiceId: "ethereum", + ServerType: config.RelayMinerServerTypeHTTP, ServiceConfig: &config.RelayMinerSupplierServiceConfig{ - Url: &url.URL{Scheme: "http", Host: "anvil.servicer:8545"}, + BackendUrl: &url.URL{Scheme: "http", Host: "anvil.servicer:8545"}, Authentication: &config.RelayMinerSupplierServiceAuthentication{ Username: "user", Password: "pwd", }, Headers: map[string]string{}, }, - Hosts: []string{ + PubliclyExposedEndpoints: []string{ "ethereum.devnet1.poktroll.com", "ethereum", }, }, "7b-llm-model": { - ServiceId: "7b-llm-model", - Type: config.ProxyTypeHTTP, + ServiceId: "7b-llm-model", + ServerType: config.RelayMinerServerTypeHTTP, ServiceConfig: &config.RelayMinerSupplierServiceConfig{ - Url: &url.URL{Scheme: "http", Host: "llama-endpoint"}, + BackendUrl: &url.URL{Scheme: "http", Host: "llama-endpoint"}, }, - Hosts: []string{ + PubliclyExposedEndpoints: []string{ "7b-llm-model.devnet1.poktroll.com", "7b-llm-model", }, @@ -175,84 +168,6 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { }, }, }, - { - desc: "valid: multiple proxies for a single supplier, no auth", - - inputConfigYAML: ` - pocket_node: - query_node_rpc_url: tcp://127.0.0.1:36657 - query_node_grpc_url: tcp://127.0.0.1:36658 - tx_node_rpc_url: tcp://127.0.0.1:36659 - signing_key_name: supplier1 - smt_store_path: smt_stores - proxies: - - proxy_name: first-proxy - host: 127.0.0.1:8080 - type: http - - proxy_name: second-proxy - host: 127.0.0.1:8081 - type: http - suppliers: - - service_id: ethereum - type: http - service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - first-proxy - - second-proxy - `, - - expectedErr: nil, - expectedConfig: &config.RelayMinerConfig{ - PocketNode: &config.RelayMinerPocketNodeConfig{ - QueryNodeRPCUrl: &url.URL{Scheme: "tcp", Host: "127.0.0.1:36657"}, - QueryNodeGRPCUrl: &url.URL{Scheme: "tcp", Host: "127.0.0.1:36658"}, - TxNodeRPCUrl: &url.URL{Scheme: "tcp", Host: "127.0.0.1:36659"}, - }, - SigningKeyName: "supplier1", - SmtStorePath: "smt_stores", - Proxies: map[string]*config.RelayMinerProxyConfig{ - "first-proxy": { - ProxyName: "first-proxy", - Host: "127.0.0.1:8080", - Type: config.ProxyTypeHTTP, - XForwardedHostLookup: false, - Suppliers: map[string]*config.RelayMinerSupplierConfig{ - "ethereum": { - ServiceId: "ethereum", - Type: config.ProxyTypeHTTP, - ServiceConfig: &config.RelayMinerSupplierServiceConfig{ - Url: &url.URL{Scheme: "http", Host: "anvil.servicer:8545"}, - }, - Hosts: []string{ - "ethereum.devnet1.poktroll.com", - }, - }, - }, - }, - "second-proxy": { - ProxyName: "second-proxy", - Host: "127.0.0.1:8081", - Type: config.ProxyTypeHTTP, - XForwardedHostLookup: false, - Suppliers: map[string]*config.RelayMinerSupplierConfig{ - "ethereum": { - ServiceId: "ethereum", - Type: config.ProxyTypeHTTP, - ServiceConfig: &config.RelayMinerSupplierServiceConfig{ - Url: &url.URL{Scheme: "http", Host: "anvil.servicer:8545"}, - }, - Hosts: []string{ - "ethereum.devnet1.poktroll.com", - }, - }, - }, - }, - }, - }, - }, { desc: "valid: relay miner config with query node rpc url defaulting to tx node rpc url", @@ -262,20 +177,15 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { tx_node_rpc_url: tcp://127.0.0.1:36659 signing_key_name: supplier1 smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http suppliers: - service_id: ethereum - type: http + listen_url: http://127.0.0.1:8080 service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - - tcp://ethereum - proxy_names: - - http-example + backend_url: http://anvil.servicer:8545 + publicly_exposed_endpoints: + - ethereum.devnet1.poktroll.com + - ethereum + `, expectedErr: nil, @@ -287,20 +197,19 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { }, SigningKeyName: "supplier1", SmtStorePath: "smt_stores", - Proxies: map[string]*config.RelayMinerProxyConfig{ - "http-example": { - ProxyName: "http-example", - Host: "127.0.0.1:8080", - Type: config.ProxyTypeHTTP, + Servers: map[string]*config.RelayMinerServerConfig{ + "http://127.0.0.1:8080": { + ListenAddress: "127.0.0.1:8080", + ServerType: config.RelayMinerServerTypeHTTP, XForwardedHostLookup: false, - Suppliers: map[string]*config.RelayMinerSupplierConfig{ + SupplierConfigsMap: map[string]*config.RelayMinerSupplierConfig{ "ethereum": { - ServiceId: "ethereum", - Type: config.ProxyTypeHTTP, + ServiceId: "ethereum", + ServerType: config.RelayMinerServerTypeHTTP, ServiceConfig: &config.RelayMinerSupplierServiceConfig{ - Url: &url.URL{Scheme: "http", Host: "anvil.servicer:8545"}, + BackendUrl: &url.URL{Scheme: "http", Host: "anvil.servicer:8545"}, }, - Hosts: []string{ + PubliclyExposedEndpoints: []string{ "ethereum.devnet1.poktroll.com", "ethereum", }, @@ -320,21 +229,14 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { tx_node_rpc_url: tcp://127.0.0.1:36659 signing_key_name: supplier1 smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http - x_forwarded_host_lookup: true suppliers: - service_id: ethereum - type: http + x_forwarded_host_lookup: true + listen_url: http://127.0.0.1:8080 service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - - tcp://ethereum - proxy_names: - - http-example + backend_url: http://anvil.servicer:8545 + publicly_exposed_endpoints: + - ethereum.devnet1.poktroll.com `, expectedErr: nil, @@ -346,20 +248,19 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { }, SigningKeyName: "supplier1", SmtStorePath: "smt_stores", - Proxies: map[string]*config.RelayMinerProxyConfig{ - "http-example": { - ProxyName: "http-example", - Host: "127.0.0.1:8080", - Type: config.ProxyTypeHTTP, + Servers: map[string]*config.RelayMinerServerConfig{ + "http://127.0.0.1:8080": { + ListenAddress: "127.0.0.1:8080", + ServerType: config.RelayMinerServerTypeHTTP, XForwardedHostLookup: true, - Suppliers: map[string]*config.RelayMinerSupplierConfig{ + SupplierConfigsMap: map[string]*config.RelayMinerSupplierConfig{ "ethereum": { - ServiceId: "ethereum", - Type: config.ProxyTypeHTTP, + ServiceId: "ethereum", + ServerType: config.RelayMinerServerTypeHTTP, ServiceConfig: &config.RelayMinerSupplierServiceConfig{ - Url: &url.URL{Scheme: "http", Host: "anvil.servicer:8545"}, + BackendUrl: &url.URL{Scheme: "http", Host: "anvil.servicer:8545"}, }, - Hosts: []string{ + PubliclyExposedEndpoints: []string{ "ethereum.devnet1.poktroll.com", "ethereum", }, @@ -380,19 +281,13 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { tx_node_rpc_url: &tcp://127.0.0.1:36659 signing_key_name: supplier1 smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http suppliers: - service_id: ethereum - type: http + listen_url: http://127.0.0.1:8080 service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example + backend_url: http://anvil.servicer:8545 + publicly_exposed_endpoints: + - ethereum.devnet1.poktroll.com `, expectedErr: config.ErrRelayMinerConfigInvalidNodeUrl, @@ -407,19 +302,12 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { query_node_grpc_url: tcp://127.0.0.1:36658 signing_key_name: supplier1 smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http suppliers: - service_id: ethereum - type: http service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example + backend_url: http://anvil.servicer:8545 + publicly_exposed_endpoints: + - ethereum.devnet1.poktroll.com `, expectedErr: config.ErrRelayMinerConfigInvalidNodeUrl, @@ -434,19 +322,14 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { tx_node_rpc_url: tcp://127.0.0.1:36659 signing_key_name: supplier1 smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http suppliers: - service_id: ethereum - type: http + listen_url: http://127.0.0.1:8080 service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example + backend_url: http://anvil.servicer:8545 + publicly_exposed_endpoints: + - ethereum.devnet1.poktroll.com + `, expectedErr: config.ErrRelayMinerConfigInvalidNodeUrl, @@ -461,19 +344,14 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { tx_node_rpc_url: tcp://127.0.0.1:36659 signing_key_name: supplier1 smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http suppliers: - service_id: ethereum - type: http + listen_url: http://127.0.0.1:8080 service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example + backend_url: http://anvil.servicer:8545 + publicly_exposed_endpoints: + - ethereum.devnet1.poktroll.com + `, expectedErr: config.ErrRelayMinerConfigInvalidNodeUrl, @@ -488,19 +366,14 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { tx_node_rpc_url: tcp://127.0.0.1:36659 signing_key_name: supplier1 smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http suppliers: - service_id: ethereum - type: http + listen_url: http://127.0.0.1:8080 service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example + backend_url: http://anvil.servicer:8545 + publicly_exposed_endpoints: + - ethereum.devnet1.poktroll.com + `, expectedErr: config.ErrRelayMinerConfigInvalidNodeUrl, @@ -515,19 +388,14 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { tx_node_rpc_url: tcp://127.0.0.1:36659 # explicitly omitted signing key name smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http suppliers: - service_id: ethereum - type: http + listen_url: http://127.0.0.1:8080 service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example + backend_url: http://anvil.servicer:8545 + publicly_exposed_endpoints: + - ethereum.devnet1.poktroll.com + `, expectedErr: config.ErrRelayMinerConfigInvalidSigningKeyName, @@ -542,181 +410,20 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { tx_node_rpc_url: tcp://127.0.0.1:36659 signing_key_name: supplier1 # explicitly omitted smt store path - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http - suppliers: - - service_id: ethereum - type: http - service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example - `, - - expectedErr: config.ErrRelayMinerConfigInvalidSmtStorePath, - }, - { - desc: "invalid: missing proxies section", - - inputConfigYAML: ` - pocket_node: - query_node_rpc_url: tcp://127.0.0.1:36657 - query_node_grpc_url: tcp://127.0.0.1:36658 - tx_node_rpc_url: tcp://127.0.0.1:36659 - signing_key_name: supplier1 - smt_store_path: smt_stores - # explicitly omitted proxies section - suppliers: - - proxy_name: ethereum - type: http - service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example - `, - - expectedErr: config.ErrRelayMinerConfigInvalidProxy, - }, - { - desc: "invalid: empty proxies section", - - inputConfigYAML: ` - pocket_node: - query_node_rpc_url: tcp://127.0.0.1:36657 - query_node_grpc_url: tcp://127.0.0.1:36658 - tx_node_rpc_url: tcp://127.0.0.1:36659 - signing_key_name: supplier1 - smt_store_path: smt_stores - proxies: # explicitly empty proxies section - suppliers: - - service_id: ethereum - type: http - service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example - `, - - expectedErr: config.ErrRelayMinerConfigInvalidProxy, - }, - { - desc: "invalid: omitted proxy name", - - inputConfigYAML: ` - pocket_node: - query_node_rpc_url: tcp://127.0.0.1:36657 - query_node_grpc_url: tcp://127.0.0.1:36658 - tx_node_rpc_url: tcp://127.0.0.1:36659 - signing_key_name: supplier1 - smt_store_path: smt_stores - proxies: - # explicitly omitted proxy name - - host: 127.0.0.1:8080 - type: http - suppliers: - - service_id: ethereum - type: http - service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example - `, - - expectedErr: config.ErrRelayMinerConfigInvalidProxy, - }, - { - desc: "invalid: empty proxy name", - - inputConfigYAML: ` - pocket_node: - query_node_rpc_url: tcp://127.0.0.1:36657 - query_node_grpc_url: tcp://127.0.0.1:36658 - tx_node_rpc_url: tcp://127.0.0.1:36659 - signing_key_name: supplier1 - smt_store_path: smt_stores - proxies: - - proxy_name: # explicitly empty proxy name - host: 127.0.0.1:8080 - type: http suppliers: - service_id: ethereum - type: http + listen_url: http://127.0.0.1:8080 service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example - `, + backend_url: http://anvil.servicer:8545 + publicly_exposed_endpoints: + - ethereum.devnet1.poktroll.com - expectedErr: config.ErrRelayMinerConfigInvalidProxy, - }, - { - desc: "invalid: missing http proxy host", - - inputConfigYAML: ` - pocket_node: - query_node_rpc_url: tcp://127.0.0.1:36657 - query_node_grpc_url: tcp://127.0.0.1:36658 - tx_node_rpc_url: tcp://127.0.0.1:36659 - signing_key_name: supplier1 - smt_store_path: smt_stores - proxies: - - proxy_name: http-example - # explicitly missing proxy host - type: http - suppliers: - - service_id: ethereum - type: http - service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example `, - expectedErr: config.ErrRelayMinerConfigInvalidProxy, - }, - { - desc: "invalid: empty http proxy host", - - inputConfigYAML: ` - pocket_node: - query_node_rpc_url: tcp://127.0.0.1:36657 - query_node_grpc_url: tcp://127.0.0.1:36658 - tx_node_rpc_url: tcp://127.0.0.1:36659 - signing_key_name: supplier1 - smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: # explicitly empty proxy host - type: http - suppliers: - - service_id: ethereum - type: http - service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example - `, - - expectedErr: config.ErrRelayMinerConfigInvalidProxy, + expectedErr: config.ErrRelayMinerConfigInvalidSmtStorePath, }, { - desc: "invalid: missing proxy type", + desc: "invalid: empty listen address", inputConfigYAML: ` pocket_node: @@ -725,52 +432,20 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { tx_node_rpc_url: tcp://127.0.0.1:36659 signing_key_name: supplier1 smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - # explicitly missing proxy type suppliers: - service_id: ethereum - type: http + listen_url: http:// # explicitly empty listen url service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example - `, - - expectedErr: config.ErrRelayMinerConfigInvalidProxy, - }, - { - desc: "invalid: empty proxy type", + backend_url: http://anvil.servicer:8545 + publicly_exposed_endpoints: + - ethereum.devnet1.poktroll.com - inputConfigYAML: ` - pocket_node: - query_node_rpc_url: tcp://127.0.0.1:36657 - query_node_grpc_url: tcp://127.0.0.1:36658 - tx_node_rpc_url: tcp://127.0.0.1:36659 - signing_key_name: supplier1 - smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: # explicitly empty proxy type - suppliers: - - service_id: ethereum - type: http - service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example `, - expectedErr: config.ErrRelayMinerConfigInvalidProxy, + expectedErr: config.ErrRelayMinerConfigInvalidServer, }, { - desc: "invalid: unsupported proxy type", + desc: "invalid: unsupported server type", inputConfigYAML: ` pocket_node: @@ -779,22 +454,16 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { tx_node_rpc_url: tcp://127.0.0.1:36659 signing_key_name: supplier1 smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: unsupported suppliers: - service_id: ethereum - type: http + listen_url: unsupported://127.0.0.1:8080 service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example + backend_url: http://anvil.servicer:8545 + publicly_exposed_endpoints: + - ethereum.devnet1.poktroll.com `, - expectedErr: config.ErrRelayMinerConfigInvalidProxy, + expectedErr: config.ErrRelayMinerConfigInvalidServer, }, { desc: "invalid: missing supplier name", @@ -806,19 +475,13 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { tx_node_rpc_url: tcp://127.0.0.1:36659 signing_key_name: supplier1 smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http suppliers: - # explicitly missing supplier name - - type: http + - listen_url: http://127.0.0.1:8080 + # explicitly missing supplier name service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example + backend_url: http://anvil.servicer:8545 + publicly_exposed_endpoints: + - ethereum.devnet1.poktroll.com `, expectedErr: config.ErrRelayMinerConfigInvalidSupplier, @@ -833,52 +496,20 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { tx_node_rpc_url: tcp://127.0.0.1:36659 signing_key_name: supplier1 smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http suppliers: - service_id: # explicitly empty supplier name - type: http + listen_url: http://127.0.0.1:8080 service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example - `, - - expectedErr: config.ErrRelayMinerConfigInvalidSupplier, - }, - { - desc: "invalid: unsupported supplier type", + backend_url: http://anvil.servicer:8545 + publicly_exposed_endpoints: + - ethereum.devnet1.poktroll.com - inputConfigYAML: ` - pocket_node: - query_node_rpc_url: tcp://127.0.0.1:36657 - query_node_grpc_url: tcp://127.0.0.1:36658 - tx_node_rpc_url: tcp://127.0.0.1:36659 - signing_key_name: supplier1 - smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http - suppliers: - - service_id: ethereum - type: unsupported - service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example `, expectedErr: config.ErrRelayMinerConfigInvalidSupplier, }, { - desc: "invalid: missing supplier type", + desc: "invalid: bad supplier service config backend url", inputConfigYAML: ` pocket_node: @@ -887,52 +518,20 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { tx_node_rpc_url: tcp://127.0.0.1:36659 signing_key_name: supplier1 smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http suppliers: - service_id: ethereum - # explicitly missing supplier type + listen_url: http://127.0.0.1:8080 service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example - `, - - expectedErr: config.ErrRelayMinerConfigInvalidSupplier, - }, - { - desc: "invalid: empty supplier type", + backend_url: &http://anvil.servicer:8545 + publicly_exposed_endpoints: + - ethereum.devnet1.poktroll.com - inputConfigYAML: ` - pocket_node: - query_node_rpc_url: tcp://127.0.0.1:36657 - query_node_grpc_url: tcp://127.0.0.1:36658 - tx_node_rpc_url: tcp://127.0.0.1:36659 - signing_key_name: supplier1 - smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http - suppliers: - - service_id: ethereum - type: # explicitly empty supplier type - service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example `, expectedErr: config.ErrRelayMinerConfigInvalidSupplier, }, { - desc: "invalid: bad supplier service config url", + desc: "invalid: empty supplier service config backend url", inputConfigYAML: ` pocket_node: @@ -941,52 +540,20 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { tx_node_rpc_url: tcp://127.0.0.1:36659 signing_key_name: supplier1 smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http suppliers: - service_id: ethereum - type: http + listen_url: http://127.0.0.1:8080 service_config: - url: &http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example - `, + backend_url: # explicitly empty supplier service config backend url + publicly_exposed_endpoints: + - ethereum.devnet1.poktroll.com - expectedErr: config.ErrRelayMinerConfigInvalidSupplier, - }, - { - desc: "invalid: empty supplier service config url", - - inputConfigYAML: ` - pocket_node: - query_node_rpc_url: tcp://127.0.0.1:36657 - query_node_grpc_url: tcp://127.0.0.1:36658 - tx_node_rpc_url: tcp://127.0.0.1:36659 - signing_key_name: supplier1 - smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http - suppliers: - - service_id: ethereum - type: http - service_config: - url: # explicitly empty supplier service config url - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example `, expectedErr: config.ErrRelayMinerConfigInvalidSupplier, }, { - desc: "invalid: missing supplier service config url", + desc: "invalid: missing supplier service config backend url", inputConfigYAML: ` pocket_node: @@ -995,25 +562,19 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { tx_node_rpc_url: tcp://127.0.0.1:36659 signing_key_name: supplier1 smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http suppliers: - service_id: ethereum - type: http + listen_url: http://127.0.0.1:8080 service_config: - # explicitly missing supplier service config url - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example + publicly_exposed_endpoints: + - ethereum.devnet1.poktroll.com + # explicitly missing supplier service config backend url `, expectedErr: config.ErrRelayMinerConfigInvalidSupplier, }, { - desc: "invalid: bad supplier host", + desc: "invalid: blank supplier exposed endpoint", inputConfigYAML: ` pocket_node: @@ -1022,112 +583,17 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { tx_node_rpc_url: tcp://127.0.0.1:36659 signing_key_name: supplier1 smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http suppliers: - service_id: ethereum - type: http + listen_url: http://127.0.0.1:8080 service_config: - url: http://anvil.servicer:8545 - hosts: - - &tcp://ethereum.devnet1.poktroll.com - proxy_names: - - http-example + backend_url: http://anvil.servicer:8545 + publicly_exposed_endpoints: + - # explicitly blank supplier host `, expectedErr: config.ErrRelayMinerConfigInvalidSupplier, }, - { - desc: "invalid: blank supplier host", - - inputConfigYAML: ` - pocket_node: - query_node_rpc_url: tcp://127.0.0.1:36657 - query_node_grpc_url: tcp://127.0.0.1:36658 - tx_node_rpc_url: tcp://127.0.0.1:36659 - signing_key_name: supplier1 - smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http - suppliers: - - service_id: ethereum - type: http - service_config: - url: http://anvil.servicer:8545 - hosts: - - # explicitly blank supplier host - proxy_names: - - http-example - `, - - expectedErr: config.ErrRelayMinerConfigInvalidSupplier, - }, - { - desc: "invalid: empty supplier proxy references", - - inputConfigYAML: ` - pocket_node: - query_node_rpc_url: tcp://127.0.0.1:36657 - query_node_grpc_url: tcp://127.0.0.1:36658 - tx_node_rpc_url: tcp://127.0.0.1:36659 - signing_key_name: supplier1 - smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http - suppliers: - - service_id: ethereum - type: http - service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://ethereum.devnet1.poktroll.com - proxy_names: - - bad-proxy-name - `, - - expectedErr: config.ErrRelayMinerConfigInvalidSupplier, - }, - { - desc: "invalid: empty supplier proxy references", - - inputConfigYAML: ` - pocket_node: - query_node_rpc_url: tcp://127.0.0.1:36657 - query_node_grpc_url: tcp://127.0.0.1:36658 - tx_node_rpc_url: tcp://127.0.0.1:36659 - signing_key_name: supplier1 - smt_store_path: smt_stores - proxies: - - proxy_name: http-example - host: 127.0.0.1:8080 - type: http - suppliers: - - service_id: ethereum - type: http - service_config: - url: http://anvil.servicer:8545 - hosts: - - tcp://devnet1.poktroll.com # hosts for both suppliers are the same - proxy_names: - - http-example - - service_id: avax - type: http - service_config: - url: http://avax.servicer:8545 - hosts: - - tcp://devnet1.poktroll.com # hosts for both suppliers are the same - proxy_names: - - http-example - `, - - expectedErr: config.ErrRelayMinerConfigInvalidProxy, - }, { desc: "invalid: empty RelayMiner config file", @@ -1135,8 +601,8 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { expectedErr: config.ErrRelayMinerConfigEmpty, }, - // TODO_NB: Test for supplier and proxy types mismatch once we have more - // than one proxy type. + // TODO_NB: Test for supplier and server types mismatch once we have more + // than one server type. } for _, test := range tests { @@ -1144,6 +610,7 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { normalizedConfig := yaml.NormalizeYAMLIndentation(test.inputConfigYAML) config, err := config.ParseRelayMinerConfigs([]byte(normalizedConfig)) + // Invalid configuration if test.expectedErr != nil { require.ErrorIs(t, err, test.expectedErr) require.Nil(t, config) @@ -1154,6 +621,7 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { return } + // Valid configuration require.NoError(t, err) require.Equal( @@ -1186,60 +654,54 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { config.PocketNode.TxNodeRPCUrl.String(), ) - for proxyName, proxy := range test.expectedConfig.Proxies { - require.Equal( - t, - proxy.ProxyName, - config.Proxies[proxyName].ProxyName, - ) - + for listenAddress, server := range test.expectedConfig.Servers { require.Equal( t, - proxy.Host, - config.Proxies[proxyName].Host, + server.ListenAddress, + config.Servers[listenAddress].ListenAddress, ) require.Equal( t, - proxy.Type, - config.Proxies[proxyName].Type, + server.ServerType, + config.Servers[listenAddress].ServerType, ) - for supplierName, supplier := range proxy.Suppliers { + for supplierName, supplier := range server.SupplierConfigsMap { require.Equal( t, supplier.ServiceId, - config.Proxies[proxyName].Suppliers[supplierName].ServiceId, + config.Servers[listenAddress].SupplierConfigsMap[supplierName].ServiceId, ) require.Equal( t, - supplier.Type, - config.Proxies[proxyName].Suppliers[supplierName].Type, + supplier.ServerType, + config.Servers[listenAddress].SupplierConfigsMap[supplierName].ServerType, ) require.Equal( t, - supplier.ServiceConfig.Url.String(), - config.Proxies[proxyName].Suppliers[supplierName].ServiceConfig.Url.String(), + supplier.ServiceConfig.BackendUrl.String(), + config.Servers[listenAddress].SupplierConfigsMap[supplierName].ServiceConfig.BackendUrl.String(), ) if supplier.ServiceConfig.Authentication != nil { require.NotNil( t, - config.Proxies[proxyName].Suppliers[supplierName].ServiceConfig.Authentication, + config.Servers[listenAddress].SupplierConfigsMap[supplierName].ServiceConfig.Authentication, ) require.Equal( t, supplier.ServiceConfig.Authentication.Username, - config.Proxies[proxyName].Suppliers[supplierName].ServiceConfig.Authentication.Username, + config.Servers[listenAddress].SupplierConfigsMap[supplierName].ServiceConfig.Authentication.Username, ) require.Equal( t, supplier.ServiceConfig.Authentication.Password, - config.Proxies[proxyName].Suppliers[supplierName].ServiceConfig.Authentication.Password, + config.Servers[listenAddress].SupplierConfigsMap[supplierName].ServiceConfig.Authentication.Password, ) } @@ -1247,15 +709,15 @@ func Test_ParseRelayMinerConfigs(t *testing.T) { require.Equal( t, headerValue, - config.Proxies[proxyName].Suppliers[supplierName].ServiceConfig.Headers[headerKey], + config.Servers[listenAddress].SupplierConfigsMap[supplierName].ServiceConfig.Headers[headerKey], ) } - for i, host := range supplier.Hosts { + for i, host := range supplier.PubliclyExposedEndpoints { require.Contains( t, host, - config.Proxies[proxyName].Suppliers[supplierName].Hosts[i], + config.Servers[listenAddress].SupplierConfigsMap[supplierName].PubliclyExposedEndpoints[i], ) } } diff --git a/pkg/relayer/config/servers_config_hydrator.go b/pkg/relayer/config/servers_config_hydrator.go new file mode 100644 index 000000000..87d0378b3 --- /dev/null +++ b/pkg/relayer/config/servers_config_hydrator.go @@ -0,0 +1,60 @@ +package config + +import "net/url" + +// HydrateServers populates the servers fields of the RelayMinerConfig. +func (relayMinerConfig *RelayMinerConfig) HydrateServers( + yamlSupplierConfigs []YAMLRelayMinerSupplierConfig, +) error { + // At least one server is required + if len(yamlSupplierConfigs) == 0 { + return ErrRelayMinerConfigInvalidSupplier.Wrap("no suppliers provided") + } + + relayMinerConfig.Servers = make(map[string]*RelayMinerServerConfig) + + for _, yamlSupplierConfig := range yamlSupplierConfigs { + listenUrl, err := url.Parse(yamlSupplierConfig.ListenUrl) + if err != nil { + return ErrRelayMinerConfigInvalidServer.Wrapf( + "invalid listen url %q", + yamlSupplierConfig.ListenUrl, + ) + } + + if listenUrl.Scheme == "" { + return ErrRelayMinerConfigInvalidServer.Wrapf( + "missing scheme in listen url %q", + yamlSupplierConfig.ListenUrl, + ) + } + + if _, ok := relayMinerConfig.Servers[yamlSupplierConfig.ListenUrl]; ok { + continue + } + + serverConfig := &RelayMinerServerConfig{ + XForwardedHostLookup: yamlSupplierConfig.XForwardedHostLookup, + SupplierConfigsMap: make(map[string]*RelayMinerSupplierConfig), + } + + // Populate the server fields that are relevant to each supported server type + switch listenUrl.Scheme { + case "http", "ws": + if err := serverConfig.parseHTTPServerConfig(yamlSupplierConfig); err != nil { + return err + } + serverConfig.ServerType = RelayMinerServerTypeHTTP + default: + // Fail if the relay miner server type is not supported + return ErrRelayMinerConfigInvalidServer.Wrapf( + "invalid relay miner server type %q", + listenUrl.Scheme, + ) + } + + relayMinerConfig.Servers[yamlSupplierConfig.ListenUrl] = serverConfig + } + + return nil +} diff --git a/pkg/relayer/config/supplier_hydrator.go b/pkg/relayer/config/supplier_hydrator.go index 0b54a36f0..1fbb8e979 100644 --- a/pkg/relayer/config/supplier_hydrator.go +++ b/pkg/relayer/config/supplier_hydrator.go @@ -13,41 +13,53 @@ func (supplierConfig *RelayMinerSupplierConfig) HydrateSupplier( } supplierConfig.ServiceId = yamlSupplierConfig.ServiceId - // Supplier hosts - supplierConfig.Hosts = []string{} - existingHosts := make(map[string]bool) - for _, host := range yamlSupplierConfig.Hosts { + // Supplier public endpoints + supplierConfig.PubliclyExposedEndpoints = []string{} + existingEndpoints := make(map[string]bool) + for _, host := range yamlSupplierConfig.ServiceConfig.PubliclyExposedEndpoints { // Check if the supplier host is empty if len(host) == 0 { - return ErrRelayMinerConfigInvalidSupplier.Wrap("empty supplier host") + return ErrRelayMinerConfigInvalidSupplier.Wrap("empty supplier public endpoint") } - // Check if the supplier host is a valid URL - supplierHost, err := url.Parse(host) - if err != nil { + // Check if the supplier public endpoint is unique + if _, ok := existingEndpoints[host]; ok { return ErrRelayMinerConfigInvalidSupplier.Wrapf( - "invalid supplier host %s", + "duplicate supplier public endpoint %s", host, ) } + existingEndpoints[host] = true - // Check if the supplier host is unique - if _, ok := existingHosts[supplierHost.Host]; ok { - return ErrRelayMinerConfigInvalidSupplier.Wrapf( - "duplicate supplier host %s", - host, - ) - } - existingHosts[supplierHost.Host] = true - - // Add the supplier host to the suppliers list - supplierConfig.Hosts = append(supplierConfig.Hosts, supplierHost.Host) + // Add the supplier public endpoint to the suppliers list + supplierConfig.PubliclyExposedEndpoints = append( + supplierConfig.PubliclyExposedEndpoints, + host, + ) } - // Add a default host which corresponds to the supplier name if it is not + // Add a default endpoint which corresponds to the supplier name if it is not // already in the list - if _, ok := existingHosts[supplierConfig.ServiceId]; !ok { - supplierConfig.Hosts = append(supplierConfig.Hosts, supplierConfig.ServiceId) + if _, ok := existingEndpoints[supplierConfig.ServiceId]; !ok { + supplierConfig.PubliclyExposedEndpoints = append( + supplierConfig.PubliclyExposedEndpoints, + supplierConfig.ServiceId, + ) + } + + backendUrl, err := url.Parse(yamlSupplierConfig.ServiceConfig.BackendUrl) + if err != nil { + return ErrRelayMinerConfigInvalidSupplier.Wrapf( + "invalid supplier backend url %s", + err.Error(), + ) + } + + if backendUrl.Scheme == "" { + return ErrRelayMinerConfigInvalidSupplier.Wrapf( + "missing scheme in supplier backend url %s", + yamlSupplierConfig.ServiceConfig.BackendUrl, + ) } // Populate the supplier service fields that are relevant to each supported @@ -55,9 +67,9 @@ func (supplierConfig *RelayMinerSupplierConfig) HydrateSupplier( // If other supplier types are added in the future, they should be handled // by their own functions. supplierConfig.ServiceConfig = &RelayMinerSupplierServiceConfig{} - switch yamlSupplierConfig.Type { + switch backendUrl.Scheme { case "http": - supplierConfig.Type = ProxyTypeHTTP + supplierConfig.ServerType = RelayMinerServerTypeHTTP if err := supplierConfig.ServiceConfig. parseHTTPSupplierConfig(yamlSupplierConfig.ServiceConfig); err != nil { return err @@ -66,15 +78,7 @@ func (supplierConfig *RelayMinerSupplierConfig) HydrateSupplier( // Fail if the supplier type is not supported return ErrRelayMinerConfigInvalidSupplier.Wrapf( "invalid supplier type %s", - yamlSupplierConfig.Type, - ) - } - - // Check if the supplier has proxies - if len(yamlSupplierConfig.ProxyNames) == 0 { - return ErrRelayMinerConfigInvalidSupplier.Wrapf( - "supplier %s has no proxies", - supplierConfig.ServiceId, + backendUrl.Scheme, ) } diff --git a/pkg/relayer/config/suppliers_config_hydrator.go b/pkg/relayer/config/suppliers_config_hydrator.go index b8f0f4fa7..b5e1d6e2a 100644 --- a/pkg/relayer/config/suppliers_config_hydrator.go +++ b/pkg/relayer/config/suppliers_config_hydrator.go @@ -13,7 +13,7 @@ func (relayMinerConfig *RelayMinerConfig) HydrateSuppliers( return err } - // Supplier name should not be unique + // Supplier name should be unique if _, ok := existingSuppliers[yamlSupplierConfig.ServiceId]; ok { return ErrRelayMinerConfigInvalidSupplier.Wrapf( "duplicate supplier name %s", @@ -23,28 +23,9 @@ func (relayMinerConfig *RelayMinerConfig) HydrateSuppliers( // Mark the supplier as existing existingSuppliers[yamlSupplierConfig.ServiceId] = true - // Add the supplier config to the referenced proxies - for _, proxyName := range yamlSupplierConfig.ProxyNames { - // If the proxy name is referencing a non-existent proxy, fail - if _, ok := relayMinerConfig.Proxies[proxyName]; !ok { - return ErrRelayMinerConfigInvalidSupplier.Wrapf( - "no matching proxy %s for supplier %s", - supplierConfig.ServiceId, - proxyName, - ) - } - - // If the proxy name is referencing a proxy of a different type, fail - if supplierConfig.Type != relayMinerConfig.Proxies[proxyName].Type { - return ErrRelayMinerConfigInvalidSupplier.Wrapf( - "supplier %s and proxy %s have different types", - supplierConfig.ServiceId, - proxyName, - ) - } - - relayMinerConfig.Proxies[proxyName].Suppliers[supplierConfig.ServiceId] = supplierConfig - } + relayMinerConfig. + Servers[yamlSupplierConfig.ListenUrl]. + SupplierConfigsMap[supplierConfig.ServiceId] = supplierConfig } return nil diff --git a/pkg/relayer/config/types.go b/pkg/relayer/config/types.go index edd63ff54..7b273292e 100644 --- a/pkg/relayer/config/types.go +++ b/pkg/relayer/config/types.go @@ -2,44 +2,39 @@ package config import "net/url" -type ProxyType int +type RelayMinerServerType int const ( - ProxyTypeHTTP ProxyType = iota - // TODO: Support other proxy types: HTTPS, TCP, UNIX socket, UDP, QUIC, WebRTC ... + RelayMinerServerTypeHTTP RelayMinerServerType = iota + // TODO: Support other RelayMinerServerType: + // RelayMinerServerTypeHTTPS + // RelayMinerServerTypeTCP + // RelayMinerServerTypeUDP + // RelayMinerServerTypeQUIC + // RelayMinerServerTypeWebRTC + // RelayMinerServerTypeUNIXSocket + // Etc... ) // YAMLRelayMinerConfig is the structure used to unmarshal the RelayMiner config file -// TODO_DOCUMENT(@red-0ne): Add proper README documentation for yaml config files -// and update inline comments accordingly. type YAMLRelayMinerConfig struct { PocketNode YAMLRelayMinerPocketNodeConfig `yaml:"pocket_node"` SigningKeyName string `yaml:"signing_key_name"` SmtStorePath string `yaml:"smt_store_path"` Metrics YAMLRelayMinerMetricsConfig `yaml:"metrics"` - Proxies []YAMLRelayMinerProxyConfig `yaml:"proxies"` Suppliers []YAMLRelayMinerSupplierConfig `yaml:"suppliers"` } // YAMLRelayMinerPocketNodeConfig is the structure used to unmarshal the pocket -// node URLs section of the RelayMiner config file +// node URLs section of the RelayMiner config file. type YAMLRelayMinerPocketNodeConfig struct { QueryNodeRPCUrl string `yaml:"query_node_rpc_url"` QueryNodeGRPCUrl string `yaml:"query_node_grpc_url"` TxNodeRPCUrl string `yaml:"tx_node_rpc_url"` } -// YAMLRelayMinerProxyConfig is the structure used to unmarshal the proxy -// section of the RelayMiner config file -type YAMLRelayMinerProxyConfig struct { - ProxyName string `yaml:"proxy_name"` - Type string `yaml:"type"` - Host string `yaml:"host"` - XForwardedHostLookup bool `yaml:"x_forwarded_host_lookup"` -} - // YAMLRelayMinerMetricsConfig is the structure used to unmarshal the metrics -// section of the RelayMiner config file +// section of the RelayMiner config file. type YAMLRelayMinerMetricsConfig struct { Enabled bool `yaml:"enabled"` Addr string `yaml:"addr"` @@ -48,19 +43,19 @@ type YAMLRelayMinerMetricsConfig struct { // YAMLRelayMinerSupplierConfig is the structure used to unmarshal the supplier // section of the RelayMiner config file type YAMLRelayMinerSupplierConfig struct { - ServiceId string `yaml:"service_id"` - Type string `yaml:"type"` - Hosts []string `yaml:"hosts"` - ServiceConfig YAMLRelayMinerSupplierServiceConfig `yaml:"service_config"` - ProxyNames []string `yaml:"proxy_names"` + ServiceId string `yaml:"service_id"` + ListenUrl string `yaml:"listen_url"` + XForwardedHostLookup bool `yaml:"x_forwarded_host_lookup"` + ServiceConfig YAMLRelayMinerSupplierServiceConfig `yaml:"service_config"` } // YAMLRelayMinerSupplierServiceConfig is the structure used to unmarshal the supplier -// service sub-section of the RelayMiner config file +// service sub-section of the RelayMiner config file. type YAMLRelayMinerSupplierServiceConfig struct { - Url string `yaml:"url"` - Authentication YAMLRelayMinerSupplierServiceAuthentication `yaml:"authentication,omitempty"` - Headers map[string]string `yaml:"headers,omitempty"` + BackendUrl string `yaml:"backend_url"` + Authentication YAMLRelayMinerSupplierServiceAuthentication `yaml:"authentication,omitempty"` + Headers map[string]string `yaml:"headers,omitempty"` + PubliclyExposedEndpoints []string `yaml:"publicly_exposed_endpoints"` } // YAMLRelayMinerSupplierServiceAuthentication is the structure used to unmarshal @@ -74,7 +69,7 @@ type YAMLRelayMinerSupplierServiceAuthentication struct { // RelayMinerConfig is the structure describing the RelayMiner config type RelayMinerConfig struct { PocketNode *RelayMinerPocketNodeConfig - Proxies map[string]*RelayMinerProxyConfig + Servers map[string]*RelayMinerServerConfig Metrics *RelayMinerMetricsConfig SigningKeyName string SmtStorePath string @@ -88,25 +83,22 @@ type RelayMinerPocketNodeConfig struct { TxNodeRPCUrl *url.URL } -// RelayMinerProxyConfig is the structure resulting from parsing the proxy -// section of the RelayMiner config file. -// Each proxy embeds a map of supplier configs that are associated with it. -// Other proxy types may embed other fields in the future. eg. "https" may -// embed a TLS config. -type RelayMinerProxyConfig struct { - // ProxyName is the name of the proxy server, used to identify it in the config - ProxyName string - // Type is the transport protocol used by the proxy server like (http, https, etc.) - Type ProxyType - // Host is the host on which the proxy server will listen for incoming - // relay requests - Host string - // XForwardedHostLookup is a flag that indicates whether the proxy server +// RelayMinerServerConfig is the structure resulting from parsing the supplier's +// server section of the RelayMiner config file. +// Each server section embeds a map of supplier configs that are associated with it. +// TODO_IMPROVE: Other server types may embed other fields in the future; eg. "https" may embed a TLS config. +type RelayMinerServerConfig struct { + // ServerType is the transport protocol used by the server like (http, https, etc.) + ServerType RelayMinerServerType + // ListenAddress is the host on which the relay miner server will listen + // for incoming relay requests + ListenAddress string + // XForwardedHostLookup is a flag that indicates whether the relay miner server // should lookup the host from the X-Forwarded-Host header before falling // back to the Host header. XForwardedHostLookup bool - // Suppliers is a map of serviceIds -> RelayMinerSupplierConfig - Suppliers map[string]*RelayMinerSupplierConfig + // SupplierConfigsMap is a map of serviceIds -> RelayMinerSupplierConfig + SupplierConfigsMap map[string]*RelayMinerSupplierConfig } // RelayMinerMetricsConfig is the structure resulting from parsing the metrics @@ -121,12 +113,12 @@ type RelayMinerMetricsConfig struct { type RelayMinerSupplierConfig struct { // ServiceId is the serviceId corresponding to the current configuration. ServiceId string - // Type is the transport protocol used by the supplier, it must match the - // type of the proxy it is associated with. - Type ProxyType - // Hosts is a list of hosts advertised on-chain by the supplier, the corresponding - // proxy server will accept relay requests for these hosts. - Hosts []string + // ServerType is the transport protocol used by the supplier, it must match the + // type of the relay miner server it is associated with. + ServerType RelayMinerServerType + // PubliclyExposedEndpoints is a list of hosts advertised on-chain by the supplier, + // the corresponding relay miner server will accept relay requests for these hosts. + PubliclyExposedEndpoints []string // ServiceConfig is the config of the service that relays will be proxied to. // Other supplier types may embed other fields in the future. eg. "https" may // embed a TLS config. @@ -136,10 +128,10 @@ type RelayMinerSupplierConfig struct { // RelayMinerSupplierServiceConfig is the structure resulting from parsing the supplier // service sub-section of the RelayMiner config file. type RelayMinerSupplierServiceConfig struct { - // Url is the URL of the service that relays will be proxied to. - Url *url.URL + // BackendUrl is the URL of the service that relays will be proxied to. + BackendUrl *url.URL // Authentication is the basic auth structure used to authenticate to the - // request being proxied from the current proxy server. + // request being proxied from the current relay miner server. // If the service the relay requests are forwarded to requires basic auth // then this field must be populated. // TODO_TECHDEBT(@red-0ne): Pass the authentication to the service instance diff --git a/pkg/relayer/proxy/error_reply.go b/pkg/relayer/proxy/error_reply.go index 9653941a8..4ecee6270 100644 --- a/pkg/relayer/proxy/error_reply.go +++ b/pkg/relayer/proxy/error_reply.go @@ -11,12 +11,12 @@ import ( // replyWithError builds the appropriate error format according to the payload // using the passed in error and writes it to the writer. // NOTE: This method is used to reply with an "internal" error that is related -// to the proxy itself and not to the relayed request. +// to the server itself and not to the relayed request. func (sync *synchronousRPCServer) replyWithError( ctx context.Context, payloadBz []byte, writer http.ResponseWriter, - proxyName string, + listenAddress string, serviceId string, err error, ) { @@ -24,22 +24,27 @@ func (sync *synchronousRPCServer) replyWithError( responseBz, err := partials.GetErrorReply(ctx, payloadBz, err) if err != nil { - sync.logger.Error().Err(err).Str("service_id", serviceId).Str("proxy_name", proxyName).Msg( + sync.logger.Error().Err(err). + Str("service_id", serviceId). + Str("listen_address", listenAddress).Msg( "failed getting error reply") return } relayResponse := &types.RelayResponse{Payload: responseBz} - relayResponseBz, err := relayResponse.Marshal() if err != nil { - sync.logger.Error().Err(err).Str("service_id", serviceId).Str("proxy_name", proxyName).Msg( + sync.logger.Error().Err(err). + Str("service_id", serviceId). + Str("listen_address", listenAddress).Msg( "failed marshaling relay response") return } if _, err = writer.Write(relayResponseBz); err != nil { - sync.logger.Error().Err(err).Str("service_id", serviceId).Str("proxy_name", proxyName).Msg( + sync.logger.Error().Err(err). + Str("service_id", serviceId). + Str("listen_address", listenAddress).Msg( "failed writing relay response") return } diff --git a/pkg/relayer/proxy/errors.go b/pkg/relayer/proxy/errors.go index 74e259502..b1b5753e9 100644 --- a/pkg/relayer/proxy/errors.go +++ b/pkg/relayer/proxy/errors.go @@ -5,14 +5,14 @@ import ( ) var ( - codespace = "relayer_proxy" - ErrRelayerProxyUnsupportedRPCType = sdkerrors.Register(codespace, 1, "unsupported rpc type") - ErrRelayerProxyInvalidSession = sdkerrors.Register(codespace, 2, "invalid session in relayer request") - ErrRelayerProxyInvalidSupplier = sdkerrors.Register(codespace, 3, "supplier does not belong to session") - ErrRelayerProxyUndefinedSigningKeyName = sdkerrors.Register(codespace, 4, "supplier signing key name is undefined") - ErrRelayerProxyUndefinedProxiedServicesEndpoints = sdkerrors.Register(codespace, 5, "undefined proxied services endpoints for relayer proxy") - ErrRelayerProxyInvalidRelayRequest = sdkerrors.Register(codespace, 6, "invalid relay request") - ErrRelayerProxyInvalidRelayResponse = sdkerrors.Register(codespace, 7, "invalid relay response") - ErrRelayerProxyServiceEndpointNotHandled = sdkerrors.Register(codespace, 8, "service endpoint not handled by relayer proxy") - ErrRelayerProxyUnsupportedTransportType = sdkerrors.Register(codespace, 9, "unsupported proxy transport type") + codespace = "relayer_proxy" + ErrRelayerProxyUnsupportedRPCType = sdkerrors.Register(codespace, 1, "unsupported rpc type") + ErrRelayerProxyInvalidSession = sdkerrors.Register(codespace, 2, "invalid session in relayer request") + ErrRelayerProxyInvalidSupplier = sdkerrors.Register(codespace, 3, "supplier does not belong to session") + ErrRelayerProxyUndefinedSigningKeyName = sdkerrors.Register(codespace, 4, "supplier signing key name is undefined") + ErrRelayerServicesConfigsUndefined = sdkerrors.Register(codespace, 5, "services configurations are undefined") + ErrRelayerProxyInvalidRelayRequest = sdkerrors.Register(codespace, 6, "invalid relay request") + ErrRelayerProxyInvalidRelayResponse = sdkerrors.Register(codespace, 7, "invalid relay response") + ErrRelayerProxyServiceEndpointNotHandled = sdkerrors.Register(codespace, 8, "service endpoint not handled by relayer proxy") + ErrRelayerProxyUnsupportedTransportType = sdkerrors.Register(codespace, 9, "unsupported proxy transport type") ) diff --git a/pkg/relayer/proxy/metrics.go b/pkg/relayer/proxy/metrics.go index 8b7b3011a..d42f09c4b 100644 --- a/pkg/relayer/proxy/metrics.go +++ b/pkg/relayer/proxy/metrics.go @@ -1,9 +1,8 @@ package proxy import ( - stdprometheus "github.com/prometheus/client_golang/prometheus" - "github.com/go-kit/kit/metrics/prometheus" + stdprometheus "github.com/prometheus/client_golang/prometheus" ) const ( @@ -20,7 +19,7 @@ const ( var ( // relaysTotal is a Counter metric for the total requests processed by the relay miner. - // It increments to track proxy requests and is labeled by 'proxy_name' and 'service_id', + // It increments to track proxy requests and is labeled by 'service_id', // essential for monitoring load and traffic on different proxies and services. // // Usage: @@ -29,11 +28,11 @@ var ( relaysTotal = prometheus.NewCounterFrom(stdprometheus.CounterOpts{ Subsystem: relayMinerProcess, Name: requestsTotal, - Help: "Total number of requests processed, labeled by proxy name and service ID.", + Help: "Total number of requests processed, labeled by service ID.", }, []string{"service_id"}) // relaysErrorsTotal is a Counter for total error events in the relay miner. - // It increments with each error, labeled by 'proxy_name' and 'service_id', + // It increments with each error, labeled by 'service_id', // crucial for pinpointing error-prone areas for reliability improvement. // // Usage: @@ -46,15 +45,15 @@ var ( }, []string{"service_id"}) // relaysSuccessTotal is a Counter metric for successful requests in the relay miner. - // It increments with each successful request, labeled by 'proxy_name' and 'service_id'. + // It increments with each successful request, labeled by ''service_id'. relaysSuccessTotal = prometheus.NewCounterFrom(stdprometheus.CounterOpts{ Subsystem: relayMinerProcess, Name: requestsSuccessTotal, - Help: "Total number of successful requests processed, labeled by proxy name and service ID.", + Help: "Total number of successful requests processed, labeled by service ID.", }, []string{"service_id"}) // relaysDurationSeconds observes request durations in the relay miner. - // This histogram, labeled by 'proxy_name' and 'service_id', measures response times, + // This histogram, labeled by 'service_id', measures response times, // vital for performance analysis under different loads. // // Buckets: @@ -70,7 +69,7 @@ var ( Buckets: []float64{0.1, 0.5, 1, 2, 5, 15}, }, []string{"service_id"}) - // relayResponseSizeBytes is a histogram metric for observing proxy response size distribution. + // relayResponseSizeBytes is a histogram metric for observing response size distribution. // It counts responses in bytes, with buckets: // - 100 bytes to 50,000 bytes, capturing a range from small to large responses. // This data helps in accurately representing response size distribution and is vital @@ -85,7 +84,7 @@ var ( Buckets: []float64{100, 500, 1000, 5000, 10000, 50000}, }, []string{"service_id"}) - // relayRequestSizeBytes is a histogram metric for observing proxy request size distribution. + // relayRequestSizeBytes is a histogram metric for observing request size distribution. // It counts requests in bytes, with buckets: // - 100 bytes to 50,000 bytes, capturing a range from small to large requests. // This data helps in accurately representing request size distribution and is vital diff --git a/pkg/relayer/proxy/options.go b/pkg/relayer/proxy/options.go index ab9344c95..84f77e291 100644 --- a/pkg/relayer/proxy/options.go +++ b/pkg/relayer/proxy/options.go @@ -13,9 +13,12 @@ func WithSigningKeyName(keyName string) relayer.RelayerProxyOption { } } -// WithProxiedServicesEndpoints sets the endpoints of the proxied services. -func WithProxiedServicesEndpoints(proxyConfig map[string]*config.RelayMinerProxyConfig) relayer.RelayerProxyOption { +// WithServicesConfigMap updates the configurations of all the services +// the RelayMiner proxies requests to. +// servicesConfigMap is a map of server endpoints to their respective +// parsed configurations. +func WithServicesConfigMap(servicesConfigMap map[string]*config.RelayMinerServerConfig) relayer.RelayerProxyOption { return func(relProxy relayer.RelayerProxy) { - relProxy.(*relayerProxy).proxyConfigs = proxyConfig + relProxy.(*relayerProxy).serverConfigs = servicesConfigMap } } diff --git a/pkg/relayer/proxy/proxy.go b/pkg/relayer/proxy/proxy.go index ec9f5a1c4..8e8f8b0fa 100644 --- a/pkg/relayer/proxy/proxy.go +++ b/pkg/relayer/proxy/proxy.go @@ -18,11 +18,10 @@ import ( var _ relayer.RelayerProxy = (*relayerProxy)(nil) -// relayerProxy is the main relayer proxy that takes relay requests of supported services from the client -// and proxies them to the supported proxied services. -// It is responsible for notifying the miner about the relays that have been served so they can be counted -// when the miner enters the claim/proof phase. -// TODO_TEST: Have tests for the relayer proxy. +// relayerProxy is the main relayer proxy that takes relay requests of supported +// services from the client and proxies them to the supported backend services. +// It is responsible for notifying the miner about the relays that have been +// served so they can be counted when the miner enters the claim/proof phase. type relayerProxy struct { logger polylog.Logger @@ -43,15 +42,15 @@ type relayerProxy struct { // which is needed to check if the relay proxy should be serving an incoming relay request. sessionQuerier client.SessionQueryClient - // proxyServers is a map of proxyName -> RelayServer provided by the relayer proxy, - // where proxyName is the name of the proxy defined in the config file and + // servers is a map of listenAddress -> RelayServer provided by the relayer proxy, + // where listenAddress is the address of the server defined in the config file and // RelayServer is the server that listens for incoming relay requests. - proxyServers map[string]relayer.RelayServer + servers map[string]relayer.RelayServer - // proxyConfigs is a map of proxyName -> RelayMinerProxyConfig where proxyName - // is the name of the proxy defined in the config file and RelayMinerProxyConfig - // is the configuration of the proxy. - proxyConfigs map[string]*config.RelayMinerProxyConfig + // serverConfigs is a map of listenAddress -> RelayMinerServerConfig where listenAddress + // is the address of the server defined in the config file and RelayMinerServerConfig + // is its configuration. + serverConfigs map[string]*config.RelayMinerServerConfig // servedRelays is an observable that notifies the miner about the relays that have been served. servedRelays relayer.RelaysObservable @@ -76,7 +75,7 @@ type relayerProxy struct { // // Available options: // - WithSigningKeyName -// - WithProxiedServicesEndpoints +// - WithServicesConfigMap func NewRelayerProxy( deps depinject.Config, opts ...relayer.RelayerProxyOption, @@ -113,7 +112,7 @@ func NewRelayerProxy( // Start concurrently starts all advertised relay services and returns an error // if any of them errors. -// This method IS BLOCKING until all RelayServers are stopped. +// NB: This method IS BLOCKING until all RelayServers are stopped. func (rp *relayerProxy) Start(ctx context.Context) error { // The provided services map is built from the supplier's on-chain advertised information, // which is a runtime parameter that can be changed by the supplier. @@ -128,7 +127,7 @@ func (rp *relayerProxy) Start(ctx context.Context) error { startGroup, ctx := errgroup.WithContext(ctx) - for _, relayServer := range rp.proxyServers { + for _, relayServer := range rp.servers { server := relayServer // create a new variable scoped to the anonymous function startGroup.Go(func() error { return server.Start(ctx) }) } @@ -141,7 +140,7 @@ func (rp *relayerProxy) Start(ctx context.Context) error { func (rp *relayerProxy) Stop(ctx context.Context) error { stopGroup, ctx := errgroup.WithContext(ctx) - for _, relayServer := range rp.proxyServers { + for _, relayServer := range rp.servers { // Create a new object (i.e. deep copy) variable scoped to the anonymous function below server := relayServer stopGroup.Go(func() error { return server.Stop(ctx) }) @@ -164,8 +163,8 @@ func (rp *relayerProxy) validateConfig() error { return ErrRelayerProxyUndefinedSigningKeyName } - if rp.proxyConfigs == nil || len(rp.proxyConfigs) == 0 { - return ErrRelayerProxyUndefinedProxiedServicesEndpoints + if rp.serverConfigs == nil || len(rp.serverConfigs) == 0 { + return ErrRelayerServicesConfigsUndefined } return nil diff --git a/pkg/relayer/proxy/proxy_test.go b/pkg/relayer/proxy/proxy_test.go index b223f726e..2ed0f7415 100644 --- a/pkg/relayer/proxy/proxy_test.go +++ b/pkg/relayer/proxy/proxy_test.go @@ -22,12 +22,12 @@ import ( ) const ( - blockHeight = 1 - defaultService = "service1" - secondaryService = "service2" - thirdService = "service3" - defaultProxyServer = "server1" - secondaryProxyServer = "server2" + blockHeight = 1 + defaultService = "service1" + secondaryService = "service2" + thirdService = "service3" + defaultRelayMinerServer = "127.0.0.1:8080" + secondaryRelayMinerServer = "127.0.0.1:8081" ) var ( @@ -46,10 +46,11 @@ var ( // application. appPrivateKey *secp256k1.PrivKey - // proxiedServices is the parsed configuration of the RelayMinerProxyConfig - proxiedServices map[string]*config.RelayMinerProxyConfig + // servicesConfigMap is a map from the service endpoint to its respective + // respective parsed RelayMiner configuration. + servicesConfigMap map[string]*config.RelayMinerServerConfig - // defaultRelayerProxyBehavior is the list of functions that are used to + // defaultRelayerServerBehavior is the list of functions that are used to // define the behavior of the RelayerProxy in the tests. defaultRelayerProxyBehavior []func(*testproxy.TestBehavior) ) @@ -68,53 +69,51 @@ func init() { }, secondaryService: { { - Url: "http://supplier:8546/", + Url: "http://supplier1:8546/", RpcType: sharedtypes.RPCType_GRPC, }, }, thirdService: { { - Url: "http://supplier:8547/", + Url: "http://supplier2:8547/", RpcType: sharedtypes.RPCType_GRPC, }, }, } - proxiedServices = map[string]*config.RelayMinerProxyConfig{ - defaultProxyServer: { - ProxyName: defaultProxyServer, - Type: config.ProxyTypeHTTP, - Host: "127.0.0.1:8080", - Suppliers: map[string]*config.RelayMinerSupplierConfig{ + servicesConfigMap = map[string]*config.RelayMinerServerConfig{ + defaultRelayMinerServer: { + ServerType: config.RelayMinerServerTypeHTTP, + ListenAddress: defaultRelayMinerServer, + SupplierConfigsMap: map[string]*config.RelayMinerSupplierConfig{ defaultService: { - ServiceId: defaultService, - Type: config.ProxyTypeHTTP, - Hosts: []string{"supplier:8545"}, + ServiceId: defaultService, + ServerType: config.RelayMinerServerTypeHTTP, + PubliclyExposedEndpoints: []string{"supplier"}, ServiceConfig: &config.RelayMinerSupplierServiceConfig{ - Url: &url.URL{Scheme: "http", Host: "127.0.0.1:8545", Path: "/"}, + BackendUrl: &url.URL{Scheme: "http", Host: "127.0.0.1:8545", Path: "/"}, }, }, secondaryService: { - ServiceId: secondaryService, - Type: config.ProxyTypeHTTP, - Hosts: []string{"supplier:8546"}, + ServiceId: secondaryService, + ServerType: config.RelayMinerServerTypeHTTP, + PubliclyExposedEndpoints: []string{"supplier1"}, ServiceConfig: &config.RelayMinerSupplierServiceConfig{ - Url: &url.URL{Scheme: "http", Host: "127.0.0.1:8546", Path: "/"}, + BackendUrl: &url.URL{Scheme: "http", Host: "127.0.0.1:8546", Path: "/"}, }, }, }, }, - secondaryProxyServer: { - ProxyName: secondaryProxyServer, - Type: config.ProxyTypeHTTP, - Host: "127.0.0.1:8081", - Suppliers: map[string]*config.RelayMinerSupplierConfig{ + secondaryRelayMinerServer: { + ServerType: config.RelayMinerServerTypeHTTP, + ListenAddress: secondaryRelayMinerServer, + SupplierConfigsMap: map[string]*config.RelayMinerSupplierConfig{ thirdService: { - ServiceId: thirdService, - Type: config.ProxyTypeHTTP, - Hosts: []string{"supplier:8547"}, + ServiceId: thirdService, + ServerType: config.RelayMinerServerTypeHTTP, + PubliclyExposedEndpoints: []string{"supplier2"}, ServiceConfig: &config.RelayMinerSupplierServiceConfig{ - Url: &url.URL{Scheme: "http", Host: "127.0.0.1:8547", Path: "/"}, + BackendUrl: &url.URL{Scheme: "http", Host: "127.0.0.1:8547", Path: "/"}, }, }, }, @@ -123,7 +122,7 @@ func init() { defaultRelayerProxyBehavior = []func(*testproxy.TestBehavior){ testproxy.WithRelayerProxyDependenciesForBlockHeight(supplierKeyName, blockHeight), - testproxy.WithRelayerProxiedServices(proxiedServices), + testproxy.WithServicesConfigMap(servicesConfigMap), testproxy.WithDefaultSupplier(supplierKeyName, supplierEndpoints), testproxy.WithDefaultApplication(appPrivateKey), testproxy.WithDefaultSessionSupplier(supplierKeyName, defaultService, appPrivateKey), @@ -140,7 +139,7 @@ func TestRelayerProxy_StartAndStop(t *testing.T) { rp, err := proxy.NewRelayerProxy( test.Deps, proxy.WithSigningKeyName(supplierKeyName), - proxy.WithProxiedServicesEndpoints(proxiedServices), + proxy.WithServicesConfigMap(servicesConfigMap), ) require.NoError(t, err) @@ -150,12 +149,12 @@ func TestRelayerProxy_StartAndStop(t *testing.T) { time.Sleep(100 * time.Millisecond) // Test that RelayerProxy is handling requests (ignoring the actual response content) - res, err := http.DefaultClient.Get(fmt.Sprintf("http://%s/", proxiedServices[defaultProxyServer].Host)) + res, err := http.DefaultClient.Get(fmt.Sprintf("http://%s/", servicesConfigMap[defaultRelayMinerServer].ListenAddress)) require.NoError(t, err) require.NotNil(t, res) // Test that RelayerProxy is handling requests from the other server - res, err = http.DefaultClient.Get(fmt.Sprintf("http://%s/", proxiedServices[secondaryProxyServer].Host)) + res, err = http.DefaultClient.Get(fmt.Sprintf("http://%s/", servicesConfigMap[secondaryRelayMinerServer].ListenAddress)) require.NoError(t, err) require.NotNil(t, res) @@ -172,7 +171,7 @@ func TestRelayerProxy_InvalidSupplierKeyName(t *testing.T) { rp, err := proxy.NewRelayerProxy( test.Deps, proxy.WithSigningKeyName("wrongKeyName"), - proxy.WithProxiedServicesEndpoints(proxiedServices), + proxy.WithServicesConfigMap(servicesConfigMap), ) require.NoError(t, err) @@ -188,13 +187,13 @@ func TestRelayerProxy_MissingSupplierKeyName(t *testing.T) { _, err := proxy.NewRelayerProxy( test.Deps, proxy.WithSigningKeyName(""), - proxy.WithProxiedServicesEndpoints(proxiedServices), + proxy.WithServicesConfigMap(servicesConfigMap), ) require.Error(t, err) } -// RelayerProxy should fail to build if the proxied services endpoints are not provided -func TestRelayerProxy_NoProxiedServices(t *testing.T) { +// RelayerProxy should fail to build if the service configs are not provided +func TestRelayerProxy_EmptyServicesConfigMap(t *testing.T) { ctx := context.TODO() test := testproxy.NewRelayerProxyTestBehavior(ctx, t, defaultRelayerProxyBehavior...) @@ -202,7 +201,7 @@ func TestRelayerProxy_NoProxiedServices(t *testing.T) { _, err := proxy.NewRelayerProxy( test.Deps, proxy.WithSigningKeyName(supplierKeyName), - proxy.WithProxiedServicesEndpoints(make(map[string]*config.RelayMinerProxyConfig)), + proxy.WithServicesConfigMap(make(map[string]*config.RelayMinerServerConfig)), ) require.Error(t, err) } @@ -224,7 +223,7 @@ func TestRelayerProxy_UnsupportedRpcType(t *testing.T) { unsupportedRPCTypeBehavior := []func(*testproxy.TestBehavior){ testproxy.WithRelayerProxyDependenciesForBlockHeight(supplierKeyName, blockHeight), - testproxy.WithRelayerProxiedServices(proxiedServices), + testproxy.WithServicesConfigMap(servicesConfigMap), // The supplier is staked on-chain but the service it provides is not supported by the proxy testproxy.WithDefaultSupplier(supplierKeyName, unsupportedSupplierEndpoint), @@ -237,7 +236,7 @@ func TestRelayerProxy_UnsupportedRpcType(t *testing.T) { rp, err := proxy.NewRelayerProxy( test.Deps, proxy.WithSigningKeyName(supplierKeyName), - proxy.WithProxiedServicesEndpoints(proxiedServices), + proxy.WithServicesConfigMap(servicesConfigMap), ) require.NoError(t, err) @@ -257,20 +256,19 @@ func TestRelayerProxy_UnsupportedTransportType(t *testing.T) { }, } - unsupportedTransportProxy := map[string]*config.RelayMinerProxyConfig{ - defaultProxyServer: { - ProxyName: defaultProxyServer, + unsupportedTransportProxy := map[string]*config.RelayMinerServerConfig{ + defaultRelayMinerServer: { // The proxy is configured with an unsupported transport type - Type: config.ProxyType(100), - Host: "127.0.0.1:8080", - Suppliers: map[string]*config.RelayMinerSupplierConfig{ + ServerType: config.RelayMinerServerType(100), + ListenAddress: defaultRelayMinerServer, + SupplierConfigsMap: map[string]*config.RelayMinerSupplierConfig{ defaultService: { ServiceId: defaultService, // The proxy is configured with an unsupported transport type - Type: config.ProxyType(100), - Hosts: []string{"supplier:8545"}, + ServerType: config.RelayMinerServerType(100), + PubliclyExposedEndpoints: []string{"supplier"}, ServiceConfig: &config.RelayMinerSupplierServiceConfig{ - Url: &url.URL{Scheme: "http", Host: "127.0.0.1:8545", Path: "/"}, + BackendUrl: &url.URL{Scheme: "http", Host: "127.0.0.1:8545", Path: "/"}, }, }, }, @@ -281,7 +279,7 @@ func TestRelayerProxy_UnsupportedTransportType(t *testing.T) { testproxy.WithRelayerProxyDependenciesForBlockHeight(supplierKeyName, blockHeight), // The proxy is configured with an unsupported transport type for the proxy - testproxy.WithRelayerProxiedServices(unsupportedTransportProxy), + testproxy.WithServicesConfigMap(unsupportedTransportProxy), testproxy.WithDefaultSupplier(supplierKeyName, badTransportSupplierEndpoints), testproxy.WithDefaultApplication(appPrivateKey), testproxy.WithDefaultSessionSupplier(supplierKeyName, defaultService, appPrivateKey), @@ -292,7 +290,7 @@ func TestRelayerProxy_UnsupportedTransportType(t *testing.T) { rp, err := proxy.NewRelayerProxy( test.Deps, proxy.WithSigningKeyName(supplierKeyName), - proxy.WithProxiedServicesEndpoints(unsupportedTransportProxy), + proxy.WithServicesConfigMap(unsupportedTransportProxy), ) require.NoError(t, err) @@ -303,18 +301,17 @@ func TestRelayerProxy_UnsupportedTransportType(t *testing.T) { func TestRelayerProxy_NonConfiguredSupplierServices(t *testing.T) { ctx := context.TODO() - missingServicesProxy := map[string]*config.RelayMinerProxyConfig{ - defaultProxyServer: { - ProxyName: defaultProxyServer, - Type: config.ProxyTypeHTTP, - Host: "127.0.0.1:8080", - Suppliers: map[string]*config.RelayMinerSupplierConfig{ + missingServicesProxy := map[string]*config.RelayMinerServerConfig{ + defaultRelayMinerServer: { + ServerType: config.RelayMinerServerTypeHTTP, + ListenAddress: defaultRelayMinerServer, + SupplierConfigsMap: map[string]*config.RelayMinerSupplierConfig{ defaultService: { - ServiceId: defaultService, - Type: config.ProxyTypeHTTP, - Hosts: []string{"supplier:8545"}, + ServiceId: defaultService, + ServerType: config.RelayMinerServerTypeHTTP, + PubliclyExposedEndpoints: []string{"supplier"}, ServiceConfig: &config.RelayMinerSupplierServiceConfig{ - Url: &url.URL{Scheme: "http", Host: "127.0.0.1:8545", Path: "/"}, + BackendUrl: &url.URL{Scheme: "http", Host: "127.0.0.1:8545", Path: "/"}, }, }, }, @@ -325,7 +322,7 @@ func TestRelayerProxy_NonConfiguredSupplierServices(t *testing.T) { testproxy.WithRelayerProxyDependenciesForBlockHeight(supplierKeyName, blockHeight), // The proxy is configured with an unsupported transport type for the proxy - testproxy.WithRelayerProxiedServices(missingServicesProxy), + testproxy.WithServicesConfigMap(missingServicesProxy), testproxy.WithDefaultSupplier(supplierKeyName, supplierEndpoints), testproxy.WithDefaultApplication(appPrivateKey), testproxy.WithDefaultSessionSupplier(supplierKeyName, defaultService, appPrivateKey), @@ -336,7 +333,7 @@ func TestRelayerProxy_NonConfiguredSupplierServices(t *testing.T) { rp, err := proxy.NewRelayerProxy( test.Deps, proxy.WithSigningKeyName(supplierKeyName), - proxy.WithProxiedServicesEndpoints(missingServicesProxy), + proxy.WithServicesConfigMap(missingServicesProxy), ) require.NoError(t, err) @@ -391,7 +388,7 @@ func TestRelayerProxy_Relays(t *testing.T) { inputScenario: sendRequestWithMissingSignature, expectedErrCode: -32000, - expectedErrMsg: "missing signature from relay request", + expectedErrMsg: "missing application signature", }, { desc: "Invalid signature associated with relay request", @@ -443,7 +440,7 @@ func TestRelayerProxy_Relays(t *testing.T) { relayerProxyBehavior: []func(*testproxy.TestBehavior){ testproxy.WithRelayerProxyDependenciesForBlockHeight(supplierKeyName, blockHeight), - testproxy.WithRelayerProxiedServices(proxiedServices), + testproxy.WithServicesConfigMap(servicesConfigMap), testproxy.WithDefaultSupplier(supplierKeyName, supplierEndpoints), testproxy.WithDefaultApplication(appPrivateKey), // Missing session supplier @@ -481,7 +478,7 @@ func TestRelayerProxy_Relays(t *testing.T) { supplierKeyName, blockWithinSessionGracePeriod, ), - testproxy.WithRelayerProxiedServices(proxiedServices), + testproxy.WithServicesConfigMap(servicesConfigMap), testproxy.WithDefaultSupplier(supplierKeyName, supplierEndpoints), testproxy.WithDefaultApplication(appPrivateKey), // Add 2 sessions, with the first one being within the withing grace period @@ -503,7 +500,7 @@ func TestRelayerProxy_Relays(t *testing.T) { // Set the current block height value returned by the block provider blockOutsideSessionGracePeriod, ), - testproxy.WithRelayerProxiedServices(proxiedServices), + testproxy.WithServicesConfigMap(servicesConfigMap), testproxy.WithDefaultSupplier(supplierKeyName, supplierEndpoints), testproxy.WithDefaultApplication(appPrivateKey), // Add 3 sessions, with the first one that is no longer within its @@ -527,7 +524,7 @@ func TestRelayerProxy_Relays(t *testing.T) { rp, err := proxy.NewRelayerProxy( testBehavior.Deps, proxy.WithSigningKeyName(supplierKeyName), - proxy.WithProxiedServicesEndpoints(proxiedServices), + proxy.WithServicesConfigMap(servicesConfigMap), ) require.NoError(t, err) @@ -552,7 +549,7 @@ func sendRequestWithUnparsableBody( reader := io.NopCloser(bytes.NewReader([]byte("invalid request"))) res, err := http.DefaultClient.Post( - fmt.Sprintf("http://%s", proxiedServices[defaultProxyServer].Host), + fmt.Sprintf("http://%s", servicesConfigMap[defaultRelayMinerServer].ListenAddress), "application/json", reader, ) @@ -574,7 +571,7 @@ func sendRequestWithMissingSignature( testproxy.PrepareJsonRPCRequestPayload(), ) req.Meta.Signature = nil - return testproxy.MarshalAndSend(test, proxiedServices, defaultProxyServer, defaultService, req) + return testproxy.MarshalAndSend(test, servicesConfigMap, defaultRelayMinerServer, defaultService, req) } func sendRequestWithInvalidSignature( @@ -590,7 +587,7 @@ func sendRequestWithInvalidSignature( ) req.Meta.Signature = []byte("invalid signature") - return testproxy.MarshalAndSend(test, proxiedServices, defaultProxyServer, defaultService, req) + return testproxy.MarshalAndSend(test, servicesConfigMap, defaultRelayMinerServer, defaultService, req) } func sendRequestWithMissingSessionHeaderApplicationAddress( @@ -613,7 +610,7 @@ func sendRequestWithMissingSessionHeaderApplicationAddress( // before looking at the application address req.Meta.Signature = testproxy.GetApplicationRingSignature(t, req, randomPrivKey) - return testproxy.MarshalAndSend(test, proxiedServices, defaultProxyServer, defaultService, req) + return testproxy.MarshalAndSend(test, servicesConfigMap, defaultRelayMinerServer, defaultService, req) } func sendRequestWithNonStakedApplicationAddress( @@ -632,7 +629,7 @@ func sendRequestWithNonStakedApplicationAddress( // Have a valid signature from the non staked key req.Meta.Signature = testproxy.GetApplicationRingSignature(t, req, randomPrivKey) - return testproxy.MarshalAndSend(test, proxiedServices, defaultProxyServer, defaultService, req) + return testproxy.MarshalAndSend(test, servicesConfigMap, defaultRelayMinerServer, defaultService, req) } func sendRequestWithRingSignatureMismatch( @@ -651,24 +648,25 @@ func sendRequestWithRingSignatureMismatch( randomPrivKey := secp256k1.GenPrivKey() req.Meta.Signature = testproxy.GetApplicationRingSignature(t, req, randomPrivKey) - return testproxy.MarshalAndSend(test, proxiedServices, defaultProxyServer, defaultService, req) + return testproxy.MarshalAndSend(test, servicesConfigMap, defaultRelayMinerServer, defaultService, req) } func sendRequestWithDifferentSession( t *testing.T, test *testproxy.TestBehavior, ) (errCode int32, errorMessage string) { - // Use secondaryService instead of service1 so the session IDs don't match + // Use a block height that generates a different session ID + blockHeightAfterSessionGracePeriod := blockHeight + sessionkeeper.GetSessionGracePeriodBlockCount() req := testproxy.GenerateRelayRequest( test, appPrivateKey, - secondaryService, - blockHeight, + defaultService, + blockHeightAfterSessionGracePeriod, testproxy.PrepareJsonRPCRequestPayload(), ) req.Meta.Signature = testproxy.GetApplicationRingSignature(t, req, appPrivateKey) - return testproxy.MarshalAndSend(test, proxiedServices, defaultProxyServer, defaultService, req) + return testproxy.MarshalAndSend(test, servicesConfigMap, defaultRelayMinerServer, defaultService, req) } func sendRequestWithInvalidRelaySupplier( @@ -684,7 +682,7 @@ func sendRequestWithInvalidRelaySupplier( ) req.Meta.Signature = testproxy.GetApplicationRingSignature(t, req, appPrivateKey) - return testproxy.MarshalAndSend(test, proxiedServices, defaultProxyServer, defaultService, req) + return testproxy.MarshalAndSend(test, servicesConfigMap, defaultRelayMinerServer, defaultService, req) } func sendRequestWithSignatureForDifferentPayload( @@ -702,7 +700,7 @@ func sendRequestWithSignatureForDifferentPayload( // Alter the request payload so the hash doesn't match the one used by the signature req.Payload = []byte(`{"method":"someMethod","id":1,"jsonrpc":"2.0","params":["alteredParam"]}`) - return testproxy.MarshalAndSend(test, proxiedServices, defaultProxyServer, defaultService, req) + return testproxy.MarshalAndSend(test, servicesConfigMap, defaultRelayMinerServer, defaultService, req) } func sendRequestWithSuccessfulReply( @@ -718,7 +716,7 @@ func sendRequestWithSuccessfulReply( ) req.Meta.Signature = testproxy.GetApplicationRingSignature(t, req, appPrivateKey) - return testproxy.MarshalAndSend(test, proxiedServices, defaultProxyServer, defaultService, req) + return testproxy.MarshalAndSend(test, servicesConfigMap, defaultRelayMinerServer, defaultService, req) } // sendRequestWithCustomSessionHeight is a helper function that generates a `RelayRequest` @@ -737,6 +735,6 @@ func sendRequestWithCustomSessionHeight( ) req.Meta.Signature = testproxy.GetApplicationRingSignature(t, req, appPrivateKey) - return testproxy.MarshalAndSend(test, proxiedServices, defaultProxyServer, defaultService, req) + return testproxy.MarshalAndSend(test, servicesConfigMap, defaultRelayMinerServer, defaultService, req) } } diff --git a/pkg/relayer/proxy/server_builder.go b/pkg/relayer/proxy/server_builder.go index ea3731f47..5146902a7 100644 --- a/pkg/relayer/proxy/server_builder.go +++ b/pkg/relayer/proxy/server_builder.go @@ -42,7 +42,7 @@ func (rp *relayerProxy) BuildProvidedServices(ctx context.Context) error { } // Check that the supplier's advertised services' endpoints are present in - // the proxy config and handled by a proxy host + // the server config and handled by a server // Iterate over the supplier's advertised services then iterate over each // service's endpoint for _, service := range supplier.Services { @@ -52,11 +52,11 @@ func (rp *relayerProxy) BuildProvidedServices(ctx context.Context) error { return err } found := false - // Iterate over the proxy configs and check if `endpointUrl` is present - // in any of the proxy config's suppliers' service's hosts - for _, proxyConfig := range rp.proxyConfigs { - supplierService, ok := proxyConfig.Suppliers[service.Service.Id] - if ok && slices.Contains(supplierService.Hosts, endpointUrl.Host) { + // Iterate over the server configs and check if `endpointUrl` is present + // in any of the server config's suppliers' service's PubliclyExposedEndpoints + for _, serverConfig := range rp.serverConfigs { + supplierService, ok := serverConfig.SupplierConfigsMap[service.Service.Id] + if ok && slices.Contains(supplierService.PubliclyExposedEndpoints, endpointUrl.Hostname()) { found = true break } @@ -64,7 +64,7 @@ func (rp *relayerProxy) BuildProvidedServices(ctx context.Context) error { if !found { return ErrRelayerProxyServiceEndpointNotHandled.Wrapf( - "service endpoint %s not handled by proxy", + "service endpoint %s not handled by the relay miner", endpoint.Url, ) } @@ -73,14 +73,14 @@ func (rp *relayerProxy) BuildProvidedServices(ctx context.Context) error { rp.supplierAddress = supplier.Address - if rp.proxyServers, err = rp.initializeProxyServers(supplier.Services); err != nil { + if rp.servers, err = rp.initializeProxyServers(supplier.Services); err != nil { return err } return nil } -// initializeProxyServers initializes the proxy servers for each proxy config. +// initializeProxyServers initializes the proxy servers for each server config. func (rp *relayerProxy) initializeProxyServers( supplierServices []*sharedtypes.SupplierServiceConfig, ) (proxyServerMap map[string]relayer.RelayServer, err error) { @@ -90,21 +90,21 @@ func (rp *relayerProxy) initializeProxyServers( supplierServiceMap[service.Service.Id] = service.Service } - // Build a map of proxyName -> RelayServer for each proxy defined in the config file - proxyServers := make(map[string]relayer.RelayServer) + // Build a map of listenAddress -> RelayServer for each server defined in the config file + servers := make(map[string]relayer.RelayServer) - for _, proxyConfig := range rp.proxyConfigs { - rp.logger.Info().Str("proxy host", proxyConfig.Host).Msg("starting relay proxy server") + for _, serverConfig := range rp.serverConfigs { + rp.logger.Info().Str("server host", serverConfig.ListenAddress).Msg("starting relay proxy server") // TODO(@h5law): Implement a switch that handles all synchronous // RPC types in one server type and asynchronous RPC types in another // to create the appropriate RelayServer. - // Initialize the proxy server according to the proxy type defined in the config file - switch proxyConfig.Type { - case config.ProxyTypeHTTP: - proxyServers[proxyConfig.ProxyName] = NewSynchronousServer( + // Initialize the server according to the server type defined in the config file + switch serverConfig.ServerType { + case config.RelayMinerServerTypeHTTP: + servers[serverConfig.ListenAddress] = NewSynchronousServer( rp.logger, - proxyConfig, + serverConfig, supplierServiceMap, rp.servedRelaysPublishCh, rp, @@ -114,7 +114,7 @@ func (rp *relayerProxy) initializeProxyServers( } } - return proxyServers, nil + return servers, nil } // waitForSupplierToStake waits in a loop until it gets the on-chain supplier's diff --git a/pkg/relayer/proxy/synchronous.go b/pkg/relayer/proxy/synchronous.go index d7cdb3f2c..75d76679b 100644 --- a/pkg/relayer/proxy/synchronous.go +++ b/pkg/relayer/proxy/synchronous.go @@ -4,6 +4,7 @@ import ( "bytes" "compress/gzip" "context" + "fmt" "io" "net/http" "net/url" @@ -35,10 +36,10 @@ type synchronousRPCServer struct { // representing the supplier's advertised services. supplierServiceMap map[string]*sharedtypes.Service - // proxyConfig is the configuration of the proxy server. It contains the + // serverConfig is the configuration of the proxy server. It contains the // host address of the server, the service endpoint, and the advertised service. // endpoints it gets relay requests from. - proxyConfig *config.RelayMinerProxyConfig + serverConfig *config.RelayMinerServerConfig // server is the HTTP server that listens for incoming relay requests. server *http.Server @@ -57,7 +58,7 @@ type synchronousRPCServer struct { // and returns a RelayServer that listens to incoming RelayRequests. func NewSynchronousServer( logger polylog.Logger, - proxyConfig *config.RelayMinerProxyConfig, + serverConfig *config.RelayMinerServerConfig, supplierServiceMap map[string]*sharedtypes.Service, servedRelaysProducer chan<- *types.Relay, proxy relayer.RelayerProxy, @@ -65,10 +66,10 @@ func NewSynchronousServer( return &synchronousRPCServer{ logger: logger, supplierServiceMap: supplierServiceMap, - server: &http.Server{Addr: proxyConfig.Host}, + server: &http.Server{Addr: serverConfig.ListenAddress}, relayerProxy: proxy, servedRelaysProducer: servedRelaysProducer, - proxyConfig: proxyConfig, + serverConfig: serverConfig, } } @@ -101,55 +102,80 @@ func (sync *synchronousRPCServer) ServeHTTP(writer http.ResponseWriter, request ctx := request.Context() defer request.Body.Close() - var originHost string + sync.logger.Debug().Msg("serving synchronous relay request") + listenAddress := sync.serverConfig.ListenAddress + + // Extract the relay request from the request body. + sync.logger.Debug().Msg("extracting relay request from request body") + relayRequest, err := sync.newRelayRequest(request) + if err != nil { + sync.replyWithError(ctx, []byte{}, writer, listenAddress, "", err) + sync.logger.Warn().Err(err).Msg("failed serving relay request") + return + } + + if err := relayRequest.ValidateBasic(); err != nil { + sync.replyWithError(ctx, relayRequest.Payload, writer, listenAddress, "", err) + sync.logger.Warn().Err(err).Msg("failed validating relay response") + return + } + + supplierService := relayRequest.Meta.SessionHeader.Service + requestPayload := relayRequest.Payload + + originHost := request.Host // When the proxy is behind a reverse proxy, or is getting its requests from // a CDN or a load balancer, the host header may not contain the on-chain // advertized address needed to determine the service that the relay request is for. // These CDNs and reverse proxies usually set the X-Forwarded-Host header // to the original host. // RelayMiner operators that have such a setup can set the XForwardedHostLookup - // option to true in the proxy config to enable the proxy to look up the + // option to true in the server config to enable the proxy to look up the // original host from the X-Forwarded-Host header. - // Get the original host from X-Forwarded-Host header if specified in the proxy + // Get the original host from X-Forwarded-Host header if specified in the supplier // config and fall back to the Host header if it is not specified. - if sync.proxyConfig.XForwardedHostLookup { + if sync.serverConfig.XForwardedHostLookup { originHost = request.Header.Get("X-Forwarded-Host") } - if originHost == "" { - originHost = request.Host + // Extract the hostname from the request's Host header to match it with the + // publicly exposed endpoints of the supplier service which are hostnames + // (i.e. hosts without the port number). + // Add the http scheme to the originHost to parse it as a URL. + originHostUrl, err := url.Parse(fmt.Sprintf("http://%s", originHost)) + if err != nil { + sync.replyWithError(ctx, requestPayload, writer, listenAddress, supplierService.Id, err) + return } - var supplierService *sharedtypes.Service - var serviceUrl *url.URL + var serviceConfig *config.RelayMinerSupplierServiceConfig // Get the Service and serviceUrl corresponding to the originHost. // TODO_IMPROVE(red-0ne): Checking that the originHost is currently done by - // iterating over the proxy config's suppliers and checking if the originHost + // iterating over the server config's suppliers and checking if the originHost // is present in any of the supplier's service's hosts. We could improve this // by building a map at the server initialization level with originHost as the // key so that we can get the service and serviceUrl in O(1) time. - for _, supplierServiceConfig := range sync.proxyConfig.Suppliers { - for _, host := range supplierServiceConfig.Hosts { - if host == originHost { - supplierService = sync.supplierServiceMap[supplierServiceConfig.ServiceId] - serviceUrl = supplierServiceConfig.ServiceConfig.Url + for _, supplierServiceConfig := range sync.serverConfig.SupplierConfigsMap { + for _, host := range supplierServiceConfig.PubliclyExposedEndpoints { + if host == originHostUrl.Hostname() && supplierService.Id == supplierServiceConfig.ServiceId { + serviceConfig = supplierServiceConfig.ServiceConfig break } } - if serviceUrl != nil { + if serviceConfig != nil { break } } - if supplierService == nil || serviceUrl == nil { + if serviceConfig == nil { sync.replyWithError( ctx, - []byte{}, + requestPayload, writer, - sync.proxyConfig.ProxyName, - "unknown", + listenAddress, + supplierService.Id, ErrRelayerProxyServiceEndpointNotHandled, ) return @@ -164,32 +190,20 @@ func (sync *synchronousRPCServer) ServeHTTP(writer http.ResponseWriter, request relaysDurationSeconds.With("service_id", supplierService.Id).Observe(duration) }() - sync.logger.Debug().Msg("serving synchronous relay request") - - // Extract the relay request from the request body. - sync.logger.Debug().Msg("extracting relay request from request body") - relayRequest, err := sync.newRelayRequest(request) - if err != nil { - sync.replyWithError(ctx, []byte{}, writer, sync.proxyConfig.ProxyName, supplierService.Id, err) - sync.logger.Warn().Err(err).Msg("failed serving relay request") - return - } - relayRequestSizeBytes.With("service_id", supplierService.Id). Observe(float64(relayRequest.Size())) - // Relay the request to the proxied service and build the response that will be sent back to the client. - relay, err := sync.serveHTTP(ctx, serviceUrl, supplierService, request, relayRequest) + relay, err := sync.serveHTTP(ctx, serviceConfig, supplierService, request, relayRequest) if err != nil { // Reply with an error if the relay could not be served. - sync.replyWithError(ctx, relayRequest.Payload, writer, sync.proxyConfig.ProxyName, supplierService.Id, err) + sync.replyWithError(ctx, requestPayload, writer, listenAddress, supplierService.Id, err) sync.logger.Warn().Err(err).Msg("failed serving relay request") return } // Send the relay response to the client. if err := sync.sendRelayResponse(relay.Res, writer); err != nil { - sync.replyWithError(ctx, relayRequest.Payload, writer, sync.proxyConfig.ProxyName, supplierService.Id, err) + sync.replyWithError(ctx, requestPayload, writer, listenAddress, supplierService.Id, err) sync.logger.Warn().Err(err).Msg("failed sending relay response") return } @@ -213,7 +227,7 @@ func (sync *synchronousRPCServer) ServeHTTP(writer http.ResponseWriter, request // serveHTTP holds the underlying logic of ServeHTTP. func (sync *synchronousRPCServer) serveHTTP( ctx context.Context, - serviceUrl *url.URL, + serviceConfig *config.RelayMinerSupplierServiceConfig, supplierService *sharedtypes.Service, request *http.Request, relayRequest *types.RelayRequest, @@ -240,17 +254,29 @@ func (sync *synchronousRPCServer) serveHTTP( // Build the request to be sent to the native service by substituting // the destination URL's host with the native service's listen address. sync.logger.Debug(). - Str("destination_url", serviceUrl.String()). + Str("destination_url", serviceConfig.BackendUrl.String()). Msg("building relay request payload to service") relayHTTPRequest := &http.Request{ Method: request.Method, Header: request.Header, - URL: serviceUrl, - Host: serviceUrl.Host, + URL: serviceConfig.BackendUrl, + Host: serviceConfig.BackendUrl.Host, Body: requestBodyReader, } + if serviceConfig.Authentication != nil { + relayHTTPRequest.SetBasicAuth( + serviceConfig.Authentication.Username, + serviceConfig.Authentication.Password, + ) + } + + // Add the headers to the request. + for key, value := range serviceConfig.Headers { + relayHTTPRequest.Header.Add(key, value) + } + // Send the relay request to the native service. httpResponse, err := http.DefaultClient.Do(relayHTTPRequest) if err != nil { diff --git a/pkg/relayer/session/proof.go b/pkg/relayer/session/proof.go index 5a237af40..228b50ebc 100644 --- a/pkg/relayer/session/proof.go +++ b/pkg/relayer/session/proof.go @@ -2,6 +2,7 @@ package session import ( "context" + "fmt" "github.com/pokt-network/poktroll/pkg/either" "github.com/pokt-network/poktroll/pkg/observable" @@ -101,6 +102,10 @@ func (rs *relayerSessionsManager) newMapProveSessionFn( latestBlock := rs.blockClient.LastNBlocks(ctx, 1)[0] // TODO_BLOCKER(@red-0ne, @Olshansk): Update the path given to `ProveClosest` // from `BlockHash` to `Foo(BlockHash, SessionId)` + + // TODO: Investigate "proof for the path provided does not match one expected by the on-chain protocol" + // error that may occur due to latestBlock height differing. + fmt.Println("E2E_DEBUG: height for block hash when generating the path", latestBlock.Height(), session.GetSessionHeader().GetSessionId()) path := proofkeeper.GetPathForProof(latestBlock.Hash(), session.GetSessionHeader().GetSessionId()) proof, err := session.ProveClosest(path) if err != nil { diff --git a/testutil/testproxy/relayerproxy.go b/testutil/testproxy/relayerproxy.go index 64d21ee38..0eadee2d7 100644 --- a/testutil/testproxy/relayerproxy.go +++ b/testutil/testproxy/relayerproxy.go @@ -44,11 +44,14 @@ import ( type TestBehavior struct { ctx context.Context t *testing.T + // Deps is exported so it can be used by the dependency injection framework // from the pkg/relayer/proxy/proxy_test.go Deps depinject.Config - proxiedServices map[string]*http.Server + // proxyServersMap is a map from ServiceId to the actual Server that handles + // processing of incoming RPC requests. + proxyServersMap map[string]*http.Server } // blockHeight is the default block height used in the tests. @@ -75,7 +78,7 @@ func NewRelayerProxyTestBehavior( test := &TestBehavior{ ctx: ctx, t: t, - proxiedServices: make(map[string]*http.Server), + proxyServersMap: make(map[string]*http.Server), } for _, behavior := range behaviors { @@ -122,17 +125,17 @@ func WithRelayerProxyDependenciesForBlockHeight( } } -// WithRelayerProxiedServices creates the services that the relayer proxy will +// WithServicesConfigMap creates the services that the relayer proxy will // proxy requests to. // It creates an HTTP server for each service and starts listening on the // provided host. -func WithRelayerProxiedServices( - proxiedServices map[string]*config.RelayMinerProxyConfig, +func WithServicesConfigMap( + servicesConfigMap map[string]*config.RelayMinerServerConfig, ) func(*TestBehavior) { return func(test *TestBehavior) { - for _, proxy := range proxiedServices { - for serviceId, service := range proxy.Suppliers { - server := &http.Server{Addr: service.ServiceConfig.Url.Host} + for _, serviceConfig := range servicesConfigMap { + for serviceId, supplierConfig := range serviceConfig.SupplierConfigsMap { + server := &http.Server{Addr: supplierConfig.ServiceConfig.BackendUrl.Host} server.Handler = http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Write(prepareJsonRPCResponsePayload()) }) @@ -142,7 +145,7 @@ func WithRelayerProxiedServices( server.Shutdown(test.ctx) }() - test.proxiedServices[serviceId] = server + test.proxyServersMap[serviceId] = server } } } @@ -282,30 +285,37 @@ func WithSuccessiveSessions( // MarshalAndSend marshals the request and sends it to the provided service. func MarshalAndSend( test *TestBehavior, - proxiedServices map[string]*config.RelayMinerProxyConfig, - proxyServeName string, + servicesConfigMap map[string]*config.RelayMinerServerConfig, + serviceEndpoint string, serviceId string, request *servicetypes.RelayRequest, ) (errCode int32, errorMessage string) { - reqBz, err := request.Marshal() - require.NoError(test.t, err) - var scheme string - switch proxiedServices[proxyServeName].Type { - case config.ProxyTypeHTTP: + switch servicesConfigMap[serviceEndpoint].ServerType { + case config.RelayMinerServerTypeHTTP: scheme = "http" default: - require.FailNow(test.t, "unsupported proxy type") + require.FailNow(test.t, "unsupported server type") } + // originHost is the endpoint that the client will retrieve from the on-chain supplier record. + // The supplier may have multiple endpoints (e.g. for load geo-balancing, host failover, etc.). + // In the current test setup, we only have one endpoint per supplier, which is why we are accessing `[0]`. + // In a real-world scenario, the publicly exposed endpoint would reach a load balancer + // or a reverse proxy that would route the request to the address specified by ListenAddress. + originHost := servicesConfigMap[serviceEndpoint].SupplierConfigsMap[serviceId].PubliclyExposedEndpoints[0] + + reqBz, err := request.Marshal() + require.NoError(test.t, err) reader := io.NopCloser(bytes.NewReader(reqBz)) req := &http.Request{ + Method: http.MethodPost, Header: http.Header{ "Content-Type": []string{"application/json"}, }, - URL: &url.URL{Scheme: scheme, Host: proxiedServices[proxyServeName].Host}, - Host: proxiedServices[proxyServeName].Suppliers[serviceId].Hosts[0], + URL: &url.URL{Scheme: scheme, Host: servicesConfigMap[serviceEndpoint].ListenAddress}, + Host: originHost, Body: reader, } res, err := http.DefaultClient.Do(req) diff --git a/x/proof/keeper/msg_server_submit_proof.go b/x/proof/keeper/msg_server_submit_proof.go index 57b4cd216..547f15a9a 100644 --- a/x/proof/keeper/msg_server_submit_proof.go +++ b/x/proof/keeper/msg_server_submit_proof.go @@ -444,6 +444,10 @@ func (k msgServer) validateClosestPath( sessionkeeper.GetSessionGracePeriodBlockCount() blockHash := k.sessionKeeper.GetBlockHash(ctx, sessionEndBlockHeightWithGracePeriod) + // TODO: Investigate "proof for the path provided does not match one expected by the on-chain protocol" + // error that may occur due to block height differing from the off-chain part. + fmt.Println("E2E_DEBUG: height for block hash when verifying the proof", sessionEndBlockHeightWithGracePeriod, sessionHeader.GetSessionId()) + expectedProofPath := GetPathForProof(blockHash, sessionHeader.GetSessionId()) if !bytes.Equal(proof.Path, expectedProofPath) { return types.ErrProofInvalidProof.Wrapf( diff --git a/x/supplier/config/supplier_configs_reader.go b/x/supplier/config/supplier_configs_reader.go index de9549553..535387397 100644 --- a/x/supplier/config/supplier_configs_reader.go +++ b/x/supplier/config/supplier_configs_reader.go @@ -10,32 +10,34 @@ import ( sharedtypes "github.com/pokt-network/poktroll/x/shared/types" ) -// YAMLStakeConfig is the structure describing the supplier stake config file +// YAMLStakeConfig is the structure describing the supplier stake config file. type YAMLStakeConfig struct { StakeAmount string `yaml:"stake_amount"` Services []*YAMLStakeService `yaml:"services"` } -// YAMLStakeService is the structure describing a single service stake entry in the stake config file +// YAMLStakeService is the structure describing a single service entry in the +// stake config file. type YAMLStakeService struct { ServiceId string `yaml:"service_id"` Endpoints []YAMLServiceEndpoint `yaml:"endpoints"` } -// YAMLServiceEndpoint is the structure describing a single service endpoint in the stake config file +// YAMLServiceEndpoint is the structure describing a single service endpoint in +// the service section of the stake config file. type YAMLServiceEndpoint struct { - Url string `yaml:"url"` - RPCType string `yaml:"rpc_type"` - Config map[string]string `yaml:"config,omitempty"` + PubliclyExposedUrl string `yaml:"publicly_exposed_url"` + RPCType string `yaml:"rpc_type"` + Config map[string]string `yaml:"config,omitempty"` } -// SupplierStakeConfig is the structure describing the parsed supplier stake config +// SupplierStakeConfig is the structure describing the parsed supplier stake config. type SupplierStakeConfig struct { StakeAmount sdk.Coin Services []*sharedtypes.SupplierServiceConfig } -// ParseSupplierServiceConfig parses the stake config file into a SupplierServiceConfig +// ParseSupplierServiceConfig parses the stake config file into a SupplierServiceConfig. func ParseSupplierConfigs(configContent []byte) (*SupplierStakeConfig, error) { var stakeConfig *YAMLStakeConfig @@ -137,18 +139,8 @@ func parseEndpointEntry(endpoint YAMLServiceEndpoint) (*sharedtypes.SupplierEndp return endpointEntry, nil } -// validateEndpointURL validates the endpoint URL, making sure that the string provided is a valid URL -func validateEndpointURL(endpoint YAMLServiceEndpoint) (string, error) { - // Validate the endpoint URL - if _, err := url.Parse(endpoint.Url); err != nil { - return "", ErrSupplierConfigInvalidURL.Wrapf("%s", err) - } - - return endpoint.Url, nil -} - -// parseEndpointConfigs parses the endpoint config entries into a slice of ConfigOption -// compatible with the SupplierEndpointConfig. +// parseEndpointConfigs parses the endpoint config entries into a slice of +// ConfigOption compatible with the SupplierEndpointConfig. // It accepts a nil config entry or a map of valid config keys. func parseEndpointConfigs(endpoint YAMLServiceEndpoint) ([]*sharedtypes.ConfigOption, error) { // Prepare the endpoint configs slice @@ -190,3 +182,13 @@ func parseEndpointRPCType(endpoint YAMLServiceEndpoint) (sharedtypes.RPCType, er return sharedtypes.RPCType_UNKNOWN_RPC, ErrSupplierConfigInvalidRPCType.Wrapf("%s", endpoint.RPCType) } } + +// validateEndpointURL validates the endpoint URL, making sure that the string provided is a valid URL +func validateEndpointURL(endpoint YAMLServiceEndpoint) (string, error) { + // Validate the endpoint URL + if _, err := url.Parse(endpoint.PubliclyExposedUrl); err != nil { + return "", ErrSupplierConfigInvalidURL.Wrapf("%s", err) + } + + return endpoint.PubliclyExposedUrl, nil +} diff --git a/x/supplier/config/supplier_configs_reader_test.go b/x/supplier/config/supplier_configs_reader_test.go index 28d567fbb..90ff18a03 100644 --- a/x/supplier/config/supplier_configs_reader_test.go +++ b/x/supplier/config/supplier_configs_reader_test.go @@ -4,70 +4,378 @@ import ( "testing" sdkerrors "cosmossdk.io/errors" - "cosmossdk.io/math" + math "cosmossdk.io/math" sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/gogo/status" "github.com/stretchr/testify/require" "github.com/pokt-network/poktroll/testutil/yaml" - "github.com/pokt-network/poktroll/x/gateway/module/config" + "github.com/pokt-network/poktroll/x/shared/types" + "github.com/pokt-network/poktroll/x/supplier/config" ) -func Test_ParseGatewayStakeConfig(t *testing.T) { +func Test_ParseSupplierConfigs_Services(t *testing.T) { tests := []struct { - desc string - expectedErr *sdkerrors.Error - expectedConfig *config.GatewayStakeConfig - inputConfig string + desc string + inputConfig string + + expectedError *sdkerrors.Error + expectedConfig *config.SupplierStakeConfig }{ // Valid Configs { - desc: "valid gateway stake config", + desc: "valid full service config", + inputConfig: ` + stake_amount: 1000upokt + services: + - service_id: svc + endpoints: + - publicly_exposed_url: http://pokt.network:8081 + rpc_type: json_rpc + config: + timeout: 10 + `, + expectedError: nil, + expectedConfig: &config.SupplierStakeConfig{ + StakeAmount: sdk.NewCoin("upokt", math.NewInt(1000)), + Services: []*types.SupplierServiceConfig{ + { + Service: &types.Service{Id: "svc"}, + Endpoints: []*types.SupplierEndpoint{ + { + Url: "http://pokt.network:8081", + RpcType: types.RPCType_JSON_RPC, + Configs: []*types.ConfigOption{ + { + Key: types.ConfigOptions_TIMEOUT, + Value: "10", + }, + }, + }, + }, + }, + }, + }, + }, + { + desc: "valid service config without endpoint specific config", inputConfig: ` stake_amount: 1000upokt + services: + - service_id: svc + endpoints: + - publicly_exposed_url: http://pokt.network:8081 + rpc_type: json_rpc `, - expectedErr: nil, - expectedConfig: &config.GatewayStakeConfig{ + expectedError: nil, + expectedConfig: &config.SupplierStakeConfig{ StakeAmount: sdk.NewCoin("upokt", math.NewInt(1000)), + Services: []*types.SupplierServiceConfig{ + { + Service: &types.Service{Id: "svc"}, + Endpoints: []*types.SupplierEndpoint{ + { + Url: "http://pokt.network:8081", + RpcType: types.RPCType_JSON_RPC, + }, + }, + }, + }, + }, + }, + { + desc: "valid service config with empty endpoint specific config", + inputConfig: ` + stake_amount: 1000upokt + services: + - service_id: svc + endpoints: + - publicly_exposed_url: http://pokt.network:8081 + rpc_type: json_rpc + config: + `, + expectedError: nil, + expectedConfig: &config.SupplierStakeConfig{ + StakeAmount: sdk.NewCoin("upokt", math.NewInt(1000)), + Services: []*types.SupplierServiceConfig{ + { + Service: &types.Service{Id: "svc"}, + Endpoints: []*types.SupplierEndpoint{ + { + Url: "http://pokt.network:8081", + RpcType: types.RPCType_JSON_RPC, + Configs: []*types.ConfigOption{}, + }, + }, + }, + }, + }, + }, + { + desc: "valid service config with multiple endpoints", + inputConfig: ` + stake_amount: 1000upokt + services: + - service_id: svc + endpoints: + - publicly_exposed_url: http://pokt.network:8081 + rpc_type: json_rpc + config: + timeout: 10 + - publicly_exposed_url: http://pokt.network:8082 + rpc_type: json_rpc + config: + timeout: 11 + `, + expectedError: nil, + expectedConfig: &config.SupplierStakeConfig{ + StakeAmount: sdk.NewCoin("upokt", math.NewInt(1000)), + Services: []*types.SupplierServiceConfig{ + { + Service: &types.Service{Id: "svc"}, + Endpoints: []*types.SupplierEndpoint{ + { + Url: "http://pokt.network:8081", + RpcType: types.RPCType_JSON_RPC, + Configs: []*types.ConfigOption{ + { + Key: types.ConfigOptions_TIMEOUT, + Value: "10", + }, + }, + }, + { + Url: "http://pokt.network:8082", + RpcType: types.RPCType_JSON_RPC, + Configs: []*types.ConfigOption{ + { + Key: types.ConfigOptions_TIMEOUT, + Value: "11", + }, + }, + }, + }, + }, + }, + }, + }, + { + desc: "valid service config with multiple services", + expectedError: nil, + inputConfig: ` + stake_amount: 1000upokt + services: + - service_id: svc1 + endpoints: + - publicly_exposed_url: http://pokt.network:8081 + rpc_type: json_rpc + config: + timeout: 10 + - service_id: svc2 + endpoints: + - publicly_exposed_url: http://pokt.network:8081 + rpc_type: json_rpc + config: + timeout: 10 + `, + expectedConfig: &config.SupplierStakeConfig{ + StakeAmount: sdk.NewCoin("upokt", math.NewInt(1000)), + Services: []*types.SupplierServiceConfig{ + { + Service: &types.Service{Id: "svc1"}, + Endpoints: []*types.SupplierEndpoint{ + { + Url: "http://pokt.network:8081", + RpcType: types.RPCType_JSON_RPC, + Configs: []*types.ConfigOption{ + { + Key: types.ConfigOptions_TIMEOUT, + Value: "10", + }, + }, + }, + }, + }, + { + Service: &types.Service{Id: "svc2"}, + Endpoints: []*types.SupplierEndpoint{ + { + Url: "http://pokt.network:8081", + RpcType: types.RPCType_JSON_RPC, + Configs: []*types.ConfigOption{ + { + Key: types.ConfigOptions_TIMEOUT, + Value: "10", + }, + }, + }, + }, + }, + }, }, }, // Invalid Configs { - desc: "services_test: invalid service config with empty content", - expectedErr: config.ErrGatewayConfigEmptyContent, - inputConfig: ``, + desc: "invalid service config without service ID", + inputConfig: ` + stake_amount: 1000upokt + services: + - endpoints: + - publicly_exposed_url: http://pokt.network:8081 + rpc_type: json_rpc + config: + timeout: 10 + `, + expectedError: config.ErrSupplierConfigInvalidServiceId, + }, + { + desc: "invalid service config with empty service ID", + inputConfig: ` + stake_amount: 1000upokt + services: + - service_id: + endpoints: + - publicly_exposed_url: http://pokt.network:8081 + rpc_type: json_rpc + config: + timeout: 10 + `, + expectedError: config.ErrSupplierConfigInvalidServiceId, + }, + { + desc: "invalid service config without endpoints", + inputConfig: ` + stake_amount: 1000upokt + services: + - service_id: svc + `, + expectedError: config.ErrSupplierConfigNoEndpoints, + }, + { + desc: "invalid service config with empty endpoints", + inputConfig: ` + stake_amount: 1000upokt + services: + - service_id: svc + endpoints: + `, + expectedError: config.ErrSupplierConfigNoEndpoints, + }, + { + desc: "invalid service config with unknown endpoint config key", + inputConfig: ` + stake_amount: 1000upokt + services: + - service_id: svc + endpoints: + - publicly_exposed_url: http://pokt.network:8081 + rpc_type: json_rpc + config: + somekey: 10 + `, + expectedError: config.ErrSupplierConfigInvalidEndpointConfig, + }, + { + desc: "invalid service config with unknown endpoint rpc type", + inputConfig: ` + stake_amount: 1000upokt + services: + - service_id: svc + endpoints: + - publicly_exposed_url: http://pokt.network:8081 + rpc_type: somerpc + config: + timeout: 10 + `, + expectedError: config.ErrSupplierConfigInvalidRPCType, + }, + { + desc: "invalid service config with invalid endpoint url", + inputConfig: ` + stake_amount: 1000upokt + services: + - service_id: svc + endpoints: + - publicly_exposed_url: ::invalid_url + rpc_type: json_rpc + config: + timeout: 10 + `, + expectedError: config.ErrSupplierConfigInvalidURL, + }, + { + desc: "invalid service config with empty content", + expectedError: config.ErrSupplierConfigEmptyContent, + inputConfig: ``, + }, + { + desc: "missing stake amount", + inputConfig: ` + services: + - service_id: svc + endpoints: + - publicly_exposed_url: http://pokt.network:8081 + rpc_type: json_rpc + config: + timeout: 10 + `, + expectedError: config.ErrSupplierConfigInvalidStake, }, { desc: "invalid stake denom", inputConfig: ` stake_amount: 1000invalid + services: + - service_id: svc + endpoints: + - publicly_exposed_url: http://pokt.network:8081 + rpc_type: json_rpc + config: + timeout: 10 `, - expectedErr: config.ErrGatewayConfigInvalidStake, + expectedError: config.ErrSupplierConfigInvalidStake, }, { desc: "negative stake amount", inputConfig: ` stake_amount: -1000upokt + services: + - service_id: svc + endpoints: + - publicly_exposed_url: http://pokt.network:8081 + rpc_type: json_rpc + config: + timeout: 10 `, - expectedErr: config.ErrGatewayConfigInvalidStake, + expectedError: config.ErrSupplierConfigInvalidStake, }, { desc: "zero stake amount", inputConfig: ` stake_amount: 0upokt + services: + - service_id: svc + endpoints: + - publicly_exposed_url: http://pokt.network:8081 + rpc_type: json_rpc + config: + timeout: 10 `, - expectedErr: config.ErrGatewayConfigInvalidStake, + expectedError: config.ErrSupplierConfigInvalidStake, }, } for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { normalizedConfig := yaml.NormalizeYAMLIndentation(tt.inputConfig) - supplierServiceConfig, err := config.ParseGatewayConfig([]byte(normalizedConfig)) + supplierServiceConfig, err := config.ParseSupplierConfigs([]byte(normalizedConfig)) - if tt.expectedErr != nil { + if tt.expectedError != nil { require.Error(t, err) - require.ErrorIs(t, err, tt.expectedErr) - require.Contains(t, err.Error(), tt.expectedErr.Error()) + require.ErrorIs(t, err, tt.expectedError) + stat, ok := status.FromError(tt.expectedError) + require.True(t, ok) + require.Contains(t, stat.Message(), tt.expectedError.Error()) require.Nil(t, supplierServiceConfig) return } @@ -76,6 +384,29 @@ func Test_ParseGatewayStakeConfig(t *testing.T) { require.Equal(t, tt.expectedConfig.StakeAmount, supplierServiceConfig.StakeAmount) require.Equal(t, tt.expectedConfig.StakeAmount.Denom, supplierServiceConfig.StakeAmount.Denom) + + require.Equal(t, len(tt.expectedConfig.Services), len(supplierServiceConfig.Services)) + for svcIdx, expectedService := range tt.expectedConfig.Services { + service := supplierServiceConfig.Services[svcIdx] + + require.Equal(t, expectedService.Service.Id, service.Service.Id) + + require.Equal(t, len(expectedService.Endpoints), len(service.Endpoints)) + for endpointIdx, expectedEndpoint := range expectedService.Endpoints { + endpoint := service.Endpoints[endpointIdx] + + require.Equal(t, expectedEndpoint.Url, endpoint.Url) + require.Equal(t, expectedEndpoint.RpcType, endpoint.RpcType) + + require.Equal(t, len(expectedEndpoint.Configs), len(endpoint.Configs)) + for configIdx, expectedConfig := range expectedEndpoint.Configs { + config := endpoint.Configs[configIdx] + + require.Equal(t, expectedConfig.Key, config.Key) + require.Equal(t, expectedConfig.Value, config.Value) + } + } + } }) } } diff --git a/x/supplier/module/tx_stake_supplier_test.go b/x/supplier/module/tx_stake_supplier_test.go index 44b0c5122..c19088a8b 100644 --- a/x/supplier/module/tx_stake_supplier_test.go +++ b/x/supplier/module/tx_stake_supplier_test.go @@ -47,7 +47,7 @@ func TestCLI_StakeSupplier(t *testing.T) { services: - service_id: svc1 endpoints: - - url: http://pokt.network:8081 + - publicly_exposed_url: http://pokt.network:8081 rpc_type: json_rpc ` @@ -88,7 +88,7 @@ func TestCLI_StakeSupplier(t *testing.T) { services: - service_id: svc1 endpoints: - - url: http://pokt.network:8081 + - publicly_exposed_url: http://pokt.network:8081 rpc_type: json_rpc `, }, @@ -101,7 +101,7 @@ func TestCLI_StakeSupplier(t *testing.T) { services: - service_id: svc1 endpoints: - - url: http://pokt.network:8081 + - publicly_exposed_url: http://pokt.network:8081 rpc_type: json_rpc `, }, @@ -114,7 +114,7 @@ func TestCLI_StakeSupplier(t *testing.T) { services: - service_id: svc1 endpoints: - - url: http://pokt.network:8081 + - publicly_exposed_url: http://pokt.network:8081 rpc_type: json_rpc `, }, @@ -127,7 +127,7 @@ func TestCLI_StakeSupplier(t *testing.T) { services: - service_id: svc1 endpoints: - - url: http://pokt.network:8081 + - publicly_exposed_url: http://pokt.network:8081 rpc_type: json_rpc `, }, @@ -141,11 +141,11 @@ func TestCLI_StakeSupplier(t *testing.T) { services: - service_id: svc1 endpoints: - - url: http://pokt.network:8081 + - publicly_exposed_url: http://pokt.network:8081 rpc_type: json_rpc - service_id: svc2 endpoints: - - url: http://pokt.network:8082 + - publicly_exposed_url: http://pokt.network:8082 rpc_type: json_rpc `, }, @@ -157,7 +157,7 @@ func TestCLI_StakeSupplier(t *testing.T) { services: - service_id: svc1 endpoints: - - url: http://127.0.0.1:8082 + - publicly_exposed_url: http://127.0.0.1:8082 rpc_type: json_rpc `, }, @@ -169,19 +169,19 @@ func TestCLI_StakeSupplier(t *testing.T) { services: - service_id: svc1 endpoints: - - url: http://localhost:8082 + - publicly_exposed_url: http://localhost:8082 rpc_type: json_rpc `, }, { - desc: "services_test: valid without a pork", + desc: "services_test: valid without a port", address: supplierAccount.Address.String(), config: ` stake_amount: 1000upokt services: - service_id: svc1 endpoints: - - url: http://pokt.network + - publicly_exposed_url: http://pokt.network rpc_type: json_rpc `, }, @@ -214,7 +214,7 @@ func TestCLI_StakeSupplier(t *testing.T) { services: - service_id: svc1 endpoints: - - url: bad_url + - publicly_exposed_url: bad_url rpc_type: json_rpc `, }, @@ -237,10 +237,10 @@ func TestCLI_StakeSupplier(t *testing.T) { stake_amount: 1000upokt services: - endpoints: - - url: localhost:8081 + - publicly_exposed_url: localhost:8081 rpc_type: json_rpc - endpoints: - - url: localhost:8082 + - publicly_exposed_url: localhost:8082 rpc_type: json_rpc `, }, @@ -253,7 +253,7 @@ func TestCLI_StakeSupplier(t *testing.T) { services: - service_id: svc1 endpoints: - - url: localhost:8082 + - publicly_exposed_url: localhost:8082 `, }, } From 6680149e1285eddf33d9d90a9d414f1dfd05ab12 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Wed, 24 Apr 2024 10:59:21 +0200 Subject: [PATCH 20/28] [Tooling] Add claim/proof/settlement dashboard & link (#479) --- Makefile | 2 +- .../grafana-dashboards/claim_proof_logs.json | 154 ++++++++++++++++++ localnet/grafana-dashboards/cometbft.json | 16 +- 3 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 localnet/grafana-dashboards/claim_proof_logs.json diff --git a/Makefile b/Makefile index 8a34004a1..18f07b58a 100644 --- a/Makefile +++ b/Makefile @@ -316,7 +316,7 @@ test_e2e_env: acc_initialize_pubkeys_warn_message ## Setup the default env vars .PHONY: test_e2e test_e2e: test_e2e_env ## Run all E2E tests - go test -v ./e2e/tests/... -tags=e2e,test + go test -count=1 -v ./e2e/tests/... -tags=e2e,test .PHONY: test_e2e_app test_e2e_app: diff --git a/localnet/grafana-dashboards/claim_proof_logs.json b/localnet/grafana-dashboards/claim_proof_logs.json new file mode 100644 index 000000000..031895035 --- /dev/null +++ b/localnet/grafana-dashboards/claim_proof_logs.json @@ -0,0 +1,154 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "builder", + "expr": "{container=\"poktrolld-validator\"} | json | method = `CreateClaim`", + "queryType": "range", + "refId": "Claim Creation" + } + ], + "title": "Claim Creation", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 1, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "builder", + "expr": "{container=\"poktrolld-validator\"} | json | method = `SubmitProof`", + "queryType": "range", + "refId": "Proof Submission" + } + ], + "title": "Proof Submission", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 3, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "builder", + "expr": "{container=\"poktrolld-validator\"} | json | method = `SettleSessionAccounting`", + "queryType": "range", + "refId": "Claim Settlement" + } + ], + "title": "Claim Settlement", + "type": "logs" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [ + "claim_proof_settlement_logs" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Protocol / Claim/Proof/Settlement Logs", + "uid": "claim_proof_settlement_logs", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/localnet/grafana-dashboards/cometbft.json b/localnet/grafana-dashboards/cometbft.json index d0d2aae77..f07128aca 100644 --- a/localnet/grafana-dashboards/cometbft.json +++ b/localnet/grafana-dashboards/cometbft.json @@ -34,6 +34,20 @@ "tooltip": "", "type": "dashboards", "url": "" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [ + "claim_proof_settlement_logs" + ], + "targetBlank": false, + "title": "Claim/Proof/Settlement Logs", + "tooltip": "", + "type": "dashboards", + "url": "" } ], "panels": [ @@ -1428,6 +1442,6 @@ "timezone": "", "title": "Protocol / CometBFT Dashboard", "uid": "cosmoscometbft", - "version": 4, + "version": 5, "weekStart": "" } \ No newline at end of file From d991a18741d92894e66e79019083992682314f76 Mon Sep 17 00:00:00 2001 From: Eric Nielson Date: Fri, 26 Apr 2024 10:23:55 -0400 Subject: [PATCH 21/28] lint fix --- pkg/client/block/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/client/block/client.go b/pkg/client/block/client.go index 9ffee411d..a0ef3323a 100644 --- a/pkg/client/block/client.go +++ b/pkg/client/block/client.go @@ -47,7 +47,7 @@ func NewBlockClient( latestBlockReplayObs, latestBlockPublishCh := channel.NewReplayObservable[client.Block](ctx, 10) bClient := &blockReplayClient{ latestBlockReplayObs: latestBlockReplayObs, - close: cancel, + close: cancel, } for _, opt := range opts { From 09f5fd78a74569ea46776067378315dcef1f64cc Mon Sep 17 00:00:00 2001 From: Eric Nielson Date: Fri, 26 Apr 2024 10:37:31 -0400 Subject: [PATCH 22/28] Type coercion update --- pkg/client/block/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/client/block/options.go b/pkg/client/block/options.go index b793306fa..83a52357b 100644 --- a/pkg/client/block/options.go +++ b/pkg/client/block/options.go @@ -8,6 +8,6 @@ import "github.com/pokt-network/poktroll/pkg/client" // If connRetryLimit is < 0, it will retry indefinitely. func WithConnRetryLimit(limit int) client.BlockClientOption { return func(client client.BlockClient) { - client.(*blockClient).connRetryLimit = limit + client.(*blockReplayClient).connRetryLimit = limit } } From 89076c0933669c68174731dacda5a4184a3adffe Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 26 Apr 2024 20:08:15 +0200 Subject: [PATCH 23/28] fix: tests --- pkg/client/block/client.go | 2 +- pkg/relayer/session/session.go | 7 ++++++- pkg/relayer/session/session_test.go | 22 +++++++++++++++++++++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/pkg/client/block/client.go b/pkg/client/block/client.go index a0ef3323a..0192b5ee8 100644 --- a/pkg/client/block/client.go +++ b/pkg/client/block/client.go @@ -67,7 +67,7 @@ func NewBlockClient( return nil, err } - if err := depinject.Inject(deps, bClient.onStartQueryClient); err != nil { + if err := depinject.Inject(deps, &bClient.onStartQueryClient); err != nil { return nil, err } diff --git a/pkg/relayer/session/session.go b/pkg/relayer/session/session.go index d356ee44b..8b030cc89 100644 --- a/pkg/relayer/session/session.go +++ b/pkg/relayer/session/session.go @@ -5,6 +5,7 @@ import ( "sync" "cosmossdk.io/depinject" + cosmosclient "github.com/cosmos/cosmos-sdk/client" "github.com/pokt-network/poktroll/pkg/client" "github.com/pokt-network/poktroll/pkg/observable" @@ -43,6 +44,9 @@ type relayerSessionsManager struct { // supplierClient is used to create claims and submit proofs for sessions. supplierClient client.SupplierClient + // TODO_IN_THIS_COMMIT: godoc comment. + blockQueryClient cosmosclient.CometRPC + // storesDirectory points to a path on disk where KVStore data files are created. storesDirectory string } @@ -59,7 +63,7 @@ func NewRelayerSessions( ctx context.Context, deps depinject.Config, opts ...relayer.RelayerSessionsManagerOption, -) (relayer.RelayerSessionsManager, error) { +) (_ relayer.RelayerSessionsManager, err error) { rs := &relayerSessionsManager{ logger: polylog.Ctx(ctx), sessionsTrees: make(sessionsTreesMap), @@ -69,6 +73,7 @@ func NewRelayerSessions( if err := depinject.Inject( deps, &rs.blockClient, + &rs.blockQueryClient, &rs.supplierClient, ); err != nil { return nil, err diff --git a/pkg/relayer/session/session_test.go b/pkg/relayer/session/session_test.go index 642456ada..fee2f486c 100644 --- a/pkg/relayer/session/session_test.go +++ b/pkg/relayer/session/session_test.go @@ -7,6 +7,9 @@ import ( "time" "cosmossdk.io/depinject" + coretypes "github.com/cometbft/cometbft/rpc/core/types" + cmttypes "github.com/cometbft/cometbft/types" + "github.com/golang/mock/gomock" "github.com/pokt-network/smt" "github.com/stretchr/testify/require" @@ -15,6 +18,7 @@ import ( "github.com/pokt-network/poktroll/pkg/polylog/polyzero" "github.com/pokt-network/poktroll/pkg/relayer" "github.com/pokt-network/poktroll/pkg/relayer/session" + "github.com/pokt-network/poktroll/testutil/mockclient" "github.com/pokt-network/poktroll/testutil/testclient/testblock" "github.com/pokt-network/poktroll/testutil/testclient/testsupplier" "github.com/pokt-network/poktroll/testutil/testpolylog" @@ -40,7 +44,23 @@ func TestRelayerSessionsManager_Start(t *testing.T) { blockClient := testblock.NewAnyTimesCommittedBlocksSequenceBlockClient(t, emptyBlockHash, blocksObs) supplierClient := testsupplier.NewOneTimeClaimProofSupplierClient(ctx, t) - deps := depinject.Supply(blockClient, supplierClient) + ctrl := gomock.NewController(t) + blockQueryClientMock := mockclient.NewMockCometRPC(ctrl) + blockQueryClientMock.EXPECT(). + Block(gomock.Any(), gomock.AssignableToTypeOf((*int64)(nil))). + Return(&coretypes.ResultBlock{ + BlockID: cmttypes.BlockID{ + Hash: []byte("expected block hash"), + }, + Block: &cmttypes.Block{ + Header: cmttypes.Header{ + Height: 1, + }, + }, + }, nil). + Times(2) + + deps := depinject.Supply(blockClient, blockQueryClientMock, supplierClient) storesDirectoryOpt := testrelayer.WithTempStoresDirectory(t) // Create a new relayer sessions manager. From 008842e14e98695255f5da21cc6a95455b385c1c Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 26 Apr 2024 20:12:14 +0200 Subject: [PATCH 24/28] fix: tests --- pkg/client/block/client_test.go | 3 ++- pkg/relayer/session/session_test.go | 15 ++------------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/pkg/client/block/client_test.go b/pkg/client/block/client_test.go index cd860d6c6..0817eb905 100644 --- a/pkg/client/block/client_test.go +++ b/pkg/client/block/client_test.go @@ -81,7 +81,8 @@ func TestBlockClient(t *testing.T) { Hash: expectedHash, }, }, nil - }) + }). + AnyTimes() deps := depinject.Supply(eventsQueryClient, cometClientMock) diff --git a/pkg/relayer/session/session_test.go b/pkg/relayer/session/session_test.go index fee2f486c..5b8e13ff9 100644 --- a/pkg/relayer/session/session_test.go +++ b/pkg/relayer/session/session_test.go @@ -7,8 +7,6 @@ import ( "time" "cosmossdk.io/depinject" - coretypes "github.com/cometbft/cometbft/rpc/core/types" - cmttypes "github.com/cometbft/cometbft/types" "github.com/golang/mock/gomock" "github.com/pokt-network/smt" "github.com/stretchr/testify/require" @@ -48,17 +46,8 @@ func TestRelayerSessionsManager_Start(t *testing.T) { blockQueryClientMock := mockclient.NewMockCometRPC(ctrl) blockQueryClientMock.EXPECT(). Block(gomock.Any(), gomock.AssignableToTypeOf((*int64)(nil))). - Return(&coretypes.ResultBlock{ - BlockID: cmttypes.BlockID{ - Hash: []byte("expected block hash"), - }, - Block: &cmttypes.Block{ - Header: cmttypes.Header{ - Height: 1, - }, - }, - }, nil). - Times(2) + Return(nil, nil). + AnyTimes() deps := depinject.Supply(blockClient, blockQueryClientMock, supplierClient) storesDirectoryOpt := testrelayer.WithTempStoresDirectory(t) From 4bf968a910834549e62a41f319f9fe0b9ff97785 Mon Sep 17 00:00:00 2001 From: Eric Nielson Date: Thu, 2 May 2024 12:10:59 -0400 Subject: [PATCH 25/28] Godoc comment update --- pkg/relayer/session/session.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/relayer/session/session.go b/pkg/relayer/session/session.go index 8b030cc89..277b9721d 100644 --- a/pkg/relayer/session/session.go +++ b/pkg/relayer/session/session.go @@ -44,7 +44,7 @@ type relayerSessionsManager struct { // supplierClient is used to create claims and submit proofs for sessions. supplierClient client.SupplierClient - // TODO_IN_THIS_COMMIT: godoc comment. + // blockQueryClient is the CometBFT RPC client used to query blocks blockQueryClient cosmosclient.CometRPC // storesDirectory points to a path on disk where KVStore data files are created. From 0d607ae96c68af0d86b7fbdcf588bcdbc00021e0 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Fri, 3 May 2024 09:37:44 +0200 Subject: [PATCH 26/28] Empty commit From 1921dd8e7b40850a2a06827e5839555fdf0de3a2 Mon Sep 17 00:00:00 2001 From: Bryan White Date: Mon, 6 May 2024 10:39:08 +0200 Subject: [PATCH 27/28] Empty commit From 0693ca19c6abb48120515880fd04350d6ea568f9 Mon Sep 17 00:00:00 2001 From: Daniel Olshansky Date: Mon, 6 May 2024 14:58:16 -0700 Subject: [PATCH 28/28] Empty commit