From cd6d51dd0343016ea4fa91a13d82207e512a7b35 Mon Sep 17 00:00:00 2001 From: Martin Majlis Date: Wed, 22 Nov 2023 11:50:35 +0100 Subject: [PATCH 01/12] [datasetexporter]: Upgrade library to 0.16.0 --- cmd/configschema/go.mod | 2 +- cmd/configschema/go.sum | 4 +- cmd/otelcontribcol/go.mod | 2 +- cmd/otelcontribcol/go.sum | 4 +- exporter/datasetexporter/config.go | 8 +- exporter/datasetexporter/config_test.go | 3 +- exporter/datasetexporter/datasetexporter.go | 7 +- exporter/datasetexporter/factory.go | 1 + exporter/datasetexporter/factory_test.go | 3 +- exporter/datasetexporter/go.mod | 2 +- exporter/datasetexporter/go.sum | 4 +- .../datasetexporter/logs_exporter_test.go | 329 +++++++++++------- exporter/datasetexporter/testdata/config.yaml | 1 + go.mod | 2 +- go.sum | 4 +- 15 files changed, 223 insertions(+), 153 deletions(-) diff --git a/cmd/configschema/go.mod b/cmd/configschema/go.mod index 4f22b4aa74d1..a832b255182d 100644 --- a/cmd/configschema/go.mod +++ b/cmd/configschema/go.mod @@ -568,7 +568,7 @@ require ( github.com/relvacode/iso8601 v1.3.0 // indirect github.com/rs/cors v1.10.1 // indirect github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 // indirect - github.com/scalyr/dataset-go v0.14.0 // indirect + github.com/scalyr/dataset-go v0.16.0 // indirect github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 // indirect github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect github.com/segmentio/asm v1.2.0 // indirect diff --git a/cmd/configschema/go.sum b/cmd/configschema/go.sum index 4f735484da28..f56a1eacb254 100644 --- a/cmd/configschema/go.sum +++ b/cmd/configschema/go.sum @@ -1378,8 +1378,8 @@ github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 h1:yWfiTPwYxB0l5fGMhl/G+liULugVIHD9AU77iNLrURQ= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= -github.com/scalyr/dataset-go v0.14.0 h1:uRkoUt6LcMcH3VUTjOakQ4aq+1ooJB2t47oqYRUkV/k= -github.com/scalyr/dataset-go v0.14.0/go.mod h1:+a4BvVyS7mKquK7ySuWh4vygyDBREQrdikdcnABYKFw= +github.com/scalyr/dataset-go v0.16.0 h1:69FIJzavveWAgaH4kzngI6xxxI5qn3MdevXCXmJEqo4= +github.com/scalyr/dataset-go v0.16.0/go.mod h1:ehHlPsZSgFWxOkud1eKwmKd5bLF9LcUFrU01XuCnh+8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds= diff --git a/cmd/otelcontribcol/go.mod b/cmd/otelcontribcol/go.mod index 66c631642b35..f9c33febbc3b 100644 --- a/cmd/otelcontribcol/go.mod +++ b/cmd/otelcontribcol/go.mod @@ -586,7 +586,7 @@ require ( github.com/rs/cors v1.10.1 // indirect github.com/samber/lo v1.37.0 // indirect github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 // indirect - github.com/scalyr/dataset-go v0.14.0 // indirect + github.com/scalyr/dataset-go v0.16.0 // indirect github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 // indirect github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect github.com/segmentio/asm v1.2.0 // indirect diff --git a/cmd/otelcontribcol/go.sum b/cmd/otelcontribcol/go.sum index 5bd276872212..1bdebdba92cd 100644 --- a/cmd/otelcontribcol/go.sum +++ b/cmd/otelcontribcol/go.sum @@ -1375,8 +1375,8 @@ github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 h1:yWfiTPwYxB0l5fGMhl/G+liULugVIHD9AU77iNLrURQ= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= -github.com/scalyr/dataset-go v0.14.0 h1:uRkoUt6LcMcH3VUTjOakQ4aq+1ooJB2t47oqYRUkV/k= -github.com/scalyr/dataset-go v0.14.0/go.mod h1:+a4BvVyS7mKquK7ySuWh4vygyDBREQrdikdcnABYKFw= +github.com/scalyr/dataset-go v0.16.0 h1:69FIJzavveWAgaH4kzngI6xxxI5qn3MdevXCXmJEqo4= +github.com/scalyr/dataset-go v0.16.0/go.mod h1:ehHlPsZSgFWxOkud1eKwmKd5bLF9LcUFrU01XuCnh+8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds= diff --git a/exporter/datasetexporter/config.go b/exporter/datasetexporter/config.go index c497086434be..1bad6606d20d 100644 --- a/exporter/datasetexporter/config.go +++ b/exporter/datasetexporter/config.go @@ -5,6 +5,7 @@ package datasetexporter // import "github.com/open-telemetry/opentelemetry-colle import ( "fmt" + "strings" "time" "github.com/cenkalti/backoff/v4" @@ -144,9 +145,12 @@ func newDefaultServerHostSettings() ServerHostSettings { } } +const debugDefault = false + type Config struct { DatasetURL string `mapstructure:"dataset_url"` APIKey configopaque.String `mapstructure:"api_key"` + Debug bool `mapstructure:"debug"` BufferSettings `mapstructure:"buffer"` TracesSettings `mapstructure:"traces"` LogsSettings `mapstructure:"logs"` @@ -182,6 +186,8 @@ func (c *Config) Validate() error { func (c *Config) String() string { s := "" s += fmt.Sprintf("%s: %s; ", "DatasetURL", c.DatasetURL) + s += fmt.Sprintf("%s: %s (%d); ", "APIKey", strings.Repeat("*", len(c.APIKey)), len(c.APIKey)) + s += fmt.Sprintf("%s: %t; ", "Debug", c.Debug) s += fmt.Sprintf("%s: %+v; ", "BufferSettings", c.BufferSettings) s += fmt.Sprintf("%s: %+v; ", "LogsSettings", c.LogsSettings) s += fmt.Sprintf("%s: %+v; ", "TracesSettings", c.TracesSettings) @@ -189,7 +195,6 @@ func (c *Config) String() string { s += fmt.Sprintf("%s: %+v; ", "RetrySettings", c.RetrySettings) s += fmt.Sprintf("%s: %+v; ", "QueueSettings", c.QueueSettings) s += fmt.Sprintf("%s: %+v", "TimeoutSettings", c.TimeoutSettings) - return s } @@ -218,6 +223,7 @@ func (c *Config) convert() (*ExporterConfig, error) { UseHostName: c.ServerHostSettings.UseHostName, ServerHost: c.ServerHostSettings.ServerHost, }, + Debug: c.Debug, }, tracesSettings: c.TracesSettings, logsSettings: c.LogsSettings, diff --git a/exporter/datasetexporter/config_test.go b/exporter/datasetexporter/config_test.go index af99f0cf7f93..b0191d383a14 100644 --- a/exporter/datasetexporter/config_test.go +++ b/exporter/datasetexporter/config_test.go @@ -108,6 +108,7 @@ func TestConfigString(t *testing.T) { config := Config{ DatasetURL: "https://example.com", APIKey: "secret", + Debug: true, BufferSettings: BufferSettings{ MaxLifetime: 123, GroupBy: []string{"field1", "field2"}, @@ -140,7 +141,7 @@ func TestConfigString(t *testing.T) { } assert.Equal(t, - "DatasetURL: https://example.com; BufferSettings: {MaxLifetime:123ns GroupBy:[field1 field2] RetryInitialInterval:0s RetryMaxInterval:0s RetryMaxElapsedTime:0s RetryShutdownTimeout:0s}; LogsSettings: {ExportResourceInfo:true ExportResourcePrefix:AAA ExportScopeInfo:true ExportScopePrefix:BBB DecomposeComplexMessageField:true DecomposedComplexMessagePrefix:EEE exportSettings:{ExportSeparator:CCC ExportDistinguishingSuffix:DDD}}; TracesSettings: {exportSettings:{ExportSeparator:TTT ExportDistinguishingSuffix:UUU}}; ServerHostSettings: {UseHostName:false ServerHost:foo-bar}; RetrySettings: {Enabled:true InitialInterval:5s RandomizationFactor:0.5 Multiplier:1.5 MaxInterval:30s MaxElapsedTime:5m0s}; QueueSettings: {Enabled:true NumConsumers:10 QueueSize:1000 StorageID:}; TimeoutSettings: {Timeout:5s}", + "DatasetURL: https://example.com; APIKey: ****** (6); Debug: true; BufferSettings: {MaxLifetime:123ns GroupBy:[field1 field2] RetryInitialInterval:0s RetryMaxInterval:0s RetryMaxElapsedTime:0s RetryShutdownTimeout:0s}; LogsSettings: {ExportResourceInfo:true ExportResourcePrefix:AAA ExportScopeInfo:true ExportScopePrefix:BBB DecomposeComplexMessageField:true DecomposedComplexMessagePrefix:EEE exportSettings:{ExportSeparator:CCC ExportDistinguishingSuffix:DDD}}; TracesSettings: {exportSettings:{ExportSeparator:TTT ExportDistinguishingSuffix:UUU}}; ServerHostSettings: {UseHostName:false ServerHost:foo-bar}; RetrySettings: {Enabled:true InitialInterval:5s RandomizationFactor:0.5 Multiplier:1.5 MaxInterval:30s MaxElapsedTime:5m0s}; QueueSettings: {Enabled:true NumConsumers:10 QueueSize:1000 StorageID:}; TimeoutSettings: {Timeout:5s}", config.String(), ) } diff --git a/exporter/datasetexporter/datasetexporter.go b/exporter/datasetexporter/datasetexporter.go index 4767c588d4c4..806062fdf88e 100644 --- a/exporter/datasetexporter/datasetexporter.go +++ b/exporter/datasetexporter/datasetexporter.go @@ -17,12 +17,10 @@ import ( "go.opentelemetry.io/collector/exporter" "go.opentelemetry.io/collector/pdata/pcommon" "go.uber.org/zap" - "golang.org/x/time/rate" ) type DatasetExporter struct { client *client.DataSetClient - limiter *rate.Limiter logger *zap.Logger session string exporterCfg *ExporterConfig @@ -48,11 +46,15 @@ func newDatasetExporter(entity string, config *Config, set exporter.CreateSettin set.BuildInfo.Version, entity, ) + + meter := set.MeterProvider.Meter("datasetexporter") + client, err := client.NewClient( exporterCfg.datasetConfig, &http.Client{Timeout: time.Second * 60}, logger, &userAgent, + &meter, ) if err != nil { logger.Error("Cannot create DataSetClient: ", zap.Error(err)) @@ -61,7 +63,6 @@ func newDatasetExporter(entity string, config *Config, set exporter.CreateSettin return &DatasetExporter{ client: client, - limiter: rate.NewLimiter(100*rate.Every(1*time.Minute), 100), // 100 requests / minute session: uuid.New().String(), logger: logger, exporterCfg: exporterCfg, diff --git a/exporter/datasetexporter/factory.go b/exporter/datasetexporter/factory.go index 600cc0f1fa83..5864f52cfc75 100644 --- a/exporter/datasetexporter/factory.go +++ b/exporter/datasetexporter/factory.go @@ -32,6 +32,7 @@ func createDefaultConfig() component.Config { RetrySettings: exporterhelper.NewDefaultRetrySettings(), QueueSettings: exporterhelper.NewDefaultQueueSettings(), TimeoutSettings: exporterhelper.NewDefaultTimeoutSettings(), + Debug: debugDefault, } } diff --git a/exporter/datasetexporter/factory_test.go b/exporter/datasetexporter/factory_test.go index 2e3d3f0dab65..d07a65cffd7d 100644 --- a/exporter/datasetexporter/factory_test.go +++ b/exporter/datasetexporter/factory_test.go @@ -84,6 +84,7 @@ func TestLoadConfig(t *testing.T) { expected: &Config{ DatasetURL: "https://app.scalyr.com", APIKey: "key-full", + Debug: true, BufferSettings: BufferSettings{ MaxLifetime: 3456 * time.Millisecond, GroupBy: []string{"body.map.kubernetes.pod_id", "body.map.kubernetes.docker_id", "body.map.stream"}, @@ -160,7 +161,7 @@ func createExporterTests() []CreateTest { { name: "broken", config: &Config{}, - expectedError: fmt.Errorf("cannot get DataSetExpoter: cannot convert config: DatasetURL: ; BufferSettings: {MaxLifetime:0s GroupBy:[] RetryInitialInterval:0s RetryMaxInterval:0s RetryMaxElapsedTime:0s RetryShutdownTimeout:0s}; LogsSettings: {ExportResourceInfo:false ExportResourcePrefix: ExportScopeInfo:false ExportScopePrefix: DecomposeComplexMessageField:false DecomposedComplexMessagePrefix: exportSettings:{ExportSeparator: ExportDistinguishingSuffix:}}; TracesSettings: {exportSettings:{ExportSeparator: ExportDistinguishingSuffix:}}; ServerHostSettings: {UseHostName:false ServerHost:}; RetrySettings: {Enabled:false InitialInterval:0s RandomizationFactor:0 Multiplier:0 MaxInterval:0s MaxElapsedTime:0s}; QueueSettings: {Enabled:false NumConsumers:0 QueueSize:0 StorageID:}; TimeoutSettings: {Timeout:0s}; config is not valid: api_key is required"), + expectedError: fmt.Errorf("cannot get DataSetExpoter: cannot convert config: DatasetURL: ; APIKey: (0); Debug: false; BufferSettings: {MaxLifetime:0s GroupBy:[] RetryInitialInterval:0s RetryMaxInterval:0s RetryMaxElapsedTime:0s RetryShutdownTimeout:0s}; LogsSettings: {ExportResourceInfo:false ExportResourcePrefix: ExportScopeInfo:false ExportScopePrefix: DecomposeComplexMessageField:false DecomposedComplexMessagePrefix: exportSettings:{ExportSeparator: ExportDistinguishingSuffix:}}; TracesSettings: {exportSettings:{ExportSeparator: ExportDistinguishingSuffix:}}; ServerHostSettings: {UseHostName:false ServerHost:}; RetrySettings: {Enabled:false InitialInterval:0s RandomizationFactor:0 Multiplier:0 MaxInterval:0s MaxElapsedTime:0s}; QueueSettings: {Enabled:false NumConsumers:0 QueueSize:0 StorageID:}; TimeoutSettings: {Timeout:0s}; config is not valid: api_key is required"), }, { name: "valid", diff --git a/exporter/datasetexporter/go.mod b/exporter/datasetexporter/go.mod index 14b1ee25f130..80ebdc473166 100644 --- a/exporter/datasetexporter/go.mod +++ b/exporter/datasetexporter/go.mod @@ -6,7 +6,7 @@ require ( github.com/google/uuid v1.4.0 // github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage v0.77.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.89.0 - github.com/scalyr/dataset-go v0.14.0 + github.com/scalyr/dataset-go v0.16.0 github.com/stretchr/testify v1.8.4 go.opentelemetry.io/collector/component v0.89.0 go.opentelemetry.io/collector/confmap v0.89.0 diff --git a/exporter/datasetexporter/go.sum b/exporter/datasetexporter/go.sum index 03bddd7eae36..bb73a22ab9bc 100644 --- a/exporter/datasetexporter/go.sum +++ b/exporter/datasetexporter/go.sum @@ -87,8 +87,8 @@ github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lne github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/scalyr/dataset-go v0.14.0 h1:uRkoUt6LcMcH3VUTjOakQ4aq+1ooJB2t47oqYRUkV/k= -github.com/scalyr/dataset-go v0.14.0/go.mod h1:+a4BvVyS7mKquK7ySuWh4vygyDBREQrdikdcnABYKFw= +github.com/scalyr/dataset-go v0.16.0 h1:69FIJzavveWAgaH4kzngI6xxxI5qn3MdevXCXmJEqo4= +github.com/scalyr/dataset-go v0.16.0/go.mod h1:ehHlPsZSgFWxOkud1eKwmKd5bLF9LcUFrU01XuCnh+8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= diff --git a/exporter/datasetexporter/logs_exporter_test.go b/exporter/datasetexporter/logs_exporter_test.go index 49eb45b47740..57b2ad699af0 100644 --- a/exporter/datasetexporter/logs_exporter_test.go +++ b/exporter/datasetexporter/logs_exporter_test.go @@ -11,7 +11,9 @@ import ( "io" "net/http" "net/http/httptest" + "sort" "strconv" + "sync" "sync/atomic" "testing" "time" @@ -780,12 +782,15 @@ func TestConsumeLogsShouldSucceed(t *testing.T) { attempt := atomic.Uint64{} wasSuccessful := atomic.Bool{} - addRequest := add_events.AddEventsRequest{} + addRequests := []add_events.AddEventsRequest{} + lock := sync.Mutex{} server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { attempt.Add(1) cer, err := extract(req) - addRequest = cer + lock.Lock() + addRequests = append(addRequests, cer) + lock.Unlock() assert.NoError(t, err, "Error reading request: %v", err) @@ -804,8 +809,9 @@ func TestConsumeLogsShouldSucceed(t *testing.T) { config := &Config{ DatasetURL: server.URL, APIKey: "key-lib", + Debug: true, BufferSettings: BufferSettings{ - MaxLifetime: 500 * time.Millisecond, + MaxLifetime: 2 * time.Second, GroupBy: []string{"attributes.container_id"}, RetryInitialInterval: time.Second, RetryMaxInterval: time.Minute, @@ -881,151 +887,204 @@ func TestConsumeLogsShouldSucceed(t *testing.T) { } assert.True(t, wasSuccessful.Load()) + assert.Equal(t, uint64(4), attempt.Load()) + + sort.SliceStable(addRequests, func(i, j int) bool { + if addRequests[i].Session == addRequests[j].Session { + return len(addRequests[i].Events) < len(addRequests[j].Events) + } + return addRequests[i].Session < addRequests[j].Session + }) + assert.Equal(t, - add_events.AddEventsRequest{ - AuthParams: request.AuthParams{ - Token: "key-lib", - }, - AddEventsRequestParams: add_events.AddEventsRequestParams{ - Session: addRequest.Session, - SessionInfo: addRequest.SessionInfo, - Events: []*add_events.Event{ - { - Thread: testLEventReq.Thread, - Log: testLEventReq.Log, - Sev: testLEventReq.Sev, - Ts: testLEventReq.Ts, - Attrs: map[string]any{ - add_events.AttrOrigServerHost: testServerHost, - "app": "server", - "instance_num": float64(1), - "dropped_attributes_count": float64(1), - "message": "This is a log message", - "span_id": "0102040800000000", - "trace_id": "08040201000000000000000000000000", - "bundle_key": "d41d8cd98f00b204e9800998ecf8427e", - - "R#resource-attr": "resource-attr-val-1", - }, + []add_events.AddEventsRequest{ + { + AuthParams: request.AuthParams{ + Token: "key-lib", + }, + AddEventsRequestParams: add_events.AddEventsRequestParams{ + Session: addRequests[0].Session, + SessionInfo: &add_events.SessionInfo{ + add_events.AttrServerHost: "serverHostFromAttribute", + add_events.AttrSessionKey: "0296b9a57cb379df0f35aaf2d23500d3", }, - { - Thread: testLEventReq.Thread, - Log: testLEventReq.Log, - Sev: testLEventReq.Sev, - Ts: testLEventReq.Ts, - Attrs: map[string]any{ - add_events.AttrOrigServerHost: "serverHostFromAttribute", - "app": "server", - "instance_num": float64(1), - "dropped_attributes_count": float64(1), - "message": "This is a log message", - "span_id": "0102040800000000", - "trace_id": "08040201000000000000000000000000", - "bundle_key": "d41d8cd98f00b204e9800998ecf8427e", - - "R#resource-attr": "resource-attr-val-1", - "R#serverHost": "serverHostFromResource", + Events: []*add_events.Event{ + { + Thread: testLEventReq.Thread, + Log: testLEventReq.Log, + Sev: testLEventReq.Sev, + Ts: testLEventReq.Ts, + Attrs: map[string]any{ + "app": "server", + "instance_num": float64(1), + "dropped_attributes_count": float64(1), + "message": "This is a log message", + "span_id": "0102040800000000", + "trace_id": "08040201000000000000000000000000", + + "R#resource-attr": "resource-attr-val-1", + "R#serverHost": "serverHostFromResource", + }, + ServerHost: "", }, }, - { - Thread: testLEventReq.Thread, - Log: testLEventReq.Log, - Sev: testLEventReq.Sev, - Ts: testLEventReq.Ts, - Attrs: map[string]any{ - add_events.AttrOrigServerHost: "serverHostFromResourceServer", - "app": "server", - "instance_num": float64(1), - "dropped_attributes_count": float64(1), - "message": "This is a log message", - "span_id": "0102040800000000", - "trace_id": "08040201000000000000000000000000", - "bundle_key": "d41d8cd98f00b204e9800998ecf8427e", - - "R#resource-attr": "resource-attr-val-1", - "R#host.name": "serverHostFromResourceHost", - "R#serverHost": "serverHostFromResourceServer", + Threads: []*add_events.Thread{testLThread}, + Logs: []*add_events.Log{testLLog}, + }, + }, + { + AuthParams: request.AuthParams{ + Token: "key-lib", + }, + AddEventsRequestParams: add_events.AddEventsRequestParams{ + Session: addRequests[1].Session, + SessionInfo: &add_events.SessionInfo{ + add_events.AttrServerHost: "serverHostFromResourceHost", + add_events.AttrSessionKey: "73b97897d80d89c9a09a3ee6ed178650", + }, + Events: []*add_events.Event{ + { + Thread: testLEventReq.Thread, + Log: testLEventReq.Log, + Sev: testLEventReq.Sev, + Ts: testLEventReq.Ts, + Attrs: map[string]any{ + "app": "server", + "instance_num": float64(1), + "dropped_attributes_count": float64(1), + "message": "This is a log message", + "span_id": "0102040800000000", + "trace_id": "08040201000000000000000000000000", + + "R#resource-attr": "resource-attr-val-1", + "R#host.name": "serverHostFromResourceHost", + }, }, }, - { - Thread: testLEventReq.Thread, - Log: testLEventReq.Log, - Sev: testLEventReq.Sev, - Ts: testLEventReq.Ts, - Attrs: map[string]any{ - add_events.AttrOrigServerHost: "serverHostFromResourceHost", - "app": "server", - "instance_num": float64(1), - "dropped_attributes_count": float64(1), - "message": "This is a log message", - "span_id": "0102040800000000", - "trace_id": "08040201000000000000000000000000", - "bundle_key": "d41d8cd98f00b204e9800998ecf8427e", - - "R#resource-attr": "resource-attr-val-1", - "R#host.name": "serverHostFromResourceHost", + Threads: []*add_events.Thread{testLThread}, + Logs: []*add_events.Log{testLLog}, + }, + }, + { + AuthParams: request.AuthParams{ + Token: "key-lib", + }, + AddEventsRequestParams: add_events.AddEventsRequestParams{ + Session: addRequests[2].Session, + SessionInfo: &add_events.SessionInfo{ + add_events.AttrServerHost: "serverHostFromResourceServer", + add_events.AttrSessionKey: "770e22b433d2e9a31fa9a81abf3b9b87", + }, + Events: []*add_events.Event{ + { + Thread: testLEventReq.Thread, + Log: testLEventReq.Log, + Sev: testLEventReq.Sev, + Ts: testLEventReq.Ts, + Attrs: map[string]any{ + "app": "server", + "instance_num": float64(1), + "dropped_attributes_count": float64(1), + "message": "This is a log message", + "span_id": "0102040800000000", + "trace_id": "08040201000000000000000000000000", + + "R#resource-attr": "resource-attr-val-1", + "R#host.name": "serverHostFromResourceHost", + "R#serverHost": "serverHostFromResourceServer", + }, }, }, - { - Thread: testLEventReq.Thread, - Log: testLEventReq.Log, - Sev: testLEventReq.Sev, - Ts: testLEventReq.Ts, - Attrs: map[string]any{ - add_events.AttrOrigServerHost: testServerHost, - "app": "server", - "instance_num": float64(1), - "dropped_attributes_count": float64(1), - "message": "This is a log message", - "span_id": "0102040800000000", - "trace_id": "08040201000000000000000000000000", - "bundle_key": "d41d8cd98f00b204e9800998ecf8427e", - - "string": "stringA", - "double": 2.0, - "bool": true, - "empty": nil, - "int": float64(3), - "map#map_empty": nil, - "map#map_string": "map_stringA", - "map#map_map#map_map_string": "map_map_stringA", - "slice#0": "slice_stringA", - "name": "filled_nameA", - "span_id_": "filled_span_idA", - - "S#string": "stringS", - "S#double": 2.0, - "S#bool": true, - "S#empty": nil, - "S#int": float64(3), - "S#map#map_empty": nil, - "S#map#map_string": "map_stringS", - "S#map#map_map#map_map_string": "map_map_stringS", - "S#slice#0": "slice_stringS", - "S#name": "filled_nameS", - "S#span_id": "filled_span_idS", - - "R#string": "stringR", - "R#double": 2.0, - "R#bool": true, - "R#empty": nil, - "R#int": float64(3), - "R#map#map_empty": nil, - "R#map#map_string": "map_stringR", - "R#map#map_map#map_map_string": "map_map_stringR", - "R#slice#0": "slice_stringR", - "R#name": "filled_nameR", - "R#span_id": "filled_span_idR", - - "R#resource-attr": "resource-attr-val-1", + Threads: []*add_events.Thread{testLThread}, + Logs: []*add_events.Log{testLLog}, + }, + }, + { + AuthParams: request.AuthParams{ + Token: "key-lib", + }, + AddEventsRequestParams: add_events.AddEventsRequestParams{ + Session: addRequests[3].Session, + SessionInfo: &add_events.SessionInfo{ + add_events.AttrServerHost: "foo", + add_events.AttrSessionKey: "caedd419dc354c24a69aac7508890ec1", + }, + Events: []*add_events.Event{ + { + Thread: testLEventReq.Thread, + Log: testLEventReq.Log, + Sev: testLEventReq.Sev, + Ts: testLEventReq.Ts, + Attrs: map[string]any{ + "app": "server", + "instance_num": float64(1), + "dropped_attributes_count": float64(1), + "message": "This is a log message", + "span_id": "0102040800000000", + "trace_id": "08040201000000000000000000000000", + + "R#resource-attr": "resource-attr-val-1", + }, + }, + { + Thread: testLEventReq.Thread, + Log: testLEventReq.Log, + Sev: testLEventReq.Sev, + Ts: testLEventReq.Ts, + Attrs: map[string]any{ + "app": "server", + "instance_num": float64(1), + "dropped_attributes_count": float64(1), + "message": "This is a log message", + "span_id": "0102040800000000", + "trace_id": "08040201000000000000000000000000", + + "string": "stringA", + "double": 2.0, + "bool": true, + "empty": nil, + "int": float64(3), + "map#map_empty": nil, + "map#map_string": "map_stringA", + "map#map_map#map_map_string": "map_map_stringA", + "slice#0": "slice_stringA", + "name": "filled_nameA", + "span_id_": "filled_span_idA", + + "S#string": "stringS", + "S#double": 2.0, + "S#bool": true, + "S#empty": nil, + "S#int": float64(3), + "S#map#map_empty": nil, + "S#map#map_string": "map_stringS", + "S#map#map_map#map_map_string": "map_map_stringS", + "S#slice#0": "slice_stringS", + "S#name": "filled_nameS", + "S#span_id": "filled_span_idS", + + "R#string": "stringR", + "R#double": 2.0, + "R#bool": true, + "R#empty": nil, + "R#int": float64(3), + "R#map#map_empty": nil, + "R#map#map_string": "map_stringR", + "R#map#map_map#map_map_string": "map_map_stringR", + "R#slice#0": "slice_stringR", + "R#name": "filled_nameR", + "R#span_id": "filled_span_idR", + + "R#resource-attr": "resource-attr-val-1", + }, }, }, + Threads: []*add_events.Thread{testLThread}, + Logs: []*add_events.Log{testLLog}, }, - Threads: []*add_events.Thread{testLThread}, - Logs: []*add_events.Log{testLLog}, }, }, - addRequest, + addRequests, ) } diff --git a/exporter/datasetexporter/testdata/config.yaml b/exporter/datasetexporter/testdata/config.yaml index 9ffe4d29a733..262597ba4bf4 100644 --- a/exporter/datasetexporter/testdata/config.yaml +++ b/exporter/datasetexporter/testdata/config.yaml @@ -14,6 +14,7 @@ dataset/lib: dataset/full: dataset_url: https://app.scalyr.com api_key: key-full + debug: true buffer: max_lifetime: 3456ms group_by: diff --git a/go.mod b/go.mod index 77c8537fe184..b40dac36e9f7 100644 --- a/go.mod +++ b/go.mod @@ -571,7 +571,7 @@ require ( github.com/rs/cors v1.10.1 // indirect github.com/samber/lo v1.37.0 // indirect github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 // indirect - github.com/scalyr/dataset-go v0.14.0 // indirect + github.com/scalyr/dataset-go v0.16.0 // indirect github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 // indirect github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect github.com/segmentio/asm v1.2.0 // indirect diff --git a/go.sum b/go.sum index 71fe0b951cea..ae63a5269287 100644 --- a/go.sum +++ b/go.sum @@ -1383,8 +1383,8 @@ github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 h1:yWfiTPwYxB0l5fGMhl/G+liULugVIHD9AU77iNLrURQ= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= -github.com/scalyr/dataset-go v0.14.0 h1:uRkoUt6LcMcH3VUTjOakQ4aq+1ooJB2t47oqYRUkV/k= -github.com/scalyr/dataset-go v0.14.0/go.mod h1:+a4BvVyS7mKquK7ySuWh4vygyDBREQrdikdcnABYKFw= +github.com/scalyr/dataset-go v0.16.0 h1:69FIJzavveWAgaH4kzngI6xxxI5qn3MdevXCXmJEqo4= +github.com/scalyr/dataset-go v0.16.0/go.mod h1:ehHlPsZSgFWxOkud1eKwmKd5bLF9LcUFrU01XuCnh+8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds= From ea6bec5abc593e79d3f820f7b12beab7fe67dead Mon Sep 17 00:00:00 2001 From: Martin Majlis Date: Wed, 22 Nov 2023 12:25:21 +0100 Subject: [PATCH 02/12] Add changelog --- .../datasetexporter-update-to-0.16.0.yaml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 .chloggen/datasetexporter-update-to-0.16.0.yaml diff --git a/.chloggen/datasetexporter-update-to-0.16.0.yaml b/.chloggen/datasetexporter-update-to-0.16.0.yaml new file mode 100755 index 000000000000..d34faea08c28 --- /dev/null +++ b/.chloggen/datasetexporter-update-to-0.16.0.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: datasetexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Collect usage metrics with Otel and sent grouped attributes in session info. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [27650, 27652] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] From de9282c575e6cf47295b411355561da22831bba8 Mon Sep 17 00:00:00 2001 From: Martin Majlis Date: Wed, 22 Nov 2023 12:27:20 +0100 Subject: [PATCH 03/12] Make `go mod tidy -compat=1.20` happy --- exporter/datasetexporter/go.mod | 1 - exporter/datasetexporter/go.sum | 2 -- 2 files changed, 3 deletions(-) diff --git a/exporter/datasetexporter/go.mod b/exporter/datasetexporter/go.mod index 80ebdc473166..3b881c2095d4 100644 --- a/exporter/datasetexporter/go.mod +++ b/exporter/datasetexporter/go.mod @@ -13,7 +13,6 @@ require ( go.opentelemetry.io/collector/exporter v0.89.0 go.opentelemetry.io/collector/pdata v1.0.0-rcv0018 go.uber.org/zap v1.26.0 - golang.org/x/time v0.4.0 ) diff --git a/exporter/datasetexporter/go.sum b/exporter/datasetexporter/go.sum index bb73a22ab9bc..2ca57df0ef52 100644 --- a/exporter/datasetexporter/go.sum +++ b/exporter/datasetexporter/go.sum @@ -176,8 +176,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.4.0 h1:Z81tqI5ddIoXDPvVQ7/7CC9TnLM7ubaFG2qXYd5BbYY= -golang.org/x/time v0.4.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= From 3ea0348f81e039b85b7e8af3ef967bb394f135f0 Mon Sep 17 00:00:00 2001 From: Martin Majlis Date: Wed, 22 Nov 2023 14:19:16 +0100 Subject: [PATCH 04/12] Update documentation --- exporter/datasetexporter/README.md | 43 +++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/exporter/datasetexporter/README.md b/exporter/datasetexporter/README.md index 8ac481490d15..425e35329ce6 100644 --- a/exporter/datasetexporter/README.md +++ b/exporter/datasetexporter/README.md @@ -45,9 +45,10 @@ Make sure to provide the appropriate server host value in the `serverHost` attri ### Optional Settings +- `debug` (default = false): Adds `session_key` to the server fields. It's useful for debugging throughput issues. - `buffer`: - `max_lifetime` (default = 5s): The maximum delay between sending batches from the same source. - - `group_by` (default = []): The list of attributes based on which events should be grouped. + - `group_by` (default = []): The list of attributes based on which events should be grouped. They are moved from the event attributes to the session info and shown as server fields in the UI. - `retry_initial_interval` (default = 5s): Time to wait after the first failure before retrying. - `retry_max_interval` (default = 30s): Is the upper bound on backoff. - `retry_max_elapsed_time` (default = 300s): Is the maximum amount of time spent trying to send a buffer. @@ -259,8 +260,6 @@ service: exporters: [dataset/traces] ``` -## Examples - ### Handling `serverHost` Attribute Based on the given configuration and scenarios, here's the expected behavior: @@ -280,3 +279,41 @@ Based on the given configuration and scenarios, here's the expected behavior: 5. Resource: `{}`, Log: `{'attribute.foo': 'Bar'}`, Env: `SERVER_HOST=''`, Hostname: `ip-172-31-27-19` * Since the attribute `container_id` is not set and the environmental variable `SERVER_HOST` is empty, the `hostname` of the node (`ip-172-31-27-19`) will be used as the fallback value for `serverHost`. * Used `serverHost` will be `ip-172-31-27-19`. + +## Metrics + +To enable metrics you have to: +1. Run collector with enabled feature gate `telemetry.useOtelForInternalMetrics`. This can be done by executing it with one additional parameter - `--feature-gates=telemetry.useOtelForInternalMetrics`. +2. Enable metrics scraping as part of the configuration with and add receiver into services: + ```yaml + receivers: + prometheus: + config: + scrape_configs: + - job_name: 'otel-collector' + scrape_interval: 5s + static_configs: + - targets: ['0.0.0.0:8888'] + ... + service: + pipelines: + metrics: + # add prometheus among metrics receivers + receivers: [prometheus] + processors: [batch] + exporters: [otlphttp/prometheus, debug] + ``` + +#### Available Metrics + +Available metrics contain `dataset` in their name. There are counters related to the +number of processed events (`events`), buffers (`buffer`), and transferred bytes (`bytes`). +There are also histograms related to response times (`responseTime`) and payload size (`payloadSize`). + +There are several counters related to events/buffers: +* `enqueued` - the number of received entities +* `processed` - the number of entities that were accepted by the next layer +* `dropped` - the number of entities that were not accepted by the next layer +* `broken` - the number of entities that were somehow corrupted during processing (should be 0) + +The number of entities, that are still in the queue can be computed as `enqueued - (processed + dropped + broken)`. From 2cf16e53c011d58491564e0e244f948913feec0d Mon Sep 17 00:00:00 2001 From: Martin Majlis Date: Thu, 23 Nov 2023 10:09:33 +0100 Subject: [PATCH 05/12] Fix typos --- exporter/datasetexporter/factory_test.go | 2 +- exporter/datasetexporter/logs_exporter.go | 2 +- exporter/datasetexporter/traces_exporter.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/exporter/datasetexporter/factory_test.go b/exporter/datasetexporter/factory_test.go index d07a65cffd7d..932c84f7aead 100644 --- a/exporter/datasetexporter/factory_test.go +++ b/exporter/datasetexporter/factory_test.go @@ -161,7 +161,7 @@ func createExporterTests() []CreateTest { { name: "broken", config: &Config{}, - expectedError: fmt.Errorf("cannot get DataSetExpoter: cannot convert config: DatasetURL: ; APIKey: (0); Debug: false; BufferSettings: {MaxLifetime:0s GroupBy:[] RetryInitialInterval:0s RetryMaxInterval:0s RetryMaxElapsedTime:0s RetryShutdownTimeout:0s}; LogsSettings: {ExportResourceInfo:false ExportResourcePrefix: ExportScopeInfo:false ExportScopePrefix: DecomposeComplexMessageField:false DecomposedComplexMessagePrefix: exportSettings:{ExportSeparator: ExportDistinguishingSuffix:}}; TracesSettings: {exportSettings:{ExportSeparator: ExportDistinguishingSuffix:}}; ServerHostSettings: {UseHostName:false ServerHost:}; RetrySettings: {Enabled:false InitialInterval:0s RandomizationFactor:0 Multiplier:0 MaxInterval:0s MaxElapsedTime:0s}; QueueSettings: {Enabled:false NumConsumers:0 QueueSize:0 StorageID:}; TimeoutSettings: {Timeout:0s}; config is not valid: api_key is required"), + expectedError: fmt.Errorf("cannot get DataSetExporter: cannot convert config: DatasetURL: ; APIKey: (0); Debug: false; BufferSettings: {MaxLifetime:0s GroupBy:[] RetryInitialInterval:0s RetryMaxInterval:0s RetryMaxElapsedTime:0s RetryShutdownTimeout:0s}; LogsSettings: {ExportResourceInfo:false ExportResourcePrefix: ExportScopeInfo:false ExportScopePrefix: DecomposeComplexMessageField:false DecomposedComplexMessagePrefix: exportSettings:{ExportSeparator: ExportDistinguishingSuffix:}}; TracesSettings: {exportSettings:{ExportSeparator: ExportDistinguishingSuffix:}}; ServerHostSettings: {UseHostName:false ServerHost:}; RetrySettings: {Enabled:false InitialInterval:0s RandomizationFactor:0 Multiplier:0 MaxInterval:0s MaxElapsedTime:0s}; QueueSettings: {Enabled:false NumConsumers:0 QueueSize:0 StorageID:}; TimeoutSettings: {Timeout:0s}; config is not valid: api_key is required"), }, { name: "valid", diff --git a/exporter/datasetexporter/logs_exporter.go b/exporter/datasetexporter/logs_exporter.go index 8ff1a98892e1..43dec4ab96e5 100644 --- a/exporter/datasetexporter/logs_exporter.go +++ b/exporter/datasetexporter/logs_exporter.go @@ -43,7 +43,7 @@ func createLogsExporter(ctx context.Context, set exporter.CreateSettings, config cfg := castConfig(config) e, err := newDatasetExporter("logs", cfg, set) if err != nil { - return nil, fmt.Errorf("cannot get DataSetExpoter: %w", err) + return nil, fmt.Errorf("cannot get DataSetExporter: %w", err) } return exporterhelper.NewLogsExporter( diff --git a/exporter/datasetexporter/traces_exporter.go b/exporter/datasetexporter/traces_exporter.go index 5ee1024e591c..6eb0df6ef913 100644 --- a/exporter/datasetexporter/traces_exporter.go +++ b/exporter/datasetexporter/traces_exporter.go @@ -23,7 +23,7 @@ func createTracesExporter(ctx context.Context, set exporter.CreateSettings, conf cfg := castConfig(config) e, err := newDatasetExporter("logs", cfg, set) if err != nil { - return nil, fmt.Errorf("cannot get DataSetExpoter: %w", err) + return nil, fmt.Errorf("cannot get DataSetExporter: %w", err) } return exporterhelper.NewTracesExporter( From 9423b1978b945f30440900ee376ae9657ffe4589 Mon Sep 17 00:00:00 2001 From: Martin Majlis <122797378+martin-majlis-s1@users.noreply.github.com> Date: Mon, 27 Nov 2023 10:05:35 +0100 Subject: [PATCH 06/12] Update exporter/datasetexporter/README.md Co-authored-by: Andrzej Stencel --- exporter/datasetexporter/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/datasetexporter/README.md b/exporter/datasetexporter/README.md index 425e35329ce6..f9464490d8a2 100644 --- a/exporter/datasetexporter/README.md +++ b/exporter/datasetexporter/README.md @@ -284,7 +284,7 @@ Based on the given configuration and scenarios, here's the expected behavior: To enable metrics you have to: 1. Run collector with enabled feature gate `telemetry.useOtelForInternalMetrics`. This can be done by executing it with one additional parameter - `--feature-gates=telemetry.useOtelForInternalMetrics`. -2. Enable metrics scraping as part of the configuration with and add receiver into services: +2. Enable metrics scraping as part of the configuration and add receiver into services: ```yaml receivers: prometheus: From 1b4d15f847a0ed80b836c5b8adb1126b363989f6 Mon Sep 17 00:00:00 2001 From: Martin Majlis <122797378+martin-majlis-s1@users.noreply.github.com> Date: Mon, 27 Nov 2023 10:05:45 +0100 Subject: [PATCH 07/12] Update .chloggen/datasetexporter-update-to-0.16.0.yaml Co-authored-by: Andrzej Stencel --- .chloggen/datasetexporter-update-to-0.16.0.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/datasetexporter-update-to-0.16.0.yaml b/.chloggen/datasetexporter-update-to-0.16.0.yaml index d34faea08c28..5b4053cc98cc 100755 --- a/.chloggen/datasetexporter-update-to-0.16.0.yaml +++ b/.chloggen/datasetexporter-update-to-0.16.0.yaml @@ -7,7 +7,7 @@ change_type: enhancement component: datasetexporter # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: Collect usage metrics with Otel and sent grouped attributes in session info. +note: Collect usage metrics with Otel and send grouped attributes in session info. # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. issues: [27650, 27652] From 78e874b4023e93e73088a6b431f57b99b7758947 Mon Sep 17 00:00:00 2001 From: Martin Majlis <122797378+martin-majlis-s1@users.noreply.github.com> Date: Mon, 27 Nov 2023 10:06:27 +0100 Subject: [PATCH 08/12] Update exporter/datasetexporter/README.md Co-authored-by: Andrzej Stencel --- exporter/datasetexporter/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/datasetexporter/README.md b/exporter/datasetexporter/README.md index f9464490d8a2..e2db8f31ff06 100644 --- a/exporter/datasetexporter/README.md +++ b/exporter/datasetexporter/README.md @@ -304,7 +304,7 @@ To enable metrics you have to: exporters: [otlphttp/prometheus, debug] ``` -#### Available Metrics +### Available Metrics Available metrics contain `dataset` in their name. There are counters related to the number of processed events (`events`), buffers (`buffer`), and transferred bytes (`bytes`). From c45046e29feba46f1922ed2061c8a95bfd28865c Mon Sep 17 00:00:00 2001 From: Martin Majlis Date: Mon, 27 Nov 2023 13:48:15 +0100 Subject: [PATCH 09/12] Update library to enable attributes --- cmd/configschema/go.mod | 2 +- cmd/configschema/go.sum | 4 ++-- cmd/otelcontribcol/go.mod | 2 +- cmd/otelcontribcol/go.sum | 4 ++-- exporter/datasetexporter/datasetexporter.go | 10 +++++++++- exporter/datasetexporter/go.mod | 2 +- exporter/datasetexporter/go.sum | 2 ++ exporter/datasetexporter/traces_exporter.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- 10 files changed, 22 insertions(+), 12 deletions(-) diff --git a/cmd/configschema/go.mod b/cmd/configschema/go.mod index c1d8f554f9af..85d543aae8d3 100644 --- a/cmd/configschema/go.mod +++ b/cmd/configschema/go.mod @@ -572,7 +572,7 @@ require ( github.com/sagikazarmark/locafero v0.3.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 // indirect - github.com/scalyr/dataset-go v0.16.0 // indirect + github.com/scalyr/dataset-go v0.17.0 // indirect github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 // indirect github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect github.com/segmentio/asm v1.2.0 // indirect diff --git a/cmd/configschema/go.sum b/cmd/configschema/go.sum index a26896fb1245..aa37e73dc408 100644 --- a/cmd/configschema/go.sum +++ b/cmd/configschema/go.sum @@ -1399,8 +1399,8 @@ github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 h1:yWfiTPwYxB0l5fGMhl/G+liULugVIHD9AU77iNLrURQ= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= -github.com/scalyr/dataset-go v0.16.0 h1:69FIJzavveWAgaH4kzngI6xxxI5qn3MdevXCXmJEqo4= -github.com/scalyr/dataset-go v0.16.0/go.mod h1:ehHlPsZSgFWxOkud1eKwmKd5bLF9LcUFrU01XuCnh+8= +github.com/scalyr/dataset-go v0.17.0 h1:5YI/VlbLHr4Ui6SegWm0yjZYioypWB68U7nFQBKNOn8= +github.com/scalyr/dataset-go v0.17.0/go.mod h1:ehHlPsZSgFWxOkud1eKwmKd5bLF9LcUFrU01XuCnh+8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds= diff --git a/cmd/otelcontribcol/go.mod b/cmd/otelcontribcol/go.mod index 51c0e3bfb313..d286e8d505b8 100644 --- a/cmd/otelcontribcol/go.mod +++ b/cmd/otelcontribcol/go.mod @@ -591,7 +591,7 @@ require ( github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/samber/lo v1.38.1 // indirect github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 // indirect - github.com/scalyr/dataset-go v0.16.0 // indirect + github.com/scalyr/dataset-go v0.17.0 // indirect github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 // indirect github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect github.com/segmentio/asm v1.2.0 // indirect diff --git a/cmd/otelcontribcol/go.sum b/cmd/otelcontribcol/go.sum index 4c0fa2a3817d..eb995dad9fbe 100644 --- a/cmd/otelcontribcol/go.sum +++ b/cmd/otelcontribcol/go.sum @@ -1395,8 +1395,8 @@ github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 h1:yWfiTPwYxB0l5fGMhl/G+liULugVIHD9AU77iNLrURQ= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= -github.com/scalyr/dataset-go v0.16.0 h1:69FIJzavveWAgaH4kzngI6xxxI5qn3MdevXCXmJEqo4= -github.com/scalyr/dataset-go v0.16.0/go.mod h1:ehHlPsZSgFWxOkud1eKwmKd5bLF9LcUFrU01XuCnh+8= +github.com/scalyr/dataset-go v0.17.0 h1:5YI/VlbLHr4Ui6SegWm0yjZYioypWB68U7nFQBKNOn8= +github.com/scalyr/dataset-go v0.17.0/go.mod h1:ehHlPsZSgFWxOkud1eKwmKd5bLF9LcUFrU01XuCnh+8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds= diff --git a/exporter/datasetexporter/datasetexporter.go b/exporter/datasetexporter/datasetexporter.go index 806062fdf88e..0c3362bcbf2a 100644 --- a/exporter/datasetexporter/datasetexporter.go +++ b/exporter/datasetexporter/datasetexporter.go @@ -6,6 +6,7 @@ package datasetexporter // import "github.com/open-telemetry/opentelemetry-colle import ( "context" "fmt" + "github.com/scalyr/dataset-go/pkg/meter_config" "net/http" "reflect" "strconv" @@ -32,6 +33,8 @@ func newDatasetExporter(entity string, config *Config, set exporter.CreateSettin logger.Info("Creating new DataSetExporter", zap.String("config", config.String()), zap.String("entity", entity), + zap.String("id.string", set.ID.String()), + zap.String("id.name", set.ID.Name()), ) exporterCfg, err := config.convert() if err != nil { @@ -48,13 +51,18 @@ func newDatasetExporter(entity string, config *Config, set exporter.CreateSettin ) meter := set.MeterProvider.Meter("datasetexporter") + meterConfig := meter_config.NewMeterConfig( + &meter, + entity, + set.ID.Name(), + ) client, err := client.NewClient( exporterCfg.datasetConfig, &http.Client{Timeout: time.Second * 60}, logger, &userAgent, - &meter, + meterConfig, ) if err != nil { logger.Error("Cannot create DataSetClient: ", zap.Error(err)) diff --git a/exporter/datasetexporter/go.mod b/exporter/datasetexporter/go.mod index 3b881c2095d4..1b18b7c5089a 100644 --- a/exporter/datasetexporter/go.mod +++ b/exporter/datasetexporter/go.mod @@ -6,7 +6,7 @@ require ( github.com/google/uuid v1.4.0 // github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage v0.77.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.89.0 - github.com/scalyr/dataset-go v0.16.0 + github.com/scalyr/dataset-go v0.17.0 github.com/stretchr/testify v1.8.4 go.opentelemetry.io/collector/component v0.89.0 go.opentelemetry.io/collector/confmap v0.89.0 diff --git a/exporter/datasetexporter/go.sum b/exporter/datasetexporter/go.sum index 2ca57df0ef52..aab1f8fee767 100644 --- a/exporter/datasetexporter/go.sum +++ b/exporter/datasetexporter/go.sum @@ -89,6 +89,8 @@ github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/scalyr/dataset-go v0.16.0 h1:69FIJzavveWAgaH4kzngI6xxxI5qn3MdevXCXmJEqo4= github.com/scalyr/dataset-go v0.16.0/go.mod h1:ehHlPsZSgFWxOkud1eKwmKd5bLF9LcUFrU01XuCnh+8= +github.com/scalyr/dataset-go v0.17.0 h1:5YI/VlbLHr4Ui6SegWm0yjZYioypWB68U7nFQBKNOn8= +github.com/scalyr/dataset-go v0.17.0/go.mod h1:ehHlPsZSgFWxOkud1eKwmKd5bLF9LcUFrU01XuCnh+8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= diff --git a/exporter/datasetexporter/traces_exporter.go b/exporter/datasetexporter/traces_exporter.go index 6eb0df6ef913..ea352b7e02f5 100644 --- a/exporter/datasetexporter/traces_exporter.go +++ b/exporter/datasetexporter/traces_exporter.go @@ -21,7 +21,7 @@ const ServiceNameKey = "service.name" func createTracesExporter(ctx context.Context, set exporter.CreateSettings, config component.Config) (exporter.Traces, error) { cfg := castConfig(config) - e, err := newDatasetExporter("logs", cfg, set) + e, err := newDatasetExporter("traces", cfg, set) if err != nil { return nil, fmt.Errorf("cannot get DataSetExporter: %w", err) } diff --git a/go.mod b/go.mod index fcd96d8931ef..375bd47f8406 100644 --- a/go.mod +++ b/go.mod @@ -575,7 +575,7 @@ require ( github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/samber/lo v1.38.1 // indirect github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 // indirect - github.com/scalyr/dataset-go v0.16.0 // indirect + github.com/scalyr/dataset-go v0.17.0 // indirect github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 // indirect github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect github.com/segmentio/asm v1.2.0 // indirect diff --git a/go.sum b/go.sum index 2fe72a27798f..f83bb5a6eb15 100644 --- a/go.sum +++ b/go.sum @@ -1404,8 +1404,8 @@ github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21 h1:yWfiTPwYxB0l5fGMhl/G+liULugVIHD9AU77iNLrURQ= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.21/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= -github.com/scalyr/dataset-go v0.16.0 h1:69FIJzavveWAgaH4kzngI6xxxI5qn3MdevXCXmJEqo4= -github.com/scalyr/dataset-go v0.16.0/go.mod h1:ehHlPsZSgFWxOkud1eKwmKd5bLF9LcUFrU01XuCnh+8= +github.com/scalyr/dataset-go v0.17.0 h1:5YI/VlbLHr4Ui6SegWm0yjZYioypWB68U7nFQBKNOn8= +github.com/scalyr/dataset-go v0.17.0/go.mod h1:ehHlPsZSgFWxOkud1eKwmKd5bLF9LcUFrU01XuCnh+8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds= From d1b89227d26b71f1bdc60d74c2292fba29c2e8bb Mon Sep 17 00:00:00 2001 From: Martin Majlis Date: Mon, 27 Nov 2023 14:00:26 +0100 Subject: [PATCH 10/12] Run `make gotidy` one more time --- exporter/datasetexporter/go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/exporter/datasetexporter/go.sum b/exporter/datasetexporter/go.sum index aab1f8fee767..02281ad89c27 100644 --- a/exporter/datasetexporter/go.sum +++ b/exporter/datasetexporter/go.sum @@ -87,8 +87,6 @@ github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lne github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/scalyr/dataset-go v0.16.0 h1:69FIJzavveWAgaH4kzngI6xxxI5qn3MdevXCXmJEqo4= -github.com/scalyr/dataset-go v0.16.0/go.mod h1:ehHlPsZSgFWxOkud1eKwmKd5bLF9LcUFrU01XuCnh+8= github.com/scalyr/dataset-go v0.17.0 h1:5YI/VlbLHr4Ui6SegWm0yjZYioypWB68U7nFQBKNOn8= github.com/scalyr/dataset-go v0.17.0/go.mod h1:ehHlPsZSgFWxOkud1eKwmKd5bLF9LcUFrU01XuCnh+8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From c60935ed913afe2429b99cceabd45dc559d86f2a Mon Sep 17 00:00:00 2001 From: Martin Majlis Date: Mon, 27 Nov 2023 15:04:55 +0100 Subject: [PATCH 11/12] Fix imports order --- exporter/datasetexporter/datasetexporter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/datasetexporter/datasetexporter.go b/exporter/datasetexporter/datasetexporter.go index 0c3362bcbf2a..70c962730b14 100644 --- a/exporter/datasetexporter/datasetexporter.go +++ b/exporter/datasetexporter/datasetexporter.go @@ -6,7 +6,6 @@ package datasetexporter // import "github.com/open-telemetry/opentelemetry-colle import ( "context" "fmt" - "github.com/scalyr/dataset-go/pkg/meter_config" "net/http" "reflect" "strconv" @@ -15,6 +14,7 @@ import ( "github.com/google/uuid" "github.com/scalyr/dataset-go/pkg/api/add_events" "github.com/scalyr/dataset-go/pkg/client" + "github.com/scalyr/dataset-go/pkg/meter_config" "go.opentelemetry.io/collector/exporter" "go.opentelemetry.io/collector/pdata/pcommon" "go.uber.org/zap" From ecadde1b919f05c5880742d1729a3177c6b18a5d Mon Sep 17 00:00:00 2001 From: Martin Majlis Date: Tue, 28 Nov 2023 10:03:16 +0100 Subject: [PATCH 12/12] Incorporate comments from PR --- exporter/datasetexporter/config.go | 4 ++-- exporter/datasetexporter/config_test.go | 2 +- exporter/datasetexporter/factory_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/exporter/datasetexporter/config.go b/exporter/datasetexporter/config.go index 1bad6606d20d..4db73a648bd1 100644 --- a/exporter/datasetexporter/config.go +++ b/exporter/datasetexporter/config.go @@ -5,7 +5,6 @@ package datasetexporter // import "github.com/open-telemetry/opentelemetry-colle import ( "fmt" - "strings" "time" "github.com/cenkalti/backoff/v4" @@ -184,9 +183,10 @@ func (c *Config) Validate() error { // String returns a string representation of the Config object. // It includes all the fields and their values in the format "field_name: field_value". func (c *Config) String() string { + apiKey, _ := c.APIKey.MarshalText() s := "" s += fmt.Sprintf("%s: %s; ", "DatasetURL", c.DatasetURL) - s += fmt.Sprintf("%s: %s (%d); ", "APIKey", strings.Repeat("*", len(c.APIKey)), len(c.APIKey)) + s += fmt.Sprintf("%s: %s (%d); ", "APIKey", apiKey, len(c.APIKey)) s += fmt.Sprintf("%s: %t; ", "Debug", c.Debug) s += fmt.Sprintf("%s: %+v; ", "BufferSettings", c.BufferSettings) s += fmt.Sprintf("%s: %+v; ", "LogsSettings", c.LogsSettings) diff --git a/exporter/datasetexporter/config_test.go b/exporter/datasetexporter/config_test.go index b0191d383a14..3a019974b039 100644 --- a/exporter/datasetexporter/config_test.go +++ b/exporter/datasetexporter/config_test.go @@ -141,7 +141,7 @@ func TestConfigString(t *testing.T) { } assert.Equal(t, - "DatasetURL: https://example.com; APIKey: ****** (6); Debug: true; BufferSettings: {MaxLifetime:123ns GroupBy:[field1 field2] RetryInitialInterval:0s RetryMaxInterval:0s RetryMaxElapsedTime:0s RetryShutdownTimeout:0s}; LogsSettings: {ExportResourceInfo:true ExportResourcePrefix:AAA ExportScopeInfo:true ExportScopePrefix:BBB DecomposeComplexMessageField:true DecomposedComplexMessagePrefix:EEE exportSettings:{ExportSeparator:CCC ExportDistinguishingSuffix:DDD}}; TracesSettings: {exportSettings:{ExportSeparator:TTT ExportDistinguishingSuffix:UUU}}; ServerHostSettings: {UseHostName:false ServerHost:foo-bar}; RetrySettings: {Enabled:true InitialInterval:5s RandomizationFactor:0.5 Multiplier:1.5 MaxInterval:30s MaxElapsedTime:5m0s}; QueueSettings: {Enabled:true NumConsumers:10 QueueSize:1000 StorageID:}; TimeoutSettings: {Timeout:5s}", + "DatasetURL: https://example.com; APIKey: [REDACTED] (6); Debug: true; BufferSettings: {MaxLifetime:123ns GroupBy:[field1 field2] RetryInitialInterval:0s RetryMaxInterval:0s RetryMaxElapsedTime:0s RetryShutdownTimeout:0s}; LogsSettings: {ExportResourceInfo:true ExportResourcePrefix:AAA ExportScopeInfo:true ExportScopePrefix:BBB DecomposeComplexMessageField:true DecomposedComplexMessagePrefix:EEE exportSettings:{ExportSeparator:CCC ExportDistinguishingSuffix:DDD}}; TracesSettings: {exportSettings:{ExportSeparator:TTT ExportDistinguishingSuffix:UUU}}; ServerHostSettings: {UseHostName:false ServerHost:foo-bar}; RetrySettings: {Enabled:true InitialInterval:5s RandomizationFactor:0.5 Multiplier:1.5 MaxInterval:30s MaxElapsedTime:5m0s}; QueueSettings: {Enabled:true NumConsumers:10 QueueSize:1000 StorageID:}; TimeoutSettings: {Timeout:5s}", config.String(), ) } diff --git a/exporter/datasetexporter/factory_test.go b/exporter/datasetexporter/factory_test.go index 932c84f7aead..bda8aa484cdc 100644 --- a/exporter/datasetexporter/factory_test.go +++ b/exporter/datasetexporter/factory_test.go @@ -161,7 +161,7 @@ func createExporterTests() []CreateTest { { name: "broken", config: &Config{}, - expectedError: fmt.Errorf("cannot get DataSetExporter: cannot convert config: DatasetURL: ; APIKey: (0); Debug: false; BufferSettings: {MaxLifetime:0s GroupBy:[] RetryInitialInterval:0s RetryMaxInterval:0s RetryMaxElapsedTime:0s RetryShutdownTimeout:0s}; LogsSettings: {ExportResourceInfo:false ExportResourcePrefix: ExportScopeInfo:false ExportScopePrefix: DecomposeComplexMessageField:false DecomposedComplexMessagePrefix: exportSettings:{ExportSeparator: ExportDistinguishingSuffix:}}; TracesSettings: {exportSettings:{ExportSeparator: ExportDistinguishingSuffix:}}; ServerHostSettings: {UseHostName:false ServerHost:}; RetrySettings: {Enabled:false InitialInterval:0s RandomizationFactor:0 Multiplier:0 MaxInterval:0s MaxElapsedTime:0s}; QueueSettings: {Enabled:false NumConsumers:0 QueueSize:0 StorageID:}; TimeoutSettings: {Timeout:0s}; config is not valid: api_key is required"), + expectedError: fmt.Errorf("cannot get DataSetExporter: cannot convert config: DatasetURL: ; APIKey: [REDACTED] (0); Debug: false; BufferSettings: {MaxLifetime:0s GroupBy:[] RetryInitialInterval:0s RetryMaxInterval:0s RetryMaxElapsedTime:0s RetryShutdownTimeout:0s}; LogsSettings: {ExportResourceInfo:false ExportResourcePrefix: ExportScopeInfo:false ExportScopePrefix: DecomposeComplexMessageField:false DecomposedComplexMessagePrefix: exportSettings:{ExportSeparator: ExportDistinguishingSuffix:}}; TracesSettings: {exportSettings:{ExportSeparator: ExportDistinguishingSuffix:}}; ServerHostSettings: {UseHostName:false ServerHost:}; RetrySettings: {Enabled:false InitialInterval:0s RandomizationFactor:0 Multiplier:0 MaxInterval:0s MaxElapsedTime:0s}; QueueSettings: {Enabled:false NumConsumers:0 QueueSize:0 StorageID:}; TimeoutSettings: {Timeout:0s}; config is not valid: api_key is required"), }, { name: "valid",