[datasetexporter] Upgrade to dataset-go v0.20.0 (open-telemetry#33812)

**Description:** Upgrade to dataset-go v0.20.0 In the PR open-telemetry#31293 we have introduced releasing unused resources. However there was a bug that was occasional SIGSEVs. This was fixed in scalyr/dataset-go#100. Therefore this PR is: * upgrading to `dataset-go` v0.20.0 - https://github.com/scalyr/dataset-go/releases/tag/v0.20.0 - which contains the fix * introducing configuration option `buffer.max_parallel_outgoing` to control the maximum number of outgoing connections. **Link to tracking Issue:** open-telemetry#33812 **Testing:** Unit tests and stress tests **Documentation:** <img width="1490" alt="Screenshot 2024-06-27 at 12 04 24" src="https://github.com/open-telemetry/opentelemetry-collector-contrib/assets/122797378/2ebb30d7-e362-49ec-995a-5d9f43d6bb5a">
coralogix · Jul 1, 2024 · 5260327 · 5260327
1 parent 0814644
commit 5260327
Show file tree

Hide file tree

Showing 12 changed files with 46 additions and 13 deletions.
diff --git a/.chloggen/datasetexporter-update-to-0.20.0.yaml b/.chloggen/datasetexporter-update-to-0.20.0.yaml
@@ -0,0 +1,29 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: 'bug_fix'
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: datasetexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Upgrade dataset-go to v0.20.0
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [33812]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  Upgrade `dataset-go` library from v0.19.0 to v0.20.0.
+  Make number of outgoing connections configurable.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []
diff --git a/cmd/otelcontribcol/go.mod b/cmd/otelcontribcol/go.mod
@@ -429,7 +429,6 @@ require (
 	github.com/containerd/ttrpc v1.2.2 // indirect
 	github.com/coreos/go-oidc/v3 v3.10.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
-	github.com/cskr/pubsub v1.0.2 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/danieljoos/wincred v1.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
@@ -693,7 +692,7 @@ require (
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/samber/lo v1.38.1 // indirect
 	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.25 // indirect
-	github.com/scalyr/dataset-go v0.19.0 // indirect
+	github.com/scalyr/dataset-go v0.20.0 // indirect
 	github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 // indirect
 	github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect
 	github.com/segmentio/asm v1.2.0 // indirect

diff --git a/cmd/otelcontribcol/go.sum b/cmd/otelcontribcol/go.sum
diff --git a/exporter/datasetexporter/README.md b/exporter/datasetexporter/README.md
@@ -54,6 +54,7 @@ Make sure to provide the appropriate server host value in the `serverHost` attri
   - `retry_max_interval` (default = 30s): Is the upper bound on backoff.
   - `retry_max_elapsed_time` (default = 300s): Is the maximum amount of time spent trying to send a buffer.
   - `retry_shutdown_timeout` (default = 30s): The maximum time for which it will try to send data to the DataSet during shutdown. This value should be shorter than container's grace period.
+  - `max_parallel_outgoing` (default = 100): The maximum number of parallel outgoing requests.
 - `logs`:
     - `export_resource_info_on_event` (default = false): Include LogRecord resource information (if available) on the DataSet event.
     - `export_resource_prefix` (default = 'resource.attributes.'): A prefix string for the resource, if `export_resource_info_on_event` is enabled.

diff --git a/exporter/datasetexporter/config.go b/exporter/datasetexporter/config.go
@@ -111,6 +111,7 @@ const bufferRetryInitialInterval = 5 * time.Second
 const bufferRetryMaxInterval = 30 * time.Second
 const bufferRetryMaxElapsedTime = 300 * time.Second
 const bufferRetryShutdownTimeout = 30 * time.Second
+const bufferMaxParallelOutgoing = 100
 
 type BufferSettings struct {
 	MaxLifetime          time.Duration `mapstructure:"max_lifetime"`
@@ -120,6 +121,7 @@ type BufferSettings struct {
 	RetryMaxInterval     time.Duration `mapstructure:"retry_max_interval"`
 	RetryMaxElapsedTime  time.Duration `mapstructure:"retry_max_elapsed_time"`
 	RetryShutdownTimeout time.Duration `mapstructure:"retry_shutdown_timeout"`
+	MaxParallelOutgoing  int           `mapstructure:"max_parallel_outgoing"`
 }
 
 // newDefaultBufferSettings returns the default settings for BufferSettings.
@@ -132,6 +134,7 @@ func newDefaultBufferSettings() BufferSettings {
 		RetryMaxInterval:     bufferRetryMaxInterval,
 		RetryMaxElapsedTime:  bufferRetryMaxElapsedTime,
 		RetryShutdownTimeout: bufferRetryShutdownTimeout,
+		MaxParallelOutgoing:  bufferMaxParallelOutgoing,
 	}
 }
 
@@ -218,6 +221,7 @@ func (c *Config) convert() *ExporterConfig {
 				RetryMultiplier:          backoff.DefaultMultiplier,
 				RetryRandomizationFactor: backoff.DefaultRandomizationFactor,
 				RetryShutdownTimeout:     c.BufferSettings.RetryShutdownTimeout,
+				MaxParallelOutgoing:      c.BufferSettings.MaxParallelOutgoing,
 			},
 			ServerHostSettings: server_host_config.DataSetServerHostSettings{
 				UseHostName: c.ServerHostSettings.UseHostName,

diff --git a/exporter/datasetexporter/config_test.go b/exporter/datasetexporter/config_test.go
@@ -143,7 +143,7 @@ func TestConfigString(t *testing.T) {
 	}
 
 	assert.Equal(t,
-		"DatasetURL: https://example.com; APIKey: [REDACTED] (6); Debug: true; BufferSettings: {MaxLifetime:123ns PurgeOlderThan:567ns GroupBy:[field1 field2] RetryInitialInterval:0s RetryMaxInterval:0s RetryMaxElapsedTime:0s RetryShutdownTimeout:0s}; LogsSettings: {ExportResourceInfo:true ExportResourcePrefix:AAA ExportScopeInfo:true ExportScopePrefix:BBB DecomposeComplexMessageField:true DecomposedComplexMessagePrefix:EEE exportSettings:{ExportSeparator:CCC ExportDistinguishingSuffix:DDD}}; TracesSettings: {exportSettings:{ExportSeparator:TTT ExportDistinguishingSuffix:UUU}}; ServerHostSettings: {UseHostName:false ServerHost:foo-bar}; BackOffConfig: {Enabled:true InitialInterval:5s RandomizationFactor:0.5 Multiplier:1.5 MaxInterval:30s MaxElapsedTime:5m0s}; QueueSettings: {Enabled:true NumConsumers:10 QueueSize:1000 StorageID:<nil>}; TimeoutSettings: {Timeout:5s}",
+		"DatasetURL: https://example.com; APIKey: [REDACTED] (6); Debug: true; BufferSettings: {MaxLifetime:123ns PurgeOlderThan:567ns GroupBy:[field1 field2] RetryInitialInterval:0s RetryMaxInterval:0s RetryMaxElapsedTime:0s RetryShutdownTimeout:0s MaxParallelOutgoing:0}; LogsSettings: {ExportResourceInfo:true ExportResourcePrefix:AAA ExportScopeInfo:true ExportScopePrefix:BBB DecomposeComplexMessageField:true DecomposedComplexMessagePrefix:EEE exportSettings:{ExportSeparator:CCC ExportDistinguishingSuffix:DDD}}; TracesSettings: {exportSettings:{ExportSeparator:TTT ExportDistinguishingSuffix:UUU}}; ServerHostSettings: {UseHostName:false ServerHost:foo-bar}; BackOffConfig: {Enabled:true InitialInterval:5s RandomizationFactor:0.5 Multiplier:1.5 MaxInterval:30s MaxElapsedTime:5m0s}; QueueSettings: {Enabled:true NumConsumers:10 QueueSize:1000 StorageID:<nil>}; TimeoutSettings: {Timeout:5s}",
 		config.String(),
 	)
 }

diff --git a/exporter/datasetexporter/factory_test.go b/exporter/datasetexporter/factory_test.go
@@ -72,6 +72,7 @@ func TestLoadConfig(t *testing.T) {
 					RetryMaxInterval:     bufferRetryMaxInterval,
 					RetryMaxElapsedTime:  bufferRetryMaxElapsedTime,
 					RetryShutdownTimeout: bufferRetryShutdownTimeout,
+					MaxParallelOutgoing:  bufferMaxParallelOutgoing,
 				},
 				TracesSettings:     newDefaultTracesSettings(),
 				LogsSettings:       newDefaultLogsSettings(),
@@ -95,6 +96,7 @@ func TestLoadConfig(t *testing.T) {
 					RetryMaxInterval:     22 * time.Second,
 					RetryMaxElapsedTime:  23 * time.Second,
 					RetryShutdownTimeout: 24 * time.Second,
+					MaxParallelOutgoing:  25,
 				},
 				TracesSettings: TracesSettings{
 					exportSettings: exportSettings{

diff --git a/exporter/datasetexporter/go.mod b/exporter/datasetexporter/go.mod
@@ -6,7 +6,7 @@ require (
 	github.com/google/uuid v1.6.0
 	// github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage v0.103.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.103.0
-	github.com/scalyr/dataset-go v0.19.0
+	github.com/scalyr/dataset-go v0.20.0
 	github.com/stretchr/testify v1.9.0
 	go.opentelemetry.io/collector/component v0.103.0
 	go.opentelemetry.io/collector/confmap v0.103.0
@@ -27,7 +27,6 @@ require (
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/cskr/pubsub v1.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect

diff --git a/exporter/datasetexporter/go.sum b/exporter/datasetexporter/go.sum
diff --git a/exporter/datasetexporter/logs_exporter_stress_test.go b/exporter/datasetexporter/logs_exporter_stress_test.go
@@ -85,6 +85,7 @@ func TestConsumeLogsManyLogsShouldSucceed(t *testing.T) {
 			RetryMaxElapsedTime:  50 * maxDelay,
 			RetryShutdownTimeout: time.Minute,
 			PurgeOlderThan:       100 * maxDelay,
+			MaxParallelOutgoing:  bufferMaxParallelOutgoing,
 		},
 		BackOffConfig:   configretry.NewDefaultBackOffConfig(),
 		QueueSettings:   exporterhelper.NewDefaultQueueSettings(),

diff --git a/exporter/datasetexporter/logs_exporter_test.go b/exporter/datasetexporter/logs_exporter_test.go
@@ -795,6 +795,7 @@ func TestConsumeLogsShouldSucceed(t *testing.T) {
 			RetryMaxInterval:     time.Minute,
 			RetryMaxElapsedTime:  time.Hour,
 			RetryShutdownTimeout: time.Minute,
+			MaxParallelOutgoing:  100,
 		},
 		LogsSettings: LogsSettings{
 			ExportResourceInfo:   true,

diff --git a/exporter/datasetexporter/testdata/config.yaml b/exporter/datasetexporter/testdata/config.yaml
@@ -26,6 +26,7 @@ dataset/full:
     retry_max_interval: 22s
     retry_max_elapsed_time: 23s
     retry_shutdown_timeout: 24s
+    max_parallel_outgoing: 25
   logs:
     export_resource_info_on_event: true
     export_resource_prefix: "_resource_"