Merge pull request #1712 from nak0f/elastic-compression

elasticsearch output compression_level option
kube-logging · Apr 2, 2024 · 73901c8 · 73901c8
2 parents 4e4283f + 9b07b28
commit 73901c8
Show file tree

Hide file tree

Showing 7 changed files with 42 additions and 0 deletions.
diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_clusteroutputs.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_clusteroutputs.yaml
@@ -279,6 +279,8 @@ spec:
                             type: object
                         type: object
                     type: object
+                  compression_level:
+                    type: string
                   content_type:
                     type: string
                   custom_headers:
@@ -1569,6 +1571,8 @@ spec:
                             type: object
                         type: object
                     type: object
+                  compression_level:
+                    type: string
                   content_type:
                     type: string
                   custom_headers:
@@ -7207,6 +7211,8 @@ spec:
                             type: object
                         type: object
                     type: object
+                  compression_level:
+                    type: string
                   content_type:
                     type: string
                   custom_headers:
@@ -8497,6 +8503,8 @@ spec:
                             type: object
                         type: object
                     type: object
+                  compression_level:
+                    type: string
                   content_type:
                     type: string
                   custom_headers:

diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_outputs.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_outputs.yaml
@@ -279,6 +279,8 @@ spec:
                             type: object
                         type: object
                     type: object
+                  compression_level:
+                    type: string
                   content_type:
                     type: string
                   custom_headers:
@@ -1569,6 +1571,8 @@ spec:
                             type: object
                         type: object
                     type: object
+                  compression_level:
+                    type: string
                   content_type:
                     type: string
                   custom_headers:
@@ -6519,6 +6523,8 @@ spec:
                             type: object
                         type: object
                     type: object
+                  compression_level:
+                    type: string
                   content_type:
                     type: string
                   custom_headers:
@@ -7809,6 +7815,8 @@ spec:
                             type: object
                         type: object
                     type: object
+                  compression_level:
+                    type: string
                   content_type:
                     type: string
                   custom_headers:

diff --git a/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml b/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml
@@ -279,6 +279,8 @@ spec:
                             type: object
                         type: object
                     type: object
+                  compression_level:
+                    type: string
                   content_type:
                     type: string
                   custom_headers:
@@ -1569,6 +1571,8 @@ spec:
                             type: object
                         type: object
                     type: object
+                  compression_level:
+                    type: string
                   content_type:
                     type: string
                   custom_headers:
@@ -7207,6 +7211,8 @@ spec:
                             type: object
                         type: object
                     type: object
+                  compression_level:
+                    type: string
                   content_type:
                     type: string
                   custom_headers:
@@ -8497,6 +8503,8 @@ spec:
                             type: object
                         type: object
                     type: object
+                  compression_level:
+                    type: string
                   content_type:
                     type: string
                   custom_headers:

diff --git a/config/crd/bases/logging.banzaicloud.io_outputs.yaml b/config/crd/bases/logging.banzaicloud.io_outputs.yaml
@@ -279,6 +279,8 @@ spec:
                             type: object
                         type: object
                     type: object
+                  compression_level:
+                    type: string
                   content_type:
                     type: string
                   custom_headers:
@@ -1569,6 +1571,8 @@ spec:
                             type: object
                         type: object
                     type: object
+                  compression_level:
+                    type: string
                   content_type:
                     type: string
                   custom_headers:
@@ -6519,6 +6523,8 @@ spec:
                             type: object
                         type: object
                     type: object
+                  compression_level:
+                    type: string
                   content_type:
                     type: string
                   custom_headers:
@@ -7809,6 +7815,8 @@ spec:
                             type: object
                         type: object
                     type: object
+                  compression_level:
+                    type: string
                   content_type:
                     type: string
                   custom_headers:

diff --git a/docs/configuration/plugins/outputs/elasticsearch.md b/docs/configuration/plugins/outputs/elasticsearch.md
@@ -54,6 +54,12 @@ Configure bulk_message request splitting threshold size. Default value is 20MB.
 
 Default: 20MB
 
+### compression_level (string, optional) {#elasticsearch-compression_level}
+
+Option for adding gzip compression of output data. Valid options: default_compression, best_compression, best_speed, no_compression.
+
+Default: no_compression
+
 ### content_type (string, optional) {#elasticsearch-content_type}
 
 With content_type application/x-ndjson, elasticsearch plugin adds application/x-ndjson as Content-Profile in payload.

diff --git a/pkg/sdk/logging/model/output/elasticsearch.go b/pkg/sdk/logging/model/output/elasticsearch.go
@@ -263,6 +263,8 @@ type ElasticsearchOutput struct {
 	// If set to true, the output uses the [legacy index template format](https://www.elastic.co/guide/en/elasticsearch/reference/7.13/indices-templates-v1.html). Otherwise, it uses the [composable index template](https://www.elastic.co/guide/en/elasticsearch/reference/7.13/index-templates.html) format. (default: true)
 	// +kubebuilder:validation:Optional
 	UseLegacyTemplate *bool `json:"use_legacy_template,omitempty"`
+	// Option for adding gzip compression of output data. Valid options: default_compression, best_compression, best_speed, no_compression. (default: no_compression)
+	CompressionLevel string `json:"compression_level,omitempty"`
 }
 
 func (e *ElasticsearchOutput) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) {

diff --git a/pkg/sdk/logging/model/output/elasticsearch_test.go b/pkg/sdk/logging/model/output/elasticsearch_test.go
@@ -34,11 +34,13 @@ buffer:
   timekey: 1m
   timekey_wait: 30s
   timekey_use_utc: true
+compression_level: default_compression
 `)
 	expected := `
   <match **>
 	@type elasticsearch
 	@id test
+	compression_level default_compression
 	exception_backup true
 	fail_on_detecting_es_version_retry_exceed true
 	fail_on_putting_template_retry_exceed true