From 4750319356256478d293018052ee06c38ece2648 Mon Sep 17 00:00:00 2001 From: Veprim Krasnici Date: Wed, 20 Nov 2024 17:54:51 +0000 Subject: [PATCH 1/8] Added support for base64 encoded ieee float32 types --- plugins/processors/converter/converter.go | 79 +++++++++++++++---- .../processors/converter/converter_test.go | 28 +++++++ 2 files changed, 90 insertions(+), 17 deletions(-) diff --git a/plugins/processors/converter/converter.go b/plugins/processors/converter/converter.go index 8ca1c021fe16f..3c5b6907b731d 100644 --- a/plugins/processors/converter/converter.go +++ b/plugins/processors/converter/converter.go @@ -3,9 +3,12 @@ package converter import ( _ "embed" + "encoding/base64" "errors" + "fmt" "math" "math/big" + "strconv" "strings" "github.com/influxdata/telegraf" @@ -18,15 +21,16 @@ import ( var sampleConfig string type Conversion struct { - Measurement []string `toml:"measurement"` - Tag []string `toml:"tag"` - String []string `toml:"string"` - Integer []string `toml:"integer"` - Unsigned []string `toml:"unsigned"` - Boolean []string `toml:"boolean"` - Float []string `toml:"float"` - Timestamp []string `toml:"timestamp"` - TimestampFormat string `toml:"timestamp_format"` + Measurement []string `toml:"measurement"` + Tag []string `toml:"tag"` + String []string `toml:"string"` + Integer []string `toml:"integer"` + Unsigned []string `toml:"unsigned"` + Boolean []string `toml:"boolean"` + Float []string `toml:"float"` + Timestamp []string `toml:"timestamp"` + TimestampFormat string `toml:"timestamp_format"` + Base64IEEEFloat32 []string `toml:"base64_ieee_float32"` } type Converter struct { @@ -39,14 +43,15 @@ type Converter struct { } type ConversionFilter struct { - Measurement filter.Filter - Tag filter.Filter - String filter.Filter - Integer filter.Filter - Unsigned filter.Filter - Boolean filter.Filter - Float filter.Filter - Timestamp filter.Filter + Measurement filter.Filter + Tag filter.Filter + String filter.Filter + Integer filter.Filter + Unsigned filter.Filter + Boolean filter.Filter + Float filter.Filter + Timestamp filter.Filter + Base64IEEEFloat32 filter.Filter } func (*Converter) SampleConfig() string { @@ -132,6 +137,11 @@ func compileFilter(conv *Conversion) (*ConversionFilter, error) { return nil, err } + cf.Base64IEEEFloat32, err = filter.Compile(conv.Base64IEEEFloat32) + if err != nil { + return nil, err + } + return cf, nil } @@ -249,6 +259,14 @@ func (p *Converter) convertFields(metric telegraf.Metric) { metric.SetTime(time) metric.RemoveField(key) } + + case p.fieldConversions.Base64IEEEFloat32 != nil && p.fieldConversions.Base64IEEEFloat32.Match(key): + if v, err := base64ToFloat32(value.(string)); err != nil { + p.Log.Errorf("Converting to base64_ieee_float32 [%T] failed: %v", value, err) + metric.RemoveField(key) + } else { + metric.AddField(key, v) + } } } } @@ -345,6 +363,33 @@ func toFloat(v interface{}) (float64, error) { return internal.ToFloat64(v) } +func base64ToFloat32(encoded string) (float32, error) { + // Decode the Base64 string to bytes + decodedBytes, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return 0, err + } + + // Check if the byte length matches a float32 (4 bytes) + if len(decodedBytes) != 4 { + return 0, errors.New("decoded byte length is not 4 bytes") + } + + // Convert the bytes to a string representation as per IEEE 754 of the bits + bits_str_representation := fmt.Sprintf("%08b%08b%08b%08b", decodedBytes[0], decodedBytes[1], decodedBytes[2], decodedBytes[3]) + + // Convert the bits to a uint32 + bits, err := strconv.ParseUint(bits_str_representation, 2, 32) + + if err != nil { + return 0, err + } + + // Convert the uint32 (bits) to a float32 based on IEEE 754 binary representation + return math.Float32frombits(uint32(bits)), nil + +} + func init() { processors.Add("converter", func() telegraf.Processor { return &Converter{} diff --git a/plugins/processors/converter/converter_test.go b/plugins/processors/converter/converter_test.go index 7ab9ea79cae1e..45a539454de6e 100644 --- a/plugins/processors/converter/converter_test.go +++ b/plugins/processors/converter/converter_test.go @@ -770,6 +770,34 @@ func TestMeasurement(t *testing.T) { ), }, }, + { + name: "float32 from ieee754 float32 encoded as base64", + converter: &Converter{ + Fields: &Conversion{ + Base64IEEEFloat32: []string{"a", "b"}, + }, + }, + input: testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "a": "QlAAAA==", + "b": "QlgAAA==", + }, + time.Unix(0, 0), + ), + expected: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "a": float32(52), + "b": float32(54), + }, + time.Unix(0, 0), + ), + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 13702c383feda3dfa003c4b75ec860dba897ec86 Mon Sep 17 00:00:00 2001 From: Veprim Krasnici Date: Wed, 20 Nov 2024 18:47:18 +0000 Subject: [PATCH 2/8] feat: updated readme for converter --- plugins/processors/converter/README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/plugins/processors/converter/README.md b/plugins/processors/converter/README.md index 19977dabf1879..f5c1b38047bc7 100644 --- a/plugins/processors/converter/README.md +++ b/plugins/processors/converter/README.md @@ -1,12 +1,12 @@ # Converter Processor Plugin -The converter processor is used to change the type of tag or field values. In +The converter processor is used to change the type of tag or field values. In addition to changing field types it can convert between fields and tags. Values that cannot be converted are dropped. **Note:** When converting tags to fields, take care to ensure the series is -still uniquely identifiable. Fields with the same series key (measurement + +still uniquely identifiable. Fields with the same series key (measurement + tags) will overwrite one another. **Note on large strings being converted to numeric types:** When converting a @@ -67,6 +67,12 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. boolean = [] float = [] + # Optional field to use for converting base64 encoding of IEEE 754 Float32 values + # i.e. data_json_content_state_openconfig-platform-psu:output-power":"RKeAAA==" + # into a float32 value 1340 + + # base64_ieee_float32 = [] + ## Optional field to use as metric timestamp # timestamp = [] From 58a71c9f2e9500d56c12baea4bcaceb2f3097d97 Mon Sep 17 00:00:00 2001 From: Veprim Krasnici Date: Fri, 22 Nov 2024 07:00:47 +0000 Subject: [PATCH 3/8] run make docs --- plugins/processors/converter/README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/plugins/processors/converter/README.md b/plugins/processors/converter/README.md index f5c1b38047bc7..036f085560af1 100644 --- a/plugins/processors/converter/README.md +++ b/plugins/processors/converter/README.md @@ -67,12 +67,6 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. boolean = [] float = [] - # Optional field to use for converting base64 encoding of IEEE 754 Float32 values - # i.e. data_json_content_state_openconfig-platform-psu:output-power":"RKeAAA==" - # into a float32 value 1340 - - # base64_ieee_float32 = [] - ## Optional field to use as metric timestamp # timestamp = [] From c95d470c38241a1d229226b676feed99ec68d978 Mon Sep 17 00:00:00 2001 From: Veprim Krasnici Date: Fri, 22 Nov 2024 11:01:32 +0000 Subject: [PATCH 4/8] FIxed the linting error --- docs/LICENSE_OF_DEPENDENCIES.md | 2 ++ plugins/processors/converter/converter.go | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/LICENSE_OF_DEPENDENCIES.md b/docs/LICENSE_OF_DEPENDENCIES.md index ae4696ad3cee3..3c32be6ea0d99 100644 --- a/docs/LICENSE_OF_DEPENDENCIES.md +++ b/docs/LICENSE_OF_DEPENDENCIES.md @@ -8,6 +8,7 @@ following works: - collectd.org [ISC License](https://github.com/collectd/go-collectd/blob/master/LICENSE) - dario.cat/mergo [BSD 3-Clause "New" or "Revised" License](https://github.com/imdario/mergo/blob/master/LICENSE) - filippo.io/edwards25519 [BSD 3-Clause "New" or "Revised" License](https://github.com/FiloSottile/edwards25519/blob/main/LICENSE) +- github.com/99designs/go-keychain [MIT License](https://github.com/99designs/go-keychain/blob/master/LICENSE) - github.com/99designs/keyring [MIT License](https://github.com/99designs/keyring/blob/master/LICENSE) - github.com/Azure/azure-amqp-common-go [MIT License](https://github.com/Azure/azure-amqp-common-go/blob/master/LICENSE) - github.com/Azure/azure-event-hubs-go [MIT License](https://github.com/Azure/azure-event-hubs-go/blob/master/LICENSE) @@ -139,6 +140,7 @@ following works: - github.com/gabriel-vasile/mimetype [MIT License](https://github.com/gabriel-vasile/mimetype/blob/master/LICENSE) - github.com/go-asn1-ber/asn1-ber [MIT License](https://github.com/go-asn1-ber/asn1-ber/blob/v1.3/LICENSE) - github.com/go-chi/chi [MIT License](https://github.com/go-chi/chi/blob/master/LICENSE) +- github.com/go-darwin/apfs [BSD 3-Clause "New" or "Revised" License] (https://github.com/go-darwin/apfs/blob/main/LICENSE) - github.com/go-git/go-billy [Apache License 2.0](https://github.com/go-git/go-billy/blob/master/LICENSE) - github.com/go-ldap/ldap [MIT License](https://github.com/go-ldap/ldap/blob/v3.4.1/LICENSE) - github.com/go-logfmt/logfmt [MIT License](https://github.com/go-logfmt/logfmt/blob/master/LICENSE) diff --git a/plugins/processors/converter/converter.go b/plugins/processors/converter/converter.go index 3c5b6907b731d..76081424b9e57 100644 --- a/plugins/processors/converter/converter.go +++ b/plugins/processors/converter/converter.go @@ -376,10 +376,10 @@ func base64ToFloat32(encoded string) (float32, error) { } // Convert the bytes to a string representation as per IEEE 754 of the bits - bits_str_representation := fmt.Sprintf("%08b%08b%08b%08b", decodedBytes[0], decodedBytes[1], decodedBytes[2], decodedBytes[3]) + bitsStrRepresentation := fmt.Sprintf("%08b%08b%08b%08b", decodedBytes[0], decodedBytes[1], decodedBytes[2], decodedBytes[3]) // Convert the bits to a uint32 - bits, err := strconv.ParseUint(bits_str_representation, 2, 32) + bits, err := strconv.ParseUint(bitsStrRepresentation, 2, 32) if err != nil { return 0, err @@ -387,7 +387,6 @@ func base64ToFloat32(encoded string) (float32, error) { // Convert the uint32 (bits) to a float32 based on IEEE 754 binary representation return math.Float32frombits(uint32(bits)), nil - } func init() { From aac2045b87bc60aa257a7331838c601f02204a55 Mon Sep 17 00:00:00 2001 From: Veprim Krasnici Date: Sun, 24 Nov 2024 21:31:31 +0000 Subject: [PATCH 5/8] Updated the files and all checks passed locally --- docs/LICENSE_OF_DEPENDENCIES.md | 2 +- plugins/processors/converter/README.md | 6 ++++++ plugins/processors/converter/sample.conf | 6 ++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/LICENSE_OF_DEPENDENCIES.md b/docs/LICENSE_OF_DEPENDENCIES.md index 3c32be6ea0d99..be3ca9a44f4bf 100644 --- a/docs/LICENSE_OF_DEPENDENCIES.md +++ b/docs/LICENSE_OF_DEPENDENCIES.md @@ -140,7 +140,7 @@ following works: - github.com/gabriel-vasile/mimetype [MIT License](https://github.com/gabriel-vasile/mimetype/blob/master/LICENSE) - github.com/go-asn1-ber/asn1-ber [MIT License](https://github.com/go-asn1-ber/asn1-ber/blob/v1.3/LICENSE) - github.com/go-chi/chi [MIT License](https://github.com/go-chi/chi/blob/master/LICENSE) -- github.com/go-darwin/apfs [BSD 3-Clause "New" or "Revised" License] (https://github.com/go-darwin/apfs/blob/main/LICENSE) +- github.com/go-darwin/apfs [BSD 3-Clause "New" or "Revised" License](https://github.com/go-darwin/apfs/blob/main/LICENSE) - github.com/go-git/go-billy [Apache License 2.0](https://github.com/go-git/go-billy/blob/master/LICENSE) - github.com/go-ldap/ldap [MIT License](https://github.com/go-ldap/ldap/blob/v3.4.1/LICENSE) - github.com/go-logfmt/logfmt [MIT License](https://github.com/go-logfmt/logfmt/blob/master/LICENSE) diff --git a/plugins/processors/converter/README.md b/plugins/processors/converter/README.md index 036f085560af1..f5c1b38047bc7 100644 --- a/plugins/processors/converter/README.md +++ b/plugins/processors/converter/README.md @@ -67,6 +67,12 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. boolean = [] float = [] + # Optional field to use for converting base64 encoding of IEEE 754 Float32 values + # i.e. data_json_content_state_openconfig-platform-psu:output-power":"RKeAAA==" + # into a float32 value 1340 + + # base64_ieee_float32 = [] + ## Optional field to use as metric timestamp # timestamp = [] diff --git a/plugins/processors/converter/sample.conf b/plugins/processors/converter/sample.conf index d162a1d15fe15..71a30cf3ab775 100644 --- a/plugins/processors/converter/sample.conf +++ b/plugins/processors/converter/sample.conf @@ -35,6 +35,12 @@ boolean = [] float = [] + # Optional field to use for converting base64 encoding of IEEE 754 Float32 values + # i.e. data_json_content_state_openconfig-platform-psu:output-power":"RKeAAA==" + # into a float32 value 1340 + + # base64_ieee_float32 = [] + ## Optional field to use as metric timestamp # timestamp = [] From 30ab6d344edb923afd39af5f20babe1d39ed0851 Mon Sep 17 00:00:00 2001 From: Veprim Krasnici Date: Sun, 24 Nov 2024 21:41:03 +0000 Subject: [PATCH 6/8] removed two entries on the dependencies list --- docs/LICENSE_OF_DEPENDENCIES.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/LICENSE_OF_DEPENDENCIES.md b/docs/LICENSE_OF_DEPENDENCIES.md index be3ca9a44f4bf..ae4696ad3cee3 100644 --- a/docs/LICENSE_OF_DEPENDENCIES.md +++ b/docs/LICENSE_OF_DEPENDENCIES.md @@ -8,7 +8,6 @@ following works: - collectd.org [ISC License](https://github.com/collectd/go-collectd/blob/master/LICENSE) - dario.cat/mergo [BSD 3-Clause "New" or "Revised" License](https://github.com/imdario/mergo/blob/master/LICENSE) - filippo.io/edwards25519 [BSD 3-Clause "New" or "Revised" License](https://github.com/FiloSottile/edwards25519/blob/main/LICENSE) -- github.com/99designs/go-keychain [MIT License](https://github.com/99designs/go-keychain/blob/master/LICENSE) - github.com/99designs/keyring [MIT License](https://github.com/99designs/keyring/blob/master/LICENSE) - github.com/Azure/azure-amqp-common-go [MIT License](https://github.com/Azure/azure-amqp-common-go/blob/master/LICENSE) - github.com/Azure/azure-event-hubs-go [MIT License](https://github.com/Azure/azure-event-hubs-go/blob/master/LICENSE) @@ -140,7 +139,6 @@ following works: - github.com/gabriel-vasile/mimetype [MIT License](https://github.com/gabriel-vasile/mimetype/blob/master/LICENSE) - github.com/go-asn1-ber/asn1-ber [MIT License](https://github.com/go-asn1-ber/asn1-ber/blob/v1.3/LICENSE) - github.com/go-chi/chi [MIT License](https://github.com/go-chi/chi/blob/master/LICENSE) -- github.com/go-darwin/apfs [BSD 3-Clause "New" or "Revised" License](https://github.com/go-darwin/apfs/blob/main/LICENSE) - github.com/go-git/go-billy [Apache License 2.0](https://github.com/go-git/go-billy/blob/master/LICENSE) - github.com/go-ldap/ldap [MIT License](https://github.com/go-ldap/ldap/blob/v3.4.1/LICENSE) - github.com/go-logfmt/logfmt [MIT License](https://github.com/go-logfmt/logfmt/blob/master/LICENSE) From 9cc2d160df4afc0cf407db770964af2369c081dc Mon Sep 17 00:00:00 2001 From: Veprim Krasnici Date: Mon, 25 Nov 2024 14:44:08 +0000 Subject: [PATCH 7/8] Updated readme.md and sample.conf as per @DStrand1 suggestions --- plugins/processors/converter/README.md | 6 +++--- plugins/processors/converter/sample.conf | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/processors/converter/README.md b/plugins/processors/converter/README.md index f5c1b38047bc7..e34b7c1c22f0f 100644 --- a/plugins/processors/converter/README.md +++ b/plugins/processors/converter/README.md @@ -67,9 +67,9 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. boolean = [] float = [] - # Optional field to use for converting base64 encoding of IEEE 754 Float32 values - # i.e. data_json_content_state_openconfig-platform-psu:output-power":"RKeAAA==" - # into a float32 value 1340 + ## Optional field to use for converting base64 encoding of IEEE 754 Float32 values + ## i.e. data_json_content_state_openconfig-platform-psu:output-power":"RKeAAA==" + ## into a float32 value 1340 # base64_ieee_float32 = [] diff --git a/plugins/processors/converter/sample.conf b/plugins/processors/converter/sample.conf index 71a30cf3ab775..826bcca54f394 100644 --- a/plugins/processors/converter/sample.conf +++ b/plugins/processors/converter/sample.conf @@ -35,9 +35,9 @@ boolean = [] float = [] - # Optional field to use for converting base64 encoding of IEEE 754 Float32 values - # i.e. data_json_content_state_openconfig-platform-psu:output-power":"RKeAAA==" - # into a float32 value 1340 + ## Optional field to use for converting base64 encoding of IEEE 754 Float32 values + ## i.e. data_json_content_state_openconfig-platform-psu:output-power":"RKeAAA==" + ## into a float32 value 1340 # base64_ieee_float32 = [] From 76c57390f230eb9cad405ba966b1bdead58c4117 Mon Sep 17 00:00:00 2001 From: Veprim Krasnici Date: Mon, 2 Dec 2024 11:15:57 +0000 Subject: [PATCH 8/8] Changed the sample conf to retrigger circle-ci checks --- plugins/processors/converter/README.md | 2 +- plugins/processors/converter/sample.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/processors/converter/README.md b/plugins/processors/converter/README.md index e34b7c1c22f0f..48228a04396db 100644 --- a/plugins/processors/converter/README.md +++ b/plugins/processors/converter/README.md @@ -67,7 +67,7 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details. boolean = [] float = [] - ## Optional field to use for converting base64 encoding of IEEE 754 Float32 values + ## Optional field to use for converting base64 encoding of IEEE 754 Float32 values ## i.e. data_json_content_state_openconfig-platform-psu:output-power":"RKeAAA==" ## into a float32 value 1340 diff --git a/plugins/processors/converter/sample.conf b/plugins/processors/converter/sample.conf index 826bcca54f394..91f6c0720d28c 100644 --- a/plugins/processors/converter/sample.conf +++ b/plugins/processors/converter/sample.conf @@ -35,7 +35,7 @@ boolean = [] float = [] - ## Optional field to use for converting base64 encoding of IEEE 754 Float32 values + ## Optional field to use for converting base64 encoding of IEEE 754 Float32 values ## i.e. data_json_content_state_openconfig-platform-psu:output-power":"RKeAAA==" ## into a float32 value 1340