diff --git a/metric.go b/metric.go index f940350ea319e..b45a0ccb99b86 100644 --- a/metric.go +++ b/metric.go @@ -123,3 +123,12 @@ type Metric interface { // to any output. Drop() } + +// TemplateMetric is an interface to use in templates (e.g text/template) +// to generate complex strings from metric properties +// e.g. '{{.Neasurement}}-{{.Tag "foo"}}-{{.Field "bar"}}' +type TemplateMetric interface { + Name() string + Tag(key string) string + Field(key string) interface{} +} diff --git a/metric/metric.go b/metric/metric.go index 8e523bbac9425..0e9268c7cdda8 100644 --- a/metric/metric.go +++ b/metric/metric.go @@ -175,6 +175,11 @@ func (m *metric) GetTag(key string) (string, bool) { return "", false } +func (m *metric) Tag(key string) string { + v, _ := m.GetTag(key) + return v +} + func (m *metric) RemoveTag(key string) { for i, tag := range m.tags { if tag.Key == key { @@ -214,6 +219,13 @@ func (m *metric) GetField(key string) (interface{}, bool) { return nil, false } +func (m *metric) Field(key string) interface{} { + if v, found := m.GetField(key); found { + return v + } + return nil +} + func (m *metric) RemoveField(key string) { for i, field := range m.fields { if field.Key == key { diff --git a/plugins/processors/all/lookup.go b/plugins/processors/all/lookup.go new file mode 100644 index 0000000000000..3642907bc96f9 --- /dev/null +++ b/plugins/processors/all/lookup.go @@ -0,0 +1,5 @@ +//go:build !custom || processors || processors.lookup + +package all + +import _ "github.com/influxdata/telegraf/plugins/processors/lookup" // register plugin diff --git a/plugins/processors/lookup/README.md b/plugins/processors/lookup/README.md new file mode 100644 index 0000000000000..a1380243bf173 --- /dev/null +++ b/plugins/processors/lookup/README.md @@ -0,0 +1,159 @@ +# Lookup Processor Plugin + +The Lookup Processor allows to use one or more files containing a lookup-table +for annotating incoming metrics. The lookup is _static_ as the files are only +used on startup. The main use-case for this is to annotate metrics with +additional tags e.g. dependent on their source. Multiple tags can be added +depending on the lookup-table _files_. + +The lookup key can be generated using a Golang template with the ability to +access the metric name via `{{.Name}}`, the tag values via `{{.Tag "mytag"}}`, +with `mytag` being the tag-name and field-values via `{{.Field "myfield"}}`, +with `myfield` being the field-name. Non-existing tags and field will result +in an empty string or `nil` respectively. In case the key cannot be found, the +metric is passed-trough unchanged. By default all matching tags are added and +existing tag-values are overwritten. + +Please note: The plugin only supports the addition of tags and thus all mapped +tag-values need to be strings! + +## Global configuration options + +In addition to the plugin-specific configuration settings, plugins support +additional global and plugin configuration settings. These settings are used to +modify metrics, tags, and field or create aliases and configure ordering, etc. +See the [CONFIGURATION.md][CONFIGURATION.md] for more details. + +[CONFIGURATION.md]: ../../../docs/CONFIGURATION.md#plugins + +## Configuration + +```toml @sample.conf +# Lookup a key derived from metrics in a static file +[[processors.lookup]] + ## List of files containing the lookup-table + files = ["path/to/lut.json", "path/to/another_lut.json"] + + ## Format of the lookup file(s) + ## Available formats are: + ## json -- JSON file with 'key: {tag-key: tag-value, ...}' mapping + ## csv_key_name_value -- CSV file with 'key,tag-key,tag-value,...,tag-key,tag-value' mapping + ## csv_key_values -- CSV file with a header containing tag-names and + ## rows with 'key,tag-value,...,tag-value' mappings + # format = "json" + + ## Template for generating the lookup-key from the metric. + ## This is a Golang template (see https://pkg.go.dev/text/template) to + ## access the metric name (`{{.Name}}`), a tag value (`{{.Tag "name"}}`) or + ## a field value (`{{.Field "name"}}`). + key = '{{.Tag "host"}}' +``` + +## File formats + +The following descriptions assume `key`s to be unique identifiers used for +matching the configured `key`. The `tag-name`/`tag-value` pairs are the tags +added to a metric if the key matches. + +### `json` format + +In the `json` format, the input `files` must have the following format + +```json +{ + "keyA": { + "tag-name1": "tag-value1", + ... + "tag-nameN": "tag-valueN", + }, + ... + "keyZ": { + "tag-name1": "tag-value1", + ... + "tag-nameM": "tag-valueM", + } +} +``` + +Please note that only _strings_ are supported for all elements. + +### `csv_key_name_value` format + +The `csv_key_name_value` format specifies comma-separated-value files with +the following format + +```csv +# Optional comments +keyA,tag-name1,tag-value1,...,tag-nameN,tag-valueN +keyB,tag-name1,tag-value1 +... +keyZ,tag-name1,tag-value1,...,tag-nameM,tag-valueM +``` + +The formatting uses colons (`,`) as separators and allows for comments defined +as lines starting with a hash (`#`). All lines can have different numbers but +must at least contain three columns and follow the name/value pair format, i.e. +there cannot be a name without value. + +### `csv_key_values` format + +This setting specifies comma-separated-value files with the following format + +```csv +# Optional comments +ignored,tag-name1,...,tag-valueN +keyA,tag-value1,...,,,, +keyB,tag-value1,,,,..., +... +keyZ,tag-value1,...,tag-valueM,..., +``` + +The formatting uses colons (`,`) as separators and allows for comments defined +as lines starting with a hash (`#`). All lines __must__ contain the same number +of columns. The first non-comment line __must__ contain a header specifying the +tag-names. As the first column contains the key to match the first header value +is ignored. There have to be at least two columns. + +Please note that empty tag-values will be ignored and the tag will not be added. + +## Example + +With a lookup table of + +```json +{ + "xyzzy-green": { + "location": "eu-central", + "rack": "C12-01" + }, + "xyzzy-red": { + "location": "us-west", + "rack": "C01-42" + }, +} +``` + +in `format = "json"` and a `key` of `key = '{{.Name}}-{{.Tag "host"}}'` you get + +```diff +- xyzzy,host=green value=3.14 1502489900000000000 +- xyzzy,host=red value=2.71 1502499100000000000 ++ xyzzy,host=green,location=eu-central,rack=C12-01 value=3.14 1502489900000000000 ++ xyzzy,host=red,location=us-west,rack=C01-42 value=2.71 1502499100000000000 +xyzzy,host=blue value=6.62 1502499700000000000 +``` + +The same results can be achieved with `format = "csv_key_name_value"` and + +```csv +xyzzy-green,location,eu-central,rack,C12-01 +xyzzy-red,location,us-west,rack,C01-42 +``` + +or `format = "csv_key_values"` and + +```csv +-,location,rack +xyzzy-green,eu-central,C12-01 +xyzzy-red,us-west,C01-42 +``` diff --git a/plugins/processors/lookup/lookup.go b/plugins/processors/lookup/lookup.go new file mode 100644 index 0000000000000..46a18fd4cb2b2 --- /dev/null +++ b/plugins/processors/lookup/lookup.go @@ -0,0 +1,214 @@ +//go:generate ../../../tools/readme_config_includer/generator +package lookup + +import ( + "bytes" + _ "embed" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + "text/template" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/processors" +) + +//go:embed sample.conf +var sampleConfig string + +type Processor struct { + Filenames []string `toml:"files"` + Fileformat string `toml:"format"` + KeyTemplate string `toml:"key"` + Log telegraf.Logger `toml:"-"` + + tmpl *template.Template + mappings map[string][]telegraf.Tag +} + +func (*Processor) SampleConfig() string { + return sampleConfig +} + +func (p *Processor) Init() error { + if len(p.Filenames) < 1 { + return errors.New("missing 'files'") + } + + if p.KeyTemplate == "" { + return errors.New("missing 'key_template'") + } + + tmpl, err := template.New("key").Parse(p.KeyTemplate) + if err != nil { + return fmt.Errorf("creating template failed: %w", err) + } + p.tmpl = tmpl + + p.mappings = make(map[string][]telegraf.Tag) + switch strings.ToLower(p.Fileformat) { + case "", "json": + return p.loadJSONFiles() + case "csv_key_name_value": + return p.loadCSVKeyNameValueFiles() + case "csv_key_values": + return p.loadCSVKeyValuesFiles() + } + + return fmt.Errorf("invalid format %q", p.Fileformat) +} + +func (p *Processor) Apply(in ...telegraf.Metric) []telegraf.Metric { + out := make([]telegraf.Metric, 0, len(in)) + for _, m := range in { + var buf bytes.Buffer + if err := p.tmpl.Execute(&buf, m); err != nil { + p.Log.Errorf("generating key failed: %v", err) + p.Log.Debugf("metric was %v", m) + out = append(out, m) + continue + } + + if tags, found := p.mappings[buf.String()]; found { + for _, tag := range tags { + m.AddTag(tag.Key, tag.Value) + } + } + out = append(out, m) + } + return out +} + +func (p *Processor) loadJSONFiles() error { + for _, fn := range p.Filenames { + buf, err := os.ReadFile(fn) + if err != nil { + return fmt.Errorf("loading %q failed: %w", fn, err) + } + + var data map[string]map[string]string + if err := json.Unmarshal(buf, &data); err != nil { + return fmt.Errorf("parsing %q failed: %w", fn, err) + } + + for key, tags := range data { + for k, v := range tags { + p.mappings[key] = append(p.mappings[key], telegraf.Tag{Key: k, Value: v}) + } + } + } + return nil +} + +func (p *Processor) loadCSVKeyNameValueFiles() error { + for _, fn := range p.Filenames { + if err := p.loadCSVKeyNameValueFile(fn); err != nil { + return err + } + } + return nil +} + +func (p *Processor) loadCSVKeyNameValueFile(fn string) error { + f, err := os.Open(fn) + if err != nil { + return fmt.Errorf("loading %q failed: %w", fn, err) + } + defer f.Close() + + reader := csv.NewReader(f) + reader.Comment = '#' + reader.FieldsPerRecord = -1 + reader.TrimLeadingSpace = true + + line := 0 + for { + line++ + data, err := reader.Read() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return fmt.Errorf("reading line %d in %q failed: %w", line, fn, err) + } + if len(data) < 3 { + return fmt.Errorf("line %d in %q has not enough columns, requiring at least `key,name,value`", line, fn) + } + if len(data)%2 != 1 { + return fmt.Errorf("line %d in %q has a tag-name without value", line, fn) + } + + key := data[0] + for i := 1; i < len(data)-1; i += 2 { + k, v := data[i], data[i+1] + p.mappings[key] = append(p.mappings[key], telegraf.Tag{Key: k, Value: v}) + } + } + + return nil +} + +func (p *Processor) loadCSVKeyValuesFiles() error { + for _, fn := range p.Filenames { + if err := p.loadCSVKeyValuesFile(fn); err != nil { + return err + } + } + return nil +} + +func (p *Processor) loadCSVKeyValuesFile(fn string) error { + f, err := os.Open(fn) + if err != nil { + return fmt.Errorf("loading %q failed: %w", fn, err) + } + defer f.Close() + + reader := csv.NewReader(f) + reader.Comment = '#' + reader.TrimLeadingSpace = true + + // Read the first line which should be the header + header, err := reader.Read() + if err != nil { + if errors.Is(err, io.EOF) { + return fmt.Errorf("missing header in %q", fn) + } + return fmt.Errorf("reading header in %q failed: %w", fn, err) + } + if len(header) < 2 { + return fmt.Errorf("header in %q has not enough columns, requiring at least `key,value`", fn) + } + header = header[1:] + + line := 1 + for { + line++ + data, err := reader.Read() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return fmt.Errorf("reading line %d in %q failed: %w", line, fn, err) + } + + key := data[0] + for i, v := range data[1:] { + v = strings.TrimSpace(v) + if v != "" { + p.mappings[key] = append(p.mappings[key], telegraf.Tag{Key: header[i], Value: v}) + } + } + } + + return nil +} +func init() { + processors.Add("lookup", func() telegraf.Processor { + return &Processor{} + }) +} diff --git a/plugins/processors/lookup/lookup_test.go b/plugins/processors/lookup/lookup_test.go new file mode 100644 index 0000000000000..a3a384cd061a5 --- /dev/null +++ b/plugins/processors/lookup/lookup_test.go @@ -0,0 +1,94 @@ +package lookup + +import ( + "os" + "path/filepath" + "testing" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/config" + "github.com/influxdata/telegraf/plugins/parsers/influx" + "github.com/influxdata/telegraf/plugins/processors" + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" +) + +func TestInit(t *testing.T) { + plugin := &Processor{} + require.ErrorContains(t, plugin.Init(), "missing 'files'") + + plugin = &Processor{ + Filenames: []string{"blah.json"}, + } + require.ErrorContains(t, plugin.Init(), "missing 'key_template'") + + plugin = &Processor{ + Filenames: []string{"blah.json"}, + KeyTemplate: "lala", + } + require.ErrorIs(t, plugin.Init(), os.ErrNotExist) + + plugin = &Processor{ + Filenames: []string{"blah.json"}, + Fileformat: "foo", + KeyTemplate: "lala", + } + require.ErrorContains(t, plugin.Init(), "invalid format") +} + +func TestCases(t *testing.T) { + // Get all directories in testcases + folders, err := os.ReadDir("testcases") + require.NoError(t, err) + + // Make sure tests contains data + require.NotEmpty(t, folders) + + // Set up for file inputs + processors.Add("lookup", func() telegraf.Processor { + return &Processor{Log: testutil.Logger{}} + }) + + for _, f := range folders { + // Only handle folders + if !f.IsDir() { + continue + } + + fname := f.Name() + testdataPath := filepath.Join("testcases", fname) + configFilename := filepath.Join(testdataPath, "telegraf.conf") + inputFilename := filepath.Join(testdataPath, "input.influx") + expectedFilename := filepath.Join(testdataPath, "expected.out") + + t.Run(fname, func(t *testing.T) { + // Get parser to parse input and expected output + parser := &influx.Parser{} + require.NoError(t, parser.Init()) + + input, err := testutil.ParseMetricsFromFile(inputFilename, parser) + require.NoError(t, err) + + var expected []telegraf.Metric + if _, err := os.Stat(expectedFilename); err == nil { + var err error + expected, err = testutil.ParseMetricsFromFile(expectedFilename, parser) + require.NoError(t, err) + } + + // Configure the plugin + cfg := config.NewConfig() + require.NoError(t, cfg.LoadConfig(configFilename)) + require.Len(t, cfg.Processors, 1, "wrong number of processors") + + type unwrappable interface{ Unwrap() telegraf.Processor } + proc := cfg.Processors[0].Processor.(unwrappable) + plugin := proc.Unwrap().(*Processor) + require.NoError(t, plugin.Init()) + + // Process expected metrics and compare with resulting metrics + actual := plugin.Apply(input...) + testutil.RequireMetricsEqual(t, expected, actual) + }) + } +} diff --git a/plugins/processors/lookup/sample.conf b/plugins/processors/lookup/sample.conf new file mode 100644 index 0000000000000..c240ed755ea29 --- /dev/null +++ b/plugins/processors/lookup/sample.conf @@ -0,0 +1,18 @@ +# Lookup a key derived from metrics in a static file +[[processors.lookup]] + ## List of files containing the lookup-table + files = ["path/to/lut.json", "path/to/another_lut.json"] + + ## Format of the lookup file(s) + ## Available formats are: + ## json -- JSON file with 'key: {tag-key: tag-value, ...}' mapping + ## csv_key_name_value -- CSV file with 'key,tag-key,tag-value,...,tag-key,tag-value' mapping + ## csv_key_values -- CSV file with a header containing tag-names and + ## rows with 'key,tag-value,...,tag-value' mappings + # format = "json" + + ## Template for generating the lookup-key from the metric. + ## This is a Golang template (see https://pkg.go.dev/text/template) to + ## access the metric name (`{{.Name}}`), a tag value (`{{.Tag "name"}}`) or + ## a field value (`{{.Field "name"}}`). + key = '{{.Tag "host"}}' diff --git a/plugins/processors/lookup/testcases/multiple_files_json/expected.out b/plugins/processors/lookup/testcases/multiple_files_json/expected.out new file mode 100644 index 0000000000000..95e2f9aae285b --- /dev/null +++ b/plugins/processors/lookup/testcases/multiple_files_json/expected.out @@ -0,0 +1,5 @@ +cpu,cpu=cpu-total,host=Hugin,location=at\ home,type=desktop usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000123 +cpu,cpu=cpu-total,host=Munin,os=Android,type=mobile usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000456 +cpu,cpu=cpu-total,host=Thor,location=eu-west1,type=server,cabinet=r15-02 usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000789 +disk,device=nvme0n1p4,fstype=ext4,host=Hugin,mode=rw,path=/,type=desktop free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000111 +disk,device=nvme0n1p4,fstype=ext4,host=Munin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000222 diff --git a/plugins/processors/lookup/testcases/multiple_files_json/input.influx b/plugins/processors/lookup/testcases/multiple_files_json/input.influx new file mode 100644 index 0000000000000..44779c033d7da --- /dev/null +++ b/plugins/processors/lookup/testcases/multiple_files_json/input.influx @@ -0,0 +1,5 @@ +cpu,cpu=cpu-total,host=Hugin usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000123 +cpu,cpu=cpu-total,host=Munin usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000456 +cpu,cpu=cpu-total,host=Thor usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000789 +disk,device=nvme0n1p4,fstype=ext4,host=Hugin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000111 +disk,device=nvme0n1p4,fstype=ext4,host=Munin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000222 \ No newline at end of file diff --git a/plugins/processors/lookup/testcases/multiple_files_json/lut_hugin.json b/plugins/processors/lookup/testcases/multiple_files_json/lut_hugin.json new file mode 100644 index 0000000000000..6bbcf26495ec1 --- /dev/null +++ b/plugins/processors/lookup/testcases/multiple_files_json/lut_hugin.json @@ -0,0 +1,9 @@ +{ + "cpu-Hugin": { + "location": "at home", + "type": "desktop" + }, + "disk-Hugin": { + "type": "desktop" + } +} \ No newline at end of file diff --git a/plugins/processors/lookup/testcases/multiple_files_json/lut_munin.json b/plugins/processors/lookup/testcases/multiple_files_json/lut_munin.json new file mode 100644 index 0000000000000..856b1ccd5b46e --- /dev/null +++ b/plugins/processors/lookup/testcases/multiple_files_json/lut_munin.json @@ -0,0 +1,6 @@ +{ + "cpu-Munin": { + "os": "Android", + "type": "mobile" + } +} \ No newline at end of file diff --git a/plugins/processors/lookup/testcases/multiple_files_json/lut_thor.json b/plugins/processors/lookup/testcases/multiple_files_json/lut_thor.json new file mode 100644 index 0000000000000..2809f0cf6ef63 --- /dev/null +++ b/plugins/processors/lookup/testcases/multiple_files_json/lut_thor.json @@ -0,0 +1,7 @@ +{ + "cpu-Thor": { + "location": "eu-west1", + "type": "server", + "cabinet": "r15-02" + } +} \ No newline at end of file diff --git a/plugins/processors/lookup/testcases/multiple_files_json/telegraf.conf b/plugins/processors/lookup/testcases/multiple_files_json/telegraf.conf new file mode 100644 index 0000000000000..7533d1eafcf1e --- /dev/null +++ b/plugins/processors/lookup/testcases/multiple_files_json/telegraf.conf @@ -0,0 +1,7 @@ +[[processors.lookup]] + files = [ + "testcases/multiple_files_json/lut_hugin.json", + "testcases/multiple_files_json/lut_munin.json", + "testcases/multiple_files_json/lut_thor.json" + ] + key = '{{.Name}}-{{.Tag "host"}}' diff --git a/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/expected.out b/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/expected.out new file mode 100644 index 0000000000000..95e2f9aae285b --- /dev/null +++ b/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/expected.out @@ -0,0 +1,5 @@ +cpu,cpu=cpu-total,host=Hugin,location=at\ home,type=desktop usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000123 +cpu,cpu=cpu-total,host=Munin,os=Android,type=mobile usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000456 +cpu,cpu=cpu-total,host=Thor,location=eu-west1,type=server,cabinet=r15-02 usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000789 +disk,device=nvme0n1p4,fstype=ext4,host=Hugin,mode=rw,path=/,type=desktop free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000111 +disk,device=nvme0n1p4,fstype=ext4,host=Munin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000222 diff --git a/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/input.influx b/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/input.influx new file mode 100644 index 0000000000000..44779c033d7da --- /dev/null +++ b/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/input.influx @@ -0,0 +1,5 @@ +cpu,cpu=cpu-total,host=Hugin usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000123 +cpu,cpu=cpu-total,host=Munin usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000456 +cpu,cpu=cpu-total,host=Thor usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000789 +disk,device=nvme0n1p4,fstype=ext4,host=Hugin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000111 +disk,device=nvme0n1p4,fstype=ext4,host=Munin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000222 \ No newline at end of file diff --git a/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/lut.csv b/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/lut.csv new file mode 100644 index 0000000000000..13520ddfd4d20 --- /dev/null +++ b/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/lut.csv @@ -0,0 +1,5 @@ +# key, tag-name, tag-value,...,tag-name,tag-value +cpu-Hugin,location,at home,type,desktop +cpu-Munin,os,Android,type,mobile +cpu-Thor,location,eu-west1,type,server,cabinet,r15-02 +disk-Hugin,type,desktop diff --git a/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/telegraf.conf b/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/telegraf.conf new file mode 100644 index 0000000000000..45816148e9c93 --- /dev/null +++ b/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/telegraf.conf @@ -0,0 +1,4 @@ +[[processors.lookup]] + files = ["testcases/normal_lookup_csv_key_name_value/lut.csv"] + format = "csv_key_name_value" + key = '{{.Name}}-{{.Tag "host"}}' diff --git a/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/expected.out b/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/expected.out new file mode 100644 index 0000000000000..95e2f9aae285b --- /dev/null +++ b/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/expected.out @@ -0,0 +1,5 @@ +cpu,cpu=cpu-total,host=Hugin,location=at\ home,type=desktop usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000123 +cpu,cpu=cpu-total,host=Munin,os=Android,type=mobile usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000456 +cpu,cpu=cpu-total,host=Thor,location=eu-west1,type=server,cabinet=r15-02 usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000789 +disk,device=nvme0n1p4,fstype=ext4,host=Hugin,mode=rw,path=/,type=desktop free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000111 +disk,device=nvme0n1p4,fstype=ext4,host=Munin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000222 diff --git a/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/input.influx b/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/input.influx new file mode 100644 index 0000000000000..44779c033d7da --- /dev/null +++ b/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/input.influx @@ -0,0 +1,5 @@ +cpu,cpu=cpu-total,host=Hugin usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000123 +cpu,cpu=cpu-total,host=Munin usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000456 +cpu,cpu=cpu-total,host=Thor usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000789 +disk,device=nvme0n1p4,fstype=ext4,host=Hugin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000111 +disk,device=nvme0n1p4,fstype=ext4,host=Munin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000222 \ No newline at end of file diff --git a/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/lut.csv b/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/lut.csv new file mode 100644 index 0000000000000..aa85ba0161e65 --- /dev/null +++ b/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/lut.csv @@ -0,0 +1,7 @@ +# Some comment +# lines +host,location,type,os,cabinet +cpu-Hugin,at home,desktop,, +cpu-Munin,,mobile,Android, +cpu-Thor,eu-west1,server,,r15-02 +disk-Hugin,,desktop,, diff --git a/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/telegraf.conf b/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/telegraf.conf new file mode 100644 index 0000000000000..e1d1c00490e6c --- /dev/null +++ b/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/telegraf.conf @@ -0,0 +1,4 @@ +[[processors.lookup]] + files = ["testcases/normal_lookup_csv_key_values/lut.csv"] + format = "csv_key_values" + key = '{{.Name}}-{{.Tag "host"}}' diff --git a/plugins/processors/lookup/testcases/normal_lookup_json/expected.out b/plugins/processors/lookup/testcases/normal_lookup_json/expected.out new file mode 100644 index 0000000000000..95e2f9aae285b --- /dev/null +++ b/plugins/processors/lookup/testcases/normal_lookup_json/expected.out @@ -0,0 +1,5 @@ +cpu,cpu=cpu-total,host=Hugin,location=at\ home,type=desktop usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000123 +cpu,cpu=cpu-total,host=Munin,os=Android,type=mobile usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000456 +cpu,cpu=cpu-total,host=Thor,location=eu-west1,type=server,cabinet=r15-02 usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000789 +disk,device=nvme0n1p4,fstype=ext4,host=Hugin,mode=rw,path=/,type=desktop free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000111 +disk,device=nvme0n1p4,fstype=ext4,host=Munin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000222 diff --git a/plugins/processors/lookup/testcases/normal_lookup_json/input.influx b/plugins/processors/lookup/testcases/normal_lookup_json/input.influx new file mode 100644 index 0000000000000..44779c033d7da --- /dev/null +++ b/plugins/processors/lookup/testcases/normal_lookup_json/input.influx @@ -0,0 +1,5 @@ +cpu,cpu=cpu-total,host=Hugin usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000123 +cpu,cpu=cpu-total,host=Munin usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000456 +cpu,cpu=cpu-total,host=Thor usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000789 +disk,device=nvme0n1p4,fstype=ext4,host=Hugin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000111 +disk,device=nvme0n1p4,fstype=ext4,host=Munin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000222 \ No newline at end of file diff --git a/plugins/processors/lookup/testcases/normal_lookup_json/lut.json b/plugins/processors/lookup/testcases/normal_lookup_json/lut.json new file mode 100644 index 0000000000000..8364c3b18bba7 --- /dev/null +++ b/plugins/processors/lookup/testcases/normal_lookup_json/lut.json @@ -0,0 +1,18 @@ +{ + "cpu-Hugin": { + "location": "at home", + "type": "desktop" + }, + "cpu-Munin": { + "os": "Android", + "type": "mobile" + }, + "cpu-Thor": { + "location": "eu-west1", + "type": "server", + "cabinet": "r15-02" + }, + "disk-Hugin": { + "type": "desktop" + } +} \ No newline at end of file diff --git a/plugins/processors/lookup/testcases/normal_lookup_json/telegraf.conf b/plugins/processors/lookup/testcases/normal_lookup_json/telegraf.conf new file mode 100644 index 0000000000000..eef963464ed3e --- /dev/null +++ b/plugins/processors/lookup/testcases/normal_lookup_json/telegraf.conf @@ -0,0 +1,3 @@ +[[processors.lookup]] + files = ["testcases/normal_lookup_json/lut.json"] + key = '{{.Name}}-{{.Tag "host"}}'