From df190611663068859447b6f6585ec25d55b4017a Mon Sep 17 00:00:00 2001
From: Sven Rebhan <36194019+srebhan@users.noreply.github.com>
Date: Wed, 8 Mar 2023 18:44:23 +0100
Subject: [PATCH] feat(processors): Add lookup processor (#12809)

---
 metric.go                                     |   9 +
 metric/metric.go                              |  12 +
 plugins/processors/all/lookup.go              |   5 +
 plugins/processors/lookup/README.md           | 159 +++++++++++++
 plugins/processors/lookup/lookup.go           | 214 ++++++++++++++++++
 plugins/processors/lookup/lookup_test.go      |  94 ++++++++
 plugins/processors/lookup/sample.conf         |  18 ++
 .../multiple_files_json/expected.out          |   5 +
 .../multiple_files_json/input.influx          |   5 +
 .../multiple_files_json/lut_hugin.json        |   9 +
 .../multiple_files_json/lut_munin.json        |   6 +
 .../multiple_files_json/lut_thor.json         |   7 +
 .../multiple_files_json/telegraf.conf         |   7 +
 .../expected.out                              |   5 +
 .../input.influx                              |   5 +
 .../normal_lookup_csv_key_name_value/lut.csv  |   5 +
 .../telegraf.conf                             |   4 +
 .../normal_lookup_csv_key_values/expected.out |   5 +
 .../normal_lookup_csv_key_values/input.influx |   5 +
 .../normal_lookup_csv_key_values/lut.csv      |   7 +
 .../telegraf.conf                             |   4 +
 .../testcases/normal_lookup_json/expected.out |   5 +
 .../testcases/normal_lookup_json/input.influx |   5 +
 .../testcases/normal_lookup_json/lut.json     |  18 ++
 .../normal_lookup_json/telegraf.conf          |   3 +
 25 files changed, 621 insertions(+)
 create mode 100644 plugins/processors/all/lookup.go
 create mode 100644 plugins/processors/lookup/README.md
 create mode 100644 plugins/processors/lookup/lookup.go
 create mode 100644 plugins/processors/lookup/lookup_test.go
 create mode 100644 plugins/processors/lookup/sample.conf
 create mode 100644 plugins/processors/lookup/testcases/multiple_files_json/expected.out
 create mode 100644 plugins/processors/lookup/testcases/multiple_files_json/input.influx
 create mode 100644 plugins/processors/lookup/testcases/multiple_files_json/lut_hugin.json
 create mode 100644 plugins/processors/lookup/testcases/multiple_files_json/lut_munin.json
 create mode 100644 plugins/processors/lookup/testcases/multiple_files_json/lut_thor.json
 create mode 100644 plugins/processors/lookup/testcases/multiple_files_json/telegraf.conf
 create mode 100644 plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/expected.out
 create mode 100644 plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/input.influx
 create mode 100644 plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/lut.csv
 create mode 100644 plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/telegraf.conf
 create mode 100644 plugins/processors/lookup/testcases/normal_lookup_csv_key_values/expected.out
 create mode 100644 plugins/processors/lookup/testcases/normal_lookup_csv_key_values/input.influx
 create mode 100644 plugins/processors/lookup/testcases/normal_lookup_csv_key_values/lut.csv
 create mode 100644 plugins/processors/lookup/testcases/normal_lookup_csv_key_values/telegraf.conf
 create mode 100644 plugins/processors/lookup/testcases/normal_lookup_json/expected.out
 create mode 100644 plugins/processors/lookup/testcases/normal_lookup_json/input.influx
 create mode 100644 plugins/processors/lookup/testcases/normal_lookup_json/lut.json
 create mode 100644 plugins/processors/lookup/testcases/normal_lookup_json/telegraf.conf

diff --git a/metric.go b/metric.go
index f940350ea319e..b45a0ccb99b86 100644
--- a/metric.go
+++ b/metric.go
@@ -123,3 +123,12 @@ type Metric interface {
 	// to any output.
 	Drop()
 }
+
+// TemplateMetric is an interface to use in templates (e.g text/template)
+// to generate complex strings from metric properties
+// e.g. '{{.Neasurement}}-{{.Tag "foo"}}-{{.Field "bar"}}'
+type TemplateMetric interface {
+	Name() string
+	Tag(key string) string
+	Field(key string) interface{}
+}
diff --git a/metric/metric.go b/metric/metric.go
index 8e523bbac9425..0e9268c7cdda8 100644
--- a/metric/metric.go
+++ b/metric/metric.go
@@ -175,6 +175,11 @@ func (m *metric) GetTag(key string) (string, bool) {
 	return "", false
 }
 
+func (m *metric) Tag(key string) string {
+	v, _ := m.GetTag(key)
+	return v
+}
+
 func (m *metric) RemoveTag(key string) {
 	for i, tag := range m.tags {
 		if tag.Key == key {
@@ -214,6 +219,13 @@ func (m *metric) GetField(key string) (interface{}, bool) {
 	return nil, false
 }
 
+func (m *metric) Field(key string) interface{} {
+	if v, found := m.GetField(key); found {
+		return v
+	}
+	return nil
+}
+
 func (m *metric) RemoveField(key string) {
 	for i, field := range m.fields {
 		if field.Key == key {
diff --git a/plugins/processors/all/lookup.go b/plugins/processors/all/lookup.go
new file mode 100644
index 0000000000000..3642907bc96f9
--- /dev/null
+++ b/plugins/processors/all/lookup.go
@@ -0,0 +1,5 @@
+//go:build !custom || processors || processors.lookup
+
+package all
+
+import _ "github.com/influxdata/telegraf/plugins/processors/lookup" // register plugin
diff --git a/plugins/processors/lookup/README.md b/plugins/processors/lookup/README.md
new file mode 100644
index 0000000000000..a1380243bf173
--- /dev/null
+++ b/plugins/processors/lookup/README.md
@@ -0,0 +1,159 @@
+# Lookup Processor Plugin
+
+The Lookup Processor allows to use one or more files containing a lookup-table
+for annotating incoming metrics. The lookup is _static_ as the files are only
+used on startup. The main use-case for this is to annotate metrics with
+additional tags e.g. dependent on their source. Multiple tags can be added
+depending on the lookup-table _files_.
+
+The lookup key can be generated using a Golang template with the ability to
+access the metric name via `{{.Name}}`, the tag values via `{{.Tag "mytag"}}`,
+with `mytag` being the tag-name and field-values via `{{.Field "myfield"}}`,
+with `myfield` being the field-name. Non-existing tags and field will result
+in an empty string or `nil` respectively. In case the key cannot be found, the
+metric is passed-trough unchanged. By default all matching tags are added and
+existing tag-values are overwritten.
+
+Please note: The plugin only supports the addition of tags and thus all mapped
+tag-values need to be strings!
+
+## Global configuration options <!-- @/docs/includes/plugin_config.md -->
+
+In addition to the plugin-specific configuration settings, plugins support
+additional global and plugin configuration settings. These settings are used to
+modify metrics, tags, and field or create aliases and configure ordering, etc.
+See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
+
+[CONFIGURATION.md]: ../../../docs/CONFIGURATION.md#plugins
+
+## Configuration
+
+```toml @sample.conf
+# Lookup a key derived from metrics in a static file
+[[processors.lookup]]
+  ## List of files containing the lookup-table
+  files = ["path/to/lut.json", "path/to/another_lut.json"]
+
+  ## Format of the lookup file(s)
+  ## Available formats are:
+  ##    json               -- JSON file with 'key: {tag-key: tag-value, ...}' mapping
+  ##    csv_key_name_value -- CSV file with 'key,tag-key,tag-value,...,tag-key,tag-value' mapping
+  ##    csv_key_values     -- CSV file with a header containing tag-names and
+  ##                          rows with 'key,tag-value,...,tag-value' mappings
+  # format = "json"
+
+  ## Template for generating the lookup-key from the metric.
+  ## This is a Golang template (see https://pkg.go.dev/text/template) to
+  ## access the metric name (`{{.Name}}`), a tag value (`{{.Tag "name"}}`) or
+  ## a field value (`{{.Field "name"}}`).
+  key = '{{.Tag "host"}}'
+```
+
+## File formats
+
+The following descriptions assume `key`s to be unique identifiers used for
+matching the configured `key`. The `tag-name`/`tag-value` pairs are the tags
+added to a metric if the key matches.
+
+### `json` format
+
+In the `json` format, the input `files` must have the following format
+
+```json
+{
+  "keyA": {
+    "tag-name1": "tag-value1",
+    ...
+    "tag-nameN": "tag-valueN",
+  },
+  ...
+  "keyZ": {
+    "tag-name1": "tag-value1",
+    ...
+    "tag-nameM": "tag-valueM",
+  }
+}
+```
+
+Please note that only _strings_ are supported for all elements.
+
+### `csv_key_name_value` format
+
+The `csv_key_name_value` format specifies comma-separated-value files with
+the following format
+
+```csv
+# Optional comments
+keyA,tag-name1,tag-value1,...,tag-nameN,tag-valueN
+keyB,tag-name1,tag-value1
+...
+keyZ,tag-name1,tag-value1,...,tag-nameM,tag-valueM
+```
+
+The formatting uses colons (`,`) as separators and allows for comments defined
+as lines starting with a hash (`#`). All lines can have different numbers but
+must at least contain three columns and follow the name/value pair format, i.e.
+there cannot be a name without value.
+
+### `csv_key_values` format
+
+This setting specifies comma-separated-value files with the following format
+
+```csv
+# Optional comments
+ignored,tag-name1,...,tag-valueN
+keyA,tag-value1,...,,,,
+keyB,tag-value1,,,,...,
+...
+keyZ,tag-value1,...,tag-valueM,...,
+```
+
+The formatting uses colons (`,`) as separators and allows for comments defined
+as lines starting with a hash (`#`). All lines __must__ contain the same number
+of columns. The first non-comment line __must__ contain a header specifying the
+tag-names. As the first column contains the key to match the first header value
+is ignored. There have to be at least two columns.
+
+Please note that empty tag-values will be ignored and the tag will not be added.
+
+## Example
+
+With a lookup table of
+
+```json
+{
+  "xyzzy-green": {
+    "location": "eu-central",
+    "rack": "C12-01"
+  },
+  "xyzzy-red": {
+    "location": "us-west",
+    "rack": "C01-42"
+  },
+}
+```
+
+in `format = "json"` and a `key` of `key = '{{.Name}}-{{.Tag "host"}}'` you get
+
+```diff
+- xyzzy,host=green value=3.14 1502489900000000000
+- xyzzy,host=red  value=2.71 1502499100000000000
++ xyzzy,host=green,location=eu-central,rack=C12-01 value=3.14 1502489900000000000
++ xyzzy,host=red,location=us-west,rack=C01-42 value=2.71 1502499100000000000
+xyzzy,host=blue  value=6.62 1502499700000000000
+```
+
+The same results can be achieved with `format = "csv_key_name_value"` and
+
+```csv
+xyzzy-green,location,eu-central,rack,C12-01
+xyzzy-red,location,us-west,rack,C01-42
+```
+
+or `format = "csv_key_values"` and
+
+```csv
+-,location,rack
+xyzzy-green,eu-central,C12-01
+xyzzy-red,us-west,C01-42
+```
diff --git a/plugins/processors/lookup/lookup.go b/plugins/processors/lookup/lookup.go
new file mode 100644
index 0000000000000..46a18fd4cb2b2
--- /dev/null
+++ b/plugins/processors/lookup/lookup.go
@@ -0,0 +1,214 @@
+//go:generate ../../../tools/readme_config_includer/generator
+package lookup
+
+import (
+	"bytes"
+	_ "embed"
+	"encoding/csv"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"text/template"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/plugins/processors"
+)
+
+//go:embed sample.conf
+var sampleConfig string
+
+type Processor struct {
+	Filenames   []string        `toml:"files"`
+	Fileformat  string          `toml:"format"`
+	KeyTemplate string          `toml:"key"`
+	Log         telegraf.Logger `toml:"-"`
+
+	tmpl     *template.Template
+	mappings map[string][]telegraf.Tag
+}
+
+func (*Processor) SampleConfig() string {
+	return sampleConfig
+}
+
+func (p *Processor) Init() error {
+	if len(p.Filenames) < 1 {
+		return errors.New("missing 'files'")
+	}
+
+	if p.KeyTemplate == "" {
+		return errors.New("missing 'key_template'")
+	}
+
+	tmpl, err := template.New("key").Parse(p.KeyTemplate)
+	if err != nil {
+		return fmt.Errorf("creating template failed: %w", err)
+	}
+	p.tmpl = tmpl
+
+	p.mappings = make(map[string][]telegraf.Tag)
+	switch strings.ToLower(p.Fileformat) {
+	case "", "json":
+		return p.loadJSONFiles()
+	case "csv_key_name_value":
+		return p.loadCSVKeyNameValueFiles()
+	case "csv_key_values":
+		return p.loadCSVKeyValuesFiles()
+	}
+
+	return fmt.Errorf("invalid format %q", p.Fileformat)
+}
+
+func (p *Processor) Apply(in ...telegraf.Metric) []telegraf.Metric {
+	out := make([]telegraf.Metric, 0, len(in))
+	for _, m := range in {
+		var buf bytes.Buffer
+		if err := p.tmpl.Execute(&buf, m); err != nil {
+			p.Log.Errorf("generating key failed: %v", err)
+			p.Log.Debugf("metric was %v", m)
+			out = append(out, m)
+			continue
+		}
+
+		if tags, found := p.mappings[buf.String()]; found {
+			for _, tag := range tags {
+				m.AddTag(tag.Key, tag.Value)
+			}
+		}
+		out = append(out, m)
+	}
+	return out
+}
+
+func (p *Processor) loadJSONFiles() error {
+	for _, fn := range p.Filenames {
+		buf, err := os.ReadFile(fn)
+		if err != nil {
+			return fmt.Errorf("loading %q failed: %w", fn, err)
+		}
+
+		var data map[string]map[string]string
+		if err := json.Unmarshal(buf, &data); err != nil {
+			return fmt.Errorf("parsing %q failed: %w", fn, err)
+		}
+
+		for key, tags := range data {
+			for k, v := range tags {
+				p.mappings[key] = append(p.mappings[key], telegraf.Tag{Key: k, Value: v})
+			}
+		}
+	}
+	return nil
+}
+
+func (p *Processor) loadCSVKeyNameValueFiles() error {
+	for _, fn := range p.Filenames {
+		if err := p.loadCSVKeyNameValueFile(fn); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (p *Processor) loadCSVKeyNameValueFile(fn string) error {
+	f, err := os.Open(fn)
+	if err != nil {
+		return fmt.Errorf("loading %q failed: %w", fn, err)
+	}
+	defer f.Close()
+
+	reader := csv.NewReader(f)
+	reader.Comment = '#'
+	reader.FieldsPerRecord = -1
+	reader.TrimLeadingSpace = true
+
+	line := 0
+	for {
+		line++
+		data, err := reader.Read()
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			return fmt.Errorf("reading line %d in %q failed: %w", line, fn, err)
+		}
+		if len(data) < 3 {
+			return fmt.Errorf("line %d in %q has not enough columns, requiring at least `key,name,value`", line, fn)
+		}
+		if len(data)%2 != 1 {
+			return fmt.Errorf("line %d in %q has a tag-name without value", line, fn)
+		}
+
+		key := data[0]
+		for i := 1; i < len(data)-1; i += 2 {
+			k, v := data[i], data[i+1]
+			p.mappings[key] = append(p.mappings[key], telegraf.Tag{Key: k, Value: v})
+		}
+	}
+
+	return nil
+}
+
+func (p *Processor) loadCSVKeyValuesFiles() error {
+	for _, fn := range p.Filenames {
+		if err := p.loadCSVKeyValuesFile(fn); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (p *Processor) loadCSVKeyValuesFile(fn string) error {
+	f, err := os.Open(fn)
+	if err != nil {
+		return fmt.Errorf("loading %q failed: %w", fn, err)
+	}
+	defer f.Close()
+
+	reader := csv.NewReader(f)
+	reader.Comment = '#'
+	reader.TrimLeadingSpace = true
+
+	// Read the first line which should be the header
+	header, err := reader.Read()
+	if err != nil {
+		if errors.Is(err, io.EOF) {
+			return fmt.Errorf("missing header in %q", fn)
+		}
+		return fmt.Errorf("reading header in %q failed: %w", fn, err)
+	}
+	if len(header) < 2 {
+		return fmt.Errorf("header in %q has not enough columns, requiring at least `key,value`", fn)
+	}
+	header = header[1:]
+
+	line := 1
+	for {
+		line++
+		data, err := reader.Read()
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			return fmt.Errorf("reading line %d in %q failed: %w", line, fn, err)
+		}
+
+		key := data[0]
+		for i, v := range data[1:] {
+			v = strings.TrimSpace(v)
+			if v != "" {
+				p.mappings[key] = append(p.mappings[key], telegraf.Tag{Key: header[i], Value: v})
+			}
+		}
+	}
+
+	return nil
+}
+func init() {
+	processors.Add("lookup", func() telegraf.Processor {
+		return &Processor{}
+	})
+}
diff --git a/plugins/processors/lookup/lookup_test.go b/plugins/processors/lookup/lookup_test.go
new file mode 100644
index 0000000000000..a3a384cd061a5
--- /dev/null
+++ b/plugins/processors/lookup/lookup_test.go
@@ -0,0 +1,94 @@
+package lookup
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/config"
+	"github.com/influxdata/telegraf/plugins/parsers/influx"
+	"github.com/influxdata/telegraf/plugins/processors"
+	"github.com/influxdata/telegraf/testutil"
+	"github.com/stretchr/testify/require"
+)
+
+func TestInit(t *testing.T) {
+	plugin := &Processor{}
+	require.ErrorContains(t, plugin.Init(), "missing 'files'")
+
+	plugin = &Processor{
+		Filenames: []string{"blah.json"},
+	}
+	require.ErrorContains(t, plugin.Init(), "missing 'key_template'")
+
+	plugin = &Processor{
+		Filenames:   []string{"blah.json"},
+		KeyTemplate: "lala",
+	}
+	require.ErrorIs(t, plugin.Init(), os.ErrNotExist)
+
+	plugin = &Processor{
+		Filenames:   []string{"blah.json"},
+		Fileformat:  "foo",
+		KeyTemplate: "lala",
+	}
+	require.ErrorContains(t, plugin.Init(), "invalid format")
+}
+
+func TestCases(t *testing.T) {
+	// Get all directories in testcases
+	folders, err := os.ReadDir("testcases")
+	require.NoError(t, err)
+
+	// Make sure tests contains data
+	require.NotEmpty(t, folders)
+
+	// Set up for file inputs
+	processors.Add("lookup", func() telegraf.Processor {
+		return &Processor{Log: testutil.Logger{}}
+	})
+
+	for _, f := range folders {
+		// Only handle folders
+		if !f.IsDir() {
+			continue
+		}
+
+		fname := f.Name()
+		testdataPath := filepath.Join("testcases", fname)
+		configFilename := filepath.Join(testdataPath, "telegraf.conf")
+		inputFilename := filepath.Join(testdataPath, "input.influx")
+		expectedFilename := filepath.Join(testdataPath, "expected.out")
+
+		t.Run(fname, func(t *testing.T) {
+			// Get parser to parse input and expected output
+			parser := &influx.Parser{}
+			require.NoError(t, parser.Init())
+
+			input, err := testutil.ParseMetricsFromFile(inputFilename, parser)
+			require.NoError(t, err)
+
+			var expected []telegraf.Metric
+			if _, err := os.Stat(expectedFilename); err == nil {
+				var err error
+				expected, err = testutil.ParseMetricsFromFile(expectedFilename, parser)
+				require.NoError(t, err)
+			}
+
+			// Configure the plugin
+			cfg := config.NewConfig()
+			require.NoError(t, cfg.LoadConfig(configFilename))
+			require.Len(t, cfg.Processors, 1, "wrong number of processors")
+
+			type unwrappable interface{ Unwrap() telegraf.Processor }
+			proc := cfg.Processors[0].Processor.(unwrappable)
+			plugin := proc.Unwrap().(*Processor)
+			require.NoError(t, plugin.Init())
+
+			// Process expected metrics and compare with resulting metrics
+			actual := plugin.Apply(input...)
+			testutil.RequireMetricsEqual(t, expected, actual)
+		})
+	}
+}
diff --git a/plugins/processors/lookup/sample.conf b/plugins/processors/lookup/sample.conf
new file mode 100644
index 0000000000000..c240ed755ea29
--- /dev/null
+++ b/plugins/processors/lookup/sample.conf
@@ -0,0 +1,18 @@
+# Lookup a key derived from metrics in a static file
+[[processors.lookup]]
+  ## List of files containing the lookup-table
+  files = ["path/to/lut.json", "path/to/another_lut.json"]
+
+  ## Format of the lookup file(s)
+  ## Available formats are:
+  ##    json               -- JSON file with 'key: {tag-key: tag-value, ...}' mapping
+  ##    csv_key_name_value -- CSV file with 'key,tag-key,tag-value,...,tag-key,tag-value' mapping
+  ##    csv_key_values     -- CSV file with a header containing tag-names and
+  ##                          rows with 'key,tag-value,...,tag-value' mappings
+  # format = "json"
+
+  ## Template for generating the lookup-key from the metric.
+  ## This is a Golang template (see https://pkg.go.dev/text/template) to
+  ## access the metric name (`{{.Name}}`), a tag value (`{{.Tag "name"}}`) or
+  ## a field value (`{{.Field "name"}}`).
+  key = '{{.Tag "host"}}'
diff --git a/plugins/processors/lookup/testcases/multiple_files_json/expected.out b/plugins/processors/lookup/testcases/multiple_files_json/expected.out
new file mode 100644
index 0000000000000..95e2f9aae285b
--- /dev/null
+++ b/plugins/processors/lookup/testcases/multiple_files_json/expected.out
@@ -0,0 +1,5 @@
+cpu,cpu=cpu-total,host=Hugin,location=at\ home,type=desktop usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000123
+cpu,cpu=cpu-total,host=Munin,os=Android,type=mobile usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000456
+cpu,cpu=cpu-total,host=Thor,location=eu-west1,type=server,cabinet=r15-02 usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000789
+disk,device=nvme0n1p4,fstype=ext4,host=Hugin,mode=rw,path=/,type=desktop free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000111
+disk,device=nvme0n1p4,fstype=ext4,host=Munin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000222
diff --git a/plugins/processors/lookup/testcases/multiple_files_json/input.influx b/plugins/processors/lookup/testcases/multiple_files_json/input.influx
new file mode 100644
index 0000000000000..44779c033d7da
--- /dev/null
+++ b/plugins/processors/lookup/testcases/multiple_files_json/input.influx
@@ -0,0 +1,5 @@
+cpu,cpu=cpu-total,host=Hugin usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000123
+cpu,cpu=cpu-total,host=Munin usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000456
+cpu,cpu=cpu-total,host=Thor usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000789
+disk,device=nvme0n1p4,fstype=ext4,host=Hugin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000111
+disk,device=nvme0n1p4,fstype=ext4,host=Munin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000222
\ No newline at end of file
diff --git a/plugins/processors/lookup/testcases/multiple_files_json/lut_hugin.json b/plugins/processors/lookup/testcases/multiple_files_json/lut_hugin.json
new file mode 100644
index 0000000000000..6bbcf26495ec1
--- /dev/null
+++ b/plugins/processors/lookup/testcases/multiple_files_json/lut_hugin.json
@@ -0,0 +1,9 @@
+{
+    "cpu-Hugin": {
+        "location": "at home",
+        "type": "desktop"
+    },
+    "disk-Hugin": {
+        "type": "desktop"
+    }
+}
\ No newline at end of file
diff --git a/plugins/processors/lookup/testcases/multiple_files_json/lut_munin.json b/plugins/processors/lookup/testcases/multiple_files_json/lut_munin.json
new file mode 100644
index 0000000000000..856b1ccd5b46e
--- /dev/null
+++ b/plugins/processors/lookup/testcases/multiple_files_json/lut_munin.json
@@ -0,0 +1,6 @@
+{
+    "cpu-Munin": {
+        "os": "Android",
+        "type": "mobile"
+    }
+}
\ No newline at end of file
diff --git a/plugins/processors/lookup/testcases/multiple_files_json/lut_thor.json b/plugins/processors/lookup/testcases/multiple_files_json/lut_thor.json
new file mode 100644
index 0000000000000..2809f0cf6ef63
--- /dev/null
+++ b/plugins/processors/lookup/testcases/multiple_files_json/lut_thor.json
@@ -0,0 +1,7 @@
+{
+    "cpu-Thor": {
+        "location": "eu-west1",
+        "type": "server",
+        "cabinet": "r15-02"
+    }
+}
\ No newline at end of file
diff --git a/plugins/processors/lookup/testcases/multiple_files_json/telegraf.conf b/plugins/processors/lookup/testcases/multiple_files_json/telegraf.conf
new file mode 100644
index 0000000000000..7533d1eafcf1e
--- /dev/null
+++ b/plugins/processors/lookup/testcases/multiple_files_json/telegraf.conf
@@ -0,0 +1,7 @@
+[[processors.lookup]]
+    files = [
+        "testcases/multiple_files_json/lut_hugin.json",
+        "testcases/multiple_files_json/lut_munin.json",
+        "testcases/multiple_files_json/lut_thor.json"
+    ]
+    key = '{{.Name}}-{{.Tag "host"}}'
diff --git a/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/expected.out b/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/expected.out
new file mode 100644
index 0000000000000..95e2f9aae285b
--- /dev/null
+++ b/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/expected.out
@@ -0,0 +1,5 @@
+cpu,cpu=cpu-total,host=Hugin,location=at\ home,type=desktop usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000123
+cpu,cpu=cpu-total,host=Munin,os=Android,type=mobile usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000456
+cpu,cpu=cpu-total,host=Thor,location=eu-west1,type=server,cabinet=r15-02 usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000789
+disk,device=nvme0n1p4,fstype=ext4,host=Hugin,mode=rw,path=/,type=desktop free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000111
+disk,device=nvme0n1p4,fstype=ext4,host=Munin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000222
diff --git a/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/input.influx b/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/input.influx
new file mode 100644
index 0000000000000..44779c033d7da
--- /dev/null
+++ b/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/input.influx
@@ -0,0 +1,5 @@
+cpu,cpu=cpu-total,host=Hugin usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000123
+cpu,cpu=cpu-total,host=Munin usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000456
+cpu,cpu=cpu-total,host=Thor usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000789
+disk,device=nvme0n1p4,fstype=ext4,host=Hugin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000111
+disk,device=nvme0n1p4,fstype=ext4,host=Munin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000222
\ No newline at end of file
diff --git a/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/lut.csv b/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/lut.csv
new file mode 100644
index 0000000000000..13520ddfd4d20
--- /dev/null
+++ b/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/lut.csv
@@ -0,0 +1,5 @@
+# key, tag-name, tag-value,...,tag-name,tag-value
+cpu-Hugin,location,at home,type,desktop
+cpu-Munin,os,Android,type,mobile
+cpu-Thor,location,eu-west1,type,server,cabinet,r15-02
+disk-Hugin,type,desktop
diff --git a/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/telegraf.conf b/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/telegraf.conf
new file mode 100644
index 0000000000000..45816148e9c93
--- /dev/null
+++ b/plugins/processors/lookup/testcases/normal_lookup_csv_key_name_value/telegraf.conf
@@ -0,0 +1,4 @@
+[[processors.lookup]]
+    files = ["testcases/normal_lookup_csv_key_name_value/lut.csv"]
+    format = "csv_key_name_value"
+    key = '{{.Name}}-{{.Tag "host"}}'
diff --git a/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/expected.out b/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/expected.out
new file mode 100644
index 0000000000000..95e2f9aae285b
--- /dev/null
+++ b/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/expected.out
@@ -0,0 +1,5 @@
+cpu,cpu=cpu-total,host=Hugin,location=at\ home,type=desktop usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000123
+cpu,cpu=cpu-total,host=Munin,os=Android,type=mobile usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000456
+cpu,cpu=cpu-total,host=Thor,location=eu-west1,type=server,cabinet=r15-02 usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000789
+disk,device=nvme0n1p4,fstype=ext4,host=Hugin,mode=rw,path=/,type=desktop free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000111
+disk,device=nvme0n1p4,fstype=ext4,host=Munin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000222
diff --git a/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/input.influx b/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/input.influx
new file mode 100644
index 0000000000000..44779c033d7da
--- /dev/null
+++ b/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/input.influx
@@ -0,0 +1,5 @@
+cpu,cpu=cpu-total,host=Hugin usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000123
+cpu,cpu=cpu-total,host=Munin usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000456
+cpu,cpu=cpu-total,host=Thor usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000789
+disk,device=nvme0n1p4,fstype=ext4,host=Hugin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000111
+disk,device=nvme0n1p4,fstype=ext4,host=Munin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000222
\ No newline at end of file
diff --git a/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/lut.csv b/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/lut.csv
new file mode 100644
index 0000000000000..aa85ba0161e65
--- /dev/null
+++ b/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/lut.csv
@@ -0,0 +1,7 @@
+# Some comment
+# lines
+host,location,type,os,cabinet
+cpu-Hugin,at home,desktop,,
+cpu-Munin,,mobile,Android,
+cpu-Thor,eu-west1,server,,r15-02
+disk-Hugin,,desktop,,
diff --git a/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/telegraf.conf b/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/telegraf.conf
new file mode 100644
index 0000000000000..e1d1c00490e6c
--- /dev/null
+++ b/plugins/processors/lookup/testcases/normal_lookup_csv_key_values/telegraf.conf
@@ -0,0 +1,4 @@
+[[processors.lookup]]
+    files = ["testcases/normal_lookup_csv_key_values/lut.csv"]
+    format = "csv_key_values"
+    key = '{{.Name}}-{{.Tag "host"}}'
diff --git a/plugins/processors/lookup/testcases/normal_lookup_json/expected.out b/plugins/processors/lookup/testcases/normal_lookup_json/expected.out
new file mode 100644
index 0000000000000..95e2f9aae285b
--- /dev/null
+++ b/plugins/processors/lookup/testcases/normal_lookup_json/expected.out
@@ -0,0 +1,5 @@
+cpu,cpu=cpu-total,host=Hugin,location=at\ home,type=desktop usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000123
+cpu,cpu=cpu-total,host=Munin,os=Android,type=mobile usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000456
+cpu,cpu=cpu-total,host=Thor,location=eu-west1,type=server,cabinet=r15-02 usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000789
+disk,device=nvme0n1p4,fstype=ext4,host=Hugin,mode=rw,path=/,type=desktop free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000111
+disk,device=nvme0n1p4,fstype=ext4,host=Munin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000222
diff --git a/plugins/processors/lookup/testcases/normal_lookup_json/input.influx b/plugins/processors/lookup/testcases/normal_lookup_json/input.influx
new file mode 100644
index 0000000000000..44779c033d7da
--- /dev/null
+++ b/plugins/processors/lookup/testcases/normal_lookup_json/input.influx
@@ -0,0 +1,5 @@
+cpu,cpu=cpu-total,host=Hugin usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000123
+cpu,cpu=cpu-total,host=Munin usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000456
+cpu,cpu=cpu-total,host=Thor usage_guest=0,usage_guest_nice=0,usage_idle=99.75000000049295,usage_iowait=0,usage_irq=0.1250000000007958,usage_nice=0,usage_softirq=0,usage_steal=0,usage_system=0,usage_user=0.12500000000363798 1678124473000000789
+disk,device=nvme0n1p4,fstype=ext4,host=Hugin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000111
+disk,device=nvme0n1p4,fstype=ext4,host=Munin,mode=rw,path=/ free=65652391936i,inodes_free=40445279i,inodes_total=45047808i,inodes_used=4602529i,total=725328994304i,used=622756728832i,used_percent=90.4631722684 1678124473000000222
\ No newline at end of file
diff --git a/plugins/processors/lookup/testcases/normal_lookup_json/lut.json b/plugins/processors/lookup/testcases/normal_lookup_json/lut.json
new file mode 100644
index 0000000000000..8364c3b18bba7
--- /dev/null
+++ b/plugins/processors/lookup/testcases/normal_lookup_json/lut.json
@@ -0,0 +1,18 @@
+{
+    "cpu-Hugin": {
+        "location": "at home",
+        "type": "desktop"
+    },
+    "cpu-Munin": {
+        "os": "Android",
+        "type": "mobile"
+    },
+    "cpu-Thor": {
+        "location": "eu-west1",
+        "type": "server",
+        "cabinet": "r15-02"
+    },
+    "disk-Hugin": {
+        "type": "desktop"
+    }
+}
\ No newline at end of file
diff --git a/plugins/processors/lookup/testcases/normal_lookup_json/telegraf.conf b/plugins/processors/lookup/testcases/normal_lookup_json/telegraf.conf
new file mode 100644
index 0000000000000..eef963464ed3e
--- /dev/null
+++ b/plugins/processors/lookup/testcases/normal_lookup_json/telegraf.conf
@@ -0,0 +1,3 @@
+[[processors.lookup]]
+    files = ["testcases/normal_lookup_json/lut.json"]
+    key = '{{.Name}}-{{.Tag "host"}}'