From e28369f4c689d27c7ef79f6c38d0050e84ea7a86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Simas?= Date: Sat, 27 Jul 2024 11:15:59 -0300 Subject: [PATCH 1/5] fix: label names in influxdb dashboard --- dashboards/Zettelkasten-InfluxDB.json | 95 ++++++++++++++++++++++++--- 1 file changed, 86 insertions(+), 9 deletions(-) diff --git a/dashboards/Zettelkasten-InfluxDB.json b/dashboards/Zettelkasten-InfluxDB.json index fb8ca15..48589f3 100644 --- a/dashboards/Zettelkasten-InfluxDB.json +++ b/dashboards/Zettelkasten-InfluxDB.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 2, - "id": 1, + "id": 67, "links": [], "panels": [ { @@ -82,7 +82,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.1", + "pluginVersion": "10.4.1", "targets": [ { "datasource": { @@ -144,7 +144,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.1", + "pluginVersion": "10.4.1", "targets": [ { "datasource": { @@ -206,7 +206,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.1", + "pluginVersion": "10.4.1", "targets": [ { "datasource": { @@ -269,7 +269,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.1", + "pluginVersion": "10.4.1", "targets": [ { "datasource": { @@ -355,7 +355,7 @@ } ] }, - "pluginVersion": "11.0.1", + "pluginVersion": "10.4.1", "targets": [ { "datasource": { @@ -502,6 +502,19 @@ } ], "title": "Notes", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "note_count": "Note count" + } + } + } + ], "type": "timeseries" }, { @@ -592,6 +605,19 @@ } ], "title": "Words", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "word_count": "Word count" + } + } + } + ], "type": "timeseries" }, { @@ -683,6 +709,19 @@ } ], "title": "Links", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "link_count": "Link count" + } + } + } + ], "type": "timeseries" }, { @@ -774,6 +813,19 @@ } ], "title": "Reading time", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "word_count": "Reading time" + } + } + } + ], "type": "timeseries" }, { @@ -865,6 +917,19 @@ } ], "title": "Average link count", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Average link count" + } + } + } + ], "type": "timeseries" }, { @@ -956,6 +1021,19 @@ } ], "title": "Average word count", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Average word count" + } + } + } + ], "type": "timeseries" } ], @@ -968,7 +1046,7 @@ "current": { "selected": false, "text": "influxdb", - "value": "cdqmo6y19tc74d" + "value": "cdpk3wq4tof7kc" }, "hide": 0, "includeAll": false, @@ -988,11 +1066,10 @@ "from": "now-30d", "to": "now" }, - "timeRangeUpdatedDuringEditOrView": false, "timepicker": {}, "timezone": "browser", "title": "Zettelkasten", "uid": "fdoghlpqzr5kwe", - "version": 11, + "version": 2, "weekStart": "" } \ No newline at end of file From c93ced5dd64d4b0729a2c90071212457a4aa467b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Simas?= Date: Sat, 27 Jul 2024 13:27:31 -0300 Subject: [PATCH 2/5] feat: add support for VictoriaMetrics storage Adds support for using VictoriaMetrics as a storage backend. The metrics are encoded using the InfluxDB line protocol and written to the remote VictoriaMetrics write endpoint. Using the InfluxDB line protocol allows for specifying a timestamp when writing the metrics, enabling backfilling of historical metrics. --- go.mod | 2 +- internal/config/config.go | 16 ++++++--- internal/config/config_test.go | 47 +++++++++++++++++++++++++ internal/storage/influxdb.go | 17 +++++++-- internal/storage/victoriametrics.go | 53 +++++++++++++++++++++++++++++ main.go | 9 +++-- 6 files changed, 134 insertions(+), 10 deletions(-) create mode 100644 internal/storage/victoriametrics.go diff --git a/go.mod b/go.mod index 41427d6..82b690e 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.22.2 require ( github.com/gookit/validate v1.5.2 github.com/influxdata/influxdb-client-go/v2 v2.13.0 + github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 github.com/knadh/koanf v1.5.0 github.com/stretchr/testify v1.9.0 github.com/yuin/goldmark v1.7.4 @@ -19,7 +20,6 @@ require ( github.com/google/uuid v1.3.1 // indirect github.com/gookit/filter v1.2.1 // indirect github.com/gookit/goutil v0.6.15 // indirect - github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index 7550a46..4b5b09c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -22,10 +22,11 @@ type Config struct { IgnoreFiles []string `koanf:"ignore_files"` CollectionInterval time.Duration `koanf:"collection_interval"` CollectHistoricalMetrics bool `koanf:"collect_historical_metrics"` - InfluxDBURL string `koanf:"influxdb_url" validate:"required|fullUrl"` - InfluxDBToken string `koanf:"influxdb_token" validate:"required"` - InfluxDBOrg string `koanf:"influxdb_org" validate:"required"` - InfluxDBBucket string `koanf:"influxdb_bucket" validate:"required"` + VictoriaMetricsURL string `koanf:"victoriametrics_url" validate:"fullUrl"` + InfluxDBURL string `koanf:"influxdb_url" validate:"fullUrl"` + InfluxDBToken string `koanf:"influxdb_token" validate:"requiredWith:InfluxDBURL"` + InfluxDBOrg string `koanf:"influxdb_org" validate:"requiredWith:InfluxDBURL"` + InfluxDBBucket string `koanf:"influxdb_bucket" validate:"requiredWith:InfluxDBURL"` } func LoadConfig() (Config, error) { @@ -75,6 +76,12 @@ func LoadConfig() (Config, error) { if cfg.ZettelkastenGitURL != "" && cfg.ZettelkastenDirectory != "" { return Config{}, errors.New("ZettelkastenGitURL and ZettelkastenDirectory cannot be provided together") } + if cfg.VictoriaMetricsURL != "" && cfg.InfluxDBURL != "" { + return Config{}, errors.New("InfluxDBURL and VictoriaMetricsURL cannot be provided together") + } + if cfg.VictoriaMetricsURL == "" && cfg.InfluxDBURL == "" { + return Config{}, errors.New("Either InfluxDBURL or VictoriaMetricsURL must be provided") + } return cfg, nil } @@ -89,6 +96,7 @@ func (c Config) LogValue() slog.Value { slog.Any("IgnoreFiles", c.IgnoreFiles), slog.Duration("CollectionInterval", c.CollectionInterval), slog.Bool("CollectHistoricalMetrics", c.CollectHistoricalMetrics), + slog.String("VictoriaMetricsURL", c.VictoriaMetricsURL), slog.String("InfluxDBURL", c.InfluxDBURL), slog.String("InfluxDBToken", "[REDACTED]"), slog.String("InfluxDBOrg", c.InfluxDBOrg), diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 7c8671e..0067a0f 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -118,6 +118,31 @@ func TestLoadConfig_FullEnvGit(t *testing.T) { } } +func TestLoadConfig_FullEnvGitVictoriaMetrics(t *testing.T) { + t.Setenv("VICTORIAMETRICS_URL", "http://localhost:8428") + t.Setenv("COLLECTION_INTERVAL", "15m") + t.Setenv("COLLECT_HISTORICAL_METRICS", "false") + t.Setenv("LOG_LEVEL", "ERROR") + t.Setenv("ZETTELKASTEN_GIT_URL", "https://github.com/user/zettel") + t.Setenv("ZETTELKASTEN_GIT_BRANCH", "any-branch") + t.Setenv("ZETTELKASTEN_GIT_TOKEN", "any-token") + t.Setenv("IGNORE_FILES", ".obsidian,test,/something/another,dir/file.md") + c, err := LoadConfig() + if assert.NoError(t, err) { + expected := Config{ + VictoriaMetricsURL: "http://localhost:8428", + CollectionInterval: time.Minute * 15, + CollectHistoricalMetrics: false, + LogLevel: slog.LevelError, + ZettelkastenGitURL: "https://github.com/user/zettel", + ZettelkastenGitBranch: "any-branch", + ZettelkastenGitToken: "any-token", + IgnoreFiles: []string{".obsidian", "test", "/something/another", "dir/file.md"}, + } + assert.Equal(t, expected, c) + } +} + func TestLoadConfig_Validate(t *testing.T) { data := []struct { name string @@ -140,6 +165,28 @@ func TestLoadConfig_Validate(t *testing.T) { "ZETTELKASTEN_GIT_URL": "any-string", }, }, + { + name: "missing influxdb auth", + shouldError: true, + env: map[string]string{ + "LOG_LEVEL": "INFO", + "ZETTELKASTEN_GIT_URL": "any-string", + "INFLUXDB_URL": "http://localhost:8086", + }, + }, + { + name: "both storage", + shouldError: true, + env: map[string]string{ + "LOG_LEVEL": "INFO", + "ZETTELKASTEN_GIT_URL": "any-string", + "INFLUXDB_URL": "http://localhost:8086", + "INFLUXDB_TOKEN": "any-token", + "INFLUXDB_ORG": "any-org", + "INFLUXDB_BUCKET": "any-bucket", + "VICTORIAMETRICS_URL": "httpL//localhost:8428", + }, + }, { name: "valid config", shouldError: false, diff --git a/internal/storage/influxdb.go b/internal/storage/influxdb.go index 777325c..696e856 100644 --- a/internal/storage/influxdb.go +++ b/internal/storage/influxdb.go @@ -5,6 +5,7 @@ import ( influxdb2 "github.com/influxdata/influxdb-client-go/v2" "github.com/influxdata/influxdb-client-go/v2/api" + "github.com/influxdata/influxdb-client-go/v2/api/write" "github.com/luissimas/zettelkasten-exporter/internal/metrics" ) @@ -29,6 +30,15 @@ func NewInfluxDBStorage(url, org, bucket, token string) InfluxDBStorage { // WriteMetric writes `metric` for `noteName` to the storage with `timestamp`. func (i InfluxDBStorage) WriteMetrics(zettelkastenMetrics metrics.Metrics, timestamp time.Time) { + points := createInfluxDBPoints(zettelkastenMetrics, timestamp) + for _, point := range points { + i.writeAPI.WritePoint(point) + } +} + +// createInfluxDBPoints creates a slice of InfluxDB measurement points from `zettelkastenMetrics` with the given `timestamp`. +func createInfluxDBPoints(zettelkastenMetrics metrics.Metrics, timestamp time.Time) []*write.Point { + points := make([]*write.Point, 0, len(zettelkastenMetrics.Notes)+1) // Aggregated metrics point := influxdb2.NewPoint( totalMeasurementName, @@ -40,11 +50,11 @@ func (i InfluxDBStorage) WriteMetrics(zettelkastenMetrics metrics.Metrics, times }, timestamp, ) - i.writeAPI.WritePoint(point) + points = append(points, point) // Individual note metrics for name, metric := range zettelkastenMetrics.Notes { - point := influxdb2.NewPoint( + point = influxdb2.NewPoint( notesMeasurementName, map[string]string{"name": name}, map[string]interface{}{ @@ -54,6 +64,7 @@ func (i InfluxDBStorage) WriteMetrics(zettelkastenMetrics metrics.Metrics, times }, timestamp, ) - i.writeAPI.WritePoint(point) + points = append(points, point) } + return points } diff --git a/internal/storage/victoriametrics.go b/internal/storage/victoriametrics.go new file mode 100644 index 0000000..ae7859b --- /dev/null +++ b/internal/storage/victoriametrics.go @@ -0,0 +1,53 @@ +package storage + +import ( + "bytes" + "fmt" + "log/slog" + "net/http" + "time" + + "github.com/influxdata/influxdb-client-go/v2/api/write" + lp "github.com/influxdata/line-protocol" + "github.com/luissimas/zettelkasten-exporter/internal/metrics" +) + +type VictoriaMetricsStorage struct { + writeUrl string +} + +func NewVictoriaMetricsStorage(url string) VictoriaMetricsStorage { + return VictoriaMetricsStorage{writeUrl: fmt.Sprintf("%s/api/v2/write", url)} +} + +func (v VictoriaMetricsStorage) WriteMetrics(zettelkastenMetrics metrics.Metrics, timestamp time.Time) { + // NOTE: we encode the metrics in the InfluxDB line protocol and write them to the VictoriaMetrics write endpoint. + // Reference: https://docs.victoriametrics.com/#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf + points := createInfluxDBPoints(zettelkastenMetrics, timestamp) + content, err := encodePoints(points) + if err != nil { + slog.Error("Error encoding points into line procotol", slog.Any("error", err)) + } + slog.Info("Writing metrics to endpoint", slog.String("content", string(content))) + _, err = http.Post(v.writeUrl, "application/x-www-form-urlencoded", bytes.NewBuffer(content)) + if err != nil { + slog.Error("Error sending POST request to endpoint", slog.Any("error", err), slog.String("url", v.writeUrl)) + } +} + +// encodePoints encodes the given `points` into InfluxDB's line protocol. +func encodePoints(points []*write.Point) ([]byte, error) { + var buffer bytes.Buffer + e := lp.NewEncoder(&buffer) + e.SetFieldTypeSupport(lp.UintSupport) + e.FailOnFieldErr(true) + e.SetPrecision(time.Millisecond) + slog.Info("Endcoding points", slog.Any("points", points)) + for _, point := range points { + _, err := e.Encode(point) + if err != nil { + return make([]byte, 0), err + } + } + return buffer.Bytes(), nil +} diff --git a/main.go b/main.go index 58fc474..8e57fca 100644 --- a/main.go +++ b/main.go @@ -21,8 +21,13 @@ func main() { os.Exit(1) } slog.Debug("Loaded config", slog.Any("config", cfg)) - storage := storage.NewInfluxDBStorage(cfg.InfluxDBURL, cfg.InfluxDBOrg, cfg.InfluxDBBucket, cfg.InfluxDBToken) - collector := collector.NewCollector(cfg.IgnoreFiles, storage) + var metricsStorage storage.Storage + if cfg.VictoriaMetricsURL != "" { + metricsStorage = storage.NewVictoriaMetricsStorage(cfg.VictoriaMetricsURL) + } else { + metricsStorage = storage.NewInfluxDBStorage(cfg.InfluxDBURL, cfg.InfluxDBOrg, cfg.InfluxDBBucket, cfg.InfluxDBToken) + } + collector := collector.NewCollector(cfg.IgnoreFiles, metricsStorage) var zet zettelkasten.Zettelkasten if cfg.ZettelkastenGitURL != "" { zet = zettelkasten.NewGitZettelkasten(cfg.ZettelkastenGitURL, cfg.ZettelkastenGitBranch, cfg.ZettelkastenGitToken) From acae8be83f17b44003a4925551a4a089e8447abc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Simas?= Date: Sat, 27 Jul 2024 14:01:57 -0300 Subject: [PATCH 3/5] feat: handle errors when writing to storage Adds error handling when writing metrics to the storage. --- internal/collector/collector.go | 5 ++++- internal/storage/fake.go | 3 ++- internal/storage/influxdb.go | 15 ++++++++++----- internal/storage/storage.go | 2 +- internal/storage/victoriametrics.go | 9 ++++++--- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/internal/collector/collector.go b/internal/collector/collector.go index 4aaf65e..ec44cfd 100644 --- a/internal/collector/collector.go +++ b/internal/collector/collector.go @@ -41,7 +41,10 @@ func (c *Collector) CollectMetrics(root fs.FS, collectionTime time.Time) error { return err } - c.storage.WriteMetrics(collected, collectionTime) + err = c.storage.WriteMetrics(collected, collectionTime) + if err != nil { + return err + } slog.Debug("Collected metrics", slog.Duration("duration", time.Since(start))) return nil diff --git a/internal/storage/fake.go b/internal/storage/fake.go index 8be9300..42048d3 100644 --- a/internal/storage/fake.go +++ b/internal/storage/fake.go @@ -14,7 +14,8 @@ func NewFakeStorage() FakeStorage { return FakeStorage{} } -func (f FakeStorage) WriteMetrics(zettelkastenMetrics metrics.Metrics, timestamp time.Time) { +func (f FakeStorage) WriteMetrics(zettelkastenMetrics metrics.Metrics, timestamp time.Time) error { + return nil } func (f FakeStorage) IsEmpty() bool { diff --git a/internal/storage/influxdb.go b/internal/storage/influxdb.go index 696e856..25ca572 100644 --- a/internal/storage/influxdb.go +++ b/internal/storage/influxdb.go @@ -1,6 +1,8 @@ package storage import ( + "context" + "log/slog" "time" influxdb2 "github.com/influxdata/influxdb-client-go/v2" @@ -16,24 +18,27 @@ const totalMeasurementName = "total" // InfluxDBStorage represents the implementation of a metric storage using InfluxDB. type InfluxDBStorage struct { - writeAPI api.WriteAPI + writeAPI api.WriteAPIBlocking queryAPI api.QueryAPI } // NewInfluxDBStorage creates a new `InfluxDBStorage`. func NewInfluxDBStorage(url, org, bucket, token string) InfluxDBStorage { client := influxdb2.NewClient(url, string(token)) - writeAPI := client.WriteAPI(org, bucket) + writeAPI := client.WriteAPIBlocking(org, bucket) queryAPI := client.QueryAPI(org) return InfluxDBStorage{writeAPI: writeAPI, queryAPI: queryAPI} } // WriteMetric writes `metric` for `noteName` to the storage with `timestamp`. -func (i InfluxDBStorage) WriteMetrics(zettelkastenMetrics metrics.Metrics, timestamp time.Time) { +func (i InfluxDBStorage) WriteMetrics(zettelkastenMetrics metrics.Metrics, timestamp time.Time) error { points := createInfluxDBPoints(zettelkastenMetrics, timestamp) - for _, point := range points { - i.writeAPI.WritePoint(point) + slog.Debug("Writing metrics to InfluxDB", slog.Any("points", points)) + err := i.writeAPI.WritePoint(context.Background(), points...) + if err != nil { + slog.Error("Error writing points to InfluxDB storage", slog.Any("error", err)) } + return err } // createInfluxDBPoints creates a slice of InfluxDB measurement points from `zettelkastenMetrics` with the given `timestamp`. diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 7861ec9..fa01529 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -9,5 +9,5 @@ import ( // Storage represents a storage for metrics. type Storage interface { // WriteMetric writes the `zettelkastenMetrics` to the storage. - WriteMetrics(zettelkastenMetrics metrics.Metrics, timestamp time.Time) + WriteMetrics(zettelkastenMetrics metrics.Metrics, timestamp time.Time) error } diff --git a/internal/storage/victoriametrics.go b/internal/storage/victoriametrics.go index ae7859b..7c12be9 100644 --- a/internal/storage/victoriametrics.go +++ b/internal/storage/victoriametrics.go @@ -20,19 +20,22 @@ func NewVictoriaMetricsStorage(url string) VictoriaMetricsStorage { return VictoriaMetricsStorage{writeUrl: fmt.Sprintf("%s/api/v2/write", url)} } -func (v VictoriaMetricsStorage) WriteMetrics(zettelkastenMetrics metrics.Metrics, timestamp time.Time) { +func (v VictoriaMetricsStorage) WriteMetrics(zettelkastenMetrics metrics.Metrics, timestamp time.Time) error { // NOTE: we encode the metrics in the InfluxDB line protocol and write them to the VictoriaMetrics write endpoint. // Reference: https://docs.victoriametrics.com/#how-to-send-data-from-influxdb-compatible-agents-such-as-telegraf points := createInfluxDBPoints(zettelkastenMetrics, timestamp) content, err := encodePoints(points) if err != nil { slog.Error("Error encoding points into line procotol", slog.Any("error", err)) + return err } - slog.Info("Writing metrics to endpoint", slog.String("content", string(content))) + slog.Debug("Writing metrics to VictoriaMetrics", slog.String("content", string(content))) _, err = http.Post(v.writeUrl, "application/x-www-form-urlencoded", bytes.NewBuffer(content)) if err != nil { slog.Error("Error sending POST request to endpoint", slog.Any("error", err), slog.String("url", v.writeUrl)) + return err } + return nil } // encodePoints encodes the given `points` into InfluxDB's line protocol. @@ -42,7 +45,7 @@ func encodePoints(points []*write.Point) ([]byte, error) { e.SetFieldTypeSupport(lp.UintSupport) e.FailOnFieldErr(true) e.SetPrecision(time.Millisecond) - slog.Info("Endcoding points", slog.Any("points", points)) + slog.Debug("Encoding points", slog.Any("points", points)) for _, point := range points { _, err := e.Encode(point) if err != nil { From 1753d1c15c4ba79b8850b8f650924575348cebc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Simas?= Date: Mon, 29 Jul 2024 08:59:42 -0300 Subject: [PATCH 4/5] feat: add promql dashboard Adds a Grafana dashboard querying the exporter data from a PromQL source. Currently this source is VictoriaMetrics, but the PromQL compatibility enables Prometheus use in the future. --- dashboards/Zettelkasten-PromQL.json | 1244 +++++++++++++++++++++++++++ 1 file changed, 1244 insertions(+) create mode 100644 dashboards/Zettelkasten-PromQL.json diff --git a/dashboards/Zettelkasten-PromQL.json b/dashboards/Zettelkasten-PromQL.json new file mode 100644 index 0000000..2bcc668 --- /dev/null +++ b/dashboards/Zettelkasten-PromQL.json @@ -0,0 +1,1244 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 2, + "id": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of notes created in the visualization period", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "edogaymh9y96of" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "increase(total_note_count[$__range])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"total\" and r[\"_field\"] == \"note_count\")\n |> spread()", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "New notes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of links created in the visualization period", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "edogaymh9y96of" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "increase(total_link_count[$__range])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"total\" and r[\"_field\"] == \"link_count\")\n |> spread()", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "New links", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of words written in the visualization period", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "edogaymh9y96of" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "increase(total_word_count[$__range])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"total\" and r[\"_field\"] == \"word_count\")\n |> spread()", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "New words", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Time to read all notes created in the visualization period", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "yellow", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "m" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "edogaymh9y96of" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "round(increase(total_word_count[$__range]) / 212, 1)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"total\" and r[\"_field\"] == \"word_count\")\n |> map(fn: (r) => ({r with _value: r._value / uint(v: 212)}))\n |> spread()", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Reading time", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Metrics from each markdown note in the Zettelkasten", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false, + "minWidth": 50 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Reading time" + }, + "properties": [ + { + "id": "unit", + "value": "m" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 3, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": true + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Backlinks" + } + ] + }, + "pluginVersion": "11.1.1", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "edogaymh9y96of" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "notes_link_count", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"notes\")\n |> last()\n |> pivot(rowKey: [\"name\"], columnKey: [\"_field\"], valueColumn: \"_value\")\n |> map(fn: (r) => ({\n _measurement: r._measurement,\n name: r.name,\n backlink_count: r.backlink_count,\n link_count: r.link_count,\n word_count: r.word_count,\n time_to_read: r.word_count / uint(v: 212)\n }))\n |> group()", + "range": false, + "refId": "Link", + "useBackend": false + }, + { + "datasource": { + "type": "influxdb", + "uid": "edogaymh9y96of" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "notes_backlink_count", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"notes\")\n |> last()\n |> pivot(rowKey: [\"name\"], columnKey: [\"_field\"], valueColumn: \"_value\")\n |> map(fn: (r) => ({\n _measurement: r._measurement,\n name: r.name,\n backlink_count: r.backlink_count,\n link_count: r.link_count,\n word_count: r.word_count,\n time_to_read: r.word_count / uint(v: 212)\n }))\n |> group()", + "range": false, + "refId": "Backlinks", + "useBackend": false + }, + { + "datasource": { + "type": "influxdb", + "uid": "edogaymh9y96of" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "notes_word_count", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"notes\")\n |> last()\n |> pivot(rowKey: [\"name\"], columnKey: [\"_field\"], valueColumn: \"_value\")\n |> map(fn: (r) => ({\n _measurement: r._measurement,\n name: r.name,\n backlink_count: r.backlink_count,\n link_count: r.link_count,\n word_count: r.word_count,\n time_to_read: r.word_count / uint(v: 212)\n }))\n |> group()", + "range": false, + "refId": "Words", + "useBackend": false + }, + { + "datasource": { + "type": "influxdb", + "uid": "edogaymh9y96of" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "round(notes_word_count / 212, 1)", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"notes\")\n |> last()\n |> pivot(rowKey: [\"name\"], columnKey: [\"_field\"], valueColumn: \"_value\")\n |> map(fn: (r) => ({\n _measurement: r._measurement,\n name: r.name,\n backlink_count: r.backlink_count,\n link_count: r.link_count,\n word_count: r.word_count,\n time_to_read: r.word_count / uint(v: 212)\n }))\n |> group()", + "range": false, + "refId": "Reading time", + "useBackend": false + } + ], + "title": "Notes", + "transformations": [ + { + "id": "joinByLabels", + "options": { + "value": "__name__" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "_measurement": true, + "_start": true, + "_stop": true + }, + "includeByName": {}, + "indexByName": { + "Value": 4, + "name": 0, + "notes_backlink_count": 2, + "notes_link_count": 1, + "notes_word_count": 3 + }, + "renameByName": { + "Value": "Reading time", + "_start": "", + "backlink_count": "Backlinks", + "link_count": "Links", + "name": "Name", + "notes_backlink_count": "Backlinks", + "notes_link_count": "Links", + "notes_word_count": "Words", + "time_to_read": "Reading time", + "word_count": "Words" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 12, + "panels": [], + "title": "Historical data", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Historical evolution of the number of notes in the Zettelkasten", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "total_note_count", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"total\")\n |> filter(fn: (r) => r[\"_field\"] == \"note_count\")", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Notes", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "note_count": "Note count", + "total_note_count": "Note count" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Historical evolution of words in the Zettelkasten", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "total_word_count", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"total\")\n |> filter(fn: (r) => r[\"_field\"] == \"word_count\")", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Words", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "total_word_count": "Word count", + "word_count": "Word count" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Historical evolution of the number of links in the Zettelkasten", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "total_link_count", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"total\")\n |> filter(fn: (r) => r[\"_field\"] == \"link_count\")", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Links", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "link_count": "Link count", + "total_link_count": "Link count" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Historical evolution of the total reading time for the Zettelkasten", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "yellow", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "m" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "total_word_count / 212", + "fullMetaSearch": false, + "includeNullMetadata": true, + "legendFormat": "__auto", + "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"total\")\n |> filter(fn: (r) => r[\"_field\"] == \"word_count\")\n |> map(fn: (r) => ({r with _value: r._value / uint(v: 212)}))", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Reading time", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "total_word_count / 212": "Reading time", + "word_count": "Reading time" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Historical evolution of the average number of links per note in the Zettelkasten", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.1", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(notes_link_count[$__range]) / count(notes_link_count[$__range])", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "__auto", + "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"notes\" and r[\"_field\"] == \"link_count\")\n |> group(columns: [\"__time\"])\n |> aggregateWindow(\n every: v.windowPeriod,\n fn: (column, tables=<-) => tables |> mean(),\n column: \"name\",\n createEmpty: false\n)", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Average link count", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Average link count", + "sum(notes_link_count[2592000s]) / count(notes_link_count[2592000s])": "Average link count" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Historical evolution of the average number of words per note in the Zettelkasten", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.1", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(notes_word_count[$__range]) / count(notes_word_count[$__range])", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "__auto", + "query": "from(bucket: v.defaultBucket)\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r[\"_measurement\"] == \"notes\" and r[\"_field\"] == \"word_count\")\n |> group(columns: [\"__time\"])\n |> aggregateWindow(\n every: v.windowPeriod,\n fn: (column, tables=<-) => tables |> mean(),\n column: \"name\",\n createEmpty: false\n)", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Average word count", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {}, + "includeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Average word count", + "sum(notes_word_count[2592000s]) / count(notes_word_count[2592000s])": "Average word count" + } + } + } + ], + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "prometheus", + "value": "ddt7nsuyconb4a" + }, + "hide": 0, + "includeAll": false, + "label": "Data source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Zettelkasten", + "uid": "fdoghlpqzr5kwe", + "version": 3, + "weekStart": "" +} \ No newline at end of file From 7be38de34960e1338d2b5e5cb372cb0eb0d37d9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Simas?= Date: Tue, 30 Jul 2024 08:04:01 -0300 Subject: [PATCH 5/5] docs: document victoriametrics usage Adds documentation and examples regarding the setup with a VictoriaMetrics storage backend. --- README.md | 45 ++++++++------- ...pose.yaml => docker-compose-influxdb.yaml} | 10 ++-- .../docker-compose-victoriametrics.yaml | 38 +++++++++++++ .../{manifest.yaml => manifest-influxdb.yaml} | 0 .../kubernetes/manifest-victoriametrics.yaml | 56 +++++++++++++++++++ 5 files changed, 123 insertions(+), 26 deletions(-) rename examples/compose/{docker-compose.yaml => docker-compose-influxdb.yaml} (91%) create mode 100644 examples/compose/docker-compose-victoriametrics.yaml rename examples/kubernetes/{manifest.yaml => manifest-influxdb.yaml} (100%) create mode 100644 examples/kubernetes/manifest-victoriametrics.yaml diff --git a/README.md b/README.md index 002485b..66dc98d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ # Zettelkasten exporter -An agent that collects metrics from an zettelkasten and stores them into an InfluxDB bucket. +An agent that collects metrics from an zettelkasten and lets you visualize them in Grafana. ![](./docs/assets/dashboard.png) @@ -18,6 +18,7 @@ An agent that collects metrics from an zettelkasten and stores them into an Infl - Parses both markdown and wiki links - Authenticate in private git repositories using personal access tokens - Grafana dashboards included +- Support for both InfluxDB and VictoriaMetrics as storage backends ## Usage @@ -26,23 +27,29 @@ The exporter is distributed as both a binary and a Docker image. The currently d - [Docker compose example](./examples/compose) - [Kubernetes example](./examples/kubernetes) -Note that for a complete solution, it will be necessary to setup InfluxDB and Grafana. For more information about setting them up, refer to their documentation. Here are some links that might be useful: +Note that for a complete solution, it will be necessary to setup Grafana and either InfluxDB or VictoriaMetrics as data sources. For more information about setting them up, refer to their documentation. Here are some links that might be useful: - https://grafana.com/docs/grafana/latest/getting-started/get-started-grafana-influxdb/ - https://docs.influxdata.com/influxdb/v2/get-started/setup/ +- https://docs.victoriametrics.com/ -The provided dashboard uses `Flux` as the language to query InfluxDB, so make sure to set the "Query language" option to "Flux" when creating the InfluxDB data source in Grafana. +In the `dashboards` folder there are two Grafana dashboards provided: one for InfluxDB and another for VictoriaMetrics. You'll have to import the appropriate dashboard for your storage backend. + +The provided InfluxDB dashboard uses `Flux` as the query language, so make sure to set the "Query language" option to "Flux" when creating the InfluxDB data source in Grafana. + +For both storage backends, make sure to configure the data retention period according to your needs. ## Configuration -All configuration is supplied via environment variables. You should supply at least the required variables and either the `ZETTELKASTEN_DIRECTORY` or the `ZETTELKASTEN_GIT_URL` variables. +All configuration is supplied via environment variables. You should supply at least the zettelkasten source via the `ZETTELKASTEN_DIRECTORY` or `ZETTELKASTEN_GIT_URL` variables and the storage backend via the `VICTORIAMETRICS_URL` or `INFLUXDB_*` variables. | Name | Description | Default | Required | | -------------------------- | -------------------------------------------------------------------- | ------------------------------ | -------- | -| INFLUXDB_URL | The InfluxDB URL | | Yes | -| INFLUXDB_TOKEN | The InfluxDB token to authenticate in the bucket | | Yes | -| INFLUXDB_ORG | The InfluxDB org containing the bucket | | Yes | -| INFLUXDB_BUCKET | The InfluxDB bucket to register metrics | | Yes | +| VICTORIAMETRICS_URL | The VictoriaMetrics URL | | No | +| INFLUXDB_URL | The InfluxDB URL | | No | +| INFLUXDB_TOKEN | The InfluxDB token to authenticate in the bucket | | No | +| INFLUXDB_ORG | The InfluxDB org containing the bucket | | No | +| INFLUXDB_BUCKET | The InfluxDB bucket to register metrics | | No | | ZETTELKASTEN_DIRECTORY | The local directory containing the zettelkasten | | No | | ZETTELKASTEN_GIT_URL | The URL for the git repository containing the zettelkasten | | No | | ZETTELKASTEN_GIT_TOKEN | The access token to authenticate with private repositories | | No | @@ -54,26 +61,24 @@ All configuration is supplied via environment variables. You should supply at le ## Metrics -The exporter collects metrics by parsing the contents of the markdown files present in the Zettelkasten. Currently the exporter stores metrics for individual notes and also aggregated metrics describing the entire Zettelkasten. The combination of raw and pre processed metrics allows for both flexibility and efficiency when querying the data, at the cost of a slightly higher storage usage. The two sets of metrics are stored in the same InfluxDB bucket under different [measurement names](https://docs.influxdata.com/influxdb/cloud/reference/key-concepts/data-elements/#measurement). +The exporter collects metrics by parsing the contents of the markdown files present in the Zettelkasten. Currently the exporter stores metrics for individual notes and also aggregated metrics describing the entire Zettelkasten. The combination of raw and pre processed metrics allows for both flexibility and efficiency when querying the data, at the cost of a slightly higher storage usage. When using the InfluxDB storage, the two sets of metrics are stored in the same InfluxDB bucket under different [measurement names](https://docs.influxdata.com/influxdb/cloud/reference/key-concepts/data-elements/#measurement). When using the VictoriaMetrics storage, each metric is stored under a different name. The following table describes all metrics collected by the exporter and their respective measurement names: -| Measurement | Name | Description | -|-------------|----------------|-----------------------------------------| -| notes | link_count | Number of links in the note | -| notes | word_count | Number of words in the note | -| notes | backlink_count | Number of links that reference the note | -| total | note_count | Number of notes in the Zettelkasten | -| total | link_count | Number of links in the Zettelkasten | -| total | word_count | Number of words in the Zettelkasten | +| InfluxDB measurement | InfluxDB name | VictoriaMetrics name | Description | +|----------------------|----------------|----------------------|-----------------------------------------| +| notes | link_count | notes_link_count | Number of links in the note | +| notes | word_count | notes_word_count | Number of words in the note | +| notes | backlink_count | notes_backlink_count | Number of links that reference the note | +| total | note_count | total_note_count | Number of notes in the Zettelkasten | +| total | link_count | total_link_count | Number of links in the Zettelkasten | +| total | word_count | total_word_count | Number of words in the Zettelkasten | ## Roadmap These are some features that I'd like to include in the future. -- Collect additional metrics -- Support Prometheus remote write as a storage -- Support VictoriaMetrics as a storage +- Support Prometheus remote write as a storage backend ## References diff --git a/examples/compose/docker-compose.yaml b/examples/compose/docker-compose-influxdb.yaml similarity index 91% rename from examples/compose/docker-compose.yaml rename to examples/compose/docker-compose-influxdb.yaml index 69b651f..562f522 100644 --- a/examples/compose/docker-compose.yaml +++ b/examples/compose/docker-compose-influxdb.yaml @@ -1,7 +1,5 @@ # This is a sample compose file for deploying the zettelkasten-exporter # using an InfluxDB storage. -version: '3.8' - volumes: influxdb-data: {} influxdb-config: {} @@ -10,16 +8,14 @@ volumes: services: grafana: image: grafana/grafana - depends_on: - - influxdb restart: unless-stopped volumes: - grafana-data:/var/lib/grafan ports: - 3000:3000 - influxdb: image: influxdb:2 + restart: unless-stopped environment: # We opt for an automated setup of InfluxDB for simplicity. It's # strongly recommended to change those credentials or doing a @@ -29,6 +25,8 @@ services: DOCKER_INFLUXDB_INIT_PASSWORD: password DOCKER_INFLUXDB_INIT_ORG: default DOCKER_INFLUXDB_INIT_BUCKET: zettelkasten + # In your own setup you'll probably want to specify a longer + # retention period. DOCKER_INFLUXDB_INIT_RETENTION: 1w DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: demo-auth-token volumes: @@ -36,9 +34,9 @@ services: - influxdb-config:/etc/influxdb2 ports: - 8086:8086 - zettelkasten-exporter: image: ghcr.io/luissimas/zettelkasten-exporter:latest + restart: unless-stopped depends_on: - influxdb environment: diff --git a/examples/compose/docker-compose-victoriametrics.yaml b/examples/compose/docker-compose-victoriametrics.yaml new file mode 100644 index 0000000..07850a1 --- /dev/null +++ b/examples/compose/docker-compose-victoriametrics.yaml @@ -0,0 +1,38 @@ +# This is a sample compose file for deploying the zettelkasten-exporter +# using an VictoriaMetrics storage. +volumes: + grafana-data: {} + victoriametrics-data: {} + +services: + grafana: + image: grafana/grafana + restart: unless-stopped + volumes: + - grafana-data:/var/lib/grafan + ports: + - 3000:3000 + victoriametrics: + image: victoriametrics/victoria-metrics:latest + # In your own setup you'll probably want to specify a longer + # retention period. + command: -retentionPeriod=1w + restart: unless-stopped + ports: + - 8428:8428 + volumes: + - victoriametrics-data:/victoria-metrics-data + zettelkasten-exporter: + image: ghcr.io/luissimas/zettelkasten-exporter:latest + restart: unless-stopped + depends_on: + - victoriametrics + environment: + LOG_LEVEL: INFO + # Here we use a local directory for simplicity, but check out the + # README to see how to configure different sources such as git repositories. + ZETTELKASTEN_DIRECTORY: /sample + COLLECTION_INTERVAL: 10s + VICTORIAMETRICS_URL: http://victoriametrics:8428 + volumes: + - ./sample:/sample diff --git a/examples/kubernetes/manifest.yaml b/examples/kubernetes/manifest-influxdb.yaml similarity index 100% rename from examples/kubernetes/manifest.yaml rename to examples/kubernetes/manifest-influxdb.yaml diff --git a/examples/kubernetes/manifest-victoriametrics.yaml b/examples/kubernetes/manifest-victoriametrics.yaml new file mode 100644 index 0000000..1e6650e --- /dev/null +++ b/examples/kubernetes/manifest-victoriametrics.yaml @@ -0,0 +1,56 @@ +# This is a sample manifest for deploying the zettelkasten-exporter using an VictoriaMetrics +# storage. +# To deploy VictoriaMetrics, see: https://github.com/VictoriaMetrics/helm-charts +--- +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring +--- +apiVersion: v1 +kind: Secret +metadata: + name: zettelkasten-exporter-env + namespace: monitoring +type: Opaque +data: + # These are placeholder values. Replace them with the + # appropriate values for your setup. + github-token: YW55LXRva2Vu +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: zettelkasten-exporter + namespace: monitoring + labels: + app.kubernetes.io/name: zettelkasten-exporter +spec: + selector: + matchLabels: + app.kubernetes.io/name: zettelkasten-exporter + replicas: 1 + template: + metadata: + labels: + app.kubernetes.io/name: zettelkasten-exporter + spec: + containers: + - name: zettelkasten-exporter + image: "ghcr.io/luissimas/zettelkasten-exporter:latest" + env: + - name: LOG_LEVEL + value: INFO + - name: COLLECTION_INTERVAL + value: 5m + - name: ZETTELKASTEN_GIT_URL + value: + - name: ZETTELKASTEN_GIT_BRANCH + value: master + - name: ZETTELKASTEN_GIT_TOKEN + valueFrom: + secretKeyRef: + name: zettelkasten-exporter-env + key: github-token + - name: VICTORIAMETRICS_URL + value: http://victoriametrics