From 7be38de34960e1338d2b5e5cb372cb0eb0d37d9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Simas?= Date: Tue, 30 Jul 2024 08:04:01 -0300 Subject: [PATCH] docs: document victoriametrics usage Adds documentation and examples regarding the setup with a VictoriaMetrics storage backend. --- README.md | 45 ++++++++------- ...pose.yaml => docker-compose-influxdb.yaml} | 10 ++-- .../docker-compose-victoriametrics.yaml | 38 +++++++++++++ .../{manifest.yaml => manifest-influxdb.yaml} | 0 .../kubernetes/manifest-victoriametrics.yaml | 56 +++++++++++++++++++ 5 files changed, 123 insertions(+), 26 deletions(-) rename examples/compose/{docker-compose.yaml => docker-compose-influxdb.yaml} (91%) create mode 100644 examples/compose/docker-compose-victoriametrics.yaml rename examples/kubernetes/{manifest.yaml => manifest-influxdb.yaml} (100%) create mode 100644 examples/kubernetes/manifest-victoriametrics.yaml diff --git a/README.md b/README.md index 002485b..66dc98d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ # Zettelkasten exporter -An agent that collects metrics from an zettelkasten and stores them into an InfluxDB bucket. +An agent that collects metrics from an zettelkasten and lets you visualize them in Grafana. ![](./docs/assets/dashboard.png) @@ -18,6 +18,7 @@ An agent that collects metrics from an zettelkasten and stores them into an Infl - Parses both markdown and wiki links - Authenticate in private git repositories using personal access tokens - Grafana dashboards included +- Support for both InfluxDB and VictoriaMetrics as storage backends ## Usage @@ -26,23 +27,29 @@ The exporter is distributed as both a binary and a Docker image. The currently d - [Docker compose example](./examples/compose) - [Kubernetes example](./examples/kubernetes) -Note that for a complete solution, it will be necessary to setup InfluxDB and Grafana. For more information about setting them up, refer to their documentation. Here are some links that might be useful: +Note that for a complete solution, it will be necessary to setup Grafana and either InfluxDB or VictoriaMetrics as data sources. For more information about setting them up, refer to their documentation. Here are some links that might be useful: - https://grafana.com/docs/grafana/latest/getting-started/get-started-grafana-influxdb/ - https://docs.influxdata.com/influxdb/v2/get-started/setup/ +- https://docs.victoriametrics.com/ -The provided dashboard uses `Flux` as the language to query InfluxDB, so make sure to set the "Query language" option to "Flux" when creating the InfluxDB data source in Grafana. +In the `dashboards` folder there are two Grafana dashboards provided: one for InfluxDB and another for VictoriaMetrics. You'll have to import the appropriate dashboard for your storage backend. + +The provided InfluxDB dashboard uses `Flux` as the query language, so make sure to set the "Query language" option to "Flux" when creating the InfluxDB data source in Grafana. + +For both storage backends, make sure to configure the data retention period according to your needs. ## Configuration -All configuration is supplied via environment variables. You should supply at least the required variables and either the `ZETTELKASTEN_DIRECTORY` or the `ZETTELKASTEN_GIT_URL` variables. +All configuration is supplied via environment variables. You should supply at least the zettelkasten source via the `ZETTELKASTEN_DIRECTORY` or `ZETTELKASTEN_GIT_URL` variables and the storage backend via the `VICTORIAMETRICS_URL` or `INFLUXDB_*` variables. | Name | Description | Default | Required | | -------------------------- | -------------------------------------------------------------------- | ------------------------------ | -------- | -| INFLUXDB_URL | The InfluxDB URL | | Yes | -| INFLUXDB_TOKEN | The InfluxDB token to authenticate in the bucket | | Yes | -| INFLUXDB_ORG | The InfluxDB org containing the bucket | | Yes | -| INFLUXDB_BUCKET | The InfluxDB bucket to register metrics | | Yes | +| VICTORIAMETRICS_URL | The VictoriaMetrics URL | | No | +| INFLUXDB_URL | The InfluxDB URL | | No | +| INFLUXDB_TOKEN | The InfluxDB token to authenticate in the bucket | | No | +| INFLUXDB_ORG | The InfluxDB org containing the bucket | | No | +| INFLUXDB_BUCKET | The InfluxDB bucket to register metrics | | No | | ZETTELKASTEN_DIRECTORY | The local directory containing the zettelkasten | | No | | ZETTELKASTEN_GIT_URL | The URL for the git repository containing the zettelkasten | | No | | ZETTELKASTEN_GIT_TOKEN | The access token to authenticate with private repositories | | No | @@ -54,26 +61,24 @@ All configuration is supplied via environment variables. You should supply at le ## Metrics -The exporter collects metrics by parsing the contents of the markdown files present in the Zettelkasten. Currently the exporter stores metrics for individual notes and also aggregated metrics describing the entire Zettelkasten. The combination of raw and pre processed metrics allows for both flexibility and efficiency when querying the data, at the cost of a slightly higher storage usage. The two sets of metrics are stored in the same InfluxDB bucket under different [measurement names](https://docs.influxdata.com/influxdb/cloud/reference/key-concepts/data-elements/#measurement). +The exporter collects metrics by parsing the contents of the markdown files present in the Zettelkasten. Currently the exporter stores metrics for individual notes and also aggregated metrics describing the entire Zettelkasten. The combination of raw and pre processed metrics allows for both flexibility and efficiency when querying the data, at the cost of a slightly higher storage usage. When using the InfluxDB storage, the two sets of metrics are stored in the same InfluxDB bucket under different [measurement names](https://docs.influxdata.com/influxdb/cloud/reference/key-concepts/data-elements/#measurement). When using the VictoriaMetrics storage, each metric is stored under a different name. The following table describes all metrics collected by the exporter and their respective measurement names: -| Measurement | Name | Description | -|-------------|----------------|-----------------------------------------| -| notes | link_count | Number of links in the note | -| notes | word_count | Number of words in the note | -| notes | backlink_count | Number of links that reference the note | -| total | note_count | Number of notes in the Zettelkasten | -| total | link_count | Number of links in the Zettelkasten | -| total | word_count | Number of words in the Zettelkasten | +| InfluxDB measurement | InfluxDB name | VictoriaMetrics name | Description | +|----------------------|----------------|----------------------|-----------------------------------------| +| notes | link_count | notes_link_count | Number of links in the note | +| notes | word_count | notes_word_count | Number of words in the note | +| notes | backlink_count | notes_backlink_count | Number of links that reference the note | +| total | note_count | total_note_count | Number of notes in the Zettelkasten | +| total | link_count | total_link_count | Number of links in the Zettelkasten | +| total | word_count | total_word_count | Number of words in the Zettelkasten | ## Roadmap These are some features that I'd like to include in the future. -- Collect additional metrics -- Support Prometheus remote write as a storage -- Support VictoriaMetrics as a storage +- Support Prometheus remote write as a storage backend ## References diff --git a/examples/compose/docker-compose.yaml b/examples/compose/docker-compose-influxdb.yaml similarity index 91% rename from examples/compose/docker-compose.yaml rename to examples/compose/docker-compose-influxdb.yaml index 69b651f..562f522 100644 --- a/examples/compose/docker-compose.yaml +++ b/examples/compose/docker-compose-influxdb.yaml @@ -1,7 +1,5 @@ # This is a sample compose file for deploying the zettelkasten-exporter # using an InfluxDB storage. -version: '3.8' - volumes: influxdb-data: {} influxdb-config: {} @@ -10,16 +8,14 @@ volumes: services: grafana: image: grafana/grafana - depends_on: - - influxdb restart: unless-stopped volumes: - grafana-data:/var/lib/grafan ports: - 3000:3000 - influxdb: image: influxdb:2 + restart: unless-stopped environment: # We opt for an automated setup of InfluxDB for simplicity. It's # strongly recommended to change those credentials or doing a @@ -29,6 +25,8 @@ services: DOCKER_INFLUXDB_INIT_PASSWORD: password DOCKER_INFLUXDB_INIT_ORG: default DOCKER_INFLUXDB_INIT_BUCKET: zettelkasten + # In your own setup you'll probably want to specify a longer + # retention period. DOCKER_INFLUXDB_INIT_RETENTION: 1w DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: demo-auth-token volumes: @@ -36,9 +34,9 @@ services: - influxdb-config:/etc/influxdb2 ports: - 8086:8086 - zettelkasten-exporter: image: ghcr.io/luissimas/zettelkasten-exporter:latest + restart: unless-stopped depends_on: - influxdb environment: diff --git a/examples/compose/docker-compose-victoriametrics.yaml b/examples/compose/docker-compose-victoriametrics.yaml new file mode 100644 index 0000000..07850a1 --- /dev/null +++ b/examples/compose/docker-compose-victoriametrics.yaml @@ -0,0 +1,38 @@ +# This is a sample compose file for deploying the zettelkasten-exporter +# using an VictoriaMetrics storage. +volumes: + grafana-data: {} + victoriametrics-data: {} + +services: + grafana: + image: grafana/grafana + restart: unless-stopped + volumes: + - grafana-data:/var/lib/grafan + ports: + - 3000:3000 + victoriametrics: + image: victoriametrics/victoria-metrics:latest + # In your own setup you'll probably want to specify a longer + # retention period. + command: -retentionPeriod=1w + restart: unless-stopped + ports: + - 8428:8428 + volumes: + - victoriametrics-data:/victoria-metrics-data + zettelkasten-exporter: + image: ghcr.io/luissimas/zettelkasten-exporter:latest + restart: unless-stopped + depends_on: + - victoriametrics + environment: + LOG_LEVEL: INFO + # Here we use a local directory for simplicity, but check out the + # README to see how to configure different sources such as git repositories. + ZETTELKASTEN_DIRECTORY: /sample + COLLECTION_INTERVAL: 10s + VICTORIAMETRICS_URL: http://victoriametrics:8428 + volumes: + - ./sample:/sample diff --git a/examples/kubernetes/manifest.yaml b/examples/kubernetes/manifest-influxdb.yaml similarity index 100% rename from examples/kubernetes/manifest.yaml rename to examples/kubernetes/manifest-influxdb.yaml diff --git a/examples/kubernetes/manifest-victoriametrics.yaml b/examples/kubernetes/manifest-victoriametrics.yaml new file mode 100644 index 0000000..1e6650e --- /dev/null +++ b/examples/kubernetes/manifest-victoriametrics.yaml @@ -0,0 +1,56 @@ +# This is a sample manifest for deploying the zettelkasten-exporter using an VictoriaMetrics +# storage. +# To deploy VictoriaMetrics, see: https://github.com/VictoriaMetrics/helm-charts +--- +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring +--- +apiVersion: v1 +kind: Secret +metadata: + name: zettelkasten-exporter-env + namespace: monitoring +type: Opaque +data: + # These are placeholder values. Replace them with the + # appropriate values for your setup. + github-token: YW55LXRva2Vu +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: zettelkasten-exporter + namespace: monitoring + labels: + app.kubernetes.io/name: zettelkasten-exporter +spec: + selector: + matchLabels: + app.kubernetes.io/name: zettelkasten-exporter + replicas: 1 + template: + metadata: + labels: + app.kubernetes.io/name: zettelkasten-exporter + spec: + containers: + - name: zettelkasten-exporter + image: "ghcr.io/luissimas/zettelkasten-exporter:latest" + env: + - name: LOG_LEVEL + value: INFO + - name: COLLECTION_INTERVAL + value: 5m + - name: ZETTELKASTEN_GIT_URL + value: + - name: ZETTELKASTEN_GIT_BRANCH + value: master + - name: ZETTELKASTEN_GIT_TOKEN + valueFrom: + secretKeyRef: + name: zettelkasten-exporter-env + key: github-token + - name: VICTORIAMETRICS_URL + value: http://victoriametrics