implement metric exporting via opentelemetry

Pipfile

@@ -11,6 +11,10 @@ whitenoise = "~=6.3"
 django-mathfilters = "~=0.1.2"
 simple_openid_connect = { version = "~=0.2.4", extras = ["django", "djangorestframework"] }
 environs = { version = "~=9.5", extras = ["django"] }
+opentelemetry-api = "~=1.22.0"
+opentelemetry-sdk = "~=1.22.0"
+opentelemetry-exporter-otlp-proto-http = "~=1.22.0"
+opentelemetry-instrumentation-django = "~=0.43b0"
 
 [dev-packages]
 pre-commit = "*"

README.md

@@ -1,4 +1,5 @@
 # vinywaji
+
 > Keep track of how many drinks people have bought
 
 Vinywaji is Swahili for "drinks".
@@ -19,6 +20,7 @@ A docker image is built automatically that follows the master branch of the repo
 It is available as `ghcr.io/fsinfuhh/vinywaji:dev-latest`.
 
 Simply start it via
+
 ```shell
 docker run --name vinywaji ghcr.io/fsinfuhh/vinywaji:latest
 ```
@@ -28,9 +30,11 @@ I.e. `docker run -e VW_SECRET_KEY=foobar123 …`.
 
 ### On Baremetal from source
 
-*Although this deployment works, it is not recommended. If you need to use a deployment without containers, you should serve this application via uwsgi, gunicorn or the like.*
+*Although this deployment works, it is not recommended. If you need to use a deployment without containers, you should
+serve this application via uwsgi, gunicorn or the like.*
 
 To build the application from source, follow the following steps:
+
 ```shell
 # get the code
 git clone https://github.com/fsinfuhh/vinywaji.git
@@ -41,6 +45,7 @@ pipenv install --ignore-pipfile
 ```
 
 To start it:
+
 ```shell
 pipenv shell
 ./src/manage.py check --deploy
@@ -52,14 +57,14 @@ pipenv shell
 
 The application is configured at runtime via the following environment variables:
 
-| Name                     | Default                | Description                                                                                                                   | Notes                                                                               |
-|--------------------------|------------------------|-------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------|
-| VW_DB                   | *required*             | Url that specifies the complete database connection. [Documentation](https://pypi.org/project/dj-database-url/)               | In container based deployments this preconfigured to point to `/app/data/db.sqlite` |
-| VW_SECRET_KEY           | *required*             | Django secret key. **Keep this secret!**                                                                                      ||
-| VW_ALLOWED_HOSTS        | *required*             | List of hostnames which may be used when accessing the application.                                                           ||
-| VW_SERVED_OVER_HTTPS    | `false`                | Whether the application is served over HTTPS. If enabled, automatic redirects and additional security measures are activated. ||
-| VW_HSTS_SECONDS         | `63072000`             | If larger than 0 and `BL_SERVED_OVER_HTTPS` is true, HSTS is enabled with this configured value.                              ||
-| VW_TRUST_REVERSE_PROXY  | `false`                | If true, headers set by a reverse proxy (i.e. `X-Forwarded-Proto`) are trusted.                                               ||
-||
-| VW_OPENID_CLIENT_ID     | *required*             | Mafiasi-Identity client ID. Used for authentication                                                                           ||
-| VW_OPENID_CLIENT_SECRET | *required*             | Mafiasi-Identity client secret. Used for authentication                                                                       ||
+| Name                    | Default    | Description                                                                                                                   | Notes                                                                                                                                                                       |
+|-------------------------|------------|-------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| VW_DB                   | *required* | Url that specifies the complete database connection. [Documentation](https://pypi.org/project/dj-database-url/)               | In container based deployments this preconfigured to point to `/app/data/db.sqlite`                                                                                         |
+| VW_SECRET_KEY           | *required* | Django secret key. **Keep this secret!**                                                                                      |                                                                                                                                                                             |
+| VW_ALLOWED_HOSTS        | *required* | List of hostnames which may be used when accessing the application.                                                           |                                                                                                                                                                             |
+| VW_SERVED_OVER_HTTPS    | `false`    | Whether the application is served over HTTPS. If enabled, automatic redirects and additional security measures are activated. |                                                                                                                                                                             |
+| VW_HSTS_SECONDS         | `63072000` | If larger than 0 and `BL_SERVED_OVER_HTTPS` is true, HSTS is enabled with this configured value.                              |                                                                                                                                                                             |
+| VW_TRUST_REVERSE_PROXY  | `false`    | If true, headers set by a reverse proxy (i.e. `X-Forwarded-Proto`) are trusted.                                               |                                                                                                                                                                             |
+| VW_ENABLE_METRICS       | `false`    | If true, enable metric exporting via OpenTelemetry.                                                                           | See the [Opentelemetry Docs](https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/) on how to configure the exporter i.e. to which collector it exports. |
+| VW_OPENID_CLIENT_ID     | *required* | Mafiasi-Identity client ID. Used for authentication                                                                           |                                                                                                                                                                             |
+| VW_OPENID_CLIENT_SECRET | *required* | Mafiasi-Identity client secret. Used for authentication                                                                       |                                                                                                                                                                             |

scripts/run_otlp_collector.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -e
+
+export OTEL_PORT=4318
+export PROM_PORT=8889
+
+TMP_FILE=$(mktemp -p /tmp otel-collector-config.yml.XXXXXXX)
+trap "rm -f $TMP_FILE" EXIT
+chmod a+r $TMP_FILE
+
+cat <<EOF >> $TMP_FILE
+receivers:
+  otlp:
+    protocols:
+      http:
+        endpoint: 0.0.0.0:${OTEL_PORT}
+exporters:
+  debug:
+    verbosity: detailed
+  prometheus:
+    endpoint: 0.0.0.0:${PROM_PORT}
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [debug]
+    metrics:
+      receivers: [otlp]
+      exporters: [debug, prometheus]
+    logs:
+      receivers: [otlp]
+      exporters: [debug]
+EOF
+
+docker run \
+  --rm \
+  -p ${OTEL_PORT}:${OTEL_PORT} \
+  -p ${PROM_PORT}:${PROM_PORT} \
+   -v ${TMP_FILE}:/etc/otelcol/config.yaml \
+   otel/opentelemetry-collector

src/vinywaji/metrics/apps.py

@@ -0,0 +1,36 @@
+from django.apps import AppConfig
+from django.conf import settings
+from opentelemetry import metrics
+from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
+from opentelemetry.instrumentation.django import DjangoInstrumentor
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.sdk.resources import SERVICE_NAME, SERVICE_VERSION, Resource
+
+
+class MetricsConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "vinywaji.metrics"
+    label = "vinywaji_metrics"
+
+    def ready(self):
+        super().ready()
+        self.init_instrumentation()
+
+    def init_instrumentation(self):
+        from vinywaji.metrics import async_instruments
+
+        resource = Resource(
+            attributes={
+                SERVICE_NAME: "vinywaji",
+                SERVICE_VERSION: settings.VERSION,
+            }
+        )
+        metric_reader = PeriodicExportingMetricReader(
+            OTLPMetricExporter(),
+            export_interval_millis=15_000,
+        )
+        meter_provider = MeterProvider(resource=resource, metric_readers=[metric_reader])
+        metrics.set_meter_provider(meter_provider)
+        DjangoInstrumentor().instrument()
+        async_instruments.create_async_instruments()

src/vinywaji/metrics/async_instruments.py

@@ -0,0 +1,74 @@
+from typing import Iterable
+
+from django.db.models import Avg, Sum
+from opentelemetry import metrics
+from opentelemetry.metrics import CallbackOptions, Observation
+
+from vinywaji.core import models
+
+vinywaji_meter = metrics.get_meter("vinywaji")
+
+
+def create_async_instruments():
+    vinywaji_meter.create_observable_counter(
+        "vinywaji_transactions",
+        callbacks=[count_transactions],
+        description="The total number of transactions recorded in vinywaji",
+    )
+    vinywaji_meter.create_observable_gauge(
+        "vinywaji_assets",
+        callbacks=[calc_asset_aggregates],
+        description="The aggregated value of assets recorded in vinywaji (in euro-cent) ",
+        unit="ct",
+    )
+
+
+def count_transactions(_options: CallbackOptions) -> Iterable[Observation]:
+    n_negative = models.Transaction.objects.filter(amount__lt=0).count()
+    n_positive = models.Transaction.objects.filter(amount__gt=0).count()
+    yield Observation(value=n_negative, attributes={"transaction_type": "withdrawal"})
+    yield Observation(value=n_positive, attributes={"transaction_type": "deposit"})
+    yield Observation(value=n_positive + n_negative, attributes={"transaction_type": "any"})
+
+
+def calc_asset_aggregates(_options: CallbackOptions) -> Iterable[Observation]:
+    # aggregate all negative transactions
+    negative_aggregate = models.Transaction.objects.filter(amount__lt=0).aggregate(
+        Sum("amount"), Avg("amount")
+    )
+    yield Observation(
+        value=negative_aggregate["amount__sum"] or 0,
+        attributes={"asset_type": "negative", "aggregate_type": "sum"},
+    )
+    yield Observation(
+        value=negative_aggregate["amount__avg"] or 0,
+        attributes={"asset_type": "negative", "aggregate_type": "avg"},
+    )
+
+    # aggregate all positive transactions
+    positive_aggregate = models.Transaction.objects.filter(amount__gt=0).aggregate(
+        Sum("amount"), Avg("amount")
+    )
+    yield Observation(
+        value=positive_aggregate["amount__avg"] or 0,
+        attributes={
+            "asset_type": "positive",
+            "aggregate_type": "sum",
+        },
+    )
+    yield Observation(
+        value=positive_aggregate["amount__sum"] or 0,
+        attributes={
+            "asset_type": "positive",
+            "aggregate_type": "avg",
+        },
+    )
+
+    # calculate totals based on previous data
+    yield Observation(
+        value=(positive_aggregate["amount__sum"] or 0) + (negative_aggregate["amount__sum"] or 0),
+        attributes={
+            "asset_type": "total",
+            "aggregate_type": "sum",
+        },
+    )

src/vinywaji/settings.py

@@ -7,7 +7,7 @@
 For the full list of settings and their values, see
 https://docs.djangoproject.com/en/3.2/ref/settings/
 """
-import os
+
 from pathlib import Path
 
 from environs import Env
@@ -26,27 +26,33 @@
 TRUST_REVERSE_PROXY = env.bool("VW_TRUST_REVERSE_PROXY", default=False)
 SECRET_KEY = env.str("VW_SECRET_KEY")
 ALLOWED_HOSTS = env.list("VW_ALLOWED_HOSTS")
+ENABLE_METRICS = env.bool("VW_ENABLE_METRICS", default=False)
 
 DATABASES = {"default": env.dj_db_url("VW_DB")}
 CACHES = {"default": env.dj_cache_url("VW_CACHE", default="dummy://" if DEBUG else "locmem://")}
 
 INSTALLED_APPS = [
-    "django.contrib.auth",
-    "django.contrib.admin",
-    "django.contrib.contenttypes",
-    "django.contrib.sessions",
-    "django.contrib.messages",
-    "django.contrib.staticfiles",
-    "django.contrib.humanize",
-    "whitenoise.runserver_nostatic",
-    "rest_framework",
-    "drf_spectacular",
-    "drf_spectacular_sidecar",
-    "mathfilters",
-    "simple_openid_connect.integrations.django",
-    "vinywaji.core",
-    "vinywaji.api",
-    "vinywaji.gui",
+    i
+    for i in [
+        "django.contrib.auth",
+        "django.contrib.admin",
+        "django.contrib.contenttypes",
+        "django.contrib.sessions",
+        "django.contrib.messages",
+        "django.contrib.staticfiles",
+        "django.contrib.humanize",
+        "whitenoise.runserver_nostatic",
+        "rest_framework",
+        "drf_spectacular",
+        "drf_spectacular_sidecar",
+        "mathfilters",
+        "simple_openid_connect.integrations.django",
+        "vinywaji.core",
+        "vinywaji.api",
+        "vinywaji.gui",
+        "vinywaji.metrics" if ENABLE_METRICS else None,
+    ]
+    if i is not None
 ]
 
 MIDDLEWARE = [