From fefd776b583bc53009f4e604aaf4a62abb0fed98 Mon Sep 17 00:00:00 2001 From: jack-berg <34418638+jack-berg@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:19:14 -0500 Subject: [PATCH 1/6] Add statsd colletor example (#621) --- README.md | 3 +- other-examples/collector/statsd/README.md | 60 +++++++++++++ .../collector/statsd/k8s/collector.yaml | 87 +++++++++++++++++++ .../collector/statsd/k8s/gen-statsd.yaml | 19 ++++ .../collector/statsd/k8s/secrets.yaml | 10 +++ 5 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 other-examples/collector/statsd/README.md create mode 100644 other-examples/collector/statsd/k8s/collector.yaml create mode 100644 other-examples/collector/statsd/k8s/gen-statsd.yaml create mode 100644 other-examples/collector/statsd/k8s/secrets.yaml diff --git a/README.md b/README.md index 4a846dec..ad611c19 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,10 @@ OpenTelemetry is a big ecosystem and everything doesn't fit into the goals of th * [Telemetry data processing](./other-examples/collector/nr-config) * [Host monitoring](./other-examples/collector/host-monitoring) * [Confluent cloud monitoring](./other-examples/collector/confluentcloud) - * [Singlestore monitoring](./other-examples/collector/singlestore) * [HCP Consul monitoring](./other-examples/collector/hcp-consul) * [Redis monitoring](./other-examples/collector/redis) + * [Singlestore monitoring](./other-examples/collector/singlestore) + * [StatsD monitoring](./other-examples/collector/statsd) * Java * [OpenTelemetry Agent New Relic Config](./other-examples/java/agent-nr-config) * [Micrometer Shim with OTLP Export](./other-examples/java/micrometer-shim) diff --git a/other-examples/collector/statsd/README.md b/other-examples/collector/statsd/README.md new file mode 100644 index 00000000..1e92f73b --- /dev/null +++ b/other-examples/collector/statsd/README.md @@ -0,0 +1,60 @@ +# Monitoring StatsD with OpenTelemetry Collector + +This simple example demonstrates monitoring statsd sources with the [OpenTelemetry collector](https://opentelemetry.io/docs/collector/), using the [statsd receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/statsdreceiver) and sending the data to New Relic via OTLP. A simple load statsd load generator is configured to send dummy data to the statsd receiver. + +## Requirements + +* You need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. Docker desktop [includes a standalone Kubernetes server and client](https://docs.docker.com/desktop/kubernetes/) which is useful for local testing. +* [A New Relic account](https://one.newrelic.com/) +* [A New Relic license key](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key) + +## Running the example + +1. Update the `NEW_RELIC_API_KEY` value in [secrets.yaml](./k8s/secrets.yaml) to your New Relic license key. + + ```yaml + # ...omitted for brevity + stringData: + # New Relic API key to authenticate the export requests. + # docs: https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key + NEW_RELIC_API_KEY: + ``` + + * Note, be careful to avoid inadvertent secret sharing when modifying `secrets.yaml`. To ignore changes to this file from git, run `git update-index --skip-worktree k8s/secrets.yaml`. + + * If your account is based in the EU, update the `NEW_RELIC_OTLP_ENDPOINT` value in [collector.yaml](./k8s/collector.yaml) the endpoint to: [https://otlp.eu01.nr-data.net](https://otlp.eu01.nr-data.net) + + ```yaml + # ...omitted for brevity + env: + # The default US endpoint is set here. You can change the endpoint and port based on your requirements if needed. + # docs: https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/best-practices/opentelemetry-otlp/#configure-endpoint-port-protocol + - name: NEW_RELIC_OTLP_ENDPOINT + value: https://otlp.eu01.nr-data.net + ``` + +2. Run the application with the following command. + + ```shell + kubectl apply -f k8s/ + ``` + + * When finished, cleanup resources with the following command. This is also useful to reset if modifying configuration. + + ```shell + kubectl delete -f k8s/ + ``` + +## Viewing your data + +To review your statsd data in New Relic, navigate to "New Relic -> Query Your Data". To list the metrics reported, query for: + +``` +FROM Metric SELECT uniques(metricName) WHERE otel.library.name = 'otelcol/statsdreceiver' LIMIT MAX +``` + +See [get started with querying](https://docs.newrelic.com/docs/query-your-data/explore-query-data/get-started/introduction-querying-new-relic-data/) for additional details on querying data in New Relic. + +## Additional notes + +A simple load statsd load generator is configured to send dummy data to the statsd receiver, specified in [gen-statsd.yaml](./k8s/gen-statsd.yaml). To use in production, you'll need to configure your statsd data sources to point to the collector's statsd receiver endpoint. In this example, the collector's statsd receiver is listing for UDP on `0.0.0.0:8125`, which is exposed in a [service](https://kubernetes.io/docs/concepts/services-networking/service/). The load generator is configured point to this using service env vars set by kubernetes. diff --git a/other-examples/collector/statsd/k8s/collector.yaml b/other-examples/collector/statsd/k8s/collector.yaml new file mode 100644 index 00000000..0219aeaf --- /dev/null +++ b/other-examples/collector/statsd/k8s/collector.yaml @@ -0,0 +1,87 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: nr-statsd +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: collector-config + namespace: nr-statsd + labels: + app.kubernetes.io/name: collector-config +data: + collector-config: | + receivers: + statsd: + endpoint: 0.0.0.0:8125 + is_monotonic_counter: true + + processors: + batch: + + exporters: + otlphttp: + endpoint: ${NEW_RELIC_OTLP_ENDPOINT} + headers: + api-key: ${NEW_RELIC_API_KEY} + + service: + pipelines: + metrics: + receivers: [statsd] + processors: [batch] + exporters: [otlphttp] +--- +apiVersion: v1 +kind: Pod +metadata: + name: collector + namespace: nr-statsd + labels: + app.kubernetes.io/name: collector +spec: + containers: + - name: collector + image: otel/opentelemetry-collector-contrib:0.98.0 + env: + # The default US endpoint is set here. You can change the endpoint and port based on your requirements if needed. + # docs: https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/best-practices/opentelemetry-otlp/#configure-endpoint-port-protocol + - name: NEW_RELIC_OTLP_ENDPOINT + value: https://otlp.nr-data.net/ + # The New Relic API key used to authenticate export requests. + # Defined in secrets.yaml + - name: NEW_RELIC_API_KEY + valueFrom: + secretKeyRef: + name: nr-statsd-secret + key: NEW_RELIC_API_KEY + volumeMounts: + - name: collector-config-vol + mountPath: /etc/otelcol-contrib + ports: + - containerPort: 8125 + protocol: UDP + volumes: + - name: collector-config-vol + configMap: + name: collector-config + items: + - key: collector-config + path: config.yaml +--- +apiVersion: v1 +kind: Service +metadata: + name: collector + namespace: nr-statsd + labels: + app.kubernetes.io/name: collector +spec: + ports: + - name: statsd + port: 8125 + protocol: UDP + selector: + app.kubernetes.io/name: collector diff --git a/other-examples/collector/statsd/k8s/gen-statsd.yaml b/other-examples/collector/statsd/k8s/gen-statsd.yaml new file mode 100644 index 00000000..ab032f7b --- /dev/null +++ b/other-examples/collector/statsd/k8s/gen-statsd.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: gen-statsd + namespace: nr-statsd + labels: + app.kubernetes.io/name: gen-statsd +spec: + containers: + - name: gen-statsd + image: circonus/gen-statsd:1.0.0 + env: + - name: STATSD_HOST + value: $(COLLECTOR_SERVICE_HOST):$(COLLECTOR_SERVICE_PORT_STATSD) + - name: PROTOCOL + value: udp + - name: AGENTS + value: "1" diff --git a/other-examples/collector/statsd/k8s/secrets.yaml b/other-examples/collector/statsd/k8s/secrets.yaml new file mode 100644 index 00000000..c4a8db3c --- /dev/null +++ b/other-examples/collector/statsd/k8s/secrets.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: nr-statsd-secret + namespace: nr-statsd +stringData: + # New Relic API key to authenticate the export requests. + # docs: https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key + NEW_RELIC_API_KEY: + From 8c11fac2b31cf5b9c78461aef883b82df3984b94 Mon Sep 17 00:00:00 2001 From: jack-berg <34418638+jack-berg@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:20:57 -0500 Subject: [PATCH 2/6] Migrate host monitoring example to kubernetes (#628) * Add kubernetes host monitoring example * Replace host monitoring example with kubernetes * Fix typo * Extend host monitoring example with APM correlation --- other-examples/collector/host-monitoring/.env | 7 - .../collector/host-monitoring/README.md | 68 +++++-- .../host-monitoring/docker-compose.yaml | 19 -- .../host-monitoring/k8s/_namespace.yaml | 5 + .../host-monitoring/k8s/adservice.yaml | 25 +++ .../host-monitoring/k8s/collector.yaml | 175 ++++++++++++++++++ .../host-monitoring/k8s/secrets.yaml | 9 + .../host-monitoring/otel-config.yaml | 139 -------------- 8 files changed, 262 insertions(+), 185 deletions(-) delete mode 100644 other-examples/collector/host-monitoring/.env delete mode 100644 other-examples/collector/host-monitoring/docker-compose.yaml create mode 100644 other-examples/collector/host-monitoring/k8s/_namespace.yaml create mode 100644 other-examples/collector/host-monitoring/k8s/adservice.yaml create mode 100644 other-examples/collector/host-monitoring/k8s/collector.yaml create mode 100644 other-examples/collector/host-monitoring/k8s/secrets.yaml delete mode 100644 other-examples/collector/host-monitoring/otel-config.yaml diff --git a/other-examples/collector/host-monitoring/.env b/other-examples/collector/host-monitoring/.env deleted file mode 100644 index f36e94c6..00000000 --- a/other-examples/collector/host-monitoring/.env +++ /dev/null @@ -1,7 +0,0 @@ -# New Relic API key to authenticate the call. -# docs: https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key -NEW_RELIC_API_KEY= - -# The default US endpoint is set here. You can change the endpoint and port based on your requirements if needed. -# docs: https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/get-started/opentelemetry-set-up-your-app/#review-settings -NEW_RELIC_OTLP_ENDPOINT=https://otlp.nr-data.net/ \ No newline at end of file diff --git a/other-examples/collector/host-monitoring/README.md b/other-examples/collector/host-monitoring/README.md index 3ee44787..374e6b96 100644 --- a/other-examples/collector/host-monitoring/README.md +++ b/other-examples/collector/host-monitoring/README.md @@ -1,35 +1,63 @@ -# Host Monitoring with OpenTelemetry Collector Setup +# Monitoring Hosts with OpenTelemetry Collector -This example shows a setup for running the OpenTelemetry Collector in a docker container configured to scrape host metrics, and to enrich application telemetry with resource attributes required for entity synthesis relationships between host and service entities. This type of configuration is ideal when running the collector as an agent with an instance on every node (i.e. using kubernetes [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/)) and forwarding telemetry for applications running on that node. +This example demonstrates monitoring hosts with the [OpenTelemetry collector](https://opentelemetry.io/docs/collector/), using the [host metrics receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/hostmetricsreceiver) and sending the data to New Relic via OTLP. -For more information, please see our [Collector for host monitoring](https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/collector/opentelemetry-collector-infra-hosts/). +Additionally, it demonstrates correlating APM entities with hosts, using the OpenTelemetry collector to enrich APM OTLP data with host metadata before sending to New Relic via OTLP. -## Prerequisites +## Requirements -1. You must have a Docker daemon running. -2. You must have [Docker compose](https://docs.docker.com/compose/) installed . +* You need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. Docker desktop [includes a standalone Kubernetes server and client](https://docs.docker.com/desktop/kubernetes/) which is useful for local testing. +* [A New Relic account](https://one.newrelic.com/) +* [A New Relic license key](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key) ## Running the example -First, set your environment variables in the `.env` file in this directory. For more information on the individual variables, reference the docs available below. +1. Update the `NEW_RELIC_API_KEY` value in [secrets.yaml](./k8s/secrets.yaml) to your New Relic license key. + ```yaml + # ...omitted for brevity + stringData: + # New Relic API key to authenticate the export requests. + # docs: https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key + NEW_RELIC_API_KEY: + ``` + + * Note, be careful to avoid inadvertent secret sharing when modifying `secrets.yaml`. To ignore changes to this file from git, run `git update-index --skip-worktree k8s/secrets.yaml`. + + * If your account is based in the EU, update the `NEW_RELIC_OTLP_ENDPOINT` value in [collector.yaml](./k8s/collector.yaml) the endpoint to: [https://otlp.eu01.nr-data.net](https://otlp.eu01.nr-data.net) + + ```yaml + # ...omitted for brevity + env: + # The default US endpoint is set here. You can change the endpoint and port based on your requirements if needed. + # docs: https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/best-practices/opentelemetry-otlp/#configure-endpoint-port-protocol + - name: NEW_RELIC_OTLP_ENDPOINT + value: https://otlp.eu01.nr-data.net + ``` + +3. Run the application with the following command. + + ```shell + kubectl apply -f k8s/ + ``` + + * When finished, cleanup resources with the following command. This is also useful to reset if modifying configuration. + + ```shell + kubectl delete -f k8s/ + ``` -Once the variables are set, run the following command from the root directory to start the collector. +## Viewing your data -```shell -cd ./other-examples/collector/host-monitoring +To review your host data in New Relic, navigate to "New Relic -> All Entities -> Hosts" and click on the instance with name corresponding to the collector pod name to view the instance summary. Use [NRQL](https://docs.newrelic.com/docs/query-your-data/explore-query-data/get-started/introduction-querying-new-relic-data/) to perform ad-hoc analysis. -docker compose up +``` +FROM Metric SELECT uniques(metricName) WHERE otel.library.name like 'otelcol/hostmetricsreceiver/%' ``` -Optionally, run an OpenTelemetry instrumented application and point it at the collector's OTLP endpoint via `http://localhost:4318`. The application telemetry is enriched with resource information which New Relic uses to form relationships between the host and service entity. - -## Viewing your data +See [get started with querying](https://docs.newrelic.com/docs/query-your-data/explore-query-data/get-started/introduction-querying-new-relic-data/) for additional details on querying data in New Relic. -To review your Host data in New Relic, navigate `All Entities -> Hosts` and click on the host with the name of your local machine. +## Additional notes -## Local Variable information +This example deploys the collector as a kubernetes DaemonSet to run a collector instance on each node in the kubernetes cluster. When running in this type of configuration, it's common to route application telemetry from pods to the collector instance each pod is respectively running on, and to enrich that telemetry with additional metadata via the [kubernetes attributes processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/k8sattributesprocessor). This example omits that configuration for brevity. See [important components for kubernetes](https://opentelemetry.io/docs/kubernetes/collector/components/#filelog-receiver) for common configuration running the collector in kubernetes. -| Variable | Description | Docs | -|-----------------------------|--------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **NEW_RELIC_API_KEY** | New Relic Ingest API Key | [API Key docs](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/) | -| **NEW_RELIC_OTLP_ENDPOINT** | Default US OTLP endpoint is `https://otlp.nr-data.net` | [OTLP endpoint config docs](https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/get-started/opentelemetry-set-up-your-app/#review-settings) | +In order to demonstrate correlation between OpenTelemetry APM entities and host entities, this example deploys an instance of the opentelemetry demo [AdService](https://opentelemetry.io/docs/demo/services/ad/), defined in [adservice.yaml](./k8s/adservice.yaml). The AdService application is configured to export OTLP data to the collector DaemonSet pod running on the same host. The collector enriches the AdService telemetry with `host.id` (and other attributes) which New Relic uses to create a relationship with the host entity. diff --git a/other-examples/collector/host-monitoring/docker-compose.yaml b/other-examples/collector/host-monitoring/docker-compose.yaml deleted file mode 100644 index c7f4058a..00000000 --- a/other-examples/collector/host-monitoring/docker-compose.yaml +++ /dev/null @@ -1,19 +0,0 @@ -version: '3' -services: - collector: - image: otel/opentelemetry-collector-contrib:0.92.0 - command: --config=/etc/otelcol/config.yaml - volumes: - - ./otel-config.yaml:/etc/otelcol/config.yaml - # Mount the host file system when running in docker so we can monitor the host system, - # not the docker container. For more info see: - # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/hostmetricsreceiver#collecting-host-metrics-from-inside-a-container-linux-only - - /:/hostfs - environment: - NEW_RELIC_OTLP_ENDPOINT: ${NEW_RELIC_OTLP_ENDPOINT} - NEW_RELIC_API_KEY: ${NEW_RELIC_API_KEY} - HOST_METRICS_ROOT_PATH: /hostfs - ports: - - "4317:4317" # OTLP grpc - - "4318:4318" # OTLP http - - "13133:13133" # health diff --git a/other-examples/collector/host-monitoring/k8s/_namespace.yaml b/other-examples/collector/host-monitoring/k8s/_namespace.yaml new file mode 100644 index 00000000..534f4411 --- /dev/null +++ b/other-examples/collector/host-monitoring/k8s/_namespace.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: nr-host-monitoring diff --git a/other-examples/collector/host-monitoring/k8s/adservice.yaml b/other-examples/collector/host-monitoring/k8s/adservice.yaml new file mode 100644 index 00000000..f38a638d --- /dev/null +++ b/other-examples/collector/host-monitoring/k8s/adservice.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: adservice + namespace: nr-host-monitoring + labels: + app.kubernetes.io/name: adservice +spec: + containers: + - name: adservice + image: otel/demo:1.10.0-adservice + env: + - name: AD_SERVICE_PORT + value: "8080" + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://$(HOST_IP):4318 + - name: OTEL_SERVICE_NAME + value: adservice + ports: + - containerPort: 8080 diff --git a/other-examples/collector/host-monitoring/k8s/collector.yaml b/other-examples/collector/host-monitoring/k8s/collector.yaml new file mode 100644 index 00000000..355b3919 --- /dev/null +++ b/other-examples/collector/host-monitoring/k8s/collector.yaml @@ -0,0 +1,175 @@ + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: collector-config + namespace: nr-host-monitoring + labels: + app.kubernetes.io/name: collector-config +data: + collector-config: | + receivers: + hostmetrics: + root_path: /hostfs + collection_interval: 20s + scrapers: + cpu: + metrics: + system.cpu.utilization: + enabled: true + load: + memory: + metrics: + system.memory.utilization: + enabled: true + disk: + filesystem: + metrics: + system.filesystem.utilization: + enabled: true + # Reading /containers/services causes error running in docker. + # Delete for production deployments. + exclude_mount_points: + mount_points: ["/containers/services"] + match_type: strict + network: + paging: + metrics: + system.paging.utilization: + enabled: true + processes: + process: + metrics: + process.cpu.utilization: + enabled: true + process.cpu.time: + enabled: false + # Mute various errors reading process metrics running locally in docker. + # Delete for production deployments. + mute_process_exe_error: true + mute_process_user_error: true + mute_process_io_error: true + + filelog: + include: + - /var/log/alternatives.log + - /var/log/cloud-init.log + - /var/log/auth.log + - /var/log/dpkg.log + - /var/log/syslog + - /var/log/messages + - /var/log/secure + - /var/log/yum.log + + otlp: + protocols: + http: + grpc: + + processors: + batch: + + resourcedetection: + detectors: ["env", "system"] + system: + hostname_sources: ["os"] + resource_attributes: + host.id: + enabled: true + + resourcedetection/cloud: + detectors: ["gcp", "ec2", "azure"] + timeout: 2s + override: false + + exporters: + logging: + verbosity: detailed + otlphttp: + endpoint: ${NEW_RELIC_OTLP_ENDPOINT} + headers: + api-key: ${NEW_RELIC_API_KEY} + + service: + pipelines: + metrics/host: + receivers: [hostmetrics] + processors: [resourcedetection, resourcedetection/cloud, batch] + exporters: [otlphttp] + logs/host: + receivers: [filelog] + processors: [resourcedetection, resourcedetection/cloud, batch] + exporters: [logging, otlphttp] + traces: + receivers: [otlp] + processors: [resourcedetection, resourcedetection/cloud, batch] + exporters: [otlphttp] + metrics: + receivers: [otlp] + processors: [resourcedetection, resourcedetection/cloud, batch] + exporters: [otlphttp] + logs: + receivers: [otlp] + processors: [resourcedetection, resourcedetection/cloud, batch] + exporters: [otlphttp] +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: collector + namespace: nr-host-monitoring + labels: + app.kubernetes.io/name: collector +spec: + selector: + matchLabels: + name: collector + template: + metadata: + labels: + name: collector + spec: + containers: + - name: collector + image: otel/opentelemetry-collector-contrib:0.98.0 + env: + # The default US endpoint is set here. You can change the endpoint and port based on your requirements if needed. + # docs: https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/best-practices/opentelemetry-otlp/#configure-endpoint-port-protocol + - name: NEW_RELIC_OTLP_ENDPOINT + value: https://otlp.nr-data.net/ + # The New Relic API key used to authenticate export requests. + # Defined in secrets.yaml + - name: NEW_RELIC_API_KEY + valueFrom: + secretKeyRef: + name: nr-host-monitoring-secret + key: NEW_RELIC_API_KEY + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # host.id is required for NewRelic host entity synthesis and relationships, but is not included by any resourcedetection detector when running with docker on macOS. + # We enabled the "env" resource detector and set host.id to the name of the node via env var. + - name: OTEL_RESOURCE_ATTRIBUTES + value: host.id=$(NODE_NAME) + volumeMounts: + - name: collector-config-vol + mountPath: /etc/otelcol-contrib + - name: hostfs + mountPath: /hostfs + readOnly: true + mountPropagation: HostToContainer + ports: + - containerPort: 4318 + hostPort: 4318 + volumes: + - name: collector-config-vol + configMap: + name: collector-config + items: + - key: collector-config + path: config.yaml + - name: hostfs + hostPath: + path: / diff --git a/other-examples/collector/host-monitoring/k8s/secrets.yaml b/other-examples/collector/host-monitoring/k8s/secrets.yaml new file mode 100644 index 00000000..34b848d1 --- /dev/null +++ b/other-examples/collector/host-monitoring/k8s/secrets.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: nr-host-monitoring-secret + namespace: nr-host-monitoring +stringData: + # New Relic API key to authenticate the export requests. + # docs: https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key + NEW_RELIC_API_KEY: diff --git a/other-examples/collector/host-monitoring/otel-config.yaml b/other-examples/collector/host-monitoring/otel-config.yaml deleted file mode 100644 index 658ae784..00000000 --- a/other-examples/collector/host-monitoring/otel-config.yaml +++ /dev/null @@ -1,139 +0,0 @@ -extensions: - health_check: - -receivers: - otlp: - protocols: - grpc: - http: - - hostmetrics: - # Mount the host file system when running in docker so we can monitor the host system, - # not the docker container. For more info see: - # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/hostmetricsreceiver#collecting-host-metrics-from-inside-a-container-linux-only - # Delete for production deployments. - root_path: ${HOST_METRICS_ROOT_PATH} - collection_interval: 20s - scrapers: - cpu: - metrics: - system.cpu.utilization: - enabled: true - load: - memory: - metrics: - system.memory.utilization: - enabled: true - disk: - filesystem: - metrics: - system.filesystem.utilization: - enabled: true - # Reading /containers/services causes error running in docker. - # Delete for production deployments. - exclude_mount_points: - mount_points: ["/containers/services"] - match_type: strict - network: - paging: - metrics: - system.paging.utilization: - enabled: true - processes: - process: - metrics: - process.cpu.utilization: - enabled: true - process.cpu.time: - enabled: false - # Mute various errors reading process metrics running locally in docker. - # Delete for production deployments. - mute_process_exe_error: true - mute_process_user_error: true - mute_process_io_error: true - - filelog: - include: - - /var/log/alternatives.log - - /var/log/cloud-init.log - - /var/log/auth.log - - /var/log/dpkg.log - - /var/log/syslog - - /var/log/messages - - /var/log/secure - - /var/log/yum.log - -processors: - - transform/truncate: - trace_statements: - - context: span - statements: - - truncate_all(attributes, 4095) - - truncate_all(resource.attributes, 4095) - log_statements: - - context: log - statements: - - truncate_all(attributes, 4095) - - truncate_all(resource.attributes, 4095) - - memory_limiter: - check_interval: 1s - limit_mib: 1000 - spike_limit_mib: 200 - - batch: - - resourcedetection: - detectors: ["env", "system"] - system: - hostname_sources: ["os"] - resource_attributes: - host.id: - enabled: true - - resourcedetection/cloud: - detectors: ["gcp", "ec2", "azure"] - timeout: 2s - override: false - - # host.id is required for NewRelic host entity synthesis and relationships, but is - # not included by any resourcedetection detector when running with docker on macOS. - # We include a fallback value for demonstration purposes. - # Delete for production deployments. - resource: - attributes: - - key: host.id - value: localhost - action: upsert - -exporters: - otlphttp: - endpoint: $NEW_RELIC_OTLP_ENDPOINT - headers: - api-key: $NEW_RELIC_API_KEY - -service: - pipelines: - - metrics/hostmetrics: - receivers: [hostmetrics] - processors: [memory_limiter, resourcedetection, resourcedetection/cloud, resource, batch] - exporters: [otlphttp] - - metrics: - receivers: [otlp] - processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, resource, batch] - exporters: [otlphttp] - - traces: - receivers: [otlp] - processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, resource, batch] - exporters: [otlphttp] - - logs: - receivers: [otlp, filelog] - processors: [memory_limiter, transform/truncate, resourcedetection, resourcedetection/cloud, resource, batch] - exporters: [otlphttp] - - extensions: [health_check] \ No newline at end of file From 88e730df1b5b8a72e8d2db735098b528dfd5dd5f Mon Sep 17 00:00:00 2001 From: jack-berg <34418638+jack-berg@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:21:20 -0500 Subject: [PATCH 3/6] Convert collector hcp consul example to kubernetes (#633) --- other-examples/collector/hcp-consul/.env | 14 --- other-examples/collector/hcp-consul/README.md | 83 +++++++++++----- .../collector/hcp-consul/collector.yaml | 36 ------- .../collector/hcp-consul/docker-compose.yaml | 14 --- .../collector/hcp-consul/k8s/collector.yaml | 94 +++++++++++++++++++ .../collector/hcp-consul/k8s/secrets.yaml | 12 +++ 6 files changed, 167 insertions(+), 86 deletions(-) delete mode 100644 other-examples/collector/hcp-consul/.env delete mode 100644 other-examples/collector/hcp-consul/collector.yaml delete mode 100644 other-examples/collector/hcp-consul/docker-compose.yaml create mode 100644 other-examples/collector/hcp-consul/k8s/collector.yaml create mode 100644 other-examples/collector/hcp-consul/k8s/secrets.yaml diff --git a/other-examples/collector/hcp-consul/.env b/other-examples/collector/hcp-consul/.env deleted file mode 100644 index 0a4a08a0..00000000 --- a/other-examples/collector/hcp-consul/.env +++ /dev/null @@ -1,14 +0,0 @@ -# New Relic API key to authenticate the call. -# docs: https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key -NEW_RELIC_API_KEY= - -# The default US endpoint is set here. You can change the endpoint and port based on your requirements if needed. -# docs: https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/get-started/opentelemetry-set-up-your-app/#review-settings -NEW_RELIC_OTLP_ENDPOINT=https://otlp.nr-data.net/ - -# Set your authentication token for your HCP managed Consul enviroment. -# docs: https://developer.hashicorp.com/hcp/docs/consul/hcp-managed/access#generate-admin-token -HCP_ACCESS_TOKEN= -# Set your access URL to your HCP managed Consul environment. -# docs: https://developer.hashicorp.com/hcp/docs/consul/hcp-managed/access#get-access-url -HCP_ACCESS_URL= \ No newline at end of file diff --git a/other-examples/collector/hcp-consul/README.md b/other-examples/collector/hcp-consul/README.md index b6dbbe4c..07f98cea 100644 --- a/other-examples/collector/hcp-consul/README.md +++ b/other-examples/collector/hcp-consul/README.md @@ -1,36 +1,75 @@ -# HCP Consul OpenTelemetry metrics example setup +# Monitoring HCP Consul with OpenTelemetry Collector -This example shows a setup for running a prometheus OpenTelemetry Collector in a docker container to scrape metrics from [HCP Consul](https://developer.hashicorp.com/hcp/docs/consul) and post them the New Relic OTLP Collector Endpoint. +This simple example demonstrates monitoring [HCP Consul](https://developer.hashicorp.com/hcp/docs/consul) prometheus metrics with the [OpenTelemetry collector](https://opentelemetry.io/docs/collector/), using the [prometheus receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/prometheusreceiver) and sending the data to New Relic via OTLP. -**NOTE**: This is *not* for self-managed Consul, and will only work with HCP managed Consul. For self-managed Consul you will have to communicate with the [agent API](https://developer.hashicorp.com/consul/api-docs/agent). +## Requirements -## Prerequisites - -1. You must have a Docker daemon running. -2. You must have [Docker compose](https://docs.docker.com/compose/) installed . -3. You must have a [Consul cluster](https://developer.hashicorp.com/hcp/docs/consul) running in Consul Cloud. +* You need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. Docker desktop [includes a standalone Kubernetes server and client](https://docs.docker.com/desktop/kubernetes/) which is useful for local testing. +* [A New Relic account](https://one.newrelic.com/) +* [A New Relic license key](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key) +* [An HCP Consul Account](https://developer.hashicorp.com/sign-up) with a running [cluster](https://developer.hashicorp.com/hcp/docs/consul) +* [A HCP Consul admin access token](https://developer.hashicorp.com/hcp/docs/consul/dedicated/access#generate-admin-token) ## Running the example -First, set your environment variables in the `.env` file in this directory. For more information on the individual variables, reference the docs available below. +1. Update the `NEW_RELIC_API_KEY` values `HCP_ACCESS_TOKEN` values in [secrets.yaml](./k8s/secrets.yaml) to your New Relic license key, and HCP Consule admin access token respectively. See [HCP Consule docs](https://developer.hashicorp.com/hcp/docs/consul/hcp-managed/access#generate-admin-token) for obtaining an access token. -Once the variables are set, run the following command from the root directory to start the collector. + ```yaml + # ...omitted for brevity + stringData: + # New Relic API key to authenticate the export requests. + # docs: https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key + NEW_RELIC_API_KEY: + # Set your HCP Access Token API Key. + # docs: https://developer.hashicorp.com/hcp/docs/consul/hcp-managed/access#generate-admin-token + HCP_ACCESS_TOKEN: > + ``` + + * Note, be careful to avoid inadvertent secret sharing when modifying `secrets.yaml`. To ignore changes to this file from git, run `git update-index --skip-worktree k8s/secrets.yaml`. -```shell -cd ./other-examples/collector/hcp-consul + * If your account is based in the EU, update the `NEW_RELIC_OTLP_ENDPOINT` value in [collector.yaml](./k8s/collector.yaml) the endpoint to: [https://otlp.eu01.nr-data.net](https://otlp.eu01.nr-data.net) -docker compose up -``` + ```yaml + # ...omitted for brevity + env: + # The default US endpoint is set here. You can change the endpoint and port based on your requirements if needed. + # docs: https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/best-practices/opentelemetry-otlp/#configure-endpoint-port-protocol + - name: NEW_RELIC_OTLP_ENDPOINT + value: https://otlp.eu01.nr-data.net + ``` + +2. Set the `HCP_ACCESS_URL` env var value in [collector.yaml](./k8s/collector.yaml). See [HCP Consul docs](https://developer.hashicorp.com/hcp/docs/consul/hcp-managed/access#get-access-url) for details on obtaining org and workspace group ids. + + ```yaml + # ...omitted for brevity + # The HCP Consul access url. + # docs: https://developer.hashicorp.com/hcp/docs/consul/hcp-managed/access#get-access-url + - name: HCP_ACCESS_URL + value: + ``` + +3. Run the application with the following command. + + ```shell + kubectl apply -f k8s/ + ``` + + * When finished, cleanup resources with the following command. This is also useful to reset if modifying configuration. + + ```shell + kubectl delete -f k8s/ + ``` ## Viewing your data -To review your HCP Consul data in New Relic, navigate to your [metrics explorer](https://docs.newrelic.com/docs/query-your-data/explore-query-data/browse-data/introduction-data-explorer/) and filter for `consul`. +To review your HCP Consul data in New Relic, navigate to "New Relic -> Query Your Data". To list the metrics reported, query for: + +``` +FROM Metric SELECT uniques(metricName) WHERE otel.library.name = 'otelcol/prometheusreceiver' AND metricName LIKE 'consul%' +``` + +See [get started with querying](https://docs.newrelic.com/docs/query-your-data/explore-query-data/get-started/introduction-querying-new-relic-data/) for additional details on querying data in New Relic. -## Local Variable information +## Additional notes -| Variable | Description | Docs | -| -------- | ----------- | ---- | -| **NEW_RELIC_API_KEY** |New Relic Ingest API Key |[API Key docs](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/) | -| **NEW_RELIC_OTLP_ENDPOINT** |Default US OTLP endpoint is `https://otlp.nr-data.net` | [OTLP endpoint config docs](https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/get-started/opentelemetry-set-up-your-app/#review-settings) | -| **HCP_ACCESS_URL** | URL for communicating with your HCP managed Consul cluster |[Consul URL docs](https://developer.hashicorp.com/hcp/docs/consul/hcp-managed/access#get-access-url)| -| **HCP_ACCESS_TOKEN** | Consul admin token to authenticate with your HCP managed Consul Cluster| [Consul token docs](https://developer.hashicorp.com/hcp/docs/consul/hcp-managed/access#generate-admin-token) | +The prometheus receiver includes `service.name` and `service.instance.id` resource attributes derived from job name and target configured in `.receivers.prometheus.config.scrape_configs`. As documented [here](https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/best-practices/opentelemetry-best-practices-resources/#services), New Relic considers any data with `service.name` as a service despite the fact that not all prometheus data sources are services. As a result, you can find a `hcp-consul` entity under "New Relic -> All Entities -> Services - OpenTelemetry", although the panels will not contain data because the scraped metrics do not represent APM data. diff --git a/other-examples/collector/hcp-consul/collector.yaml b/other-examples/collector/hcp-consul/collector.yaml deleted file mode 100644 index 74968603..00000000 --- a/other-examples/collector/hcp-consul/collector.yaml +++ /dev/null @@ -1,36 +0,0 @@ -receivers: - prometheus: - config: - global: - scrape_interval: "60s" - scrape_configs: - - job_name: "hcp-consul-cluster" - scheme: "https" - dns_sd_configs: - - names: - - "$HCP_ACCESS_URL" - type: "A" - port: 443 - authorization: - credentials: "$HCP_ACCESS_TOKEN" - metrics_path: "/v1/agent/metrics" - - # Skipping TLS verification based on Consul docs(https://developer.hashicorp.com/hcp/docs/consul/monitor/metrics#prometheus). - tls_config: - insecure_skip_verify: true - -processors: - batch: - -exporters: - otlphttp: - endpoint: $NEW_RELIC_OTLP_ENDPOINT - headers: - api-key: $NEW_RELIC_API_KEY - -service: - pipelines: - metrics: - receivers: [prometheus] - processors: [batch] - exporters: [otlphttp] \ No newline at end of file diff --git a/other-examples/collector/hcp-consul/docker-compose.yaml b/other-examples/collector/hcp-consul/docker-compose.yaml deleted file mode 100644 index bc456615..00000000 --- a/other-examples/collector/hcp-consul/docker-compose.yaml +++ /dev/null @@ -1,14 +0,0 @@ -version: "3" - -services: - - otel-collector: - image: otel/opentelemetry-collector-contrib:0.92.0 - command: --config=/etc/otelcol/config.yaml - volumes: - - ./collector.yaml:/etc/otelcol/config.yaml - environment: - - NEW_RELIC_OTLP_ENDPOINT - - NEW_RELIC_API_KEY - - HCP_ACCESS_TOKEN - - HCP_ACCESS_URL diff --git a/other-examples/collector/hcp-consul/k8s/collector.yaml b/other-examples/collector/hcp-consul/k8s/collector.yaml new file mode 100644 index 00000000..b0ba1d37 --- /dev/null +++ b/other-examples/collector/hcp-consul/k8s/collector.yaml @@ -0,0 +1,94 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: nr-hcp-consul +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: collector-config + namespace: nr-hcp-consul + labels: + app.kubernetes.io/name: collector-config +data: + collector-config: | + receivers: + prometheus: + config: + scrape_configs: + - job_name: "hcp-consul" + scrape_interval: 60s # Do not go any lower than this or you'll hit rate limits + scheme: https + dns_sd_configs: + - names: + - "${HCP_ACCESS_URL}" + type: "A" + port: 443 + authorization: + credentials: "${HCP_ACCESS_TOKEN}" + metrics_path: "/v1/agent/metrics" + # Skipping TLS verification based on Consul docs(https://developer.hashicorp.com/hcp/docs/consul/monitor/metrics#prometheus). + tls_config: + insecure_skip_verify: true + + processors: + batch: + + exporters: + otlphttp: + endpoint: ${NEW_RELIC_OTLP_ENDPOINT} + headers: + api-key: ${NEW_RELIC_API_KEY} + + service: + pipelines: + metrics: + receivers: [prometheus] + processors: [batch] + exporters: [otlphttp] +--- +apiVersion: v1 +kind: Pod +metadata: + name: collector + namespace: nr-hcp-consul + labels: + app.kubernetes.io/name: collector +spec: + containers: + - name: collector + image: otel/opentelemetry-collector-contrib:0.98.0 + env: + # The default US endpoint is set here. You can change the endpoint and port based on your requirements if needed. + # docs: https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/best-practices/opentelemetry-otlp/#configure-endpoint-port-protocol + - name: NEW_RELIC_OTLP_ENDPOINT + value: https://otlp.nr-data.net/ + # The New Relic API key used to authenticate export requests. + # Defined in secrets.yaml + - name: NEW_RELIC_API_KEY + valueFrom: + secretKeyRef: + name: nr-hcp-consul-secret + key: NEW_RELIC_API_KEY + # The HCP Consul access token. + # Defined in secrets.yaml + - name: HCP_ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: nr-hcp-consul-secret + key: HCP_ACCESS_TOKEN + # The HCP Consul access url. + # docs: https://developer.hashicorp.com/hcp/docs/consul/hcp-managed/access#get-access-url + - name: HCP_ACCESS_URL + value: + volumeMounts: + - name: collector-config-vol + mountPath: /etc/otelcol-contrib + volumes: + - name: collector-config-vol + configMap: + name: collector-config + items: + - key: collector-config + path: config.yaml diff --git a/other-examples/collector/hcp-consul/k8s/secrets.yaml b/other-examples/collector/hcp-consul/k8s/secrets.yaml new file mode 100644 index 00000000..533d27d7 --- /dev/null +++ b/other-examples/collector/hcp-consul/k8s/secrets.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: nr-hcp-consul-secret + namespace: nr-hcp-consul +stringData: + # New Relic API key to authenticate the export requests. + # docs: https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key + NEW_RELIC_API_KEY: + # Set your HCP Access Token API Key. + # docs: https://developer.hashicorp.com/hcp/docs/consul/hcp-managed/access#generate-admin-token + HCP_ACCESS_TOKEN: From b2122a9af9f4de55f69d548132001306c396c79c Mon Sep 17 00:00:00 2001 From: jack-berg <34418638+jack-berg@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:21:40 -0500 Subject: [PATCH 4/6] Add docker collector example (#634) * wip * Add docker collector example * Add readme entry --- README.md | 1 + other-examples/collector/docker/.env | 11 +++ other-examples/collector/docker/README.md | 84 +++++++++++++++++++ other-examples/collector/docker/config.yaml | 56 +++++++++++++ .../collector/docker/docker-compose.yaml | 20 +++++ 5 files changed, 172 insertions(+) create mode 100644 other-examples/collector/docker/.env create mode 100644 other-examples/collector/docker/README.md create mode 100644 other-examples/collector/docker/config.yaml create mode 100644 other-examples/collector/docker/docker-compose.yaml diff --git a/README.md b/README.md index ad611c19..f3c52129 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ The [Getting Started Guides](./getting-started-guides/README.md) demonstrate how OpenTelemetry is a big ecosystem and everything doesn't fit into the goals of the [getting started guides](#getting-started-guides). These "other examples" demonstrate how other areas of OpenTelemetry fit in with New Relic. * Collector + * [Docker monitoring](./other-examples/collector/docker) * [Telemetry data processing](./other-examples/collector/nr-config) * [Host monitoring](./other-examples/collector/host-monitoring) * [Confluent cloud monitoring](./other-examples/collector/confluentcloud) diff --git a/other-examples/collector/docker/.env b/other-examples/collector/docker/.env new file mode 100644 index 00000000..c1dbbcc3 --- /dev/null +++ b/other-examples/collector/docker/.env @@ -0,0 +1,11 @@ +# New Relic API key to authenticate the call. +# docs: https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key +NEW_RELIC_API_KEY= + +# The default US endpoint is set here. You can change the endpoint and port based on your requirements if needed. +# docs: https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/get-started/opentelemetry-set-up-your-app/#review-settings +NEW_RELIC_OTLP_ENDPOINT=https://otlp.nr-data.net/ + +# User ID used to run the collector. Must have permission to access the docker socket. +# Obtain for a given user via "id -g " +HOST_USER_ID=0 diff --git a/other-examples/collector/docker/README.md b/other-examples/collector/docker/README.md new file mode 100644 index 00000000..4fb1efc6 --- /dev/null +++ b/other-examples/collector/docker/README.md @@ -0,0 +1,84 @@ +# Monitoring Docker with OpenTelemetry Collector + +This simple example demonstrates monitoring docker with the [OpenTelemetry collector](https://opentelemetry.io/docs/collector/), using the [docker stats receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/dockerstatsreceiver) and sending the data to New Relic via OTLP. + +## Requirements + +* A linux machine with docker daemon and docker compose (docker stats receiver only supports Linux). +* [A New Relic account](https://one.newrelic.com/) +* [A New Relic license key](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key) + +## Running the example + +1. Update the `NEW_RELIC_API_KEY` value in [.env](./.env) to your New Relic license key. + + ``` + # New Relic API key to authenticate the call. + # docs: https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key + NEW_RELIC_API_KEY= + ``` + + * Note, be careful to avoid inadvertent secret sharing when modifying `.env`. To ignore changes to this file from git, run `git update-index --skip-worktree .env`. + + * If your account is based in the EU, update the `NEW_RELIC_OTLP_ENDPOINT` value in `.env` to: [https://otlp.eu01.nr-data.net](https://otlp.eu01.nr-data.net) + + ``` + # ...omitted for brevity + # The default US endpoint is set here. You can change the endpoint and port based on your requirements if needed. + # docs: https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/get-started/opentelemetry-set-up-your-app/#review-settings + NEW_RELIC_OTLP_ENDPOINT=https://otlp.nr-data.net/ + ``` + +2. Update the `HOST_USER_ID` to the id of a user with permission to access the docker socket. + + The docker stats receiver reads from docker daemon socket. This example runs the collector and docker stats receiver inside a docker container, and makes the host docker socket accessible to the container by mounting a volume with `/var/run/docker.sock` from the host machine. By default, the collector contrib image runs with a user with limited permissions. We must override the user and run it with permission to access the docker socket. + + Update the `HOST_USER_ID` value in [.env](./.env) to a user with required permissions. It's set to `0` by default, which corresponds to the root user. Its good practice to run with a user with more limited access in production. + + ``` + # ...omitted for brevity + # User ID used to run the collector. Must have permission to access the docker socket. + # Obtain for a given user via "id -g " + HOST_USER_ID=0 + ``` + +3. Run the application with the following command. + + ```shell + docker compose up + ``` + + * Optionally include `-d` to run in the background. + + * When finished, cleanup resources by exiting the command with `Ctrl-D` or `Ctrl-C`. If running in the background, run the following command to stop containers. + + ```shell + docker compose stop + ``` + +## Viewing your data + +To review your docker data in New Relic, navigate to "New Relic -> All Entities -> Containers". You should see entities named `docker-collector-1` and `docker-nginx-1` corresponding to the services defined in `docker-compose.yaml`. Click to view the container summary. + +Optionally, install the [Docker OpenTelemetry quickstart](https://newrelic.com/instant-observability/docker-otel) which includes a dashboard and alerts based on the data produced by the docker stats reciever. + +Optionally, use [NRQL](https://docs.newrelic.com/docs/query-your-data/explore-query-data/get-started/introduction-querying-new-relic-data/) to perform ad-hoc analysis. To list the metrics reported, query for: + +``` +FROM Metric SELECT uniques(metricName) WHERE otel.library.name = 'otelcol/dockerstatsreceiver' +``` + +See [get started with querying](https://docs.newrelic.com/docs/query-your-data/explore-query-data/get-started/introduction-querying-new-relic-data/) for additional details on querying data in New Relic. + +## Additional notes + +New Relic depends on docker container data including the `host.id` resource attribute. The collector [config.yaml](./config.yaml) contains several resource detectors in `.processors.resourcedetection.detectors` which attempt to fetch `host.id`. If `host.id` is not detected, you can manually set it by uncommenting and editing the `OTEL_RESOURCE_ATTRIBUTES` env var in [docker-compose.yaml](./docker-compose.yaml). + +```yaml +# ...omitted for brevity +# host.id is required for New Relic. +# Optionally manually set it if one of the resource detectors in config.yaml is unable to identify it. +# - OTEL_RESOURCE_ATTRIBUTES=host.id= +``` + +This example runs a dummy nginx image defined in [docker-compose.yaml](./docker-compose.yaml). This only exists to produce more interesting data and should be removed for production deployments. diff --git a/other-examples/collector/docker/config.yaml b/other-examples/collector/docker/config.yaml new file mode 100644 index 00000000..e0a791b6 --- /dev/null +++ b/other-examples/collector/docker/config.yaml @@ -0,0 +1,56 @@ +receivers: + docker_stats: + metrics: + container.cpu.usage.total: + enabled: true + container.cpu.throttling_data.periods: + enabled: true + container.cpu.throttling_data.throttled_periods: + enabled: true + container.cpu.utilization: + enabled: true + container.memory.usage.limit: + enabled: true + container.memory.usage.total: + enabled: true + container.memory.percent: + enabled: true + container.blockio.io_service_bytes_recursive: + enabled: true + container.network.io.usage.rx_bytes: + enabled: true + container.network.io.usage.tx_bytes: + enabled: true + container.network.io.usage.rx_dropped: + enabled: true + container.network.io.usage.tx_dropped: + enabled: true + container.network.io.usage.rx_errors: + enabled: true + container.network.io.usage.tx_errors: + enabled: true + container.network.io.usage.rx_packets: + enabled: true + container.network.io.usage.tx_packets: + enabled: true + container.pids.count: + enabled: true + +processors: + batch: + + resourcedetection: + detectors: ["env", "system" "gcp", "ec2", "azure"] + +exporters: + otlphttp: + endpoint: ${NEW_RELIC_OTLP_ENDPOINT} + headers: + api-key: ${NEW_RELIC_API_KEY} + +service: + pipelines: + metrics: + receivers: [docker_stats] + processors: [resourcedetection, batch] + exporters: [otlphttp] \ No newline at end of file diff --git a/other-examples/collector/docker/docker-compose.yaml b/other-examples/collector/docker/docker-compose.yaml new file mode 100644 index 00000000..9c2d8979 --- /dev/null +++ b/other-examples/collector/docker/docker-compose.yaml @@ -0,0 +1,20 @@ +version: '3' +services: + collector: + image: otel/opentelemetry-collector-contrib:0.98.0 + volumes: + - ./config.yaml:/etc/otelcol-contrib/config.yaml + - /var/run/docker.sock:/var/run/docker.sock + # Set the user id used to run the collector. HOST_USER_ID is defined in .env, and MUST have permission to access the docker socket. + user: ${HOST_USER_ID} + # Expose the env vars defined in .env + environment: + - NEW_RELIC_API_KEY + - NEW_RELIC_OTLP_ENDPOINT + # host.id is required for New Relic. + # Optionally manually set it if one of the resource detectors in config.yaml is unable to identify it. + # - OTEL_RESOURCE_ATTRIBUTES=host.id= + + # Run a dummy nginx image to generate more interesting data. + nginx: + image: nginx From f533b169c0f7f17406ad6e59f311cf6e7dd5417d Mon Sep 17 00:00:00 2001 From: jack-berg <34418638+jack-berg@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:22:04 -0500 Subject: [PATCH 5/6] Add prometheus collector example (#624) --- README.md | 1 + other-examples/collector/prometheus/README.md | 62 +++++++++++++ .../collector/prometheus/k8s/collector.yaml | 74 ++++++++++++++++ .../k8s/prometheus-data-generator.yaml | 88 +++++++++++++++++++ .../collector/prometheus/k8s/secrets.yaml | 10 +++ 5 files changed, 235 insertions(+) create mode 100644 other-examples/collector/prometheus/README.md create mode 100644 other-examples/collector/prometheus/k8s/collector.yaml create mode 100644 other-examples/collector/prometheus/k8s/prometheus-data-generator.yaml create mode 100644 other-examples/collector/prometheus/k8s/secrets.yaml diff --git a/README.md b/README.md index f3c52129..c03172a0 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ OpenTelemetry is a big ecosystem and everything doesn't fit into the goals of th * [Host monitoring](./other-examples/collector/host-monitoring) * [Confluent cloud monitoring](./other-examples/collector/confluentcloud) * [HCP Consul monitoring](./other-examples/collector/hcp-consul) + * [Prometheus monitoring](./other-examples/collector/prometheus) * [Redis monitoring](./other-examples/collector/redis) * [Singlestore monitoring](./other-examples/collector/singlestore) * [StatsD monitoring](./other-examples/collector/statsd) diff --git a/other-examples/collector/prometheus/README.md b/other-examples/collector/prometheus/README.md new file mode 100644 index 00000000..4cad3aac --- /dev/null +++ b/other-examples/collector/prometheus/README.md @@ -0,0 +1,62 @@ +# Monitoring Prometheus with OpenTelemetry Collector + +This simple example demonstrates monitoring prometheus sources with the [OpenTelemetry collector](https://opentelemetry.io/docs/collector/), using the [prometheus receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/prometheusreceiver) and sending the data to New Relic via OTLP. A simple prometheus data generator is configured to generate dummy metrics scraped by prometheus receiver. + +## Requirements + +* You need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. Docker desktop [includes a standalone Kubernetes server and client](https://docs.docker.com/desktop/kubernetes/) which is useful for local testing. +* [A New Relic account](https://one.newrelic.com/) +* [A New Relic license key](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key) + +## Running the example + +1. Update the `NEW_RELIC_API_KEY` value in [secrets.yaml](./k8s/secrets.yaml) to your New Relic license key. + + ```yaml + # ...omitted for brevity + stringData: + # New Relic API key to authenticate the export requests. + # docs: https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key + NEW_RELIC_API_KEY: + ``` + + * Note, be careful to avoid inadvertent secret sharing when modifying `secrets.yaml`. To ignore changes to this file from git, run `git update-index --skip-worktree k8s/secrets.yaml`. + + * If your account is based in the EU, update the `NEW_RELIC_OTLP_ENDPOINT` value in [collector.yaml](./k8s/collector.yaml) the endpoint to: [https://otlp.eu01.nr-data.net](https://otlp.eu01.nr-data.net) + + ```yaml + # ...omitted for brevity + env: + # The default US endpoint is set here. You can change the endpoint and port based on your requirements if needed. + # docs: https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/best-practices/opentelemetry-otlp/#configure-endpoint-port-protocol + - name: NEW_RELIC_OTLP_ENDPOINT + value: https://otlp.eu01.nr-data.net + ``` + +2. Run the application with the following command. + + ```shell + kubectl apply -f k8s/ + ``` + + * When finished, cleanup resources with the following command. This is also useful to reset if modifying configuration. + + ```shell + kubectl delete -f k8s/ + ``` + +## Viewing your data + +To review your statsd data in New Relic, navigate to "New Relic -> Query Your Data". To list the metrics reported, query for: + +``` +FROM Metric SELECT uniques(metricName) WHERE otel.library.name = 'otelcol/prometheusreceiver' LIMIT MAX +``` + +See [get started with querying](https://docs.newrelic.com/docs/query-your-data/explore-query-data/get-started/introduction-querying-new-relic-data/) for additional details on querying data in New Relic. + +## Additional notes + +This example monitors a simple prometheus data generator instance defined in [prometheus-data-generator.yaml](./k8s/prometheus-data-generator.yaml). To use in production, you'll need to modify the `.receivers.prometheus.config.scrape_configs` value in [collector.yaml](k8s/collector.yaml) ConfigMap to point at your prometheus sources. + +The prometheus receiver includes `service.name` and `service.instance.id` resource attributes derived from job name and target configured in `.receivers.prometheus.config.scraep_configs`. As documented [here](https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/best-practices/opentelemetry-best-practices-resources/#services), New Relic considers any data with `service.name` as a service despite the fact that not all prometheus data sources are services. As a result, you can find a `prometheus_data_generator` entity under "New Relic -> All Entities -> Services - OpenTelemetry", although the panels will not contain data because the scraped metrics do not represent APM data. diff --git a/other-examples/collector/prometheus/k8s/collector.yaml b/other-examples/collector/prometheus/k8s/collector.yaml new file mode 100644 index 00000000..c5524c87 --- /dev/null +++ b/other-examples/collector/prometheus/k8s/collector.yaml @@ -0,0 +1,74 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: nr-prometheus +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: collector-config + namespace: nr-prometheus + labels: + app.kubernetes.io/name: collector-config +data: + collector-config: | + receivers: + prometheus: + config: + scrape_configs: + - job_name: prometheus_data_generator + static_configs: + # Connect to prometheus data generator pod defined in prometheus-data-generator.yaml using service env vars set by k8s + - targets: [ "${PROMETHEUS_DATA_GENERATOR_SERVICE_HOST}:${PROMETHEUS_DATA_GENERATOR_SERVICE_PORT}" ] + metrics_path: /metrics + + processors: + batch: + + exporters: + otlphttp: + endpoint: ${NEW_RELIC_OTLP_ENDPOINT} + headers: + api-key: ${NEW_RELIC_API_KEY} + + service: + pipelines: + metrics: + receivers: [prometheus] + processors: [batch] + exporters: [otlphttp] +--- +apiVersion: v1 +kind: Pod +metadata: + name: collector + namespace: nr-prometheus + labels: + app.kubernetes.io/name: collector +spec: + containers: + - name: collector + image: otel/opentelemetry-collector-contrib:0.98.0 + env: + # The default US endpoint is set here. You can change the endpoint and port based on your requirements if needed. + # docs: https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/best-practices/opentelemetry-otlp/#configure-endpoint-port-protocol + - name: NEW_RELIC_OTLP_ENDPOINT + value: https://otlp.nr-data.net/ + # The New Relic API key used to authenticate export requests. + # Defined in secrets.yaml + - name: NEW_RELIC_API_KEY + valueFrom: + secretKeyRef: + name: nr-prometheus-secret + key: NEW_RELIC_API_KEY + volumeMounts: + - name: collector-config-vol + mountPath: /etc/otelcol-contrib + volumes: + - name: collector-config-vol + configMap: + name: collector-config + items: + - key: collector-config + path: config.yaml diff --git a/other-examples/collector/prometheus/k8s/prometheus-data-generator.yaml b/other-examples/collector/prometheus/k8s/prometheus-data-generator.yaml new file mode 100644 index 00000000..5a7c1834 --- /dev/null +++ b/other-examples/collector/prometheus/k8s/prometheus-data-generator.yaml @@ -0,0 +1,88 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-data-generator-config + namespace: nr-prometheus +data: + config: | + --- + config: + - name: number_of_fruits + description: The number of fruits we have. + type: gauge + labels: [name, color] + sequence: + - time: 5 + values: 0-20 + operation: inc + labels: + name: apple + color: red + - eval_time: 5 + operation: set + labels: + name: apple + color: red + - time: 5 + eval_time: 1 + values: 0-20 + operation: inc + labels: + name: apple + color: green + - time: 5 + eval_time: 1 + values: 0-5 + operation: dec + labels: + name: apple + color: green + - time: 5 + eval_time: 1 + value: 3 + operation: inc + labels: + name: apple + color: yellow +--- +apiVersion: v1 +kind: Pod +metadata: + name: prometheus-data-generator + namespace: nr-prometheus + labels: + app.kubernetes.io/name: prometheus-data-generator +spec: + containers: + - name: prometheus-data-generator + image: littleangryclouds/prometheus-data-generator:0.2 + env: + - name: PDG_CONFIG + value: /pdg/config.yaml + volumeMounts: + - name: prometheus-data-generator-config-vol + mountPath: /pdg/ + ports: + - containerPort: 9000 + volumes: + - name: prometheus-data-generator-config-vol + configMap: + name: prometheus-data-generator-config + items: + - key: config + path: config.yaml +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus-data-generator + namespace: nr-prometheus + labels: + app.kubernetes.io/name: prometheus-data-generator +spec: + ports: + - port: 9000 + protocol: TCP + selector: + app.kubernetes.io/name: prometheus-data-generator diff --git a/other-examples/collector/prometheus/k8s/secrets.yaml b/other-examples/collector/prometheus/k8s/secrets.yaml new file mode 100644 index 00000000..4f9340cd --- /dev/null +++ b/other-examples/collector/prometheus/k8s/secrets.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: nr-prometheus-secret + namespace: nr-prometheus +stringData: + # New Relic API key to authenticate the export requests. + # docs: https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key + NEW_RELIC_API_KEY: + From 44338dcad5d9f6bc6ca0db0072cb2f87b6ede299 Mon Sep 17 00:00:00 2001 From: jack-berg <34418638+jack-berg@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:27:22 -0500 Subject: [PATCH 6/6] Migrate confluent cloud collector example to kubernetes (#626) * Add confluent cloud kubernetes example * Replace existing confluent example * Confluent Cloud Kafka --- README.md | 9 +- other-examples/collector/confluentcloud/.env | 22 ---- .../collector/confluentcloud/README.md | 91 ++++++++++---- .../collector/confluentcloud/collector.yaml | 37 ------ .../confluentcloud/docker-compose.yaml | 17 --- .../confluentcloud/k8s/collector.yaml | 112 ++++++++++++++++++ .../collector/confluentcloud/k8s/secrets.yaml | 13 ++ 7 files changed, 198 insertions(+), 103 deletions(-) delete mode 100644 other-examples/collector/confluentcloud/.env delete mode 100644 other-examples/collector/confluentcloud/collector.yaml delete mode 100644 other-examples/collector/confluentcloud/docker-compose.yaml create mode 100644 other-examples/collector/confluentcloud/k8s/collector.yaml create mode 100644 other-examples/collector/confluentcloud/k8s/secrets.yaml diff --git a/README.md b/README.md index c03172a0..6442d9cd 100644 --- a/README.md +++ b/README.md @@ -19,15 +19,16 @@ The [Getting Started Guides](./getting-started-guides/README.md) demonstrate how OpenTelemetry is a big ecosystem and everything doesn't fit into the goals of the [getting started guides](#getting-started-guides). These "other examples" demonstrate how other areas of OpenTelemetry fit in with New Relic. -* Collector +* [Collector for data processing](./other-examples/collector/nr-config) +* Collector for infrastructure monitoring + * [Confluent cloud kafka monitoring](./other-examples/collector/confluentcloud) * [Docker monitoring](./other-examples/collector/docker) - * [Telemetry data processing](./other-examples/collector/nr-config) - * [Host monitoring](./other-examples/collector/host-monitoring) - * [Confluent cloud monitoring](./other-examples/collector/confluentcloud) * [HCP Consul monitoring](./other-examples/collector/hcp-consul) + * [Host monitoring](./other-examples/collector/host-monitoring) * [Prometheus monitoring](./other-examples/collector/prometheus) * [Redis monitoring](./other-examples/collector/redis) * [Singlestore monitoring](./other-examples/collector/singlestore) + * [Singlestore monitoring](./other-examples/collector/singlestore) * [StatsD monitoring](./other-examples/collector/statsd) * Java * [OpenTelemetry Agent New Relic Config](./other-examples/java/agent-nr-config) diff --git a/other-examples/collector/confluentcloud/.env b/other-examples/collector/confluentcloud/.env deleted file mode 100644 index ac0779ef..00000000 --- a/other-examples/collector/confluentcloud/.env +++ /dev/null @@ -1,22 +0,0 @@ -# New Relic API key to authenticate the call. -# docs: https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key -NEW_RELIC_API_KEY= - -# The default US endpoint is set here. You can change the endpoint and port based on your requirements if needed. -# docs: https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/get-started/opentelemetry-set-up-your-app/#review-settings -NEW_RELIC_OTLP_ENDPOINT=https://otlp.nr-data.net/ - - -# Set your authentication keys for the Confluent Cloud metrics API. -# docs: https://docs.confluent.io/cloud/current/monitoring/metrics-api.html -CONFLUENT_API_KEY= -CONFLUENT_API_SECRET= - - -# Set your Cluster ID here. -# docs: https://docs.confluent.io/confluent-cli/current/command-reference/kafka/cluster/confluent_kafka_cluster_list.html -CLUSTER_ID= - -# OPTIONAL - if you include these be sure to uncomment them in the docker-compose file and in the collector file. -SCHEMA_REGISTRY_ID= -CONNECTOR_ID= \ No newline at end of file diff --git a/other-examples/collector/confluentcloud/README.md b/other-examples/collector/confluentcloud/README.md index 9c35953e..77fdba7e 100644 --- a/other-examples/collector/confluentcloud/README.md +++ b/other-examples/collector/confluentcloud/README.md @@ -1,34 +1,79 @@ -# Confluent Cloud OpenTelemetry metrics example setup +# Monitoring Confluent Cloud Kafka with OpenTelemetry Collector -This example shows a setup for running a prometheus OpenTelemetry Collector in a docker container to scrape metrics from Confluent Cloud and post them the New Relic OTLP Collector Endpoint. +This simple example demonstrates monitoring Confluent Cloud prometheus metrics with the [OpenTelemetry collector](https://opentelemetry.io/docs/collector/), using the [prometheus receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/prometheusreceiver) and sending the data to New Relic via OTLP. -For more information, please see our [Kafka with Confluent documentation](https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/collector/collector-configuration-examples/opentelemetry-collector-kafka-confluentcloud/). +## Requirements -## Prerequisites +* You need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. Docker desktop [includes a standalone Kubernetes server and client](https://docs.docker.com/desktop/kubernetes/) which is useful for local testing. +* [A New Relic account](https://one.newrelic.com/) +* [A New Relic license key](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key) +* [A Confluent Cloud account](https://www.confluent.io/get-started/) with a cluster running +* [A Confluent Cloud API key and secret](https://docs.confluent.io/confluent-cli/current/command-reference/api-key/confluent_api-key_create.html) -1. You must have a Docker daemon running. -2. You must have [Docker compose](https://docs.docker.com/compose/) installed . -3. You must have a [Confluent Cloud account](https://www.confluent.io/get-started/) with a cluster running. ## Running the example -First, set your environment variables in the `.env` file in this directory. For more information on the individual variables, reference the docs available below. -Once the variables are set, run the following command from the root directory to start the collector. +1. Update the `NEW_RELIC_API_KEY`, `CONFLUENT_API_KEY`, and `CONFLUENT_API_SECRET` values in [secrets.yaml](./k8s/secrets.yaml) to your New Relic license key, and confluent API key / secret respectively. See [Confluent docs](https://docs.confluent.io/cloud/current/monitoring/metrics-api.html) for obtaining API key / secret. -```shell -cd ./other-examples/collector/confluentcloud + ```yaml + # ...omitted for brevity + stringData: + # New Relic API key to authenticate the export requests. + # docs: https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key + NEW_RELIC_API_KEY: + # Set your authentication keys for the Confluent Cloud metrics API. + # docs: https://docs.confluent.io/cloud/current/monitoring/metrics-api.html + CONFLUENT_API_KEY: + CONFLUENT_API_SECRET: + ``` + + * Note, be careful to avoid inadvertent secret sharing when modifying `secrets.yaml`. To ignore changes to this file from git, run `git update-index --skip-worktree k8s/secrets.yaml`. + + * If your account is based in the EU, update the `NEW_RELIC_OTLP_ENDPOINT` value in [collector.yaml](./k8s/collector.yaml) the endpoint to: [https://otlp.eu01.nr-data.net](https://otlp.eu01.nr-data.net) + + ```yaml + # ...omitted for brevity + env: + # The default US endpoint is set here. You can change the endpoint and port based on your requirements if needed. + # docs: https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/best-practices/opentelemetry-otlp/#configure-endpoint-port-protocol + - name: NEW_RELIC_OTLP_ENDPOINT + value: https://otlp.eu01.nr-data.net + ``` + +2. Set the `CONFLUENT_CLUSTER_ID` env var value in [collector.yaml](./k8s/collector.yaml). See [Confluent docs](https://docs.confluent.io/confluent-cli/current/command-reference/kafka/cluster/confluent_kafka_cluster_list.html#description) for details on obtaining cluster id. + + ```yaml + # ...omitted for brevity + # Set your Confluent Cluster ID here. + # docs: https://docs.confluent.io/confluent-cli/current/command-reference/kafka/cluster/confluent_kafka_cluster_list.html + - name: CONFLUENT_CLUSTER_ID + value: + ``` + + * Optionally, uncomment and set the value for `CONFLUENT_SCHEMA_REGISTRY_ID` and `CONFLUENT_CONNECTOR_ID`. If setting these, you must also uncomment the corresponding references in `.receivers.prometheus.config.scrape_configs[0].params` of the collector-config ConfigMap. + +3. Run the application with the following command. + + ```shell + kubectl apply -f k8s/ + ``` + + * When finished, cleanup resources with the following command. This is also useful to reset if modifying configuration. + + ```shell + kubectl delete -f k8s/ + ``` + +## Viewing your data + +To review your statsd data in New Relic, navigate to "New Relic -> Query Your Data". To list the metrics reported, query for: -docker compose up ``` +FROM Metric SELECT uniques(metricName) WHERE otel.library.name = 'otelcol/prometheusreceiver' AND metricName like 'confluent_kafka%' LIMIT MAX +``` + +See [get started with querying](https://docs.newrelic.com/docs/query-your-data/explore-query-data/get-started/introduction-querying-new-relic-data/) for additional details on querying data in New Relic. + +## Additional notes -## Local Variable information - -| Variable | Description | Docs | -| -------- | ----------- | ---- | -| **NEW_RELIC_API_KEY** |New Relic Ingest API Key |[API Key docs](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/) | -| **NEW_RELIC_OTLP_ENDPOINT** |Default US OTLP endpoint is https://otlp.nr-data.net | [OTLP endpoint config docs](https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/get-started/opentelemetry-set-up-your-app/#review-settings) | -| **CONFLUENT_API_KEY** |API key for Confluent Cloud, can be created via cli by following the docs |[Confluent API key docs](https://docs.confluent.io/cloud/current/monitoring/metrics-api.html)| -| **CONFLUENT_API_SECRET** | API secret for Confluent Cloud | [Confluent API key docs](https://docs.confluent.io/cloud/current/monitoring/metrics-api.html) | -| **CLUSTER_ID** | ID of the cluster from Confluent Cloud | [List cluster ID docs](https://docs.confluent.io/confluent-cli/current/command-reference/kafka/cluster/confluent_kafka_cluster_list.html#description) | -| **CONNECTOR_ID** |(Optional) ID of the connector from Confluent Cloud | [List connector ID docs](https://docs.confluent.io/confluent-cli/current/command-reference/connect/cluster/confluent_connect_cluster_list.html) | -| **SCHEMA_REGISTRY_ID** | (Optional) ID of schema registry from Confluent Cloud | [List schema-registry ID docs](https://docs.confluent.io/confluent-cli/current/command-reference/schema-registry/schema/confluent_schema-registry_schema_list.html) | +The prometheus receiver includes `service.name` and `service.instance.id` resource attributes derived from job name and target configured in `.receivers.prometheus.config.scrape_configs`. As documented [here](https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/best-practices/opentelemetry-best-practices-resources/#services), New Relic considers any data with `service.name` as a service despite the fact that not all prometheus data sources are services. As a result, you can find a `confluent` entity under "New Relic -> All Entities -> Services - OpenTelemetry", although the panels will not contain data because the scraped metrics do not represent APM data. diff --git a/other-examples/collector/confluentcloud/collector.yaml b/other-examples/collector/confluentcloud/collector.yaml deleted file mode 100644 index a493cfb9..00000000 --- a/other-examples/collector/confluentcloud/collector.yaml +++ /dev/null @@ -1,37 +0,0 @@ -receivers: - prometheus: - config: - scrape_configs: - - job_name: "confluent" - scrape_interval: 60s # Do not go any lower than this or you'll hit rate limits - static_configs: - - targets: ["api.telemetry.confluent.cloud"] - scheme: https - basic_auth: - username: $CONFLUENT_API_KEY - password: $CONFLUENT_API_SECRET - metrics_path: /v2/metrics/cloud/export - params: - "resource.kafka.id": - - $CLUSTER_ID - # OPTIONAL - You can include monitoring for Confluent connectors or schema registry's by including the ID here. - # "resource.connector.id": - # - $CONNECTOR_ID - # "resource.schema_registry.id": - # - $SCHEMA_REGISTRY_ID - -processors: - batch: - -exporters: - otlphttp: - endpoint: $NEW_RELIC_OTLP_ENDPOINT - headers: - api-key: $NEW_RELIC_API_KEY - -service: - pipelines: - metrics: - receivers: [prometheus] - processors: [batch] - exporters: [otlphttp] \ No newline at end of file diff --git a/other-examples/collector/confluentcloud/docker-compose.yaml b/other-examples/collector/confluentcloud/docker-compose.yaml deleted file mode 100644 index 1770a486..00000000 --- a/other-examples/collector/confluentcloud/docker-compose.yaml +++ /dev/null @@ -1,17 +0,0 @@ -version: "3.6" - -services: - - otel-collector: - image: otel/opentelemetry-collector-contrib:0.92.0 - command: --config=/etc/otelcol/config.yaml - volumes: - - ./collector.yaml:/etc/otelcol/config.yaml - environment: - - NEW_RELIC_OTLP_ENDPOINT - - NEW_RELIC_API_KEY - - CONFLUENT_API_KEY - - CONFLUENT_API_SECRET - - CLUSTER_ID - - CONNECTOR_ID - - SCHEMA_REGISTRY_ID diff --git a/other-examples/collector/confluentcloud/k8s/collector.yaml b/other-examples/collector/confluentcloud/k8s/collector.yaml new file mode 100644 index 00000000..1ebe4a4e --- /dev/null +++ b/other-examples/collector/confluentcloud/k8s/collector.yaml @@ -0,0 +1,112 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: nr-confluent +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: collector-config + namespace: nr-confluent + labels: + app.kubernetes.io/name: collector-config +data: + collector-config: | + receivers: + prometheus: + config: + scrape_configs: + - job_name: "confluent" + scrape_interval: 60s # Do not go any lower than this or you'll hit rate limits + static_configs: + - targets: ["api.telemetry.confluent.cloud"] + scheme: https + basic_auth: + username: $CONFLUENT_API_KEY + password: $CONFLUENT_API_SECRET + metrics_path: /v2/metrics/cloud/export + params: + "resource.kafka.id": + - $CONFLUENT_CLUSTER_ID + # OPTIONAL - You can include monitoring for Confluent connectors or schema registry's by including the ID here. + #"resource.connector.id": + # - $CONFLUENT_CONNECTOR_ID + #"resource.schema_registry.id": + # - $CONFLUENT_SCHEMA_REGISTRY_ID + + processors: + batch: + + exporters: + otlphttp: + endpoint: ${NEW_RELIC_OTLP_ENDPOINT} + headers: + api-key: ${NEW_RELIC_API_KEY} + + service: + pipelines: + metrics: + receivers: [prometheus] + processors: [batch] + exporters: [otlphttp] +--- +apiVersion: v1 +kind: Pod +metadata: + name: collector + namespace: nr-confluent + labels: + app.kubernetes.io/name: collector +spec: + containers: + - name: collector + image: otel/opentelemetry-collector-contrib:0.98.0 + env: + # The default US endpoint is set here. You can change the endpoint and port based on your requirements if needed. + # docs: https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/best-practices/opentelemetry-otlp/#configure-endpoint-port-protocol + - name: NEW_RELIC_OTLP_ENDPOINT + value: https://otlp.nr-data.net/ + # The New Relic API key used to authenticate export requests. + # Defined in secrets.yaml + - name: NEW_RELIC_API_KEY + valueFrom: + secretKeyRef: + name: nr-confluent-secret + key: NEW_RELIC_API_KEY + # The Confluent API key. + # Defined in secrets.yaml + - name: CONFLUENT_API_KEY + valueFrom: + secretKeyRef: + name: nr-confluent-secret + key: CONFLUENT_API_KEY + # The Confluent API secret. + # Defined in secrets.yaml + - name: CONFLUENT_API_SECRET + valueFrom: + secretKeyRef: + name: nr-confluent-secret + key: CONFLUENT_API_SECRET + # Set your Confluent Cluster ID here. + # docs: https://docs.confluent.io/confluent-cli/current/command-reference/kafka/cluster/confluent_kafka_cluster_list.html + - name: CONFLUENT_CLUSTER_ID + value: + # OPTIONAL: Set your Confluent Schema Registry ID here, and uncomment reference in .receivers.prometheus.config.scrape_configs[0].params + # docs: https://docs.confluent.io/confluent-cli/current/command-reference/kafka/cluster/confluent_kafka_cluster_list.html + #- name: CONFLUENT_SCHEMA_REGISTRY_ID + # value: + # OPTIONAL: Set your Confluent Connector ID here, and uncomment reference in .receivers.prometheus.config.scrape_configs[0].params + # docs: https://docs.confluent.io/confluent-cli/current/command-reference/kafka/cluster/confluent_kafka_cluster_list.html + #- name: CONFLUENT_CONNECTOR_ID + # value: + volumeMounts: + - name: collector-config-vol + mountPath: /etc/otelcol-contrib + volumes: + - name: collector-config-vol + configMap: + name: collector-config + items: + - key: collector-config + path: config.yaml diff --git a/other-examples/collector/confluentcloud/k8s/secrets.yaml b/other-examples/collector/confluentcloud/k8s/secrets.yaml new file mode 100644 index 00000000..c2579d0c --- /dev/null +++ b/other-examples/collector/confluentcloud/k8s/secrets.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Secret +metadata: + name: nr-confluent-secret + namespace: nr-confluent +stringData: + # New Relic API key to authenticate the export requests. + # docs: https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#license-key + NEW_RELIC_API_KEY: + # Set your authentication keys for the Confluent Cloud metrics API. + # docs: https://docs.confluent.io/cloud/current/monitoring/metrics-api.html + CONFLUENT_API_KEY: + CONFLUENT_API_SECRET: