diff --git a/other-examples/confluentcloud/.gitignore b/other-examples/confluentcloud/.gitignore new file mode 100644 index 00000000..4f509e52 --- /dev/null +++ b/other-examples/confluentcloud/.gitignore @@ -0,0 +1 @@ +*.env \ No newline at end of file diff --git a/other-examples/confluentcloud/README.md b/other-examples/confluentcloud/README.md new file mode 100644 index 00000000..76bda751 --- /dev/null +++ b/other-examples/confluentcloud/README.md @@ -0,0 +1,40 @@ +# Confluent Cloud OTeL metrics example setup + +This example shows a setup for running a docker OTeL collector to scrape metrics from Confluent Cloud and post them the New Relic OTeL Collector Endpoint. + + + +To run the example, set the environment variables and run `docker compose up`: + +```shell +export NEW_RELIC_API_KEY= +export NEW_RELIC_OTLP_ENDPOINT=https://otlp.nr-data.net +export CONFLUENT_API_ID= +export CONFLUENT_API_SECRET= +export CLUSTER_ID= +export CLUSTER_API_KEY= +export CLUSTER_API_SECRET= +export CLUSTER_BOOTSTRAP_SERVER= + +docker compose up +``` +
+ +# Local Variable information + +| Variable | Description | Docs | +| -------- | ----------- | ---- | +| **NEW_RELIC_API_KEY** |New Relic Ingest API Key |[API Key docs](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/) | +| **NEW_RELIC_OTLP_ENDPOINT** | OTLP endpoint is https://otlp.nr-data.net | [OTLP endpoint config docs](https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/get-started/opentelemetry-set-up-your-app/#review-settings) | +| **CONFLUENT_API_ID** |Confluent Cloud API key| [Cloud api-key confluent docs](https://docs.confluent.io/cloud/current/monitoring/metrics-api.html#metrics-quick-start) | +| **CONFLUENT_API_SECRET** | Confluent Cloud API secret key | [Cloud api-key confluent docs](https://docs.confluent.io/cloud/current/monitoring/metrics-api.html#metrics-quick-start) | +| **CLUSTER_ID** | ID of cluster from Confluent Cloud | Available in your cluster settings | +| **CLUSTER_API_KEY** | Resource API key for your Confluent cluster |[Resource API key docs](https://docs.confluent.io/cloud/current/access-management/authenticate/api-keys/api-keys.html#create-a-resource-api-key) | +| **CLUSTER_API_SECRET**| Resource API secret key from your Confluent cluster| [Resource API key docs](https://docs.confluent.io/cloud/current/access-management/authenticate/api-keys/api-keys.html#create-a-resource-api-key) | +| **CLUSTER_BOOTSTRAP_SERVER** | Bootstrap Server for cluster | Available in your cluster settings | + +
+ +# Notes + + For more information on setting up this OTeL collector for Confluent Metrics, check out our [documentation](https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/collector/collector-configuration-examples/opentelemetry-collector-kafka-confluentcloud/). \ No newline at end of file diff --git a/other-examples/confluentcloud/collector.yaml b/other-examples/confluentcloud/collector.yaml new file mode 100644 index 00000000..c2e2a3cc --- /dev/null +++ b/other-examples/confluentcloud/collector.yaml @@ -0,0 +1,57 @@ +receivers: + kafkametrics: + brokers: + - $CLUSTER_BOOTSTRAP_SERVER + protocol_version: 2.0.0 + scrapers: + - brokers + - topics + - consumers + auth: + sasl: + username: $CLUSTER_API_KEY + password: $CLUSTER_API_SECRET + mechanism: PLAIN + tls: + insecure_skip_verify: false + collection_interval: 30s + + + prometheus: + config: + scrape_configs: + - job_name: "confluent" + scrape_interval: 60s # Do not go any lower than this or you'll hit rate limits + static_configs: + - targets: ["api.telemetry.confluent.cloud"] + scheme: https + basic_auth: + username: $CONFLUENT_API_ID + password: $CONFLUENT_API_SECRET + metrics_path: /v2/metrics/cloud/export + params: + "resource.kafka.id": + - $CLUSTER_ID +exporters: + otlp: + endpoint: $NEW_RELIC_OTLP_ENDPOINT + headers: + api-key: $NEW_RELIC_API_KEY +processors: + batch: + memory_limiter: + limit_mib: 400 + spike_limit_mib: 100 + check_interval: 5s +service: + telemetry: + logs: + pipelines: + metrics: + receivers: [prometheus] + processors: [batch] + exporters: [otlp] + metrics/kafka: + receivers: [kafkametrics] + processors: [batch] + exporters: [otlp] \ No newline at end of file diff --git a/other-examples/confluentcloud/docker-compose.yaml b/other-examples/confluentcloud/docker-compose.yaml new file mode 100644 index 00000000..cc3c6d8f --- /dev/null +++ b/other-examples/confluentcloud/docker-compose.yaml @@ -0,0 +1,19 @@ +version: "3.6" + +services: + + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + command: --config=/etc/otelcol/config.yaml + volumes: + - ./collector.yaml:/etc/otelcol/config.yaml + environment: + - NEW_RELIC_OTLP_ENDPOINT + - NEW_RELIC_API_KEY + - CLUSTER_BOOTSTRAP_SERVER + - CLUSTER_API_KEY + - CLUSTER_API_SECRET + - CONFLUENT_API_ID + - CONFLUENT_API_SECRET + - CLUSTER_ID +