From e9b5d8304b2735c47873aa982152c84cd068b21a Mon Sep 17 00:00:00 2001
From: connorhaugh <49422820+connorhaugh@users.noreply.github.com>
Date: Mon, 20 May 2024 09:08:32 -0400
Subject: [PATCH] feat: add dd-trace to edxapp workers (#6)

This PR adds dd-trace to the worker boxes so we can monitor them (and celery tasks). It does not require dd to be enabled.
---------

Co-authored-by: Tim McCormack <tmccormack@edx.org>
---
 .../templates/edx/app/edxapp/worker.sh.j2      | 18 +++++++++++++++---
 .../conf.d.available/workers.conf.j2           |  2 +-
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/playbooks/roles/edxapp/templates/edx/app/edxapp/worker.sh.j2 b/playbooks/roles/edxapp/templates/edx/app/edxapp/worker.sh.j2
index ccd903b8b5b..7f640f03fbe 100644
--- a/playbooks/roles/edxapp/templates/edx/app/edxapp/worker.sh.j2
+++ b/playbooks/roles/edxapp/templates/edx/app/edxapp/worker.sh.j2
@@ -4,8 +4,10 @@
 
 {% set edxapp_venv_bin = edxapp_venv_dir + "/bin" %}
 source {{ edxapp_app_dir }}/edxapp_env
+{% set executable = edxapp_venv_bin + '/celery' %}
+
 {% if COMMON_ENABLE_NEWRELIC_APP %}
-{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + edxapp_venv_bin + '/celery' %}
+{% set executable = edxapp_venv_bin + '/newrelic-admin run-program ' + executable %}
 
 export NEW_RELIC_CONFIG_FILE="{{ edxapp_app_dir }}/newrelic.ini"
 if command -v ec2metadata >/dev/null 2>&1; then
@@ -13,10 +15,20 @@ if command -v ec2metadata >/dev/null 2>&1; then
   HOSTNAME=$(hostname)
   export NEW_RELIC_PROCESS_HOST_DISPLAY_NAME="$HOSTNAME-$INSTANCEID"
 fi
-{% else %}
-{% set executable = edxapp_venv_bin + '/celery' %}
 {% endif %}
 
+{% if EDXAPP_DATADOG_ENABLE %}
+{% set executable = edxapp_venv_bin + '/ddtrace-run ' + executable %}
+export DD_TAGS="service:edx-edxapp-workers-{{ SERVICE_VARIANT }},queue:{{QUEUE_NAME}}"
+export DD_DJANGO_USE_LEGACY_RESOURCE_FORMAT=true
+# Copied from edx_django_service playbook for consistency; Datadog
+# trace debug logging issue doesn't actually affect edxapp for some
+# reason.
+export DD_TRACE_LOG_STREAM_HANDLER=false
+# Datadog's instrumentation breaks pymongo: https://github.com/edx/edx-arch-experiments/issues/580
+export DD_TRACE_PYMONGO_ENABLED=false
+{% endif -%}
+
 # We exec so that celery is the child of supervisor and can be managed properly
 
 exec {{ executable }} $@
diff --git a/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2 b/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2
index 3dd9641c9a7..44b8aa539d8 100644
--- a/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2
+++ b/playbooks/roles/edxapp/templates/edx/app/supervisor/conf.d.available/workers.conf.j2
@@ -1,7 +1,7 @@
 {% for w in edxapp_workers %}
 [program:{{ w.service_variant }}_{{ w.queue }}_{{ w.concurrency }}]
 
-environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ EDXAPP_NEWRELIC_WORKERS_APPNAME }}-{{ w.service_variant }},NEW_RELIC_DISTRIBUTED_TRACING_ENABLED={{ EDXAPP_WORKERS_ENABLE_NEWRELIC_DISTRIBUTED_TRACING }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}CONCURRENCY={{ w.concurrency }},LOGLEVEL=info,DJANGO_SETTINGS_MODULE={{ w.service_variant }}.envs.{{ worker_django_settings_module }},LANG={{ EDXAPP_LANG }},PYTHONPATH={{ edxapp_code_dir }},SERVICE_VARIANT={{ w.service_variant }},BOTO_CONFIG="{{ edxapp_app_dir }}/.boto",EDX_REST_API_CLIENT_NAME=edx.{{ w.service_variant }}.core.{{ w.queue }}
+environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ EDXAPP_NEWRELIC_WORKERS_APPNAME }}-{{ w.service_variant }},NEW_RELIC_DISTRIBUTED_TRACING_ENABLED={{ EDXAPP_WORKERS_ENABLE_NEWRELIC_DISTRIBUTED_TRACING }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}CONCURRENCY={{ w.concurrency }},LOGLEVEL=info,DJANGO_SETTINGS_MODULE={{ w.service_variant }}.envs.{{ worker_django_settings_module }},LANG={{ EDXAPP_LANG }},PYTHONPATH={{ edxapp_code_dir }},SERVICE_VARIANT={{ w.service_variant }},BOTO_CONFIG="{{ edxapp_app_dir }}/.boto",EDX_REST_API_CLIENT_NAME=edx.{{ w.service_variant }}.core.{{ w.queue }},QUEUE_NAME={{ w.service_variant }}.core.{{ w.queue }}
 user={{ common_web_user }}
 directory={{ edxapp_code_dir }}
 stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log