From 3a64656524f8e69aaa0e23e77aca9e9cf19be82b Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Thu, 8 Aug 2024 12:07:13 +0500 Subject: [PATCH 1/2] feat: Add Datadog-Git Repository Source Code Integration --- playbooks/roles/analytics_api/defaults/main.yml | 2 ++ playbooks/roles/analytics_api/meta/main.yml | 1 + playbooks/roles/ecommerce/defaults/main.yml | 1 + playbooks/roles/ecommerce/meta/main.yml | 1 + playbooks/roles/edx_django_service/defaults/main.yml | 1 + .../roles/edx_django_service/templates/edx/app/app/app.sh.j2 | 2 ++ playbooks/roles/xqueue/defaults/main.yml | 2 ++ playbooks/roles/xqueue/templates/xqueue.conf.j2 | 2 +- 8 files changed, 11 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/analytics_api/defaults/main.yml b/playbooks/roles/analytics_api/defaults/main.yml index 57c9e64a4c1..bb72c8820e3 100644 --- a/playbooks/roles/analytics_api/defaults/main.yml +++ b/playbooks/roles/analytics_api/defaults/main.yml @@ -48,6 +48,8 @@ ANALYTICS_API_VERSION: "master" ANALYTICS_API_NGINX_PORT: '1{{ analytics_api_gunicorn_port }}' ANALYTICS_API_SSL_NGINX_PORT: '4{{ analytics_api_gunicorn_port }}' +ANALYTICS_API_GIT_URL: "https://github.com/edx/edx-analytics-data-api.git" + ANALYTICS_API_REPOS: - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}" DOMAIN: "{{ COMMON_GIT_MIRROR }}" diff --git a/playbooks/roles/analytics_api/meta/main.yml b/playbooks/roles/analytics_api/meta/main.yml index 04632efb722..f477e495808 100644 --- a/playbooks/roles/analytics_api/meta/main.yml +++ b/playbooks/roles/analytics_api/meta/main.yml @@ -52,3 +52,4 @@ dependencies: edx_django_service_decrypt_config_enabled: '{{ ANALYTICS_API_DECRYPT_CONFIG_ENABLED }}' edx_django_service_copy_config_enabled: '{{ ANALYTICS_API_COPY_CONFIG_ENABLED }}' edx_django_datadog_service: '{{ analytics_api_datadog_service_name }}' + edx_django_service_git_url: '{{ ANALYTICS_API_GIT_URL }}' diff --git a/playbooks/roles/ecommerce/defaults/main.yml b/playbooks/roles/ecommerce/defaults/main.yml index edf7b862c96..28381a2c4b6 100644 --- a/playbooks/roles/ecommerce/defaults/main.yml +++ b/playbooks/roles/ecommerce/defaults/main.yml @@ -13,6 +13,7 @@ ECOMMERCE_GIT_IDENTITY: !!null +ECOMMERCE_GIT_URL: "https://github.com/edx/ecommerce.git" ECOMMERCE_REPOS: - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}" DOMAIN: "{{ COMMON_GIT_MIRROR }}" diff --git a/playbooks/roles/ecommerce/meta/main.yml b/playbooks/roles/ecommerce/meta/main.yml index 46f74f57eb0..64cdb37977a 100644 --- a/playbooks/roles/ecommerce/meta/main.yml +++ b/playbooks/roles/ecommerce/meta/main.yml @@ -69,6 +69,7 @@ dependencies: edx_django_service_copy_config_enabled: '{{ ECOMMERCE_COPY_CONFIG_ENABLED }}' EDX_DJANGO_SERVICE_ENABLE_ADMIN_URLS_RESTRICTION: '{{ ECOMMERCE_ENABLE_ADMIN_URLS_RESTRICTION }}' EDX_DJANGO_SERVICE_ADMIN_URLS: '{{ ECOMMERCE_ADMIN_URLS }}' + edx_django_service_git_url: '{{ ECOMMERCE_GIT_URL }}' - role: antivirus ANTIVIRUS_SCAN_DIRECTORY: "{{ ECOMMERCE_ANTIVIRUS_SCAN_DIRECTORY }}" when: ECOMMERCE_ENABLE_ANTIVIRUS diff --git a/playbooks/roles/edx_django_service/defaults/main.yml b/playbooks/roles/edx_django_service/defaults/main.yml index 1ce2519c1a3..fedcfabaece 100644 --- a/playbooks/roles/edx_django_service/defaults/main.yml +++ b/playbooks/roles/edx_django_service/defaults/main.yml @@ -17,6 +17,7 @@ edx_django_service_name_devstack_logs: - '{{ supervisor_log_dir }}/{{ edx_django_service_name }}-stdout.log' - '{{ supervisor_log_dir }}/{{ edx_django_service_name }}-stderr.log' +edx_django_service_git_url: '' edx_django_service_git_protocol: '{{ COMMON_GIT_PROTOCOL }}' edx_django_service_git_domain: '{{ COMMON_GIT_MIRROR }}' edx_django_service_git_path: '{{ COMMON_GIT_PATH }}' diff --git a/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 b/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 index fd10b96f03b..156fa86651d 100644 --- a/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 +++ b/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 @@ -34,6 +34,8 @@ export DD_DJANGO_USE_HANDLER_RESOURCE_FORMAT=true # other trace-related configs that are documented seem to help, but # from testing DD_TRACE_LOG_STREAM_HANDLER=false seems to help. export DD_TRACE_LOG_STREAM_HANDLER=false +export DD_GIT_COMMIT_SHA="{{ app_version }}" +export DD_GIT_REPOSITORY_URL="{{ edx_django_service_git_url }}" {% endif -%} export EDX_REST_API_CLIENT_NAME="{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ edx_django_service_name }}" diff --git a/playbooks/roles/xqueue/defaults/main.yml b/playbooks/roles/xqueue/defaults/main.yml index 510d69f28f8..d95fbaa266b 100644 --- a/playbooks/roles/xqueue/defaults/main.yml +++ b/playbooks/roles/xqueue/defaults/main.yml @@ -131,6 +131,8 @@ XQUEUE_CONFIG: XQUEUE_VERSION: "master" XQUEUE_GIT_IDENTITY: !!null +XQUEUE_GIT_REPOSITORY_URL: "https://github.com/edx/xqueue.git" + XQUEUE_REPOS: - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}" DOMAIN: "{{ COMMON_GIT_MIRROR }}" diff --git a/playbooks/roles/xqueue/templates/xqueue.conf.j2 b/playbooks/roles/xqueue/templates/xqueue.conf.j2 index 6ccf03114d3..eadd3b2052b 100644 --- a/playbooks/roles/xqueue/templates/xqueue.conf.j2 +++ b/playbooks/roles/xqueue/templates/xqueue.conf.j2 @@ -17,7 +17,7 @@ directory={{ xqueue_code_dir }} # Copied DD_TRACE_LOG_STREAM_HANDLER config from edx_django_service. This is required # to disable Datadog trace debug logging. -environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ XQUEUE_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}DD_DJANGO_USE_HANDLER_RESOURCE_FORMAT=true,DD_TAGS="service:{{ XQUEUE_DATADOG_APPNAME }} version:{{ app_version }}",DD_TRACE_LOG_STREAM_HANDLER=false,DD_PROFILING_ENABLED=true,DD_DATA_STREAMS_ENABLED=true,{% endif -%}PID=/var/tmp/xqueue.pid,PORT={{ xqueue_gunicorn_port }},ADDRESS={{ xqueue_gunicorn_host }},LANG={{ XQUEUE_LANG }},DJANGO_SETTINGS_MODULE=xqueue.{{ XQUEUE_SETTINGS }},XQUEUE_CFG={{ COMMON_CFG_DIR }}/xqueue.yml +environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ XQUEUE_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}DD_DJANGO_USE_HANDLER_RESOURCE_FORMAT=true,DD_TAGS="service:{{ XQUEUE_DATADOG_APPNAME }} version:{{ app_version }}",DD_TRACE_LOG_STREAM_HANDLER=false,DD_PROFILING_ENABLED=true,DD_DATA_STREAMS_ENABLED=true,DD_GIT_COMMIT_SHA={{ app_version }},DD_GIT_REPOSITORY_URL={{ XQUEUE_REPOSITORY_URL }},{% endif -%}PID=/var/tmp/xqueue.pid,PORT={{ xqueue_gunicorn_port }},ADDRESS={{ xqueue_gunicorn_host }},LANG={{ XQUEUE_LANG }},DJANGO_SETTINGS_MODULE=xqueue.{{ XQUEUE_SETTINGS }},XQUEUE_CFG={{ COMMON_CFG_DIR }}/xqueue.yml stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log From cc9f9c85d3f3b4f486cbdd90e20f8f940ba2d4c3 Mon Sep 17 00:00:00 2001 From: nadeemshahzad Date: Thu, 8 Aug 2024 12:53:03 +0500 Subject: [PATCH 2/2] feat: add dd source code integration --- playbooks/roles/analytics_api/defaults/main.yml | 3 +-- playbooks/roles/analytics_api/meta/main.yml | 2 +- playbooks/roles/ecommerce/defaults/main.yml | 2 +- playbooks/roles/ecommerce/meta/main.yml | 2 +- playbooks/roles/ecomworker/defaults/main.yml | 1 + .../ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 | 2 ++ playbooks/roles/edx_django_service/defaults/main.yml | 2 +- .../roles/edx_django_service/templates/edx/app/app/app.sh.j2 | 2 +- playbooks/roles/insights/defaults/main.yml | 1 + .../roles/insights/templates/edx/app/insights/insights.sh.j2 | 2 ++ playbooks/roles/xqueue/defaults/main.yml | 2 -- playbooks/roles/xqueue/templates/xqueue.conf.j2 | 2 +- 12 files changed, 13 insertions(+), 10 deletions(-) diff --git a/playbooks/roles/analytics_api/defaults/main.yml b/playbooks/roles/analytics_api/defaults/main.yml index bb72c8820e3..5b0cd75b6c6 100644 --- a/playbooks/roles/analytics_api/defaults/main.yml +++ b/playbooks/roles/analytics_api/defaults/main.yml @@ -48,8 +48,7 @@ ANALYTICS_API_VERSION: "master" ANALYTICS_API_NGINX_PORT: '1{{ analytics_api_gunicorn_port }}' ANALYTICS_API_SSL_NGINX_PORT: '4{{ analytics_api_gunicorn_port }}' -ANALYTICS_API_GIT_URL: "https://github.com/edx/edx-analytics-data-api.git" - +ANALYTICS_API_REPOSITORY_GIT_URL: "https://github.com/edx/edx-analytics-data-api.git" ANALYTICS_API_REPOS: - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}" DOMAIN: "{{ COMMON_GIT_MIRROR }}" diff --git a/playbooks/roles/analytics_api/meta/main.yml b/playbooks/roles/analytics_api/meta/main.yml index f477e495808..de38f704023 100644 --- a/playbooks/roles/analytics_api/meta/main.yml +++ b/playbooks/roles/analytics_api/meta/main.yml @@ -52,4 +52,4 @@ dependencies: edx_django_service_decrypt_config_enabled: '{{ ANALYTICS_API_DECRYPT_CONFIG_ENABLED }}' edx_django_service_copy_config_enabled: '{{ ANALYTICS_API_COPY_CONFIG_ENABLED }}' edx_django_datadog_service: '{{ analytics_api_datadog_service_name }}' - edx_django_service_git_url: '{{ ANALYTICS_API_GIT_URL }}' + edx_django_service_repository_git_url: '{{ ANALYTICS_API_REPOSITORY_GIT_URL }}' diff --git a/playbooks/roles/ecommerce/defaults/main.yml b/playbooks/roles/ecommerce/defaults/main.yml index 28381a2c4b6..293bfecd07c 100644 --- a/playbooks/roles/ecommerce/defaults/main.yml +++ b/playbooks/roles/ecommerce/defaults/main.yml @@ -13,7 +13,7 @@ ECOMMERCE_GIT_IDENTITY: !!null -ECOMMERCE_GIT_URL: "https://github.com/edx/ecommerce.git" +ECOMMERCE_REPOSITORY_GIT_URL: "https://github.com/edx/ecommerce.git" ECOMMERCE_REPOS: - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}" DOMAIN: "{{ COMMON_GIT_MIRROR }}" diff --git a/playbooks/roles/ecommerce/meta/main.yml b/playbooks/roles/ecommerce/meta/main.yml index 64cdb37977a..705cf91b135 100644 --- a/playbooks/roles/ecommerce/meta/main.yml +++ b/playbooks/roles/ecommerce/meta/main.yml @@ -69,7 +69,7 @@ dependencies: edx_django_service_copy_config_enabled: '{{ ECOMMERCE_COPY_CONFIG_ENABLED }}' EDX_DJANGO_SERVICE_ENABLE_ADMIN_URLS_RESTRICTION: '{{ ECOMMERCE_ENABLE_ADMIN_URLS_RESTRICTION }}' EDX_DJANGO_SERVICE_ADMIN_URLS: '{{ ECOMMERCE_ADMIN_URLS }}' - edx_django_service_git_url: '{{ ECOMMERCE_GIT_URL }}' + edx_django_service_repository_git_url: '{{ ECOMMERCE_REPOSITORY_GIT_URL }}' - role: antivirus ANTIVIRUS_SCAN_DIRECTORY: "{{ ECOMMERCE_ANTIVIRUS_SCAN_DIRECTORY }}" when: ECOMMERCE_ENABLE_ANTIVIRUS diff --git a/playbooks/roles/ecomworker/defaults/main.yml b/playbooks/roles/ecomworker/defaults/main.yml index 88ca4071e5b..1e2a9ff1ff1 100644 --- a/playbooks/roles/ecomworker/defaults/main.yml +++ b/playbooks/roles/ecomworker/defaults/main.yml @@ -14,6 +14,7 @@ ECOMMERCE_WORKER_GIT_IDENTITY: !!null ECOMMERCE_WORKER_VERSION: '2u/main' ECOMMERCE_WORKER_USE_PYTHON38: true +ECOMMERCE_WORKER_REPOSITORY_GIT_URL: "https://github.com/edx/ecommerce-worker.git" ECOMMERCE_WORKER_REPOS: - PROTOCOL: '{{ COMMON_GIT_PROTOCOL }}' DOMAIN: '{{ COMMON_GIT_MIRROR }}' diff --git a/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 b/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 index bfeea9c5496..0d76d20d3af 100644 --- a/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 +++ b/playbooks/roles/ecomworker/templates/edx/app/ecomworker/ecomworker.sh.j2 @@ -24,6 +24,8 @@ export DD_DJANGO_USE_HANDLER_RESOURCE_FORMAT=true export DD_TRACE_LOG_STREAM_HANDLER=false export DD_PROFILING_ENABLED=true export DD_DATA_STREAMS_ENABLED=true +export DD_GIT_COMMIT_SHA="{{ app_version }}" +export DD_GIT_REPOSITORY_URL="{{ ECOMMERCE_WORKER_REPOSITORY_GIT_URL }}" {% endif -%} source {{ ecommerce_worker_home }}/{{ ecommerce_worker_service_name }}_env diff --git a/playbooks/roles/edx_django_service/defaults/main.yml b/playbooks/roles/edx_django_service/defaults/main.yml index fedcfabaece..e359616b3b1 100644 --- a/playbooks/roles/edx_django_service/defaults/main.yml +++ b/playbooks/roles/edx_django_service/defaults/main.yml @@ -17,7 +17,7 @@ edx_django_service_name_devstack_logs: - '{{ supervisor_log_dir }}/{{ edx_django_service_name }}-stdout.log' - '{{ supervisor_log_dir }}/{{ edx_django_service_name }}-stderr.log' -edx_django_service_git_url: '' +edx_django_service_repository_git_url: '' edx_django_service_git_protocol: '{{ COMMON_GIT_PROTOCOL }}' edx_django_service_git_domain: '{{ COMMON_GIT_MIRROR }}' edx_django_service_git_path: '{{ COMMON_GIT_PATH }}' diff --git a/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 b/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 index 156fa86651d..028c1897185 100644 --- a/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 +++ b/playbooks/roles/edx_django_service/templates/edx/app/app/app.sh.j2 @@ -35,7 +35,7 @@ export DD_DJANGO_USE_HANDLER_RESOURCE_FORMAT=true # from testing DD_TRACE_LOG_STREAM_HANDLER=false seems to help. export DD_TRACE_LOG_STREAM_HANDLER=false export DD_GIT_COMMIT_SHA="{{ app_version }}" -export DD_GIT_REPOSITORY_URL="{{ edx_django_service_git_url }}" +export DD_GIT_REPOSITORY_URL="{{ edx_django_service_repository_git_url }}" {% endif -%} export EDX_REST_API_CLIENT_NAME="{{ COMMON_ENVIRONMENT }}-{{ COMMON_DEPLOYMENT }}-{{ edx_django_service_name }}" diff --git a/playbooks/roles/insights/defaults/main.yml b/playbooks/roles/insights/defaults/main.yml index 55fc867481b..cac61e99316 100644 --- a/playbooks/roles/insights/defaults/main.yml +++ b/playbooks/roles/insights/defaults/main.yml @@ -179,6 +179,7 @@ INSIGHTS_MODULE_PREVIEW_URL: "{{ INSIGHTS_LMS_BASE }}/xblock" INSIGHTS_VERSION: "master" INSIGHTS_GIT_IDENTITY: !!null +INSIGHTS_REPOSITORY_GIT_URL: "https://github.com/edx/edx-analytics-dashboard.git" INSIGHTS_REPOS: - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}" DOMAIN: "{{ COMMON_GIT_MIRROR }}" diff --git a/playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2 b/playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2 index 0acd3ee973c..2de6bfb6b8f 100644 --- a/playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2 +++ b/playbooks/roles/insights/templates/edx/app/insights/insights.sh.j2 @@ -22,6 +22,8 @@ export DD_DJANGO_USE_HANDLER_RESOURCE_FORMAT=true # trace debug logging issue doesn't actually affect edxapp for some # reason. export DD_TRACE_LOG_STREAM_HANDLER=false +export DD_GIT_COMMIT_SHA="{{ app_version }}" +export DD_GIT_REPOSITORY_URL="{{ INSIGHTS_REPOSITORY_GIT_URL }}" # TODO: There seems to be some potential issues enabling APM profiling in both NewRelic and Datadog. # A safe bet would be to enable these after NewRelic profiling is disabled. diff --git a/playbooks/roles/xqueue/defaults/main.yml b/playbooks/roles/xqueue/defaults/main.yml index d95fbaa266b..510d69f28f8 100644 --- a/playbooks/roles/xqueue/defaults/main.yml +++ b/playbooks/roles/xqueue/defaults/main.yml @@ -131,8 +131,6 @@ XQUEUE_CONFIG: XQUEUE_VERSION: "master" XQUEUE_GIT_IDENTITY: !!null -XQUEUE_GIT_REPOSITORY_URL: "https://github.com/edx/xqueue.git" - XQUEUE_REPOS: - PROTOCOL: "{{ COMMON_GIT_PROTOCOL }}" DOMAIN: "{{ COMMON_GIT_MIRROR }}" diff --git a/playbooks/roles/xqueue/templates/xqueue.conf.j2 b/playbooks/roles/xqueue/templates/xqueue.conf.j2 index eadd3b2052b..6ccf03114d3 100644 --- a/playbooks/roles/xqueue/templates/xqueue.conf.j2 +++ b/playbooks/roles/xqueue/templates/xqueue.conf.j2 @@ -17,7 +17,7 @@ directory={{ xqueue_code_dir }} # Copied DD_TRACE_LOG_STREAM_HANDLER config from edx_django_service. This is required # to disable Datadog trace debug logging. -environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ XQUEUE_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}DD_DJANGO_USE_HANDLER_RESOURCE_FORMAT=true,DD_TAGS="service:{{ XQUEUE_DATADOG_APPNAME }} version:{{ app_version }}",DD_TRACE_LOG_STREAM_HANDLER=false,DD_PROFILING_ENABLED=true,DD_DATA_STREAMS_ENABLED=true,DD_GIT_COMMIT_SHA={{ app_version }},DD_GIT_REPOSITORY_URL={{ XQUEUE_REPOSITORY_URL }},{% endif -%}PID=/var/tmp/xqueue.pid,PORT={{ xqueue_gunicorn_port }},ADDRESS={{ xqueue_gunicorn_host }},LANG={{ XQUEUE_LANG }},DJANGO_SETTINGS_MODULE=xqueue.{{ XQUEUE_SETTINGS }},XQUEUE_CFG={{ COMMON_CFG_DIR }}/xqueue.yml +environment={% if COMMON_ENABLE_NEWRELIC_APP %}NEW_RELIC_APP_NAME={{ XQUEUE_NEWRELIC_APPNAME }},NEW_RELIC_LICENSE_KEY={{ NEWRELIC_LICENSE_KEY }},{% endif -%}{% if COMMON_ENABLE_DATADOG and COMMON_ENABLE_DATADOG_APP %}DD_DJANGO_USE_HANDLER_RESOURCE_FORMAT=true,DD_TAGS="service:{{ XQUEUE_DATADOG_APPNAME }} version:{{ app_version }}",DD_TRACE_LOG_STREAM_HANDLER=false,DD_PROFILING_ENABLED=true,DD_DATA_STREAMS_ENABLED=true,{% endif -%}PID=/var/tmp/xqueue.pid,PORT={{ xqueue_gunicorn_port }},ADDRESS={{ xqueue_gunicorn_host }},LANG={{ XQUEUE_LANG }},DJANGO_SETTINGS_MODULE=xqueue.{{ XQUEUE_SETTINGS }},XQUEUE_CFG={{ COMMON_CFG_DIR }}/xqueue.yml stdout_logfile={{ supervisor_log_dir }}/%(program_name)s-stdout.log stderr_logfile={{ supervisor_log_dir }}/%(program_name)s-stderr.log