From 1345db57f98c757d860940314000b1bd5c135dda Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Fri, 9 Aug 2024 13:58:30 +0000 Subject: [PATCH 1/3] feat: Log all span-starts for the first minute (#762) Datadog Support has asked us to log all span-starts. I'm hoping that they can get what they need if we just log the first minute of spans, since the problem seems to be related to some of the first requests after deploy or restart. Excluding function-docstring linting for the span processor, since those are essentially overrides. (pylint only started caring once one of the methods was more than 4 lines long.) --- CHANGELOG.rst | 6 ++++ edx_arch_experiments/__init__.py | 2 +- .../datadog_diagnostics/apps.py | 20 +++++++++++++ .../datadog_diagnostics/tests/test_app.py | 29 +++++++++++++++++-- 4 files changed, 54 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6e5207d..c53c86c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,12 @@ Change Log Unreleased ~~~~~~~~~~ +[4.1.0] - 2024-08-09 +~~~~~~~~~~~~~~~~~~~~ +Changed +------- +* Datadog diagnostics will now log all span-starts for the first minute after server startup + [4.0.0] - 2024-08-05 ~~~~~~~~~~~~~~~~~~~~ Changed diff --git a/edx_arch_experiments/__init__.py b/edx_arch_experiments/__init__.py index a00a1e5..bb3e7ee 100644 --- a/edx_arch_experiments/__init__.py +++ b/edx_arch_experiments/__init__.py @@ -2,4 +2,4 @@ A plugin to include applications under development by the architecture team at 2U. """ -__version__ = '4.0.0' +__version__ = '4.1.0' diff --git a/edx_arch_experiments/datadog_diagnostics/apps.py b/edx_arch_experiments/datadog_diagnostics/apps.py index de29169..3ca9401 100644 --- a/edx_arch_experiments/datadog_diagnostics/apps.py +++ b/edx_arch_experiments/datadog_diagnostics/apps.py @@ -3,6 +3,7 @@ """ import logging +import time from django.apps import AppConfig from django.conf import settings @@ -26,7 +27,18 @@ # avoids logging more data than is actually needed for diagnosis. DATADOG_DIAGNOSTICS_MAX_SPANS = getattr(settings, 'DATADOG_DIAGNOSTICS_MAX_SPANS', 100) +# .. setting_name: DATADOG_DIAGNOSTICS_LOG_ALL_SPAN_STARTS_PERIOD +# .. setting_default: 60 +# .. setting_description: Log all span starts for this many seconds after worker +# startup. +DATADOG_DIAGNOSTICS_LOG_ALL_SPAN_STARTS_PERIOD = getattr( + settings, + 'DATADOG_DIAGNOSTICS_LOG_ALL_SPAN_STARTS_PERIOD', + 60 +) + +# pylint: disable=missing-function-docstring class MissingSpanProcessor: """Datadog span processor that logs unfinished spans at shutdown.""" @@ -34,12 +46,20 @@ def __init__(self): self.spans_started = 0 self.spans_finished = 0 self.open_spans = {} + self.log_all_until = time.time() + DATADOG_DIAGNOSTICS_LOG_ALL_SPAN_STARTS_PERIOD def on_span_start(self, span): self.spans_started += 1 if len(self.open_spans) < DATADOG_DIAGNOSTICS_MAX_SPANS: self.open_spans[span.span_id] = span + # We believe that the anomalous traces always come from a + # single span that is created early in the lifetime of a + # gunicorn worker. If we log *every* span-start in this early + # period, we may be able to observe something interesting. + if time.time() <= self.log_all_until: + log.info(f"Early span-start sample: {span._pprint()}") # pylint: disable=protected-access + def on_span_finish(self, span): self.spans_finished += 1 self.open_spans.pop(span.span_id, None) # "delete if present" diff --git a/edx_arch_experiments/datadog_diagnostics/tests/test_app.py b/edx_arch_experiments/datadog_diagnostics/tests/test_app.py index b0ce611..b927499 100644 --- a/edx_arch_experiments/datadog_diagnostics/tests/test_app.py +++ b/edx_arch_experiments/datadog_diagnostics/tests/test_app.py @@ -2,7 +2,7 @@ Tests for plugin app. """ -from unittest.mock import patch +from unittest.mock import call, patch from django.test import TestCase @@ -47,5 +47,30 @@ def test_logging(self, mock_log_error, mock_log_info): proc.on_span_start(FakeSpan(17)) proc.shutdown(0) - mock_log_info.assert_called_once_with("Spans created = 1; spans finished = 0") + assert mock_log_info.call_args_list == [ + call("Early span-start sample: span_id=17"), + call("Spans created = 1; spans finished = 0"), + ] mock_log_error.assert_called_once_with("Span created but not finished: span_id=17") + + @patch('edx_arch_experiments.datadog_diagnostics.apps.log.info') + def test_early_span_logging_cutoff(self, mock_log_info): + with patch('edx_arch_experiments.datadog_diagnostics.apps.time.time', side_effect=[ + # Setup + 1700000000, + # Span-start time checks + 1700000020, + 1700000040, + 1700010000, + ]): + proc = apps.MissingSpanProcessor() + # Three spans are started + proc.on_span_start(FakeSpan(44)) + proc.on_span_start(FakeSpan(45)) + proc.on_span_start(FakeSpan(46)) + + # Just two early span-starts are logged + assert mock_log_info.call_args_list == [ + call("Early span-start sample: span_id=44"), + call("Early span-start sample: span_id=45"), + ] From 6be3a73163cd93650243d93779b3921cdf8c8254 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Tue, 13 Aug 2024 15:40:01 +0000 Subject: [PATCH 2/3] fix: Remove early span-start logging; fix loading of Django settings (#772) I still don't understand why this setting loading was broken -- maybe there's something unusual about the way plugin apps are loaded? This addresses https://github.com/edx/edx-arch-experiments/issues/771 --- CHANGELOG.rst | 11 ++++ edx_arch_experiments/__init__.py | 2 +- .../datadog_diagnostics/apps.py | 54 +++++----------- .../datadog_diagnostics/tests/test_app.py | 62 ++++++++++--------- 4 files changed, 62 insertions(+), 67 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c53c86c..12f0cf5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,11 +14,22 @@ Change Log Unreleased ~~~~~~~~~~ +[4.2.0] - 2024-08-13 +~~~~~~~~~~~~~~~~~~~~ +Fixed +----- +* Fixed loading of ``DATADOG_DIAGNOSTICS_ENABLE``, which was previously not loaded properly and therefore was always True. Also fixed loading of ``DATADOG_DIAGNOSTICS_MAX_SPANS``, which was presumably broken as well. + +Removed +------- +* Removed early span-start logging. It never worked properly, possibly because workers are continually being destroyed and created, leading to high log volume. + [4.1.0] - 2024-08-09 ~~~~~~~~~~~~~~~~~~~~ Changed ------- * Datadog diagnostics will now log all span-starts for the first minute after server startup +* **WARNING**: Do not use this version; see 4.2.0 release notes. [4.0.0] - 2024-08-05 ~~~~~~~~~~~~~~~~~~~~ diff --git a/edx_arch_experiments/__init__.py b/edx_arch_experiments/__init__.py index bb3e7ee..6192ccb 100644 --- a/edx_arch_experiments/__init__.py +++ b/edx_arch_experiments/__init__.py @@ -2,4 +2,4 @@ A plugin to include applications under development by the architecture team at 2U. """ -__version__ = '4.1.0' +__version__ = '4.2.0' diff --git a/edx_arch_experiments/datadog_diagnostics/apps.py b/edx_arch_experiments/datadog_diagnostics/apps.py index 3ca9401..e4a22b8 100644 --- a/edx_arch_experiments/datadog_diagnostics/apps.py +++ b/edx_arch_experiments/datadog_diagnostics/apps.py @@ -3,7 +3,6 @@ """ import logging -import time from django.apps import AppConfig from django.conf import settings @@ -11,33 +10,6 @@ log = logging.getLogger(__name__) -# .. toggle_name: DATADOG_DIAGNOSTICS_ENABLE -# .. toggle_implementation: DjangoSetting -# .. toggle_default: True -# .. toggle_description: Enables logging of Datadog diagnostics information. -# .. toggle_use_cases: circuit_breaker -# .. toggle_creation_date: 2024-07-11 -# .. toggle_tickets: https://github.com/edx/edx-arch-experiments/issues/692 -DATADOG_DIAGNOSTICS_ENABLE = getattr(settings, 'DATADOG_DIAGNOSTICS_ENABLE', True) - -# .. setting_name: DATADOG_DIAGNOSTICS_MAX_SPANS -# .. setting_default: 100 -# .. setting_description: Limit of how many spans to hold onto and log -# when diagnosing Datadog tracing issues. This limits memory consumption -# avoids logging more data than is actually needed for diagnosis. -DATADOG_DIAGNOSTICS_MAX_SPANS = getattr(settings, 'DATADOG_DIAGNOSTICS_MAX_SPANS', 100) - -# .. setting_name: DATADOG_DIAGNOSTICS_LOG_ALL_SPAN_STARTS_PERIOD -# .. setting_default: 60 -# .. setting_description: Log all span starts for this many seconds after worker -# startup. -DATADOG_DIAGNOSTICS_LOG_ALL_SPAN_STARTS_PERIOD = getattr( - settings, - 'DATADOG_DIAGNOSTICS_LOG_ALL_SPAN_STARTS_PERIOD', - 60 -) - - # pylint: disable=missing-function-docstring class MissingSpanProcessor: """Datadog span processor that logs unfinished spans at shutdown.""" @@ -46,20 +18,19 @@ def __init__(self): self.spans_started = 0 self.spans_finished = 0 self.open_spans = {} - self.log_all_until = time.time() + DATADOG_DIAGNOSTICS_LOG_ALL_SPAN_STARTS_PERIOD + + # .. setting_name: DATADOG_DIAGNOSTICS_MAX_SPANS + # .. setting_default: 100 + # .. setting_description: Limit of how many spans to hold onto and log + # when diagnosing Datadog tracing issues. This limits memory consumption + # avoids logging more data than is actually needed for diagnosis. + self.DATADOG_DIAGNOSTICS_MAX_SPANS = getattr(settings, 'DATADOG_DIAGNOSTICS_MAX_SPANS', 100) def on_span_start(self, span): self.spans_started += 1 - if len(self.open_spans) < DATADOG_DIAGNOSTICS_MAX_SPANS: + if len(self.open_spans) < self.DATADOG_DIAGNOSTICS_MAX_SPANS: self.open_spans[span.span_id] = span - # We believe that the anomalous traces always come from a - # single span that is created early in the lifetime of a - # gunicorn worker. If we log *every* span-start in this early - # period, we may be able to observe something interesting. - if time.time() <= self.log_all_until: - log.info(f"Early span-start sample: {span._pprint()}") # pylint: disable=protected-access - def on_span_finish(self, span): self.spans_finished += 1 self.open_spans.pop(span.span_id, None) # "delete if present" @@ -80,6 +51,15 @@ class DatadogDiagnostics(AppConfig): plugin_app = {} def ready(self): + # .. toggle_name: DATADOG_DIAGNOSTICS_ENABLE + # .. toggle_implementation: DjangoSetting + # .. toggle_default: True + # .. toggle_description: Enables logging of Datadog diagnostics information. + # .. toggle_use_cases: circuit_breaker + # .. toggle_creation_date: 2024-07-11 + # .. toggle_tickets: https://github.com/edx/edx-arch-experiments/issues/692 + DATADOG_DIAGNOSTICS_ENABLE = getattr(settings, 'DATADOG_DIAGNOSTICS_ENABLE', True) + if not DATADOG_DIAGNOSTICS_ENABLE: return diff --git a/edx_arch_experiments/datadog_diagnostics/tests/test_app.py b/edx_arch_experiments/datadog_diagnostics/tests/test_app.py index b927499..ba89b95 100644 --- a/edx_arch_experiments/datadog_diagnostics/tests/test_app.py +++ b/edx_arch_experiments/datadog_diagnostics/tests/test_app.py @@ -2,9 +2,10 @@ Tests for plugin app. """ -from unittest.mock import call, patch +from unittest.mock import patch -from django.test import TestCase +from ddtrace import tracer +from django.test import TestCase, override_settings from .. import apps @@ -21,7 +22,35 @@ def _pprint(self): class TestMissingSpanProcessor(TestCase): """Tests for MissingSpanProcessor.""" - @patch.object(apps, 'DATADOG_DIAGNOSTICS_MAX_SPANS', new=3) + def test_feature_switch(self): + """ + Regression test -- the use of override_settings ensures that we read + the setting as needed, and not once at module load time (when it's + not guaranteed to be available.) + """ + def initialize(): + apps.DatadogDiagnostics('edx_arch_experiments.datadog_diagnostics', apps).ready() + + def get_processor_list(): + # pylint: disable=protected-access + return [type(sp).__name__ for sp in tracer._span_processors] + + with override_settings(DATADOG_DIAGNOSTICS_ENABLE=False): + initialize() + assert sorted(get_processor_list()) == [ + 'EndpointCallCounterProcessor', 'TopLevelSpanProcessor', + ] + + # The True case needs to come second because the initializer + # appends to the list and there isn't an immediately obvious + # way of resetting it. + with override_settings(DATADOG_DIAGNOSTICS_ENABLE=True): + initialize() + assert sorted(get_processor_list()) == [ + 'EndpointCallCounterProcessor', 'MissingSpanProcessor', 'TopLevelSpanProcessor', + ] + + @override_settings(DATADOG_DIAGNOSTICS_MAX_SPANS=3) def test_metrics(self): proc = apps.MissingSpanProcessor() ids = [2, 4, 6, 8, 10] @@ -47,30 +76,5 @@ def test_logging(self, mock_log_error, mock_log_info): proc.on_span_start(FakeSpan(17)) proc.shutdown(0) - assert mock_log_info.call_args_list == [ - call("Early span-start sample: span_id=17"), - call("Spans created = 1; spans finished = 0"), - ] + mock_log_info.assert_called_once_with("Spans created = 1; spans finished = 0") mock_log_error.assert_called_once_with("Span created but not finished: span_id=17") - - @patch('edx_arch_experiments.datadog_diagnostics.apps.log.info') - def test_early_span_logging_cutoff(self, mock_log_info): - with patch('edx_arch_experiments.datadog_diagnostics.apps.time.time', side_effect=[ - # Setup - 1700000000, - # Span-start time checks - 1700000020, - 1700000040, - 1700010000, - ]): - proc = apps.MissingSpanProcessor() - # Three spans are started - proc.on_span_start(FakeSpan(44)) - proc.on_span_start(FakeSpan(45)) - proc.on_span_start(FakeSpan(46)) - - # Just two early span-starts are logged - assert mock_log_info.call_args_list == [ - call("Early span-start sample: span_id=44"), - call("Early span-start sample: span_id=45"), - ] From 560a6fbd50947292c772223dd0e35643e24002af Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Thu, 22 Aug 2024 16:44:25 +0000 Subject: [PATCH 3/3] feat: Add celery task lifecycle logging for Datadog diagnostics (#779) Adds celery as a testing dependency. --- CHANGELOG.rst | 6 + edx_arch_experiments/__init__.py | 2 +- .../datadog_diagnostics/apps.py | 116 ++++++++++++++++ .../datadog_diagnostics/tests/test_app.py | 125 +++++++++++++++++- requirements/base.txt | 14 +- requirements/ci.txt | 4 +- requirements/dev.txt | 106 ++++++++++----- requirements/doc.txt | 88 ++++++++---- requirements/pip-tools.txt | 4 +- requirements/pip.txt | 4 +- requirements/quality.txt | 96 ++++++++++---- requirements/scripts.txt | 18 +-- requirements/test.in | 1 + requirements/test.txt | 64 ++++++--- 14 files changed, 518 insertions(+), 130 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 12f0cf5..5ed7f41 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,12 @@ Change Log Unreleased ~~~~~~~~~~ +[4.3.0] - 2024-08-22 +~~~~~~~~~~~~~~~~~~~~ +Added +----- +* Added celery lifecycle logging for Datadog diagnostics, to be enabled using ``DATADOG_DIAGNOSTICS_CELERY_LOG_SIGNALS``. + [4.2.0] - 2024-08-13 ~~~~~~~~~~~~~~~~~~~~ Fixed diff --git a/edx_arch_experiments/__init__.py b/edx_arch_experiments/__init__.py index 6192ccb..eaffb83 100644 --- a/edx_arch_experiments/__init__.py +++ b/edx_arch_experiments/__init__.py @@ -2,4 +2,4 @@ A plugin to include applications under development by the architecture team at 2U. """ -__version__ = '4.2.0' +__version__ = '4.3.0' diff --git a/edx_arch_experiments/datadog_diagnostics/apps.py b/edx_arch_experiments/datadog_diagnostics/apps.py index e4a22b8..d57c0c6 100644 --- a/edx_arch_experiments/datadog_diagnostics/apps.py +++ b/edx_arch_experiments/datadog_diagnostics/apps.py @@ -3,6 +3,7 @@ """ import logging +import re from django.apps import AppConfig from django.conf import settings @@ -41,6 +42,112 @@ def shutdown(self, _timeout): log.error(f"Span created but not finished: {span._pprint()}") # pylint: disable=protected-access +# Dictionary of Celery signal names to a task information extractor. +# The latter is a lambda accepting the signal receiver's kwargs dict +# and returning a minimal dict of info for tracking task lifecycle. +# Celery signal params vary quite a bit in how they convey the +# information we need, so this is probably better than trying to use +# one set of heuristics to get the task ID and name from all signals. +# +# Docs: https://docs.celeryq.dev/en/stable/userguide/signals.html +CELERY_SIGNAL_CONFIG = { + 'before_task_publish': lambda kwargs: {'name': kwargs['sender']}, + 'after_task_publish': lambda kwargs: {'name': kwargs['sender']}, + 'task_prerun': lambda kwargs: {'name': kwargs['task'].name, 'id': kwargs['task_id']}, + 'task_postrun': lambda kwargs: {'name': kwargs['task'].name, 'id': kwargs['task_id']}, + 'task_retry': lambda kwargs: {'name': kwargs['sender'].name, 'id': kwargs['request'].id}, + 'task_success': lambda kwargs: {'name': kwargs['sender'].name}, + 'task_failure': lambda kwargs: {'name': kwargs['sender'].name, 'id': kwargs['task_id']}, + 'task_internal_error': lambda kwargs: {'name': kwargs['sender'].name, 'id': kwargs['task_id']}, + 'task_received': lambda kwargs: {'name': kwargs['request'].name, 'id': kwargs['request'].id}, + 'task_revoked': lambda kwargs: {'name': kwargs['sender'].name, 'id': kwargs['request'].id}, + 'task_unknown': lambda kwargs: {'name': kwargs['name'], 'id': kwargs['id']}, + 'task_rejected': lambda _kwargs: {}, +} + + +def _connect_celery_handler(signal, signal_name, extractor): + """ + Register one Celery signal receiver. + + This serves as a closure to capture the config (and some state) for one signal. + If the extractor ever throws, log the error just once and don't try calling it + again for the remaining life of the process (as it will likely continue failing + the same way.) + + Args: + signal: Django Signal instance to register a handler for + signal_name: Name of signal in Celery signals module (used for logging) + extractor: Function to take signal receiver's entire kwargs and return + a dict optionally containing 'id' and 'name' keys. + """ + errored = False + + def log_celery_signal(**kwargs): + nonlocal errored + info = None + try: + if not errored: + info = extractor(kwargs) + except BaseException: + errored = True + log.exception( + f"Error while extracting Celery signal info for '{signal_name}'; " + "will not attempt for future calls to this signal." + ) + + if info is None: + extra = "(skipped data extraction)" + else: + extra = f"with name={info.get('name')} id={info.get('id')}" + log.info(f"Celery signal called: '{signal_name}' {extra}") + + signal.connect(log_celery_signal, weak=False) + + +def connect_celery_logging(): + """ + Set up logging of configured Celery signals. + + Throws if celery is not installed. + """ + import celery.signals # pylint: disable=import-outside-toplevel + + # .. setting_name: DATADOG_DIAGNOSTICS_CELERY_LOG_SIGNALS + # .. setting_default: '' + # .. setting_description: Log calls to these Celery signals (signal name as well + # as task name and id, if available). Specify as a comma and/or whitespace delimited + # list of names from the celery.signals module. Full list of available signals: + # "after_task_publish, before_task_publish, task_failure, task_internal_error, + # task_postrun, task_prerun, task_received, task_rejected, task_retry, + # task_revoked, task_success, task_unknown" + DATADOG_DIAGNOSTICS_CELERY_LOG_SIGNALS = getattr( + settings, + 'DATADOG_DIAGNOSTICS_CELERY_LOG_SIGNALS', + '' + ) + + connected_names = [] + for signal_name in re.split(r'[,\s]+', DATADOG_DIAGNOSTICS_CELERY_LOG_SIGNALS): + if not signal_name: # artifacts from splitting + continue + + signal = getattr(celery.signals, signal_name, None) + if not signal: + log.warning(f"Could not connect receiver to unknown Celery signal '{signal_name}'") + continue + + extractor = CELERY_SIGNAL_CONFIG.get(signal_name) + if not extractor: + log.warning(f"Don't know how to extract info for Celery signal '{signal_name}'; ignoring.") + continue + + _connect_celery_handler(signal, signal_name, extractor) + connected_names.append(signal_name) + + log.info(f"Logging lifecycle info for these celery signals: {connected_names!r}") + + class DatadogDiagnostics(AppConfig): """ Django application to log diagnostic information for Datadog. @@ -72,3 +179,12 @@ def ready(self): "Unable to attach MissingSpanProcessor for Datadog diagnostics" " -- ddtrace module not found." ) + + # We think that something related to Celery instrumentation is involved + # in causing trace concatenation in Datadog. DD Support has requested that + # we log lifecycle information of Celery tasks to see if all of the needed + # signals are being emitted for their span construction. + try: + connect_celery_logging() + except BaseException: + log.exception("Unable to subscribe to Celery task signals") diff --git a/edx_arch_experiments/datadog_diagnostics/tests/test_app.py b/edx_arch_experiments/datadog_diagnostics/tests/test_app.py index ba89b95..8be4678 100644 --- a/edx_arch_experiments/datadog_diagnostics/tests/test_app.py +++ b/edx_arch_experiments/datadog_diagnostics/tests/test_app.py @@ -2,9 +2,11 @@ Tests for plugin app. """ -from unittest.mock import patch +from unittest.mock import call, patch +import celery.signals from ddtrace import tracer +from django.dispatch import Signal from django.test import TestCase, override_settings from .. import apps @@ -78,3 +80,124 @@ def test_logging(self, mock_log_error, mock_log_info): mock_log_info.assert_called_once_with("Spans created = 1; spans finished = 0") mock_log_error.assert_called_once_with("Span created but not finished: span_id=17") + + +class TestCeleryLogging(TestCase): + """ + Tests for celery signal logging. + + While it would be nice to test actual Celery tasks and signals, + it's difficult to get that all working with unit tests. We'd have + to use Celery's pytest extra, which provides fixtures, but those + don't play well with unittest's TestCase classes and even after + converting to top-level functions things just seemed to hang after + setup. + + So instead, we'll mock things out at the signal level and just + ensure that each level of functionality works in isolation. + """ + + @patch('edx_arch_experiments.datadog_diagnostics.apps.log.info') + def test_default_config_has_no_signals(self, mock_log_info): + apps.DatadogDiagnostics('edx_arch_experiments.datadog_diagnostics', apps).ready() + mock_log_info.assert_called_with("Logging lifecycle info for these celery signals: []") + + @patch('edx_arch_experiments.datadog_diagnostics.apps.log.info') + def test_registration_maximal(self, mock_log_info): + """Test that all celery signal names are actually signals.""" + all_signal_names = ', '.join(sorted(apps.CELERY_SIGNAL_CONFIG.keys())) + with override_settings(DATADOG_DIAGNOSTICS_CELERY_LOG_SIGNALS=all_signal_names): + apps.DatadogDiagnostics('edx_arch_experiments.datadog_diagnostics', apps).ready() + + mock_log_info.assert_called_with( + "Logging lifecycle info for these celery signals: ['after_task_publish', " + "'before_task_publish', 'task_failure', 'task_internal_error', " + "'task_postrun', 'task_prerun', 'task_received', 'task_rejected', " + "'task_retry', 'task_revoked', 'task_success', 'task_unknown']" + ) + + @override_settings( + DATADOG_DIAGNOSTICS_CELERY_LOG_SIGNALS=',,,task_success, task_unknown, task_rejected, fake_signal' + ) + @patch('edx_arch_experiments.datadog_diagnostics.apps.log.info') + @patch('edx_arch_experiments.datadog_diagnostics.apps.log.warning') + @patch('edx_arch_experiments.datadog_diagnostics.apps._connect_celery_handler') + def test_register(self, mock_connect, mock_log_warning, mock_log_info): + """Test that signal connector is *called* as expected.""" + + with patch.dict('edx_arch_experiments.datadog_diagnostics.apps.CELERY_SIGNAL_CONFIG'): + # Add a bad entry to the config to test the signal lookup path + apps.CELERY_SIGNAL_CONFIG['fake_signal'] = lambda kwargs: {} + # Remove a real signal from the config to test the extractor lookup path + del apps.CELERY_SIGNAL_CONFIG['task_unknown'] + apps.DatadogDiagnostics('edx_arch_experiments.datadog_diagnostics', apps).ready() + + assert mock_connect.call_args_list == [ + call( + celery.signals.task_success, + 'task_success', + apps.CELERY_SIGNAL_CONFIG['task_success'], + ), + call( + celery.signals.task_rejected, + 'task_rejected', + apps.CELERY_SIGNAL_CONFIG['task_rejected'], + ), + ] + assert mock_log_warning.call_args_list == [ + call("Don't know how to extract info for Celery signal 'task_unknown'; ignoring."), + call("Could not connect receiver to unknown Celery signal 'fake_signal'"), + ] + mock_log_info.assert_called_with( + "Logging lifecycle info for these celery signals: ['task_success', 'task_rejected']" + ) + + @patch('edx_arch_experiments.datadog_diagnostics.apps.log.info') + @patch('edx_arch_experiments.datadog_diagnostics.apps.log.exception') + def test_handler(self, mock_log_exception, mock_log_info): + """Test that signal connector *behaves* as expected.""" + # Signal A will just do a straightforward data extraction from the kwargs. + # pylint: disable=protected-access + apps._connect_celery_handler( + signal_example_a, 'signal_example_a', + lambda kwargs: {'name': kwargs['info']['name']}, + ) + + # Signal B will have an extractor that goes bad on the 2nd and 3rd calls + b_called_times = 0 + + def b_extractor(kwargs): + nonlocal b_called_times + b_called_times += 1 + if b_called_times >= 2: + raise Exception("oops") + + return {'id': kwargs['the_id']} + + # pylint: disable=protected-access + apps._connect_celery_handler(signal_example_b, 'signal_example_b', b_extractor) + + # Send to B a few times to show that error logging only happens once + signal_example_b.send(sender='some_sender', the_id=42) + signal_example_b.send(sender='some_sender', the_id=42) + signal_example_b.send(sender='some_sender', the_id=42) + # And then send to A to show it still works + signal_example_a.send( + sender='some_sender', other='whatever', info={'name': "Alice"}, name='not this one', + ) + + mock_log_exception.assert_called_once_with( + "Error while extracting Celery signal info for 'signal_example_b'; " + "will not attempt for future calls to this signal." + ) + assert b_called_times == 2 + assert mock_log_info.call_args_list == [ + call("Celery signal called: 'signal_example_b' with name=None id=42"), + call("Celery signal called: 'signal_example_b' (skipped data extraction)"), + call("Celery signal called: 'signal_example_b' (skipped data extraction)"), + call("Celery signal called: 'signal_example_a' with name=Alice id=None"), + ] + + +signal_example_a = Signal() +signal_example_b = Signal() diff --git a/requirements/base.txt b/requirements/base.txt index 049726c..17ff075 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,13 +6,13 @@ # asgiref==3.8.1 # via django -attrs==23.2.0 +attrs==24.2.0 # via # jsonschema # referencing certifi==2024.7.4 # via requests -cffi==1.16.0 +cffi==1.17.0 # via # cryptography # pynacl @@ -26,7 +26,7 @@ code-annotations==1.8.0 # via edx-toggles cryptography==43.0.0 # via pyjwt -django==4.2.14 +django==4.2.15 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in @@ -79,7 +79,7 @@ jsonschema-specifications==2023.12.1 # via jsonschema markupsafe==2.1.5 # via jinja2 -newrelic==9.12.0 +newrelic==9.13.0 # via edx-django-utils pbr==6.0.0 # via stevedore @@ -97,7 +97,7 @@ pynacl==1.5.0 # via edx-django-utils python-slugify==8.0.4 # via code-annotations -pyyaml==6.0.1 +pyyaml==6.0.2 # via code-annotations referencing==0.35.1 # via @@ -105,7 +105,7 @@ referencing==0.35.1 # jsonschema-specifications requests==2.32.3 # via edx-drf-extensions -rpds-py==0.19.1 +rpds-py==0.20.0 # via # jsonschema # referencing @@ -128,5 +128,5 @@ urllib3==2.2.2 # via requests # The following packages are considered to be unsafe in a requirements file: -setuptools==72.1.0 +setuptools==73.0.1 # via -r requirements/base.in diff --git a/requirements/ci.txt b/requirements/ci.txt index 564c0f5..e39ed77 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -4,7 +4,7 @@ # # make upgrade # -cachetools==5.4.0 +cachetools==5.5.0 # via tox chardet==5.2.0 # via tox @@ -28,7 +28,7 @@ pluggy==1.5.0 # via tox pyproject-api==1.7.1 # via tox -tox==4.16.0 +tox==4.18.0 # via -r requirements/ci.in virtualenv==20.26.3 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 3606ea3..95f9801 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -4,6 +4,10 @@ # # make upgrade # +amqp==5.2.0 + # via + # -r requirements/quality.txt + # kombu asgiref==3.8.1 # via # -r requirements/quality.txt @@ -13,17 +17,19 @@ astroid==3.2.4 # -r requirements/quality.txt # pylint # pylint-celery -attrs==23.2.0 +attrs==24.2.0 # via # -r requirements/quality.txt - # cattrs - # ddtrace # jsonschema # referencing backports-tarfile==1.2.0 # via # -r requirements/quality.txt # jaraco-context +billiard==4.2.0 + # via + # -r requirements/quality.txt + # celery build==1.2.1 # via # -r requirements/pip-tools.txt @@ -32,19 +38,17 @@ bytecode==0.15.1 # via # -r requirements/quality.txt # ddtrace -cachetools==5.4.0 +cachetools==5.5.0 # via # -r requirements/ci.txt # tox -cattrs==23.2.3 - # via - # -r requirements/quality.txt - # ddtrace +celery==5.4.0 + # via -r requirements/quality.txt certifi==2024.7.4 # via # -r requirements/quality.txt # requests -cffi==1.16.0 +cffi==1.17.0 # via # -r requirements/quality.txt # cryptography @@ -62,15 +66,31 @@ click==8.1.7 # via # -r requirements/pip-tools.txt # -r requirements/quality.txt + # celery + # click-didyoumean # click-log + # click-plugins + # click-repl # code-annotations # edx-django-utils # edx-lint # pip-tools +click-didyoumean==0.3.1 + # via + # -r requirements/quality.txt + # celery click-log==0.4.0 # via # -r requirements/quality.txt # edx-lint +click-plugins==1.1.1 + # via + # -r requirements/quality.txt + # celery +click-repl==0.3.0 + # via + # -r requirements/quality.txt + # celery code-annotations==1.8.0 # via # -r requirements/quality.txt @@ -80,7 +100,7 @@ colorama==0.4.6 # via # -r requirements/ci.txt # tox -coverage[toml]==7.6.0 +coverage[toml]==7.6.1 # via # -r requirements/quality.txt # pytest-cov @@ -89,13 +109,9 @@ cryptography==43.0.0 # -r requirements/quality.txt # pyjwt # secretstorage -ddsketch==3.0.1 - # via - # -r requirements/quality.txt - # ddtrace ddt==1.7.2 # via -r requirements/quality.txt -ddtrace==2.10.1 +ddtrace==2.11.1 # via -r requirements/quality.txt deprecated==1.2.14 # via @@ -111,7 +127,7 @@ distlib==0.3.8 # via # -r requirements/ci.txt # virtualenv -django==4.2.14 +django==4.2.15 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/quality.txt @@ -183,9 +199,8 @@ idna==3.7 # via # -r requirements/quality.txt # requests -importlib-metadata==6.11.0 +importlib-metadata==8.0.0 # via - # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/quality.txt # keyring # opentelemetry-api @@ -202,7 +217,7 @@ jaraco-classes==3.4.0 # via # -r requirements/quality.txt # keyring -jaraco-context==5.3.0 +jaraco-context==6.0.1 # via # -r requirements/quality.txt # keyring @@ -226,11 +241,15 @@ jsonschema-specifications==2023.12.1 # via # -r requirements/quality.txt # jsonschema -keyring==25.2.1 +keyring==25.3.0 # via # -r requirements/quality.txt # twine -lxml==5.2.2 +kombu==5.4.0 + # via + # -r requirements/quality.txt + # celery +lxml==5.3.0 # via edx-i18n-tools markdown-it-py==3.0.0 # via @@ -248,12 +267,12 @@ mdurl==0.1.2 # via # -r requirements/quality.txt # markdown-it-py -more-itertools==10.3.0 +more-itertools==10.4.0 # via # -r requirements/quality.txt # jaraco-classes # jaraco-functools -newrelic==9.12.0 +newrelic==9.13.0 # via # -r requirements/quality.txt # edx-django-utils @@ -302,6 +321,10 @@ pluggy==1.5.0 # tox polib==1.2.0 # via edx-i18n-tools +prompt-toolkit==3.0.47 + # via + # -r requirements/quality.txt + # click-repl protobuf==5.27.3 # via # -r requirements/quality.txt @@ -310,7 +333,7 @@ psutil==6.0.0 # via # -r requirements/quality.txt # edx-django-utils -pycodestyle==2.12.0 +pycodestyle==2.12.1 # via -r requirements/quality.txt pycparser==2.22 # via @@ -375,11 +398,15 @@ pytest-cov==5.0.0 # via -r requirements/quality.txt pytest-django==4.8.0 # via -r requirements/quality.txt +python-dateutil==2.9.0.post0 + # via + # -r requirements/quality.txt + # celery python-slugify==8.0.4 # via # -r requirements/quality.txt # code-annotations -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -r requirements/quality.txt # code-annotations @@ -411,7 +438,7 @@ rich==13.7.1 # via # -r requirements/quality.txt # twine -rpds-py==0.19.1 +rpds-py==0.20.0 # via # -r requirements/quality.txt # jsonschema @@ -427,10 +454,9 @@ semantic-version==2.10.0 six==1.16.0 # via # -r requirements/quality.txt - # ddsketch - # ddtrace # edx-codejail # edx-lint + # python-dateutil snowballstemmer==2.2.0 # via # -r requirements/quality.txt @@ -449,11 +475,11 @@ text-unidecode==1.3 # via # -r requirements/quality.txt # python-slugify -tomlkit==0.13.0 +tomlkit==0.13.2 # via # -r requirements/quality.txt # pylint -tox==4.16.0 +tox==4.18.0 # via -r requirements/ci.txt twine==5.1.1 # via -r requirements/quality.txt @@ -462,16 +488,30 @@ typing-extensions==4.12.2 # -r requirements/quality.txt # ddtrace # edx-opaque-keys +tzdata==2024.1 + # via + # -r requirements/quality.txt + # celery urllib3==2.2.2 # via # -r requirements/quality.txt # requests # twine +vine==5.1.0 + # via + # -r requirements/quality.txt + # amqp + # celery + # kombu virtualenv==20.26.3 # via # -r requirements/ci.txt # tox -wheel==0.43.0 +wcwidth==0.2.13 + # via + # -r requirements/quality.txt + # prompt-toolkit +wheel==0.44.0 # via # -r requirements/pip-tools.txt # pip-tools @@ -483,7 +523,7 @@ xmltodict==0.13.0 # via # -r requirements/quality.txt # ddtrace -zipp==3.19.2 +zipp==3.20.0 # via # -r requirements/quality.txt # importlib-metadata @@ -493,7 +533,7 @@ pip==24.2 # via # -r requirements/pip-tools.txt # pip-tools -setuptools==72.1.0 +setuptools==73.0.1 # via # -r requirements/pip-tools.txt # -r requirements/quality.txt diff --git a/requirements/doc.txt b/requirements/doc.txt index 5eac2f6..0566cd9 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -6,32 +6,36 @@ # alabaster==0.7.16 # via sphinx +amqp==5.2.0 + # via + # -r requirements/test.txt + # kombu asgiref==3.8.1 # via # -r requirements/test.txt # django -attrs==23.2.0 +attrs==24.2.0 # via # -r requirements/test.txt - # cattrs - # ddtrace # jsonschema # referencing -babel==2.15.0 +babel==2.16.0 # via sphinx -bytecode==0.15.1 +billiard==4.2.0 # via # -r requirements/test.txt - # ddtrace -cattrs==23.2.3 + # celery +bytecode==0.15.1 # via # -r requirements/test.txt # ddtrace +celery==5.4.0 + # via -r requirements/test.txt certifi==2024.7.4 # via # -r requirements/test.txt # requests -cffi==1.16.0 +cffi==1.17.0 # via # -r requirements/test.txt # cryptography @@ -43,13 +47,29 @@ charset-normalizer==3.3.2 click==8.1.7 # via # -r requirements/test.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # code-annotations # edx-django-utils +click-didyoumean==0.3.1 + # via + # -r requirements/test.txt + # celery +click-plugins==1.1.1 + # via + # -r requirements/test.txt + # celery +click-repl==0.3.0 + # via + # -r requirements/test.txt + # celery code-annotations==1.8.0 # via # -r requirements/test.txt # edx-toggles -coverage[toml]==7.6.0 +coverage[toml]==7.6.1 # via # -r requirements/test.txt # pytest-cov @@ -57,19 +77,15 @@ cryptography==43.0.0 # via # -r requirements/test.txt # pyjwt -ddsketch==3.0.1 - # via - # -r requirements/test.txt - # ddtrace ddt==1.7.2 # via -r requirements/test.txt -ddtrace==2.10.1 +ddtrace==2.11.1 # via -r requirements/test.txt deprecated==1.2.14 # via # -r requirements/test.txt # opentelemetry-api -django==4.2.14 +django==4.2.15 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt @@ -139,9 +155,8 @@ idna==3.7 # requests imagesize==1.4.1 # via sphinx -importlib-metadata==6.11.0 +importlib-metadata==8.0.0 # via - # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt # opentelemetry-api iniconfig==2.0.0 @@ -159,11 +174,15 @@ jsonschema-specifications==2023.12.1 # via # -r requirements/test.txt # jsonschema +kombu==5.4.0 + # via + # -r requirements/test.txt + # celery markupsafe==2.1.5 # via # -r requirements/test.txt # jinja2 -newrelic==9.12.0 +newrelic==9.13.0 # via # -r requirements/test.txt # edx-django-utils @@ -186,6 +205,10 @@ pluggy==1.5.0 # via # -r requirements/test.txt # pytest +prompt-toolkit==3.0.47 + # via + # -r requirements/test.txt + # click-repl protobuf==5.27.3 # via # -r requirements/test.txt @@ -225,11 +248,15 @@ pytest-cov==5.0.0 # via -r requirements/test.txt pytest-django==4.8.0 # via -r requirements/test.txt +python-dateutil==2.9.0.post0 + # via + # -r requirements/test.txt + # celery python-slugify==8.0.4 # via # -r requirements/test.txt # code-annotations -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -r requirements/test.txt # code-annotations @@ -247,7 +274,7 @@ requests==2.32.3 # sphinx restructuredtext-lint==1.4.0 # via doc8 -rpds-py==0.19.1 +rpds-py==0.20.0 # via # -r requirements/test.txt # jsonschema @@ -259,10 +286,9 @@ semantic-version==2.10.0 six==1.16.0 # via # -r requirements/test.txt - # ddsketch - # ddtrace # edx-codejail # edx-sphinx-theme + # python-dateutil snowballstemmer==2.2.0 # via sphinx sphinx==5.3.0 @@ -301,10 +327,24 @@ typing-extensions==4.12.2 # -r requirements/test.txt # ddtrace # edx-opaque-keys +tzdata==2024.1 + # via + # -r requirements/test.txt + # celery urllib3==2.2.2 # via # -r requirements/test.txt # requests +vine==5.1.0 + # via + # -r requirements/test.txt + # amqp + # celery + # kombu +wcwidth==0.2.13 + # via + # -r requirements/test.txt + # prompt-toolkit wrapt==1.16.0 # via # -r requirements/test.txt @@ -313,11 +353,11 @@ xmltodict==0.13.0 # via # -r requirements/test.txt # ddtrace -zipp==3.19.2 +zipp==3.20.0 # via # -r requirements/test.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==72.1.0 +setuptools==73.0.1 # via -r requirements/test.txt diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index b0c1866..73cb449 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -16,11 +16,11 @@ pyproject-hooks==1.1.0 # via # build # pip-tools -wheel==0.43.0 +wheel==0.44.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: pip==24.2 # via pip-tools -setuptools==72.1.0 +setuptools==73.0.1 # via pip-tools diff --git a/requirements/pip.txt b/requirements/pip.txt index 54b0571..f0fca18 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -4,11 +4,11 @@ # # make upgrade # -wheel==0.43.0 +wheel==0.44.0 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: pip==24.2 # via -r requirements/pip.in -setuptools==72.1.0 +setuptools==73.0.1 # via -r requirements/pip.in diff --git a/requirements/quality.txt b/requirements/quality.txt index e10c684..701873d 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -4,6 +4,10 @@ # # make upgrade # +amqp==5.2.0 + # via + # -r requirements/test.txt + # kombu asgiref==3.8.1 # via # -r requirements/test.txt @@ -12,28 +16,28 @@ astroid==3.2.4 # via # pylint # pylint-celery -attrs==23.2.0 +attrs==24.2.0 # via # -r requirements/test.txt - # cattrs - # ddtrace # jsonschema # referencing backports-tarfile==1.2.0 # via jaraco-context -bytecode==0.15.1 +billiard==4.2.0 # via # -r requirements/test.txt - # ddtrace -cattrs==23.2.3 + # celery +bytecode==0.15.1 # via # -r requirements/test.txt # ddtrace +celery==5.4.0 + # via -r requirements/test.txt certifi==2024.7.4 # via # -r requirements/test.txt # requests -cffi==1.16.0 +cffi==1.17.0 # via # -r requirements/test.txt # cryptography @@ -45,18 +49,34 @@ charset-normalizer==3.3.2 click==8.1.7 # via # -r requirements/test.txt + # celery + # click-didyoumean # click-log + # click-plugins + # click-repl # code-annotations # edx-django-utils # edx-lint +click-didyoumean==0.3.1 + # via + # -r requirements/test.txt + # celery click-log==0.4.0 # via edx-lint +click-plugins==1.1.1 + # via + # -r requirements/test.txt + # celery +click-repl==0.3.0 + # via + # -r requirements/test.txt + # celery code-annotations==1.8.0 # via # -r requirements/test.txt # edx-lint # edx-toggles -coverage[toml]==7.6.0 +coverage[toml]==7.6.1 # via # -r requirements/test.txt # pytest-cov @@ -65,13 +85,9 @@ cryptography==43.0.0 # -r requirements/test.txt # pyjwt # secretstorage -ddsketch==3.0.1 - # via - # -r requirements/test.txt - # ddtrace ddt==1.7.2 # via -r requirements/test.txt -ddtrace==2.10.1 +ddtrace==2.11.1 # via -r requirements/test.txt deprecated==1.2.14 # via @@ -79,7 +95,7 @@ deprecated==1.2.14 # opentelemetry-api dill==0.3.8 # via pylint -django==4.2.14 +django==4.2.15 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt @@ -141,9 +157,8 @@ idna==3.7 # via # -r requirements/test.txt # requests -importlib-metadata==6.11.0 +importlib-metadata==8.0.0 # via - # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt # keyring # opentelemetry-api @@ -158,7 +173,7 @@ isort==5.13.2 # pylint jaraco-classes==3.4.0 # via keyring -jaraco-context==5.3.0 +jaraco-context==6.0.1 # via keyring jaraco-functools==4.0.2 # via keyring @@ -176,8 +191,12 @@ jsonschema-specifications==2023.12.1 # via # -r requirements/test.txt # jsonschema -keyring==25.2.1 +keyring==25.3.0 # via twine +kombu==5.4.0 + # via + # -r requirements/test.txt + # celery markdown-it-py==3.0.0 # via rich markupsafe==2.1.5 @@ -188,11 +207,11 @@ mccabe==0.7.0 # via pylint mdurl==0.1.2 # via markdown-it-py -more-itertools==10.3.0 +more-itertools==10.4.0 # via # jaraco-classes # jaraco-functools -newrelic==9.12.0 +newrelic==9.13.0 # via # -r requirements/test.txt # edx-django-utils @@ -218,6 +237,10 @@ pluggy==1.5.0 # via # -r requirements/test.txt # pytest +prompt-toolkit==3.0.47 + # via + # -r requirements/test.txt + # click-repl protobuf==5.27.3 # via # -r requirements/test.txt @@ -226,7 +249,7 @@ psutil==6.0.0 # via # -r requirements/test.txt # edx-django-utils -pycodestyle==2.12.0 +pycodestyle==2.12.1 # via -r requirements/quality.in pycparser==2.22 # via @@ -274,11 +297,15 @@ pytest-cov==5.0.0 # via -r requirements/test.txt pytest-django==4.8.0 # via -r requirements/test.txt +python-dateutil==2.9.0.post0 + # via + # -r requirements/test.txt + # celery python-slugify==8.0.4 # via # -r requirements/test.txt # code-annotations -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -r requirements/test.txt # code-annotations @@ -301,7 +328,7 @@ rfc3986==2.0.0 # via twine rich==13.7.1 # via twine -rpds-py==0.19.1 +rpds-py==0.20.0 # via # -r requirements/test.txt # jsonschema @@ -315,10 +342,9 @@ semantic-version==2.10.0 six==1.16.0 # via # -r requirements/test.txt - # ddsketch - # ddtrace # edx-codejail # edx-lint + # python-dateutil snowballstemmer==2.2.0 # via pydocstyle sqlparse==0.5.1 @@ -335,7 +361,7 @@ text-unidecode==1.3 # via # -r requirements/test.txt # python-slugify -tomlkit==0.13.0 +tomlkit==0.13.2 # via pylint twine==5.1.1 # via -r requirements/quality.in @@ -344,11 +370,25 @@ typing-extensions==4.12.2 # -r requirements/test.txt # ddtrace # edx-opaque-keys +tzdata==2024.1 + # via + # -r requirements/test.txt + # celery urllib3==2.2.2 # via # -r requirements/test.txt # requests # twine +vine==5.1.0 + # via + # -r requirements/test.txt + # amqp + # celery + # kombu +wcwidth==0.2.13 + # via + # -r requirements/test.txt + # prompt-toolkit wrapt==1.16.0 # via # -r requirements/test.txt @@ -357,11 +397,11 @@ xmltodict==0.13.0 # via # -r requirements/test.txt # ddtrace -zipp==3.19.2 +zipp==3.20.0 # via # -r requirements/test.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==72.1.0 +setuptools==73.0.1 # via -r requirements/test.txt diff --git a/requirements/scripts.txt b/requirements/scripts.txt index c7740dd..23951ac 100644 --- a/requirements/scripts.txt +++ b/requirements/scripts.txt @@ -8,19 +8,19 @@ asgiref==3.8.1 # via # -r requirements/base.txt # django -attrs==23.2.0 +attrs==24.2.0 # via # -r requirements/base.txt # jsonschema # openedx-events # referencing -avro==1.11.3 +avro==1.12.0 # via confluent-kafka certifi==2024.7.4 # via # -r requirements/base.txt # requests -cffi==1.16.0 +cffi==1.17.0 # via # -r requirements/base.txt # cryptography @@ -44,7 +44,7 @@ cryptography==43.0.0 # via # -r requirements/base.txt # pyjwt -django==4.2.14 +django==4.2.15 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.txt @@ -94,7 +94,7 @@ edx-django-utils==5.15.0 # openedx-events edx-drf-extensions==10.3.0 # via -r requirements/base.txt -edx-event-bus-kafka==5.8.0 +edx-event-bus-kafka==5.8.1 # via -r requirements/scripts.in edx-opaque-keys[django]==2.10.0 # via @@ -128,7 +128,7 @@ markupsafe==2.1.5 # via # -r requirements/base.txt # jinja2 -newrelic==9.12.0 +newrelic==9.13.0 # via # -r requirements/base.txt # edx-django-utils @@ -163,7 +163,7 @@ python-slugify==8.0.4 # via # -r requirements/base.txt # code-annotations -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -r requirements/base.txt # code-annotations @@ -177,7 +177,7 @@ requests==2.32.3 # -r requirements/base.txt # confluent-kafka # edx-drf-extensions -rpds-py==0.19.1 +rpds-py==0.20.0 # via # -r requirements/base.txt # jsonschema @@ -215,5 +215,5 @@ urllib3==2.2.2 # requests # The following packages are considered to be unsafe in a requirements file: -setuptools==72.1.0 +setuptools==73.0.1 # via -r requirements/base.txt diff --git a/requirements/test.in b/requirements/test.in index 9974b96..45939a0 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -8,3 +8,4 @@ pytest-django # pytest extension for better Django support code-annotations # provides commands used by the pii_check make target. ddt # data-driven tests ddtrace # Required for testing datadog_diagnostics app and middleware +celery # Required for testing datadog_diagnostics app diff --git a/requirements/test.txt b/requirements/test.txt index 3d2ee70..053bcc2 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,26 +4,28 @@ # # make upgrade # +amqp==5.2.0 + # via kombu asgiref==3.8.1 # via # -r requirements/base.txt # django -attrs==23.2.0 +attrs==24.2.0 # via # -r requirements/base.txt - # cattrs - # ddtrace # jsonschema # referencing +billiard==4.2.0 + # via celery bytecode==0.15.1 # via ddtrace -cattrs==23.2.3 - # via ddtrace +celery==5.4.0 + # via -r requirements/test.in certifi==2024.7.4 # via # -r requirements/base.txt # requests -cffi==1.16.0 +cffi==1.17.0 # via # -r requirements/base.txt # cryptography @@ -35,24 +37,32 @@ charset-normalizer==3.3.2 click==8.1.7 # via # -r requirements/base.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # code-annotations # edx-django-utils +click-didyoumean==0.3.1 + # via celery +click-plugins==1.1.1 + # via celery +click-repl==0.3.0 + # via celery code-annotations==1.8.0 # via # -r requirements/base.txt # -r requirements/test.in # edx-toggles -coverage[toml]==7.6.0 +coverage[toml]==7.6.1 # via pytest-cov cryptography==43.0.0 # via # -r requirements/base.txt # pyjwt -ddsketch==3.0.1 - # via ddtrace ddt==1.7.2 # via -r requirements/test.in -ddtrace==2.10.1 +ddtrace==2.11.1 # via -r requirements/test.in deprecated==1.2.14 # via opentelemetry-api @@ -111,10 +121,8 @@ idna==3.7 # via # -r requirements/base.txt # requests -importlib-metadata==6.11.0 - # via - # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt - # opentelemetry-api +importlib-metadata==8.0.0 + # via opentelemetry-api iniconfig==2.0.0 # via pytest jinja2==3.1.4 @@ -127,11 +135,13 @@ jsonschema-specifications==2023.12.1 # via # -r requirements/base.txt # jsonschema +kombu==5.4.0 + # via celery markupsafe==2.1.5 # via # -r requirements/base.txt # jinja2 -newrelic==9.12.0 +newrelic==9.13.0 # via # -r requirements/base.txt # edx-django-utils @@ -145,6 +155,8 @@ pbr==6.0.0 # stevedore pluggy==1.5.0 # via pytest +prompt-toolkit==3.0.47 + # via click-repl protobuf==5.27.3 # via ddtrace psutil==6.0.0 @@ -176,11 +188,13 @@ pytest-cov==5.0.0 # via -r requirements/test.in pytest-django==4.8.0 # via -r requirements/test.in +python-dateutil==2.9.0.post0 + # via celery python-slugify==8.0.4 # via # -r requirements/base.txt # code-annotations -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -r requirements/base.txt # code-annotations @@ -193,7 +207,7 @@ requests==2.32.3 # via # -r requirements/base.txt # edx-drf-extensions -rpds-py==0.19.1 +rpds-py==0.20.0 # via # -r requirements/base.txt # jsonschema @@ -205,9 +219,8 @@ semantic-version==2.10.0 six==1.16.0 # via # -r requirements/base.txt - # ddsketch - # ddtrace # edx-codejail + # python-dateutil sqlparse==0.5.1 # via # -r requirements/base.txt @@ -227,17 +240,26 @@ typing-extensions==4.12.2 # -r requirements/base.txt # ddtrace # edx-opaque-keys +tzdata==2024.1 + # via celery urllib3==2.2.2 # via # -r requirements/base.txt # requests +vine==5.1.0 + # via + # amqp + # celery + # kombu +wcwidth==0.2.13 + # via prompt-toolkit wrapt==1.16.0 # via deprecated xmltodict==0.13.0 # via ddtrace -zipp==3.19.2 +zipp==3.20.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==72.1.0 +setuptools==73.0.1 # via -r requirements/base.txt