From cc08d727ff4da2075d01014acd15ea1fd08559d4 Mon Sep 17 00:00:00 2001 From: Tim McCormack Date: Tue, 20 Aug 2024 18:04:38 +0000 Subject: [PATCH] feat: Add celery task lifecycle logging for Datadog diagnostics Adds celery as a testing dependency. --- CHANGELOG.rst | 6 + edx_arch_experiments/__init__.py | 2 +- .../datadog_diagnostics/apps.py | 116 ++++++++++++++++ .../datadog_diagnostics/tests/test_app.py | 125 +++++++++++++++++- requirements/base.txt | 14 +- requirements/ci.txt | 4 +- requirements/dev.txt | 106 ++++++++++----- requirements/doc.txt | 88 ++++++++---- requirements/pip-tools.txt | 4 +- requirements/pip.txt | 4 +- requirements/quality.txt | 96 ++++++++++---- requirements/scripts.txt | 18 +-- requirements/test.in | 1 + requirements/test.txt | 64 ++++++--- 14 files changed, 518 insertions(+), 130 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 12f0cf5..5ed7f41 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,12 @@ Change Log Unreleased ~~~~~~~~~~ +[4.3.0] - 2024-08-22 +~~~~~~~~~~~~~~~~~~~~ +Added +----- +* Added celery lifecycle logging for Datadog diagnostics, to be enabled using ``DATADOG_DIAGNOSTICS_CELERY_LOG_SIGNALS``. + [4.2.0] - 2024-08-13 ~~~~~~~~~~~~~~~~~~~~ Fixed diff --git a/edx_arch_experiments/__init__.py b/edx_arch_experiments/__init__.py index 6192ccb..eaffb83 100644 --- a/edx_arch_experiments/__init__.py +++ b/edx_arch_experiments/__init__.py @@ -2,4 +2,4 @@ A plugin to include applications under development by the architecture team at 2U. """ -__version__ = '4.2.0' +__version__ = '4.3.0' diff --git a/edx_arch_experiments/datadog_diagnostics/apps.py b/edx_arch_experiments/datadog_diagnostics/apps.py index e4a22b8..d57c0c6 100644 --- a/edx_arch_experiments/datadog_diagnostics/apps.py +++ b/edx_arch_experiments/datadog_diagnostics/apps.py @@ -3,6 +3,7 @@ """ import logging +import re from django.apps import AppConfig from django.conf import settings @@ -41,6 +42,112 @@ def shutdown(self, _timeout): log.error(f"Span created but not finished: {span._pprint()}") # pylint: disable=protected-access +# Dictionary of Celery signal names to a task information extractor. +# The latter is a lambda accepting the signal receiver's kwargs dict +# and returning a minimal dict of info for tracking task lifecycle. +# Celery signal params vary quite a bit in how they convey the +# information we need, so this is probably better than trying to use +# one set of heuristics to get the task ID and name from all signals. +# +# Docs: https://docs.celeryq.dev/en/stable/userguide/signals.html +CELERY_SIGNAL_CONFIG = { + 'before_task_publish': lambda kwargs: {'name': kwargs['sender']}, + 'after_task_publish': lambda kwargs: {'name': kwargs['sender']}, + 'task_prerun': lambda kwargs: {'name': kwargs['task'].name, 'id': kwargs['task_id']}, + 'task_postrun': lambda kwargs: {'name': kwargs['task'].name, 'id': kwargs['task_id']}, + 'task_retry': lambda kwargs: {'name': kwargs['sender'].name, 'id': kwargs['request'].id}, + 'task_success': lambda kwargs: {'name': kwargs['sender'].name}, + 'task_failure': lambda kwargs: {'name': kwargs['sender'].name, 'id': kwargs['task_id']}, + 'task_internal_error': lambda kwargs: {'name': kwargs['sender'].name, 'id': kwargs['task_id']}, + 'task_received': lambda kwargs: {'name': kwargs['request'].name, 'id': kwargs['request'].id}, + 'task_revoked': lambda kwargs: {'name': kwargs['sender'].name, 'id': kwargs['request'].id}, + 'task_unknown': lambda kwargs: {'name': kwargs['name'], 'id': kwargs['id']}, + 'task_rejected': lambda _kwargs: {}, +} + + +def _connect_celery_handler(signal, signal_name, extractor): + """ + Register one Celery signal receiver. + + This serves as a closure to capture the config (and some state) for one signal. + If the extractor ever throws, log the error just once and don't try calling it + again for the remaining life of the process (as it will likely continue failing + the same way.) + + Args: + signal: Django Signal instance to register a handler for + signal_name: Name of signal in Celery signals module (used for logging) + extractor: Function to take signal receiver's entire kwargs and return + a dict optionally containing 'id' and 'name' keys. + """ + errored = False + + def log_celery_signal(**kwargs): + nonlocal errored + info = None + try: + if not errored: + info = extractor(kwargs) + except BaseException: + errored = True + log.exception( + f"Error while extracting Celery signal info for '{signal_name}'; " + "will not attempt for future calls to this signal." + ) + + if info is None: + extra = "(skipped data extraction)" + else: + extra = f"with name={info.get('name')} id={info.get('id')}" + log.info(f"Celery signal called: '{signal_name}' {extra}") + + signal.connect(log_celery_signal, weak=False) + + +def connect_celery_logging(): + """ + Set up logging of configured Celery signals. + + Throws if celery is not installed. + """ + import celery.signals # pylint: disable=import-outside-toplevel + + # .. setting_name: DATADOG_DIAGNOSTICS_CELERY_LOG_SIGNALS + # .. setting_default: '' + # .. setting_description: Log calls to these Celery signals (signal name as well + # as task name and id, if available). Specify as a comma and/or whitespace delimited + # list of names from the celery.signals module. Full list of available signals: + # "after_task_publish, before_task_publish, task_failure, task_internal_error, + # task_postrun, task_prerun, task_received, task_rejected, task_retry, + # task_revoked, task_success, task_unknown" + DATADOG_DIAGNOSTICS_CELERY_LOG_SIGNALS = getattr( + settings, + 'DATADOG_DIAGNOSTICS_CELERY_LOG_SIGNALS', + '' + ) + + connected_names = [] + for signal_name in re.split(r'[,\s]+', DATADOG_DIAGNOSTICS_CELERY_LOG_SIGNALS): + if not signal_name: # artifacts from splitting + continue + + signal = getattr(celery.signals, signal_name, None) + if not signal: + log.warning(f"Could not connect receiver to unknown Celery signal '{signal_name}'") + continue + + extractor = CELERY_SIGNAL_CONFIG.get(signal_name) + if not extractor: + log.warning(f"Don't know how to extract info for Celery signal '{signal_name}'; ignoring.") + continue + + _connect_celery_handler(signal, signal_name, extractor) + connected_names.append(signal_name) + + log.info(f"Logging lifecycle info for these celery signals: {connected_names!r}") + + class DatadogDiagnostics(AppConfig): """ Django application to log diagnostic information for Datadog. @@ -72,3 +179,12 @@ def ready(self): "Unable to attach MissingSpanProcessor for Datadog diagnostics" " -- ddtrace module not found." ) + + # We think that something related to Celery instrumentation is involved + # in causing trace concatenation in Datadog. DD Support has requested that + # we log lifecycle information of Celery tasks to see if all of the needed + # signals are being emitted for their span construction. + try: + connect_celery_logging() + except BaseException: + log.exception("Unable to subscribe to Celery task signals") diff --git a/edx_arch_experiments/datadog_diagnostics/tests/test_app.py b/edx_arch_experiments/datadog_diagnostics/tests/test_app.py index ba89b95..8be4678 100644 --- a/edx_arch_experiments/datadog_diagnostics/tests/test_app.py +++ b/edx_arch_experiments/datadog_diagnostics/tests/test_app.py @@ -2,9 +2,11 @@ Tests for plugin app. """ -from unittest.mock import patch +from unittest.mock import call, patch +import celery.signals from ddtrace import tracer +from django.dispatch import Signal from django.test import TestCase, override_settings from .. import apps @@ -78,3 +80,124 @@ def test_logging(self, mock_log_error, mock_log_info): mock_log_info.assert_called_once_with("Spans created = 1; spans finished = 0") mock_log_error.assert_called_once_with("Span created but not finished: span_id=17") + + +class TestCeleryLogging(TestCase): + """ + Tests for celery signal logging. + + While it would be nice to test actual Celery tasks and signals, + it's difficult to get that all working with unit tests. We'd have + to use Celery's pytest extra, which provides fixtures, but those + don't play well with unittest's TestCase classes and even after + converting to top-level functions things just seemed to hang after + setup. + + So instead, we'll mock things out at the signal level and just + ensure that each level of functionality works in isolation. + """ + + @patch('edx_arch_experiments.datadog_diagnostics.apps.log.info') + def test_default_config_has_no_signals(self, mock_log_info): + apps.DatadogDiagnostics('edx_arch_experiments.datadog_diagnostics', apps).ready() + mock_log_info.assert_called_with("Logging lifecycle info for these celery signals: []") + + @patch('edx_arch_experiments.datadog_diagnostics.apps.log.info') + def test_registration_maximal(self, mock_log_info): + """Test that all celery signal names are actually signals.""" + all_signal_names = ', '.join(sorted(apps.CELERY_SIGNAL_CONFIG.keys())) + with override_settings(DATADOG_DIAGNOSTICS_CELERY_LOG_SIGNALS=all_signal_names): + apps.DatadogDiagnostics('edx_arch_experiments.datadog_diagnostics', apps).ready() + + mock_log_info.assert_called_with( + "Logging lifecycle info for these celery signals: ['after_task_publish', " + "'before_task_publish', 'task_failure', 'task_internal_error', " + "'task_postrun', 'task_prerun', 'task_received', 'task_rejected', " + "'task_retry', 'task_revoked', 'task_success', 'task_unknown']" + ) + + @override_settings( + DATADOG_DIAGNOSTICS_CELERY_LOG_SIGNALS=',,,task_success, task_unknown, task_rejected, fake_signal' + ) + @patch('edx_arch_experiments.datadog_diagnostics.apps.log.info') + @patch('edx_arch_experiments.datadog_diagnostics.apps.log.warning') + @patch('edx_arch_experiments.datadog_diagnostics.apps._connect_celery_handler') + def test_register(self, mock_connect, mock_log_warning, mock_log_info): + """Test that signal connector is *called* as expected.""" + + with patch.dict('edx_arch_experiments.datadog_diagnostics.apps.CELERY_SIGNAL_CONFIG'): + # Add a bad entry to the config to test the signal lookup path + apps.CELERY_SIGNAL_CONFIG['fake_signal'] = lambda kwargs: {} + # Remove a real signal from the config to test the extractor lookup path + del apps.CELERY_SIGNAL_CONFIG['task_unknown'] + apps.DatadogDiagnostics('edx_arch_experiments.datadog_diagnostics', apps).ready() + + assert mock_connect.call_args_list == [ + call( + celery.signals.task_success, + 'task_success', + apps.CELERY_SIGNAL_CONFIG['task_success'], + ), + call( + celery.signals.task_rejected, + 'task_rejected', + apps.CELERY_SIGNAL_CONFIG['task_rejected'], + ), + ] + assert mock_log_warning.call_args_list == [ + call("Don't know how to extract info for Celery signal 'task_unknown'; ignoring."), + call("Could not connect receiver to unknown Celery signal 'fake_signal'"), + ] + mock_log_info.assert_called_with( + "Logging lifecycle info for these celery signals: ['task_success', 'task_rejected']" + ) + + @patch('edx_arch_experiments.datadog_diagnostics.apps.log.info') + @patch('edx_arch_experiments.datadog_diagnostics.apps.log.exception') + def test_handler(self, mock_log_exception, mock_log_info): + """Test that signal connector *behaves* as expected.""" + # Signal A will just do a straightforward data extraction from the kwargs. + # pylint: disable=protected-access + apps._connect_celery_handler( + signal_example_a, 'signal_example_a', + lambda kwargs: {'name': kwargs['info']['name']}, + ) + + # Signal B will have an extractor that goes bad on the 2nd and 3rd calls + b_called_times = 0 + + def b_extractor(kwargs): + nonlocal b_called_times + b_called_times += 1 + if b_called_times >= 2: + raise Exception("oops") + + return {'id': kwargs['the_id']} + + # pylint: disable=protected-access + apps._connect_celery_handler(signal_example_b, 'signal_example_b', b_extractor) + + # Send to B a few times to show that error logging only happens once + signal_example_b.send(sender='some_sender', the_id=42) + signal_example_b.send(sender='some_sender', the_id=42) + signal_example_b.send(sender='some_sender', the_id=42) + # And then send to A to show it still works + signal_example_a.send( + sender='some_sender', other='whatever', info={'name': "Alice"}, name='not this one', + ) + + mock_log_exception.assert_called_once_with( + "Error while extracting Celery signal info for 'signal_example_b'; " + "will not attempt for future calls to this signal." + ) + assert b_called_times == 2 + assert mock_log_info.call_args_list == [ + call("Celery signal called: 'signal_example_b' with name=None id=42"), + call("Celery signal called: 'signal_example_b' (skipped data extraction)"), + call("Celery signal called: 'signal_example_b' (skipped data extraction)"), + call("Celery signal called: 'signal_example_a' with name=Alice id=None"), + ] + + +signal_example_a = Signal() +signal_example_b = Signal() diff --git a/requirements/base.txt b/requirements/base.txt index 049726c..17ff075 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,13 +6,13 @@ # asgiref==3.8.1 # via django -attrs==23.2.0 +attrs==24.2.0 # via # jsonschema # referencing certifi==2024.7.4 # via requests -cffi==1.16.0 +cffi==1.17.0 # via # cryptography # pynacl @@ -26,7 +26,7 @@ code-annotations==1.8.0 # via edx-toggles cryptography==43.0.0 # via pyjwt -django==4.2.14 +django==4.2.15 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in @@ -79,7 +79,7 @@ jsonschema-specifications==2023.12.1 # via jsonschema markupsafe==2.1.5 # via jinja2 -newrelic==9.12.0 +newrelic==9.13.0 # via edx-django-utils pbr==6.0.0 # via stevedore @@ -97,7 +97,7 @@ pynacl==1.5.0 # via edx-django-utils python-slugify==8.0.4 # via code-annotations -pyyaml==6.0.1 +pyyaml==6.0.2 # via code-annotations referencing==0.35.1 # via @@ -105,7 +105,7 @@ referencing==0.35.1 # jsonschema-specifications requests==2.32.3 # via edx-drf-extensions -rpds-py==0.19.1 +rpds-py==0.20.0 # via # jsonschema # referencing @@ -128,5 +128,5 @@ urllib3==2.2.2 # via requests # The following packages are considered to be unsafe in a requirements file: -setuptools==72.1.0 +setuptools==73.0.1 # via -r requirements/base.in diff --git a/requirements/ci.txt b/requirements/ci.txt index 564c0f5..e39ed77 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -4,7 +4,7 @@ # # make upgrade # -cachetools==5.4.0 +cachetools==5.5.0 # via tox chardet==5.2.0 # via tox @@ -28,7 +28,7 @@ pluggy==1.5.0 # via tox pyproject-api==1.7.1 # via tox -tox==4.16.0 +tox==4.18.0 # via -r requirements/ci.in virtualenv==20.26.3 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 3606ea3..95f9801 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -4,6 +4,10 @@ # # make upgrade # +amqp==5.2.0 + # via + # -r requirements/quality.txt + # kombu asgiref==3.8.1 # via # -r requirements/quality.txt @@ -13,17 +17,19 @@ astroid==3.2.4 # -r requirements/quality.txt # pylint # pylint-celery -attrs==23.2.0 +attrs==24.2.0 # via # -r requirements/quality.txt - # cattrs - # ddtrace # jsonschema # referencing backports-tarfile==1.2.0 # via # -r requirements/quality.txt # jaraco-context +billiard==4.2.0 + # via + # -r requirements/quality.txt + # celery build==1.2.1 # via # -r requirements/pip-tools.txt @@ -32,19 +38,17 @@ bytecode==0.15.1 # via # -r requirements/quality.txt # ddtrace -cachetools==5.4.0 +cachetools==5.5.0 # via # -r requirements/ci.txt # tox -cattrs==23.2.3 - # via - # -r requirements/quality.txt - # ddtrace +celery==5.4.0 + # via -r requirements/quality.txt certifi==2024.7.4 # via # -r requirements/quality.txt # requests -cffi==1.16.0 +cffi==1.17.0 # via # -r requirements/quality.txt # cryptography @@ -62,15 +66,31 @@ click==8.1.7 # via # -r requirements/pip-tools.txt # -r requirements/quality.txt + # celery + # click-didyoumean # click-log + # click-plugins + # click-repl # code-annotations # edx-django-utils # edx-lint # pip-tools +click-didyoumean==0.3.1 + # via + # -r requirements/quality.txt + # celery click-log==0.4.0 # via # -r requirements/quality.txt # edx-lint +click-plugins==1.1.1 + # via + # -r requirements/quality.txt + # celery +click-repl==0.3.0 + # via + # -r requirements/quality.txt + # celery code-annotations==1.8.0 # via # -r requirements/quality.txt @@ -80,7 +100,7 @@ colorama==0.4.6 # via # -r requirements/ci.txt # tox -coverage[toml]==7.6.0 +coverage[toml]==7.6.1 # via # -r requirements/quality.txt # pytest-cov @@ -89,13 +109,9 @@ cryptography==43.0.0 # -r requirements/quality.txt # pyjwt # secretstorage -ddsketch==3.0.1 - # via - # -r requirements/quality.txt - # ddtrace ddt==1.7.2 # via -r requirements/quality.txt -ddtrace==2.10.1 +ddtrace==2.11.1 # via -r requirements/quality.txt deprecated==1.2.14 # via @@ -111,7 +127,7 @@ distlib==0.3.8 # via # -r requirements/ci.txt # virtualenv -django==4.2.14 +django==4.2.15 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/quality.txt @@ -183,9 +199,8 @@ idna==3.7 # via # -r requirements/quality.txt # requests -importlib-metadata==6.11.0 +importlib-metadata==8.0.0 # via - # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/quality.txt # keyring # opentelemetry-api @@ -202,7 +217,7 @@ jaraco-classes==3.4.0 # via # -r requirements/quality.txt # keyring -jaraco-context==5.3.0 +jaraco-context==6.0.1 # via # -r requirements/quality.txt # keyring @@ -226,11 +241,15 @@ jsonschema-specifications==2023.12.1 # via # -r requirements/quality.txt # jsonschema -keyring==25.2.1 +keyring==25.3.0 # via # -r requirements/quality.txt # twine -lxml==5.2.2 +kombu==5.4.0 + # via + # -r requirements/quality.txt + # celery +lxml==5.3.0 # via edx-i18n-tools markdown-it-py==3.0.0 # via @@ -248,12 +267,12 @@ mdurl==0.1.2 # via # -r requirements/quality.txt # markdown-it-py -more-itertools==10.3.0 +more-itertools==10.4.0 # via # -r requirements/quality.txt # jaraco-classes # jaraco-functools -newrelic==9.12.0 +newrelic==9.13.0 # via # -r requirements/quality.txt # edx-django-utils @@ -302,6 +321,10 @@ pluggy==1.5.0 # tox polib==1.2.0 # via edx-i18n-tools +prompt-toolkit==3.0.47 + # via + # -r requirements/quality.txt + # click-repl protobuf==5.27.3 # via # -r requirements/quality.txt @@ -310,7 +333,7 @@ psutil==6.0.0 # via # -r requirements/quality.txt # edx-django-utils -pycodestyle==2.12.0 +pycodestyle==2.12.1 # via -r requirements/quality.txt pycparser==2.22 # via @@ -375,11 +398,15 @@ pytest-cov==5.0.0 # via -r requirements/quality.txt pytest-django==4.8.0 # via -r requirements/quality.txt +python-dateutil==2.9.0.post0 + # via + # -r requirements/quality.txt + # celery python-slugify==8.0.4 # via # -r requirements/quality.txt # code-annotations -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -r requirements/quality.txt # code-annotations @@ -411,7 +438,7 @@ rich==13.7.1 # via # -r requirements/quality.txt # twine -rpds-py==0.19.1 +rpds-py==0.20.0 # via # -r requirements/quality.txt # jsonschema @@ -427,10 +454,9 @@ semantic-version==2.10.0 six==1.16.0 # via # -r requirements/quality.txt - # ddsketch - # ddtrace # edx-codejail # edx-lint + # python-dateutil snowballstemmer==2.2.0 # via # -r requirements/quality.txt @@ -449,11 +475,11 @@ text-unidecode==1.3 # via # -r requirements/quality.txt # python-slugify -tomlkit==0.13.0 +tomlkit==0.13.2 # via # -r requirements/quality.txt # pylint -tox==4.16.0 +tox==4.18.0 # via -r requirements/ci.txt twine==5.1.1 # via -r requirements/quality.txt @@ -462,16 +488,30 @@ typing-extensions==4.12.2 # -r requirements/quality.txt # ddtrace # edx-opaque-keys +tzdata==2024.1 + # via + # -r requirements/quality.txt + # celery urllib3==2.2.2 # via # -r requirements/quality.txt # requests # twine +vine==5.1.0 + # via + # -r requirements/quality.txt + # amqp + # celery + # kombu virtualenv==20.26.3 # via # -r requirements/ci.txt # tox -wheel==0.43.0 +wcwidth==0.2.13 + # via + # -r requirements/quality.txt + # prompt-toolkit +wheel==0.44.0 # via # -r requirements/pip-tools.txt # pip-tools @@ -483,7 +523,7 @@ xmltodict==0.13.0 # via # -r requirements/quality.txt # ddtrace -zipp==3.19.2 +zipp==3.20.0 # via # -r requirements/quality.txt # importlib-metadata @@ -493,7 +533,7 @@ pip==24.2 # via # -r requirements/pip-tools.txt # pip-tools -setuptools==72.1.0 +setuptools==73.0.1 # via # -r requirements/pip-tools.txt # -r requirements/quality.txt diff --git a/requirements/doc.txt b/requirements/doc.txt index 5eac2f6..0566cd9 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -6,32 +6,36 @@ # alabaster==0.7.16 # via sphinx +amqp==5.2.0 + # via + # -r requirements/test.txt + # kombu asgiref==3.8.1 # via # -r requirements/test.txt # django -attrs==23.2.0 +attrs==24.2.0 # via # -r requirements/test.txt - # cattrs - # ddtrace # jsonschema # referencing -babel==2.15.0 +babel==2.16.0 # via sphinx -bytecode==0.15.1 +billiard==4.2.0 # via # -r requirements/test.txt - # ddtrace -cattrs==23.2.3 + # celery +bytecode==0.15.1 # via # -r requirements/test.txt # ddtrace +celery==5.4.0 + # via -r requirements/test.txt certifi==2024.7.4 # via # -r requirements/test.txt # requests -cffi==1.16.0 +cffi==1.17.0 # via # -r requirements/test.txt # cryptography @@ -43,13 +47,29 @@ charset-normalizer==3.3.2 click==8.1.7 # via # -r requirements/test.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # code-annotations # edx-django-utils +click-didyoumean==0.3.1 + # via + # -r requirements/test.txt + # celery +click-plugins==1.1.1 + # via + # -r requirements/test.txt + # celery +click-repl==0.3.0 + # via + # -r requirements/test.txt + # celery code-annotations==1.8.0 # via # -r requirements/test.txt # edx-toggles -coverage[toml]==7.6.0 +coverage[toml]==7.6.1 # via # -r requirements/test.txt # pytest-cov @@ -57,19 +77,15 @@ cryptography==43.0.0 # via # -r requirements/test.txt # pyjwt -ddsketch==3.0.1 - # via - # -r requirements/test.txt - # ddtrace ddt==1.7.2 # via -r requirements/test.txt -ddtrace==2.10.1 +ddtrace==2.11.1 # via -r requirements/test.txt deprecated==1.2.14 # via # -r requirements/test.txt # opentelemetry-api -django==4.2.14 +django==4.2.15 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt @@ -139,9 +155,8 @@ idna==3.7 # requests imagesize==1.4.1 # via sphinx -importlib-metadata==6.11.0 +importlib-metadata==8.0.0 # via - # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt # opentelemetry-api iniconfig==2.0.0 @@ -159,11 +174,15 @@ jsonschema-specifications==2023.12.1 # via # -r requirements/test.txt # jsonschema +kombu==5.4.0 + # via + # -r requirements/test.txt + # celery markupsafe==2.1.5 # via # -r requirements/test.txt # jinja2 -newrelic==9.12.0 +newrelic==9.13.0 # via # -r requirements/test.txt # edx-django-utils @@ -186,6 +205,10 @@ pluggy==1.5.0 # via # -r requirements/test.txt # pytest +prompt-toolkit==3.0.47 + # via + # -r requirements/test.txt + # click-repl protobuf==5.27.3 # via # -r requirements/test.txt @@ -225,11 +248,15 @@ pytest-cov==5.0.0 # via -r requirements/test.txt pytest-django==4.8.0 # via -r requirements/test.txt +python-dateutil==2.9.0.post0 + # via + # -r requirements/test.txt + # celery python-slugify==8.0.4 # via # -r requirements/test.txt # code-annotations -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -r requirements/test.txt # code-annotations @@ -247,7 +274,7 @@ requests==2.32.3 # sphinx restructuredtext-lint==1.4.0 # via doc8 -rpds-py==0.19.1 +rpds-py==0.20.0 # via # -r requirements/test.txt # jsonschema @@ -259,10 +286,9 @@ semantic-version==2.10.0 six==1.16.0 # via # -r requirements/test.txt - # ddsketch - # ddtrace # edx-codejail # edx-sphinx-theme + # python-dateutil snowballstemmer==2.2.0 # via sphinx sphinx==5.3.0 @@ -301,10 +327,24 @@ typing-extensions==4.12.2 # -r requirements/test.txt # ddtrace # edx-opaque-keys +tzdata==2024.1 + # via + # -r requirements/test.txt + # celery urllib3==2.2.2 # via # -r requirements/test.txt # requests +vine==5.1.0 + # via + # -r requirements/test.txt + # amqp + # celery + # kombu +wcwidth==0.2.13 + # via + # -r requirements/test.txt + # prompt-toolkit wrapt==1.16.0 # via # -r requirements/test.txt @@ -313,11 +353,11 @@ xmltodict==0.13.0 # via # -r requirements/test.txt # ddtrace -zipp==3.19.2 +zipp==3.20.0 # via # -r requirements/test.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==72.1.0 +setuptools==73.0.1 # via -r requirements/test.txt diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index b0c1866..73cb449 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -16,11 +16,11 @@ pyproject-hooks==1.1.0 # via # build # pip-tools -wheel==0.43.0 +wheel==0.44.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: pip==24.2 # via pip-tools -setuptools==72.1.0 +setuptools==73.0.1 # via pip-tools diff --git a/requirements/pip.txt b/requirements/pip.txt index 54b0571..f0fca18 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -4,11 +4,11 @@ # # make upgrade # -wheel==0.43.0 +wheel==0.44.0 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: pip==24.2 # via -r requirements/pip.in -setuptools==72.1.0 +setuptools==73.0.1 # via -r requirements/pip.in diff --git a/requirements/quality.txt b/requirements/quality.txt index e10c684..701873d 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -4,6 +4,10 @@ # # make upgrade # +amqp==5.2.0 + # via + # -r requirements/test.txt + # kombu asgiref==3.8.1 # via # -r requirements/test.txt @@ -12,28 +16,28 @@ astroid==3.2.4 # via # pylint # pylint-celery -attrs==23.2.0 +attrs==24.2.0 # via # -r requirements/test.txt - # cattrs - # ddtrace # jsonschema # referencing backports-tarfile==1.2.0 # via jaraco-context -bytecode==0.15.1 +billiard==4.2.0 # via # -r requirements/test.txt - # ddtrace -cattrs==23.2.3 + # celery +bytecode==0.15.1 # via # -r requirements/test.txt # ddtrace +celery==5.4.0 + # via -r requirements/test.txt certifi==2024.7.4 # via # -r requirements/test.txt # requests -cffi==1.16.0 +cffi==1.17.0 # via # -r requirements/test.txt # cryptography @@ -45,18 +49,34 @@ charset-normalizer==3.3.2 click==8.1.7 # via # -r requirements/test.txt + # celery + # click-didyoumean # click-log + # click-plugins + # click-repl # code-annotations # edx-django-utils # edx-lint +click-didyoumean==0.3.1 + # via + # -r requirements/test.txt + # celery click-log==0.4.0 # via edx-lint +click-plugins==1.1.1 + # via + # -r requirements/test.txt + # celery +click-repl==0.3.0 + # via + # -r requirements/test.txt + # celery code-annotations==1.8.0 # via # -r requirements/test.txt # edx-lint # edx-toggles -coverage[toml]==7.6.0 +coverage[toml]==7.6.1 # via # -r requirements/test.txt # pytest-cov @@ -65,13 +85,9 @@ cryptography==43.0.0 # -r requirements/test.txt # pyjwt # secretstorage -ddsketch==3.0.1 - # via - # -r requirements/test.txt - # ddtrace ddt==1.7.2 # via -r requirements/test.txt -ddtrace==2.10.1 +ddtrace==2.11.1 # via -r requirements/test.txt deprecated==1.2.14 # via @@ -79,7 +95,7 @@ deprecated==1.2.14 # opentelemetry-api dill==0.3.8 # via pylint -django==4.2.14 +django==4.2.15 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt @@ -141,9 +157,8 @@ idna==3.7 # via # -r requirements/test.txt # requests -importlib-metadata==6.11.0 +importlib-metadata==8.0.0 # via - # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt # keyring # opentelemetry-api @@ -158,7 +173,7 @@ isort==5.13.2 # pylint jaraco-classes==3.4.0 # via keyring -jaraco-context==5.3.0 +jaraco-context==6.0.1 # via keyring jaraco-functools==4.0.2 # via keyring @@ -176,8 +191,12 @@ jsonschema-specifications==2023.12.1 # via # -r requirements/test.txt # jsonschema -keyring==25.2.1 +keyring==25.3.0 # via twine +kombu==5.4.0 + # via + # -r requirements/test.txt + # celery markdown-it-py==3.0.0 # via rich markupsafe==2.1.5 @@ -188,11 +207,11 @@ mccabe==0.7.0 # via pylint mdurl==0.1.2 # via markdown-it-py -more-itertools==10.3.0 +more-itertools==10.4.0 # via # jaraco-classes # jaraco-functools -newrelic==9.12.0 +newrelic==9.13.0 # via # -r requirements/test.txt # edx-django-utils @@ -218,6 +237,10 @@ pluggy==1.5.0 # via # -r requirements/test.txt # pytest +prompt-toolkit==3.0.47 + # via + # -r requirements/test.txt + # click-repl protobuf==5.27.3 # via # -r requirements/test.txt @@ -226,7 +249,7 @@ psutil==6.0.0 # via # -r requirements/test.txt # edx-django-utils -pycodestyle==2.12.0 +pycodestyle==2.12.1 # via -r requirements/quality.in pycparser==2.22 # via @@ -274,11 +297,15 @@ pytest-cov==5.0.0 # via -r requirements/test.txt pytest-django==4.8.0 # via -r requirements/test.txt +python-dateutil==2.9.0.post0 + # via + # -r requirements/test.txt + # celery python-slugify==8.0.4 # via # -r requirements/test.txt # code-annotations -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -r requirements/test.txt # code-annotations @@ -301,7 +328,7 @@ rfc3986==2.0.0 # via twine rich==13.7.1 # via twine -rpds-py==0.19.1 +rpds-py==0.20.0 # via # -r requirements/test.txt # jsonschema @@ -315,10 +342,9 @@ semantic-version==2.10.0 six==1.16.0 # via # -r requirements/test.txt - # ddsketch - # ddtrace # edx-codejail # edx-lint + # python-dateutil snowballstemmer==2.2.0 # via pydocstyle sqlparse==0.5.1 @@ -335,7 +361,7 @@ text-unidecode==1.3 # via # -r requirements/test.txt # python-slugify -tomlkit==0.13.0 +tomlkit==0.13.2 # via pylint twine==5.1.1 # via -r requirements/quality.in @@ -344,11 +370,25 @@ typing-extensions==4.12.2 # -r requirements/test.txt # ddtrace # edx-opaque-keys +tzdata==2024.1 + # via + # -r requirements/test.txt + # celery urllib3==2.2.2 # via # -r requirements/test.txt # requests # twine +vine==5.1.0 + # via + # -r requirements/test.txt + # amqp + # celery + # kombu +wcwidth==0.2.13 + # via + # -r requirements/test.txt + # prompt-toolkit wrapt==1.16.0 # via # -r requirements/test.txt @@ -357,11 +397,11 @@ xmltodict==0.13.0 # via # -r requirements/test.txt # ddtrace -zipp==3.19.2 +zipp==3.20.0 # via # -r requirements/test.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==72.1.0 +setuptools==73.0.1 # via -r requirements/test.txt diff --git a/requirements/scripts.txt b/requirements/scripts.txt index c7740dd..23951ac 100644 --- a/requirements/scripts.txt +++ b/requirements/scripts.txt @@ -8,19 +8,19 @@ asgiref==3.8.1 # via # -r requirements/base.txt # django -attrs==23.2.0 +attrs==24.2.0 # via # -r requirements/base.txt # jsonschema # openedx-events # referencing -avro==1.11.3 +avro==1.12.0 # via confluent-kafka certifi==2024.7.4 # via # -r requirements/base.txt # requests -cffi==1.16.0 +cffi==1.17.0 # via # -r requirements/base.txt # cryptography @@ -44,7 +44,7 @@ cryptography==43.0.0 # via # -r requirements/base.txt # pyjwt -django==4.2.14 +django==4.2.15 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.txt @@ -94,7 +94,7 @@ edx-django-utils==5.15.0 # openedx-events edx-drf-extensions==10.3.0 # via -r requirements/base.txt -edx-event-bus-kafka==5.8.0 +edx-event-bus-kafka==5.8.1 # via -r requirements/scripts.in edx-opaque-keys[django]==2.10.0 # via @@ -128,7 +128,7 @@ markupsafe==2.1.5 # via # -r requirements/base.txt # jinja2 -newrelic==9.12.0 +newrelic==9.13.0 # via # -r requirements/base.txt # edx-django-utils @@ -163,7 +163,7 @@ python-slugify==8.0.4 # via # -r requirements/base.txt # code-annotations -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -r requirements/base.txt # code-annotations @@ -177,7 +177,7 @@ requests==2.32.3 # -r requirements/base.txt # confluent-kafka # edx-drf-extensions -rpds-py==0.19.1 +rpds-py==0.20.0 # via # -r requirements/base.txt # jsonschema @@ -215,5 +215,5 @@ urllib3==2.2.2 # requests # The following packages are considered to be unsafe in a requirements file: -setuptools==72.1.0 +setuptools==73.0.1 # via -r requirements/base.txt diff --git a/requirements/test.in b/requirements/test.in index 9974b96..45939a0 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -8,3 +8,4 @@ pytest-django # pytest extension for better Django support code-annotations # provides commands used by the pii_check make target. ddt # data-driven tests ddtrace # Required for testing datadog_diagnostics app and middleware +celery # Required for testing datadog_diagnostics app diff --git a/requirements/test.txt b/requirements/test.txt index 3d2ee70..053bcc2 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,26 +4,28 @@ # # make upgrade # +amqp==5.2.0 + # via kombu asgiref==3.8.1 # via # -r requirements/base.txt # django -attrs==23.2.0 +attrs==24.2.0 # via # -r requirements/base.txt - # cattrs - # ddtrace # jsonschema # referencing +billiard==4.2.0 + # via celery bytecode==0.15.1 # via ddtrace -cattrs==23.2.3 - # via ddtrace +celery==5.4.0 + # via -r requirements/test.in certifi==2024.7.4 # via # -r requirements/base.txt # requests -cffi==1.16.0 +cffi==1.17.0 # via # -r requirements/base.txt # cryptography @@ -35,24 +37,32 @@ charset-normalizer==3.3.2 click==8.1.7 # via # -r requirements/base.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # code-annotations # edx-django-utils +click-didyoumean==0.3.1 + # via celery +click-plugins==1.1.1 + # via celery +click-repl==0.3.0 + # via celery code-annotations==1.8.0 # via # -r requirements/base.txt # -r requirements/test.in # edx-toggles -coverage[toml]==7.6.0 +coverage[toml]==7.6.1 # via pytest-cov cryptography==43.0.0 # via # -r requirements/base.txt # pyjwt -ddsketch==3.0.1 - # via ddtrace ddt==1.7.2 # via -r requirements/test.in -ddtrace==2.10.1 +ddtrace==2.11.1 # via -r requirements/test.in deprecated==1.2.14 # via opentelemetry-api @@ -111,10 +121,8 @@ idna==3.7 # via # -r requirements/base.txt # requests -importlib-metadata==6.11.0 - # via - # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt - # opentelemetry-api +importlib-metadata==8.0.0 + # via opentelemetry-api iniconfig==2.0.0 # via pytest jinja2==3.1.4 @@ -127,11 +135,13 @@ jsonschema-specifications==2023.12.1 # via # -r requirements/base.txt # jsonschema +kombu==5.4.0 + # via celery markupsafe==2.1.5 # via # -r requirements/base.txt # jinja2 -newrelic==9.12.0 +newrelic==9.13.0 # via # -r requirements/base.txt # edx-django-utils @@ -145,6 +155,8 @@ pbr==6.0.0 # stevedore pluggy==1.5.0 # via pytest +prompt-toolkit==3.0.47 + # via click-repl protobuf==5.27.3 # via ddtrace psutil==6.0.0 @@ -176,11 +188,13 @@ pytest-cov==5.0.0 # via -r requirements/test.in pytest-django==4.8.0 # via -r requirements/test.in +python-dateutil==2.9.0.post0 + # via celery python-slugify==8.0.4 # via # -r requirements/base.txt # code-annotations -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -r requirements/base.txt # code-annotations @@ -193,7 +207,7 @@ requests==2.32.3 # via # -r requirements/base.txt # edx-drf-extensions -rpds-py==0.19.1 +rpds-py==0.20.0 # via # -r requirements/base.txt # jsonschema @@ -205,9 +219,8 @@ semantic-version==2.10.0 six==1.16.0 # via # -r requirements/base.txt - # ddsketch - # ddtrace # edx-codejail + # python-dateutil sqlparse==0.5.1 # via # -r requirements/base.txt @@ -227,17 +240,26 @@ typing-extensions==4.12.2 # -r requirements/base.txt # ddtrace # edx-opaque-keys +tzdata==2024.1 + # via celery urllib3==2.2.2 # via # -r requirements/base.txt # requests +vine==5.1.0 + # via + # amqp + # celery + # kombu +wcwidth==0.2.13 + # via prompt-toolkit wrapt==1.16.0 # via deprecated xmltodict==0.13.0 # via ddtrace -zipp==3.19.2 +zipp==3.20.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==72.1.0 +setuptools==73.0.1 # via -r requirements/base.txt