From cc657b3df9eed8665ed45026e273b6f55404d5b1 Mon Sep 17 00:00:00 2001 From: Jeff Triplett Date: Thu, 19 Oct 2023 11:23:43 -0400 Subject: [PATCH] :gear: Adds Django 4.2 support and fixes trove classifiers (#363) * :gear: Adds Django 4.2 support and fixes trove classifiers * :shirts: Runs black on code to make CI happy * :shirts: Runs isort on code to make CI happy * :green_heart: Updates test to match exception copy * :pencil: mirrors old README.rst to make docs work Not a great solution, but a workaround. * :gear: Adds pip cache support * :fire: Removes 4.0 from grid because it's no longer supported * :gear: Updates cache-key This isn't really used, but I think having caching is a win * :gear: Adds missing file * :fire: Removes cache line * Fix storages deprecation warnings (#372) * :gear: Adds Django 4.2 support and fixes trove classifiers * :shirts: Runs black on code to make CI happy * :shirts: Runs isort on code to make CI happy * :green_heart: Updates test to match exception copy * :pencil: mirrors old README.rst to make docs work Not a great solution, but a workaround. * :gear: Adds pip cache support * :fire: Removes 4.0 from grid because it's no longer supported * :gear: Updates cache-key This isn't really used, but I think having caching is a win * :gear: Adds missing file * :fire: Removes cache line * Fix storages deprecation warnings --------- Co-authored-by: Jeff Triplett * Fix isort ordering --------- Co-authored-by: Bruno Alla Co-authored-by: Frank Wiles --- .github/workflows/ci.yml | 8 +- docs/readme.rst | 292 ++++++++++++++++++++- health_check/contrib/rabbitmq/backends.py | 4 +- health_check/db/migrations/0001_initial.py | 1 - health_check/storage/backends.py | 21 +- pyproject.toml | 9 + setup.cfg | 5 +- tests/test_cache.py | 8 +- tests/test_storage.py | 53 +++- 9 files changed, 381 insertions(+), 20 deletions(-) create mode 100644 pyproject.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aefe91b0..542961b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version: "3.x" + cache: 'pip' + cache-dependency-path: '**/pyproject.toml' - run: python -m pip install --upgrade pip build wheel twine - run: python -m build --sdist --wheel - run: python -m twine check dist/* @@ -49,6 +51,8 @@ jobs: uses: actions/setup-python@v4 with: python-version: "3.10" + cache: 'pip' + cache-dependency-path: '**/pyproject.toml' - run: python -m pip install -e .[docs] - run: python -m sphinx -b html -W docs docs/_build @@ -65,14 +69,16 @@ jobs: django-version: - "2.2" - "3.2" - - "4.0" - "4.1" + - "4.2" steps: - uses: actions/checkout@v3 - name: Setup Python version ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: '**/pyproject.toml' - run: python -m pip install .[test] - run: python -m pip install Django~="${{ matrix.django-version }}.0" - run: python -m pytest diff --git a/docs/readme.rst b/docs/readme.rst index 72a33558..11f0778f 100644 --- a/docs/readme.rst +++ b/docs/readme.rst @@ -1 +1,291 @@ -.. include:: ../README.rst +django-health-check +=================== + +|version| |pyversion| |djversion| |license| + +This project checks for various conditions and provides reports when +anomalous behavior is detected. + +The following health checks are bundled with this project: + +- cache +- database +- storage +- disk and memory utilization (via ``psutil``) +- AWS S3 storage +- Celery task queue +- Celery ping +- RabbitMQ +- Migrations + +Writing your own custom health checks is also very quick and easy. + +We also like contributions, so don’t be afraid to make a pull request. + +Use Cases +--------- + +The primary intended use case is to monitor conditions via HTTP(S), with +responses available in HTML and JSON formats. When you get back a +response that includes one or more problems, you can then decide the +appropriate course of action, which could include generating +notifications and/or automating the replacement of a failing node with a +new one. If you are monitoring health in a high-availability environment +with a load balancer that returns responses from multiple nodes, please +note that certain checks (e.g., disk and memory usage) will return +responses specific to the node selected by the load balancer. + +Supported Versions +------------------ + +We officially only support the latest version of Python as well as the +latest version of Django and the latest Django LTS version. + +Installation +------------ + +First, install the ``django-health-check`` package: + +.. code:: shell + + $ pip install django-health-check + +Add the health checker to a URL you want to use: + +.. code:: python + + urlpatterns = [ + # ... + url(r'^ht/', include('health_check.urls')), + ] + +Add the ``health_check`` applications to your ``INSTALLED_APPS``: + +.. code:: python + + INSTALLED_APPS = [ + # ... + 'health_check', # required + 'health_check.db', # stock Django health checkers + 'health_check.cache', + 'health_check.storage', + 'health_check.contrib.migrations', + 'health_check.contrib.celery', # requires celery + 'health_check.contrib.celery_ping', # requires celery + 'health_check.contrib.psutil', # disk and memory utilization; requires psutil + 'health_check.contrib.s3boto3_storage', # requires boto3 and S3BotoStorage backend + 'health_check.contrib.rabbitmq', # requires RabbitMQ broker + 'health_check.contrib.redis', # requires Redis broker + ] + +**Note:** If using ``boto 2.x.x`` use +``health_check.contrib.s3boto_storage`` + +(Optional) If using the ``psutil`` app, you can configure disk and +memory threshold settings; otherwise below defaults are assumed. If you +want to disable one of these checks, set its value to ``None``. + +.. code:: python + + HEALTH_CHECK = { + 'DISK_USAGE_MAX': 90, # percent + 'MEMORY_MIN': 100, # in MB + } + +If using the DB check, run migrations: + +.. code:: shell + + $ django-admin migrate + +To use the RabbitMQ healthcheck, please make sure that there is a +variable named ``BROKER_URL`` on django.conf.settings with the required +format to connect to your rabbit server. For example: + +.. code:: python + + BROKER_URL = "amqp://myuser:mypassword@localhost:5672/myvhost" + +To use the Redis healthcheck, please make sure that there is a variable +named ``REDIS_URL`` on django.conf.settings with the required format to +connect to your redis server. For example: + +.. code:: python + + REDIS_URL = "redis://localhost:6370" + +The cache healthcheck tries to write and read a specific key within the +cache backend. It can be customized by setting ``HEALTHCHECK_CACHE_KEY`` +to another value: + +.. code:: python + + HEALTHCHECK_CACHE_KEY = "custom_healthcheck_key" + +Setting up monitoring +--------------------- + +You can use tools like Pingdom, StatusCake or other uptime robots to +monitor service status. The ``/ht/`` endpoint will respond with an HTTP +200 if all checks passed and with an HTTP 500 if any of the tests +failed. Getting machine-readable JSON reports + +If you want machine-readable status reports you can request the ``/ht/`` +endpoint with the ``Accept`` HTTP header set to ``application/json`` or +pass ``format=json`` as a query parameter. + +The backend will return a JSON response: + +.. code:: shell + + $ curl -v -X GET -H "Accept: application/json" http://www.example.com/ht/ + + > GET /ht/ HTTP/1.1 + > Host: www.example.com + > Accept: application/json + > + < HTTP/1.1 200 OK + < Content-Type: application/json + + { + "CacheBackend": "working", + "DatabaseBackend": "working", + "S3BotoStorageHealthCheck": "working" + } + + $ curl -v -X GET http://www.example.com/ht/?format=json + + > GET /ht/?format=json HTTP/1.1 + > Host: www.example.com + > + < HTTP/1.1 200 OK + < Content-Type: application/json + + { + "CacheBackend": "working", + "DatabaseBackend": "working", + "S3BotoStorageHealthCheck": "working" + } + +Writing a custom health check +----------------------------- + +Writing a health check is quick and easy: + +.. code:: python + + from health_check.backends import BaseHealthCheckBackend + + class MyHealthCheckBackend(BaseHealthCheckBackend): + #: The status endpoints will respond with a 200 status code + #: even if the check errors. + critical_service = False + + def check_status(self): + # The test code goes here. + # You can use `self.add_error` or + # raise a `HealthCheckException`, + # similar to Django's form validation. + pass + + def identifier(self): + return self.__class__.__name__ # Display name on the endpoint. + +After writing a custom checker, register it in your app configuration: + +.. code:: python + + from django.apps import AppConfig + + from health_check.plugins import plugin_dir + + class MyAppConfig(AppConfig): + name = 'my_app' + + def ready(self): + from .backends import MyHealthCheckBackend + plugin_dir.register(MyHealthCheckBackend) + +Make sure the application you write the checker into is registered in +your ``INSTALLED_APPS``. + +Customizing output +------------------ + +You can customize HTML or JSON rendering by inheriting from ``MainView`` +in ``health_check.views`` and customizing the ``template_name``, +``get``, ``render_to_response`` and ``render_to_response_json`` +properties: + +.. code:: python + + # views.py + from health_check.views import MainView + + class HealthCheckCustomView(MainView): + template_name = 'myapp/health_check_dashboard.html' # customize the used templates + + def get(self, request, *args, **kwargs): + plugins = [] + status = 200 # needs to be filled status you need + # ... + if 'application/json' in request.META.get('HTTP_ACCEPT', ''): + return self.render_to_response_json(plugins, status) + return self.render_to_response(plugins, status) + + def render_to_response(self, plugins, status): # customize HTML output + return HttpResponse('COOL' if status == 200 else 'SWEATY', status=status) + + def render_to_response_json(self, plugins, status): # customize JSON output + return JsonResponse( + {str(p.identifier()): 'COOL' if status == 200 else 'SWEATY' for p in plugins}, + status=status + ) + + # urls.py + import views + + urlpatterns = [ + # ... + url(r'^ht/$', views.HealthCheckCustomView.as_view(), name='health_check_custom'), + ] + +Django command +-------------- + +You can run the Django command ``health_check`` to perform your health +checks via the command line, or periodically with a cron, as follow: + +.. code:: shell + + django-admin health_check + +This should yield the following output: + +:: + + DatabaseHealthCheck ... working + CustomHealthCheck ... unavailable: Something went wrong! + +Similar to the http version, a critical error will cause the command to +quit with the exit code ``1``. + +Other resources +--------------- + +- `django-watchman `__ + is a package that does some of the same things in a slightly + different way. +- See this + `weblog `__ + about configuring Django and health checking with AWS Elastic Load + Balancer. + +.. |version| image:: https://img.shields.io/pypi/v/django-health-check.svg + :target: https://pypi.python.org/pypi/django-health-check/ +.. |pyversion| image:: https://img.shields.io/pypi/pyversions/django-health-check.svg + :target: https://pypi.python.org/pypi/django-health-check/ +.. |djversion| image:: https://img.shields.io/pypi/djversions/django-health-check.svg + :target: https://pypi.python.org/pypi/django-health-check/ +.. |license| image:: https://img.shields.io/badge/license-MIT-blue.svg + :target: https://pypi.python.org/pypi/django-health-check/ diff --git a/health_check/contrib/rabbitmq/backends.py b/health_check/contrib/rabbitmq/backends.py index 63a9a82c..adaff36b 100644 --- a/health_check/contrib/rabbitmq/backends.py +++ b/health_check/contrib/rabbitmq/backends.py @@ -19,7 +19,9 @@ def check_status(self): """Check RabbitMQ service by opening and closing a broker channel.""" logger.debug("Checking for a broker_url on django settings...") - broker_url_setting_key = f"{self.namespace}_BROKER_URL" if self.namespace else "BROKER_URL" + broker_url_setting_key = ( + f"{self.namespace}_BROKER_URL" if self.namespace else "BROKER_URL" + ) broker_url = getattr(settings, broker_url_setting_key, None) logger.debug("Got %s as the broker_url. Connecting to rabbit...", broker_url) diff --git a/health_check/db/migrations/0001_initial.py b/health_check/db/migrations/0001_initial.py index 872a387b..8eda0443 100644 --- a/health_check/db/migrations/0001_initial.py +++ b/health_check/db/migrations/0001_initial.py @@ -5,7 +5,6 @@ class Migration(migrations.Migration): - initial = True replaces = [ diff --git a/health_check/storage/backends.py b/health_check/storage/backends.py index 16024bcf..5920b133 100644 --- a/health_check/storage/backends.py +++ b/health_check/storage/backends.py @@ -1,8 +1,13 @@ import uuid +import django from django.conf import settings from django.core.files.base import ContentFile -from django.core.files.storage import get_storage_class + +if django.VERSION >= (4, 2): + from django.core.files.storage import InvalidStorageError, storages +else: + from django.core.files.storage import get_storage_class from health_check.backends import BaseHealthCheckBackend from health_check.exceptions import ServiceUnavailable @@ -22,13 +27,20 @@ class MyStorageHealthCheck(StorageHealthCheck): (e.g 'django.core.files.storage.FileSystemStorage') or a Storage instance. """ + storage_alias = None storage = None def get_storage(self): - if isinstance(self.storage, str): - return get_storage_class(self.storage)() + if django.VERSION >= (4, 2): + try: + return storages[self.storage_alias] + except InvalidStorageError: + return None else: - return self.storage + if isinstance(self.storage, str): + return get_storage_class(self.storage)() + else: + return self.storage def get_file_name(self): return "health_check_storage_test/test-%s.txt" % uuid.uuid4() @@ -68,4 +80,5 @@ def check_status(self): class DefaultFileStorageHealthCheck(StorageHealthCheck): + storage_alias = "default" storage = settings.DEFAULT_FILE_STORAGE diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..cac98090 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,9 @@ +[tool.black] +target-version = ["py38"] + +[tool.pytest.ini_options] +addopts = "--nomigrations --reuse-db --strict-markers" +DJANGO_SETTINGS_MODULE = "tests.testapp.settings" +norecursedirs = ".git" +python_files = "test_*.py" +xfail_strict = true diff --git a/setup.cfg b/setup.cfg index 025b6833..fe59d1f8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,6 +16,7 @@ classifier = Framework :: Django :: 3.2 Framework :: Django :: 4.0 Framework :: Django :: 4.1 + Framework :: Django :: 4.2 Intended Audience :: Developers License :: OSI Approved :: MIT License Operating System :: OS Independent @@ -27,10 +28,6 @@ classifier = Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Topic :: Software Development :: Quality Assurance - Framework :: Django - Framework :: Django :: 2.2 - Framework :: Django :: 4.0 - Framework :: Django :: 4.1 Topic :: System :: Logging Topic :: System :: Monitoring Topic :: Utilities diff --git a/tests/test_cache.py b/tests/test_cache.py index 1d311377..121dc26d 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -71,9 +71,7 @@ def test_multiple_backends_check_broken(self): cache_backend = CacheBackend("broken") cache_backend.run_check() self.assertTrue(cache_backend.errors) - self.assertIn( - "unavailable: Cache key does not match", cache_backend.pretty_status() - ) + self.assertIn("does not match", cache_backend.pretty_status()) # check_status should raise ServiceUnavailable when values at cache key do not match @patch( @@ -83,9 +81,7 @@ def test_set_fails(self): cache_backend = CacheBackend() cache_backend.run_check() self.assertTrue(cache_backend.errors) - self.assertIn( - "unavailable: Cache key does not match", cache_backend.pretty_status() - ) + self.assertIn("does not match", cache_backend.pretty_status()) # check_status should catch generic exceptions raised by set and convert to ServiceUnavailable @patch( diff --git a/tests/test_storage.py b/tests/test_storage.py index f8eb90b3..475d98a3 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -1,5 +1,7 @@ +import unittest from unittest import mock +import django from django.core.files.storage import Storage from django.test import TestCase @@ -10,6 +12,10 @@ ) +class CustomStorage(Storage): + pass + + class MockStorage(Storage): """ A Mock Storage backend used for testing. @@ -68,6 +74,25 @@ def test_get_storage(self): default_storage = DefaultFileStorageHealthCheck() self.assertIsInstance(default_storage.get_storage(), Storage) + @unittest.skipUnless((4, 2) <= django.VERSION < (5, 0), "Only for Django 4.2 - 5.0") + def test_get_storage_django_between_42_and_50(self): + """Check that the old DEFAULT_FILE_STORAGE setting keeps being supported.""" + # Note: this test doesn't work on Django<4.2 because the setting value is + # evaluated when the class attribute DefaultFileStorageHealthCheck.store is + # read, which is at import time, before we can mock the setting. + with self.settings(DEFAULT_FILE_STORAGE="tests.test_storage.CustomStorage"): + default_storage = DefaultFileStorageHealthCheck() + self.assertIsInstance(default_storage.get_storage(), CustomStorage) + + @unittest.skipUnless((4, 2) <= django.VERSION, "Django 4.2+ required") + def test_get_storage_django_42_plus(self): + """Check that the new STORAGES setting is supported.""" + with self.settings( + STORAGES={"default": {"BACKEND": "tests.test_storage.CustomStorage"}} + ): + default_storage = DefaultFileStorageHealthCheck() + self.assertIsInstance(default_storage.get_storage(), CustomStorage) + @mock.patch( "health_check.storage.backends.DefaultFileStorageHealthCheck.storage", MockStorage(), @@ -88,21 +113,45 @@ def test_check_status_working(self): ): self.assertTrue(default_storage_health.check_status()) + @unittest.skipUnless(django.VERSION <= (4, 1), "Only for Django 4.1 and earlier") @mock.patch( "health_check.storage.backends.DefaultFileStorageHealthCheck.storage", MockStorage(saves=False), ) - def test_file_does_not_exist(self): + def test_file_does_not_exist_django_41_earlier(self): + """Test check_status raises ServiceUnavailable when file is not saved.""" + default_storage_health = DefaultFileStorageHealthCheck() + with self.assertRaises(ServiceUnavailable): + default_storage_health.check_status() + + @unittest.skipUnless((4, 2) <= django.VERSION, "Only for Django 4.2+") + @mock.patch( + "health_check.storage.backends.storages", + {"default": MockStorage(saves=False)}, + ) + def test_file_does_not_exist_django_42_plus(self): """Test check_status raises ServiceUnavailable when file is not saved.""" default_storage_health = DefaultFileStorageHealthCheck() with self.assertRaises(ServiceUnavailable): default_storage_health.check_status() + @unittest.skipUnless(django.VERSION <= (4, 1), "Only for Django 4.1 and earlier") @mock.patch( "health_check.storage.backends.DefaultFileStorageHealthCheck.storage", MockStorage(deletes=False), ) - def test_file_not_deleted(self): + def test_file_not_deleted_django_41_earlier(self): + """Test check_status raises ServiceUnavailable when file is not deleted.""" + default_storage_health = DefaultFileStorageHealthCheck() + with self.assertRaises(ServiceUnavailable): + default_storage_health.check_status() + + @unittest.skipUnless((4, 2) <= django.VERSION, "Only for Django 4.2+") + @mock.patch( + "health_check.storage.backends.storages", + {"default": MockStorage(deletes=False)}, + ) + def test_file_not_deleted_django_42_plus(self): """Test check_status raises ServiceUnavailable when file is not deleted.""" default_storage_health = DefaultFileStorageHealthCheck() with self.assertRaises(ServiceUnavailable):