From ce6b10d78cb2caf79d855fecf6b905ec0600520a Mon Sep 17 00:00:00 2001 From: Alexey Tsitkin Date: Thu, 2 May 2019 16:58:22 -0700 Subject: [PATCH 1/5] adding health check support --- eb_sqs/management/commands/healthcheck.py | 32 +++++++++++++++++++++++ eb_sqs/settings.py | 4 +++ eb_sqs/worker/service.py | 10 +++++++ 3 files changed, 46 insertions(+) create mode 100644 eb_sqs/management/commands/healthcheck.py diff --git a/eb_sqs/management/commands/healthcheck.py b/eb_sqs/management/commands/healthcheck.py new file mode 100644 index 0000000..fc07996 --- /dev/null +++ b/eb_sqs/management/commands/healthcheck.py @@ -0,0 +1,32 @@ +import sys + +import logging +from datetime import timedelta +from django.conf import settings +from django.core.management import BaseCommand +from django.utils import timezone +from django.utils.dateparse import parse_datetime + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = 'Checks the SQS worker is healthy, and if not returns a failure code' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + try: + with open(settings.HEALTHCHECK_FILE_NAME, 'r') as file: + last_healthcheck_date_str = file.readlines()[0] + + if parse_datetime(last_healthcheck_date_str) < timezone.now() - timedelta(seconds=settings.HEALTHCHECK_UNHEALTHY_PERIOD_S): + self._return_failure() + except Exception: + self._return_failure() + + @staticmethod + def _return_failure(): + logger.warning('[django-eb-sqs] Health check failed') + sys.exit(1) diff --git a/eb_sqs/settings.py b/eb_sqs/settings.py index 291ff7f..c7ae840 100644 --- a/eb_sqs/settings.py +++ b/eb_sqs/settings.py @@ -37,3 +37,7 @@ QUEUE_MESSAGE_RETENTION = getattr(settings, 'EB_SQS_QUEUE_MESSAGE_RETENTION', '1209600') # type: str QUEUE_VISIBILITY_TIMEOUT = getattr(settings, 'EB_SQS_QUEUE_VISIBILITY_TIMEOUT', '300') # type: str + +MIN_HEALTHCHECK_WRITE_PERIOD_S = getattr(settings, 'EB_SQS_MIN_HEALTHCHECK_WRITE_PERIOD_S', 10) # type: int +HEALTHCHECK_UNHEALTHY_PERIOD_S = getattr(settings, 'EB_SQS_HEALTHCHECK_UNHEALTHY_PERIOD_S', int(QUEUE_VISIBILITY_TIMEOUT)) # type: int +HEALTHCHECK_FILE_NAME = getattr(settings, 'EB_SQS_HEALTHCHECK_FILE_NAME', 'healthcheck.txt') # type: str diff --git a/eb_sqs/worker/service.py b/eb_sqs/worker/service.py index db8e34b..243fc3b 100644 --- a/eb_sqs/worker/service.py +++ b/eb_sqs/worker/service.py @@ -43,6 +43,9 @@ def process_queues(self, queue_names): static_queues = queues last_update_time = timezone.now() - timedelta(seconds=settings.REFRESH_PREFIX_QUEUES_S) + self.write_healthcheck_file() + last_healthcheck_time = timezone.now() + logger.debug('[django-eb-sqs] Connected to SQS: {}'.format(', '.join(queue_names))) worker = WorkerFactory.default().create() @@ -67,6 +70,9 @@ def process_queues(self, queue_names): logger.debug('[django-eb-sqs] Processing {} queues'.format(len(queues))) self.process_messages(queues, worker, static_queues) + if timezone.now() - timedelta(seconds=settings.MIN_HEALTHCHECK_PERIOD_S) > last_healthcheck_time: + self.write_healthcheck_file() + def process_messages(self, queues, worker, static_queues): # type: (list, Worker, list) -> None for queue in queues: @@ -144,3 +150,7 @@ def get_queues_by_prefixes(self, sqs, prefixes): queues += sqs.queues.filter(QueueNamePrefix=prefix) return queues + + def write_healthcheck_file(self): + with open(settings.HEALTHCHECK_FILE_NAME, 'w') as file: + file.write(timezone.now().isoformat()) From debac581223ab677ead691ab4fe273989cb91bd7 Mon Sep 17 00:00:00 2001 From: Alexey Tsitkin Date: Thu, 2 May 2019 17:04:03 -0700 Subject: [PATCH 2/5] bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 17af45f..3009e02 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name='django-eb-sqs', - version='1.16', + version='1.20', package_dir={'eb_sqs': 'eb_sqs'}, include_package_data=True, packages=find_packages(), From b6ed6ea815dbce3113f277035b77266a919bb086 Mon Sep 17 00:00:00 2001 From: Alexey Tsitkin Date: Thu, 2 May 2019 17:06:31 -0700 Subject: [PATCH 3/5] fix minor bug --- eb_sqs/worker/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_sqs/worker/service.py b/eb_sqs/worker/service.py index 243fc3b..9a69f3f 100644 --- a/eb_sqs/worker/service.py +++ b/eb_sqs/worker/service.py @@ -70,7 +70,7 @@ def process_queues(self, queue_names): logger.debug('[django-eb-sqs] Processing {} queues'.format(len(queues))) self.process_messages(queues, worker, static_queues) - if timezone.now() - timedelta(seconds=settings.MIN_HEALTHCHECK_PERIOD_S) > last_healthcheck_time: + if timezone.now() - timedelta(seconds=settings.MIN_HEALTHCHECK_WRITE_PERIOD_S) > last_healthcheck_time: self.write_healthcheck_file() def process_messages(self, queues, worker, static_queues): From 44badb130d60c40a221a24649f699860286dd6c0 Mon Sep 17 00:00:00 2001 From: Alexey Tsitkin Date: Thu, 2 May 2019 17:18:20 -0700 Subject: [PATCH 4/5] fix bug --- eb_sqs/worker/service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/eb_sqs/worker/service.py b/eb_sqs/worker/service.py index 9a69f3f..ec41c08 100644 --- a/eb_sqs/worker/service.py +++ b/eb_sqs/worker/service.py @@ -72,6 +72,7 @@ def process_queues(self, queue_names): if timezone.now() - timedelta(seconds=settings.MIN_HEALTHCHECK_WRITE_PERIOD_S) > last_healthcheck_time: self.write_healthcheck_file() + last_healthcheck_time = timezone.now() def process_messages(self, queues, worker, static_queues): # type: (list, Worker, list) -> None From d4d6d7186f85986b4046cfd6199e39cc3a78cf2d Mon Sep 17 00:00:00 2001 From: Alexey Tsitkin Date: Thu, 2 May 2019 17:32:11 -0700 Subject: [PATCH 5/5] fix incorrect settings import --- eb_sqs/management/commands/healthcheck.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eb_sqs/management/commands/healthcheck.py b/eb_sqs/management/commands/healthcheck.py index fc07996..e996617 100644 --- a/eb_sqs/management/commands/healthcheck.py +++ b/eb_sqs/management/commands/healthcheck.py @@ -2,11 +2,13 @@ import logging from datetime import timedelta -from django.conf import settings + from django.core.management import BaseCommand from django.utils import timezone from django.utils.dateparse import parse_datetime +from eb_sqs import settings + logger = logging.getLogger(__name__)