From 5e0969ec68f74b5c508b85e672deb991ec30f63f Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Wed, 8 May 2024 13:03:25 -0400 Subject: [PATCH] fix: Set a longer heartbeat timeout, to handle high load --- exporter/management/commands/flattener.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/exporter/management/commands/flattener.py b/exporter/management/commands/flattener.py index d92f73d..fcea39a 100644 --- a/exporter/management/commands/flattener.py +++ b/exporter/management/commands/flattener.py @@ -29,9 +29,12 @@ def handle(self, *args, **options): on_message_callback=callback, queue="flattener_init", routing_keys=["flattener_init", "flattener_file"], - # Witnessed "AMQPHeartbeatTimeout: No activity or too many missed heartbeats in the last 60 seconds." - # while using Pika's BlockingConnection when processing the largest files. - rabbit_params={"heartbeat": 0}, + # When there's high load, the heartbeat isn't sent, causing "missed heartbeats from client, timeout: 60s" + # and "closing AMQP connection" in RabbitMQ logs and ConnectionResetError(104, 'Connection reset by peer') + # in the asynchronous client. Use 1800 seconds to give time for a heartbeat. + # https://stackoverflow.com/q/70006802/244258 + # https://www.rabbitmq.com/docs/heartbeats#disabling + rabbit_params={"heartbeat": 1800}, # 30 mins decorator=decorator, )