Merge pull request #60 from tutorcruncher/reduce-logging

Reduce logging - seems to be overloading memory

chronos/main.py

@@ -44,6 +44,8 @@
     logfire.instrument_requests()
 
 logging.config.dictConfig(config)
-
+# Remove excessive sqlalchemy logging
+logging.getLogger('sqlalchemy').setLevel(logging.ERROR)
+logging.getLogger('sqlalchemy.engine.Engine').disabled = True
 app.include_router(main_router, prefix='')
 app.include_router(cronjob, prefix='')

chronos/worker.py

@@ -207,6 +207,7 @@ async def delete_old_logs_job():
     if cache.get(DELETE_JOBS_KEY):
         return
     else:
+        await cache.set(DELETE_JOBS_KEY, 'True', ex=1200)
         with logfire.span('Starting to delete old logs'):
             _delete_old_logs_job.delay()
 
@@ -228,7 +229,6 @@ def get_count(date_to_delete_before: datetime) -> int:
 @celery_app.task
 def _delete_old_logs_job():
     with logfire.span('Started to delete old logs'):
-        cache.set(DELETE_JOBS_KEY, 'True', ex=1200)
         with Session(engine) as db:
             # Get all logs older than 15 days
             date_to_delete_before = datetime.now(UTC) - timedelta(days=15)

tests/test_worker.py

@@ -303,3 +303,36 @@ def test_delete_old_logs(self, db: Session, client: TestClient, celery_session_w
         logs = db.exec(select(WebhookLog)).all()
         # The log from 15 days ago is seconds older than the check and thus sdoesn't get deleted
         assert len(logs) == 15
+
+    # Used for testing memory usage. Unnecessary for CI testing
+    # @profile and install memory-profiler to use
+    # def test_delete_old_logs_many(self, db: Session, client: TestClient, celery_session_worker):
+    #     eps = create_endpoint_from_dft_data()
+    #     for ep in eps:
+    #         db.add(ep)
+    #     db.commit()
+    #
+    #     for i in range(0, 30):
+    #         whl = create_webhook_log_from_dft_data(
+    #             webhook_endpoint_id=ep.id,
+    #             timestamp=datetime.utcnow() - timedelta(days=i),
+    #         )
+    #         db.add(whl)
+    #     db.commit()
+    #
+    #     for y in range(1000):
+    #         for i in range(1000):
+    #             whl = create_webhook_log_from_dft_data(
+    #                 webhook_endpoint_id=ep.id,
+    #                 timestamp=datetime.utcnow() - timedelta(days=20),
+    #             )
+    #             db.add(whl)
+    #         db.commit()
+    #
+    #     logs = db.exec(select(WebhookLog)).all()
+    #     assert len(logs) == 1000030
+    #
+    #     _delete_old_logs_job()
+    #     logs = db.exec(select(WebhookLog)).all()
+    #     # The log from 15 days ago is seconds older than the check and thus sdoesn't get deleted
+    #     assert len(logs) == 15