feat: delete jobs that are more than a year older than the active job

data_registry/process_manager/__init__.py

@@ -1,5 +1,7 @@
+import datetime
 import logging
 
+from dateutil.relativedelta import relativedelta
 from django.conf import settings
 from django.db import transaction
 
@@ -47,7 +49,8 @@ def process(collection: models.Collection) -> None:
        -  If it failed temporarily, log the reason
        -  If it failed permanently, fail the task and end the job
 
-    -  If all tasks succeeded, end the job and update the collection's active job and last retrieved date.
+    -  If all tasks succeeded, end the job, update the collection's active job and last retrieved date and delete jobs
+       that are more than a year older than the active job.
 
     In other words, this function advances each job by at most one task. As such, for all tasks of a job to succeed,
     this function needs to run at least as many times are there are tasks in the ``JOB_TASKS_PLAN`` setting.
@@ -119,3 +122,19 @@ def process(collection: models.Collection) -> None:
                 collection.save()
 
                 logger.debug("Job %s has succeeded (%s: %s)", job, country, collection)
+
+                jobs_to_delete = (
+                    collection.job_set.exclude(id=job.id)
+                    .filter(
+                        end__lt=datetime.datetime.now(tz=job.end.tzinfo) - relativedelta(years=1),
+                        status=models.Job.Status.COMPLETED,
+                    )
+                    .order_by("end")
+                )
+
+                # Keep the most recent complete job
+                jobs_to_delete = jobs_to_delete[-1] if len(jobs_to_delete) > 1 else []
+
+                for job_to_delete in jobs_to_delete:
+                    job_to_delete.delete()
+                    logger.debug("Old job %s has been deleted (%s: %s)", job_to_delete, country, collection)