feat: Add release_date to release and compiled_release, closes #402

process/management/commands/file_worker.py

@@ -288,12 +288,14 @@ def _store_data(collection_file, package, releases_or_records, data_type, *, upg
                     package_data=get_or_create(PackageData, package),
                     data=data,
                     ocid=release_or_record["ocid"],
-                    release_id=release_or_record.get("id", ""),
+                    release_id=release_or_record.get("id") or "",
+                    release_date=release_or_record.get("date") or "",
                 ).save()
             case Format.compiled_release:
                 CompiledRelease(
                     collection=collection_file.collection,
                     collection_file_item=collection_file_item,
                     data=data,
                     ocid=release_or_record["ocid"],
+                    release_date=release_or_record.get("date") or "",
                 ).save()

process/migrations/0041_auto_20241007_0357.py

@@ -0,0 +1,11 @@
+# Generated by Django 4.2.15 on 2024-10-07 03:57
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("process", "0040_auto_20241003_1605"),
+    ]
+
+    operations = [migrations.RunSQL("DROP INDEX data_data_date_idx")]

process/migrations/0042_compiledrelease_release_date_release_release_date.py

@@ -0,0 +1,38 @@
+# Generated by Django 4.2.15 on 2024-10-07 04:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("process", "0041_auto_20241007_0357"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="compiledrelease",
+            name="release_date",
+            field=models.TextField(blank=True),
+        ),
+        migrations.AddField(
+            model_name="release",
+            name="release_date",
+            field=models.TextField(blank=True),
+        ),
+        migrations.RunSQL(
+            """
+            UPDATE release
+            SET release_date = COALESCE(data ->> 'date', '')
+            FROM data
+            WHERE data_id = data.id
+            """
+        ),
+        migrations.RunSQL(
+            """
+            UPDATE compiled_release
+            SET release_date = COALESCE(data ->> 'date', '')
+            FROM data
+            WHERE data_id = data.id
+            """
+        ),
+    ]

process/models.py

@@ -289,7 +289,6 @@ class Meta:
             # Used by process.util.get_or_create().
             models.UniqueConstraint(name="unique_data_hash_md5", fields=["hash_md5"]),
         ]
-        # Migration 0040 adds an index on data ->> 'date'. Django would add the index on data -> 'date'.
 
     def __str__(self):
         return "{hash_md5} (id: {id})".format_map(Default(hash_md5=self.hash_md5, id=self.pk))
@@ -318,8 +317,9 @@ class Release(models.Model):
     collection = models.ForeignKey(Collection, on_delete=models.CASCADE, db_index=False)
     collection_file_item = models.ForeignKey(CollectionFileItem, on_delete=models.CASCADE, db_index=False)
 
-    release_id = models.TextField(blank=True)
     ocid = models.TextField(blank=True)
+    release_id = models.TextField(blank=True)
+    release_date = models.TextField(blank=True)
 
     data = models.ForeignKey(Data, on_delete=models.CASCADE, db_index=False)
     package_data = models.ForeignKey(PackageData, on_delete=models.CASCADE, db_index=False)
@@ -380,6 +380,7 @@ class CompiledRelease(models.Model):
     collection_file_item = models.ForeignKey(CollectionFileItem, on_delete=models.CASCADE, db_index=False)
 
     ocid = models.TextField(blank=True)
+    release_date = models.TextField(blank=True)
 
     data = models.ForeignKey(Data, on_delete=models.CASCADE, db_index=False)
 

process/processors/compiler.py

@@ -23,7 +23,13 @@ def save_compiled_release(merged, collection, ocid):
 
     data = get_or_create(Data, merged)
 
-    release = CompiledRelease(collection=collection, collection_file_item=collection_file_item, data=data, ocid=ocid)
+    release = CompiledRelease(
+        collection=collection,
+        collection_file_item=collection_file_item,
+        data=data,
+        ocid=ocid,
+        release_date=merged.get("date"),
+    )
     release.save()
 
     return release

process/views.py

@@ -186,15 +186,14 @@ def metadata(self, request, pk=None):
                 """\
                 SELECT
                     LEFT(MAX(ocid), 11) AS ocid_prefix,
-                    MIN(data ->> 'date') AS published_from,
-                    MAX(data ->> 'date') AS published_to
+                    MIN(release_date) AS published_from,
+                    MAX(release_date) AS published_to
                 FROM
                     compiled_release
-                    JOIN data ON data.id = data_id
                 WHERE
                     collection_id = %(collection_id)s
-                    AND data ->> 'date' > '1970-01-01'
-                    AND data ->> 'date' <= %(today)s
+                    AND release_date > '1970-01-01'
+                    AND release_date <= %(today)s
                 """,
                 {"collection_id": pk, "today": str(datetime.datetime.now(tz=datetime.UTC).date())},
             )