From f3cf412501154e6df6d28fbd60b551cb297d5bca Mon Sep 17 00:00:00 2001 From: James McKinney <26463+jpmckinney@users.noreply.github.com> Date: Mon, 7 Oct 2024 00:47:30 -0400 Subject: [PATCH] feat: Add release_date to release and compiled_release, closes #402 --- process/management/commands/file_worker.py | 4 +- process/migrations/0041_auto_20241007_0357.py | 11 ++++++ ...lease_release_date_release_release_date.py | 38 +++++++++++++++++++ process/models.py | 5 ++- process/processors/compiler.py | 8 +++- process/views.py | 9 ++--- 6 files changed, 66 insertions(+), 9 deletions(-) create mode 100644 process/migrations/0041_auto_20241007_0357.py create mode 100644 process/migrations/0042_compiledrelease_release_date_release_release_date.py diff --git a/process/management/commands/file_worker.py b/process/management/commands/file_worker.py index ad26f96..b622516 100644 --- a/process/management/commands/file_worker.py +++ b/process/management/commands/file_worker.py @@ -288,7 +288,8 @@ def _store_data(collection_file, package, releases_or_records, data_type, *, upg package_data=get_or_create(PackageData, package), data=data, ocid=release_or_record["ocid"], - release_id=release_or_record.get("id", ""), + release_id=release_or_record.get("id") or "", + release_date=release_or_record.get("date") or "", ).save() case Format.compiled_release: CompiledRelease( @@ -296,4 +297,5 @@ def _store_data(collection_file, package, releases_or_records, data_type, *, upg collection_file_item=collection_file_item, data=data, ocid=release_or_record["ocid"], + release_date=release_or_record.get("date") or "", ).save() diff --git a/process/migrations/0041_auto_20241007_0357.py b/process/migrations/0041_auto_20241007_0357.py new file mode 100644 index 0000000..8df60ba --- /dev/null +++ b/process/migrations/0041_auto_20241007_0357.py @@ -0,0 +1,11 @@ +# Generated by Django 4.2.15 on 2024-10-07 03:57 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("process", "0040_auto_20241003_1605"), + ] + + operations = [migrations.RunSQL("DROP INDEX data_data_date_idx")] diff --git a/process/migrations/0042_compiledrelease_release_date_release_release_date.py b/process/migrations/0042_compiledrelease_release_date_release_release_date.py new file mode 100644 index 0000000..aed419f --- /dev/null +++ b/process/migrations/0042_compiledrelease_release_date_release_release_date.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.15 on 2024-10-07 04:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("process", "0041_auto_20241007_0357"), + ] + + operations = [ + migrations.AddField( + model_name="compiledrelease", + name="release_date", + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name="release", + name="release_date", + field=models.TextField(blank=True), + ), + migrations.RunSQL( + """ + UPDATE release + SET release_date = COALESCE(data ->> 'date', '') + FROM data + WHERE data_id = data.id + """ + ), + migrations.RunSQL( + """ + UPDATE compiled_release + SET release_date = COALESCE(data ->> 'date', '') + FROM data + WHERE data_id = data.id + """ + ), + ] diff --git a/process/models.py b/process/models.py index 3566d8a..8bbff9e 100644 --- a/process/models.py +++ b/process/models.py @@ -289,7 +289,6 @@ class Meta: # Used by process.util.get_or_create(). models.UniqueConstraint(name="unique_data_hash_md5", fields=["hash_md5"]), ] - # Migration 0040 adds an index on data ->> 'date'. Django would add the index on data -> 'date'. def __str__(self): return "{hash_md5} (id: {id})".format_map(Default(hash_md5=self.hash_md5, id=self.pk)) @@ -318,8 +317,9 @@ class Release(models.Model): collection = models.ForeignKey(Collection, on_delete=models.CASCADE, db_index=False) collection_file_item = models.ForeignKey(CollectionFileItem, on_delete=models.CASCADE, db_index=False) - release_id = models.TextField(blank=True) ocid = models.TextField(blank=True) + release_id = models.TextField(blank=True) + release_date = models.TextField(blank=True) data = models.ForeignKey(Data, on_delete=models.CASCADE, db_index=False) package_data = models.ForeignKey(PackageData, on_delete=models.CASCADE, db_index=False) @@ -380,6 +380,7 @@ class CompiledRelease(models.Model): collection_file_item = models.ForeignKey(CollectionFileItem, on_delete=models.CASCADE, db_index=False) ocid = models.TextField(blank=True) + release_date = models.TextField(blank=True) data = models.ForeignKey(Data, on_delete=models.CASCADE, db_index=False) diff --git a/process/processors/compiler.py b/process/processors/compiler.py index 07f94ae..54c7cbb 100644 --- a/process/processors/compiler.py +++ b/process/processors/compiler.py @@ -23,7 +23,13 @@ def save_compiled_release(merged, collection, ocid): data = get_or_create(Data, merged) - release = CompiledRelease(collection=collection, collection_file_item=collection_file_item, data=data, ocid=ocid) + release = CompiledRelease( + collection=collection, + collection_file_item=collection_file_item, + data=data, + ocid=ocid, + release_date=merged.get("date"), + ) release.save() return release diff --git a/process/views.py b/process/views.py index 1ab7afd..1711a42 100644 --- a/process/views.py +++ b/process/views.py @@ -186,15 +186,14 @@ def metadata(self, request, pk=None): """\ SELECT LEFT(MAX(ocid), 11) AS ocid_prefix, - MIN(data ->> 'date') AS published_from, - MAX(data ->> 'date') AS published_to + MIN(release_date) AS published_from, + MAX(release_date) AS published_to FROM compiled_release - JOIN data ON data.id = data_id WHERE collection_id = %(collection_id)s - AND data ->> 'date' > '1970-01-01' - AND data ->> 'date' <= %(today)s + AND release_date > '1970-01-01' + AND release_date <= %(today)s """, {"collection_id": pk, "today": str(datetime.datetime.now(tz=datetime.UTC).date())}, )