From c0eec23a540c57fddf3d5f82067ffb1c5918beeb Mon Sep 17 00:00:00 2001 From: Ricardo Dahis Date: Sun, 8 Dec 2024 14:26:14 +1100 Subject: [PATCH] feat: raw_data_source_component --- backend/apps/api/v1/admin.py | 41 ++++++++- ...on_area_rawdatasourcecomponent_and_more.py | 67 ++++++++++++++ ...ion_rawdatasource_requires_registration.py | 18 ++++ backend/apps/api/v1/models.py | 89 ++++++++++++++++++- backend/apps/api/v1/translation.py | 7 ++ 5 files changed, 218 insertions(+), 4 deletions(-) create mode 100644 backend/apps/api/v1/migrations/0053_alter_organization_area_rawdatasourcecomponent_and_more.py create mode 100644 backend/apps/api/v1/migrations/0054_rename_required_registration_rawdatasource_requires_registration.py diff --git a/backend/apps/api/v1/admin.py b/backend/apps/api/v1/admin.py index 93cb82d5..dba38d24 100644 --- a/backend/apps/api/v1/admin.py +++ b/backend/apps/api/v1/admin.py @@ -66,6 +66,7 @@ Pipeline, QualityCheck, RawDataSource, + RawDataSourceComponent, Status, Table, TableNeighbor, @@ -222,7 +223,6 @@ class RawDataSourceInline(OrderedTranslatedInline): fields = [ "order", "move_up_down_links", - "id", "name", "description", "availability", @@ -236,6 +236,25 @@ class RawDataSourceInline(OrderedTranslatedInline): "order", ] +class RawDataSourceComponentInline(OrderedTranslatedInline): + model = RawDataSourceComponent + extra = 0 + show_change_link = True + fields = [ + "order", + "move_up_down_links", + "name", + "description", + "url", + ] + readonly_fields = [ + "order", + "move_up_down_links", + ] + ordering = [ + "order", + ] + class InformationRequestInline(OrderedTranslatedInline): model = InformationRequest @@ -925,9 +944,28 @@ class RawDataSourceAdmin(OrderedInlineModelAdminMixin, TabbedTranslationAdmin): "languages", "area_ip_address_required", ] + inlines = [ + RawDataSourceComponentInline, + CoverageInline, + ObservationLevelInline, + UpdateInline, + PollInline, + ] + +class RawDataSourceComponentAdmin(OrderedInlineModelAdminMixin, TabbedTranslationAdmin): + actions = [ + reorder_observation_levels, + ] + list_display = ["name", "raw_data_source", "created_at", "updated_at"] + search_fields = ["name", "raw_data_source__name"] + readonly_fields = ["id", "created_at", "updated_at"] + autocomplete_fields = [ + "raw_data_source", + ] inlines = [ CoverageInline, ObservationLevelInline, + UpdateInline, PollInline, ] @@ -1343,6 +1381,7 @@ class PipelineAdmin(admin.ModelAdmin): admin.site.register(Organization, OrganizationAdmin) admin.site.register(Pipeline, PipelineAdmin) admin.site.register(RawDataSource, RawDataSourceAdmin) +admin.site.register(RawDataSourceComponent, RawDataSourceComponentAdmin) admin.site.register(Status, StatusAdmin) admin.site.register(Table, TableAdmin) admin.site.register(TableNeighbor, TableNeighborAdmin) diff --git a/backend/apps/api/v1/migrations/0053_alter_organization_area_rawdatasourcecomponent_and_more.py b/backend/apps/api/v1/migrations/0053_alter_organization_area_rawdatasourcecomponent_and_more.py new file mode 100644 index 00000000..7c4ab817 --- /dev/null +++ b/backend/apps/api/v1/migrations/0053_alter_organization_area_rawdatasourcecomponent_and_more.py @@ -0,0 +1,67 @@ +# Generated by Django 4.2.16 on 2024-12-08 03:03 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('v1', '0052_remove_dataset_is_closed'), + ] + + operations = [ + migrations.AlterField( + model_name='organization', + name='area', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='organizations', to='v1.area'), + ), + migrations.CreateModel( + name='RawDataSourceComponent', + fields=[ + ('order', models.PositiveIntegerField(db_index=True, editable=False, verbose_name='order')), + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=255)), + ('name_pt', models.CharField(max_length=255, null=True)), + ('name_en', models.CharField(max_length=255, null=True)), + ('name_es', models.CharField(max_length=255, null=True)), + ('description', models.TextField(blank=True, null=True)), + ('description_pt', models.TextField(blank=True, null=True)), + ('description_en', models.TextField(blank=True, null=True)), + ('description_es', models.TextField(blank=True, null=True)), + ('url', models.URLField(blank=True, max_length=500, null=True)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('version', models.IntegerField(blank=True, null=True)), + ('raw_data_source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='raw_data_source_components', to='v1.rawdatasource')), + ('status', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='raw_data_source_components', to='v1.status')), + ], + options={ + 'verbose_name': 'Raw Data Source Component', + 'verbose_name_plural': 'Raw Data Source Components', + 'db_table': 'raw_data_source_component', + 'ordering': ['url'], + }, + ), + migrations.AddField( + model_name='coverage', + name='raw_data_source_component', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='coverages', to='v1.rawdatasourcecomponent'), + ), + migrations.AddField( + model_name='observationlevel', + name='raw_data_source_component', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='observation_levels', to='v1.rawdatasourcecomponent'), + ), + migrations.AddField( + model_name='poll', + name='raw_data_source_component', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='polls', to='v1.rawdatasourcecomponent'), + ), + migrations.AddField( + model_name='update', + name='raw_data_source_component', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='updates', to='v1.rawdatasourcecomponent'), + ), + ] diff --git a/backend/apps/api/v1/migrations/0054_rename_required_registration_rawdatasource_requires_registration.py b/backend/apps/api/v1/migrations/0054_rename_required_registration_rawdatasource_requires_registration.py new file mode 100644 index 00000000..c662f7b3 --- /dev/null +++ b/backend/apps/api/v1/migrations/0054_rename_required_registration_rawdatasource_requires_registration.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.16 on 2024-12-08 03:10 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('v1', '0053_alter_organization_area_rawdatasourcecomponent_and_more'), + ] + + operations = [ + migrations.RenameField( + model_name='rawdatasource', + old_name='required_registration', + new_name='requires_registration', + ), + ] diff --git a/backend/apps/api/v1/models.py b/backend/apps/api/v1/models.py index 46058299..6658d284 100644 --- a/backend/apps/api/v1/models.py +++ b/backend/apps/api/v1/models.py @@ -108,6 +108,13 @@ class Coverage(BaseModel): on_delete=models.CASCADE, related_name="coverages", ) + raw_data_source_component = models.ForeignKey( + "RawDataSourceComponent", + blank=True, + null=True, + on_delete=models.CASCADE, + related_name="coverages", + ) information_request = models.ForeignKey( "InformationRequest", blank=True, @@ -784,6 +791,13 @@ class Update(BaseModel): on_delete=models.CASCADE, related_name="updates", ) + raw_data_source_component = models.ForeignKey( + "RawDataSourceComponent", + blank=True, + null=True, + on_delete=models.CASCADE, + related_name="updates", + ) information_request = models.ForeignKey( "InformationRequest", blank=True, @@ -853,6 +867,13 @@ class Poll(BaseModel): on_delete=models.CASCADE, related_name="polls", ) + raw_data_source_component = models.ForeignKey( + "RawDataSourceComponent", + blank=True, + null=True, + on_delete=models.CASCADE, + related_name="polls", + ) information_request = models.ForeignKey( "InformationRequest", blank=True, @@ -1675,7 +1696,7 @@ class RawDataSource(BaseModel, OrderedModel): contains_structured_data = models.BooleanField(default=False) contains_api = models.BooleanField(default=False) is_free = models.BooleanField(default=False) - required_registration = models.BooleanField(default=False) + requires_registration = models.BooleanField(default=False) version = models.IntegerField(null=True, blank=True) status = models.ForeignKey( "Status", @@ -1701,15 +1722,70 @@ def __str__(self): @property def last_polled_at(self): - polls = [u.latest for u in self.polls.all() if u.latest] + polls = [ + poll.latest + for raw_data_source_component in self.raw_data_source_components.all() + for poll in raw_data_source_component.polls.all() + if poll.latest + ] return max(polls) if polls else None @property def last_updated_at(self): - updates = [u.latest for u in self.updates.all() if u.latest] + updates = [ + update.latest + for raw_data_source_component in self.raw_data_source_components.all() + for update in raw_data_source_component.updates.all() + if update.latest + ] return max(updates) if updates else None +class RawDataSourceComponent(BaseModel, OrderedModel): + """Model definition for RawDataSourceComponent.""" + + id = models.UUIDField(primary_key=True, default=uuid4) + name = models.CharField(max_length=255) + description = models.TextField(blank=True, null=True) + url = models.URLField(max_length=500, blank=True, null=True) + raw_data_source = models.ForeignKey( + "RawDataSource", on_delete=models.CASCADE, related_name="raw_data_source_components" + ) + created_at = models.DateTimeField(auto_now_add=True) + updated_at = models.DateTimeField(auto_now=True) + version = models.IntegerField(null=True, blank=True) + status = models.ForeignKey( + "Status", + on_delete=models.PROTECT, + related_name="raw_data_source_components", + null=True, + blank=True, + ) + order_with_respect_to = ("raw_data_source",) + + graphql_nested_filter_fields_whitelist = ["id"] + + class Meta: + """Meta definition for RawDataSource.""" + + db_table = "raw_data_source_component" + verbose_name = "Raw Data Source Component" + verbose_name_plural = "Raw Data Source Components" + ordering = ["url"] + + def __str__(self): + return f"{self.name} ({self.raw_data_source.name})" + + @property + def last_polled_at(self): + polls = [poll.latest for poll in self.polls.all() if poll.latest] + return max(polls) if polls else None + + @property + def last_updated_at(self): + updates = [update.latest for update in self.updates.all() if update.latest] + return max(updates) if updates else None + class InformationRequest(BaseModel, OrderedModel): """Model definition for InformationRequest.""" @@ -1832,6 +1908,13 @@ class ObservationLevel(BaseModel, OrderedModel): on_delete=models.CASCADE, related_name="observation_levels", ) + raw_data_source_component = models.ForeignKey( + "RawDataSourceComponent", + blank=True, + null=True, + on_delete=models.CASCADE, + related_name="observation_levels", + ) information_request = models.ForeignKey( "InformationRequest", blank=True, diff --git a/backend/apps/api/v1/translation.py b/backend/apps/api/v1/translation.py index e279d6fc..fab743e0 100644 --- a/backend/apps/api/v1/translation.py +++ b/backend/apps/api/v1/translation.py @@ -19,6 +19,7 @@ Organization, QualityCheck, RawDataSource, + RawDataSourceComponent, Status, Table, Tag, @@ -102,6 +103,11 @@ class RawDataSourceTranslationOptions(TranslationOptions): "description", ) +class RawDataSourceComponentTranslationOptions(TranslationOptions): + fields = ( + "name", + "description", + ) class StatusTranslationOptions(TranslationOptions): fields = ("name",) @@ -139,6 +145,7 @@ class ThemeTranslationOptions(TranslationOptions): translator.register(Organization, OrganizationTranslationOptions) translator.register(QualityCheck, QualityCheckTranslationOptions) translator.register(RawDataSource, RawDataSourceTranslationOptions) +translator.register(RawDataSourceComponent, RawDataSourceComponentTranslationOptions) translator.register(Status, StatusTranslationOptions) translator.register(Table, TableTranslationOptions) translator.register(Tag, TagTranslationOptions)