feat: raw_data_source_component

backend/apps/api/v1/admin.py

@@ -66,6 +66,7 @@
     Pipeline,
     QualityCheck,
     RawDataSource,
+    RawDataSourceComponent,
     Status,
     Table,
     TableNeighbor,
@@ -222,7 +223,6 @@ class RawDataSourceInline(OrderedTranslatedInline):
     fields = [
         "order",
         "move_up_down_links",
-        "id",
         "name",
         "description",
         "availability",
@@ -236,6 +236,25 @@ class RawDataSourceInline(OrderedTranslatedInline):
         "order",
     ]
 
+class RawDataSourceComponentInline(OrderedTranslatedInline):
+    model = RawDataSourceComponent
+    extra = 0
+    show_change_link = True
+    fields = [
+        "order",
+        "move_up_down_links",
+        "name",
+        "description",
+        "url",
+    ]
+    readonly_fields = [
+        "order",
+        "move_up_down_links",
+    ]
+    ordering = [
+        "order",
+    ]
+
 
 class InformationRequestInline(OrderedTranslatedInline):
     model = InformationRequest
@@ -925,9 +944,28 @@ class RawDataSourceAdmin(OrderedInlineModelAdminMixin, TabbedTranslationAdmin):
         "languages",
         "area_ip_address_required",
     ]
+    inlines = [
+        RawDataSourceComponentInline,
+        CoverageInline,
+        ObservationLevelInline,
+        UpdateInline,
+        PollInline,
+    ]
+
+class RawDataSourceComponentAdmin(OrderedInlineModelAdminMixin, TabbedTranslationAdmin):
+    actions = [
+        reorder_observation_levels,
+    ]
+    list_display = ["name", "raw_data_source", "created_at", "updated_at"]
+    search_fields = ["name", "raw_data_source__name"]
+    readonly_fields = ["id", "created_at", "updated_at"]
+    autocomplete_fields = [
+        "raw_data_source",
+    ]
     inlines = [
         CoverageInline,
         ObservationLevelInline,
+        UpdateInline,
         PollInline,
     ]
 
@@ -1343,6 +1381,7 @@ class PipelineAdmin(admin.ModelAdmin):
 admin.site.register(Organization, OrganizationAdmin)
 admin.site.register(Pipeline, PipelineAdmin)
 admin.site.register(RawDataSource, RawDataSourceAdmin)
+admin.site.register(RawDataSourceComponent, RawDataSourceComponentAdmin)
 admin.site.register(Status, StatusAdmin)
 admin.site.register(Table, TableAdmin)
 admin.site.register(TableNeighbor, TableNeighborAdmin)

backend/apps/api/v1/migrations/0053_alter_organization_area_rawdatasourcecomponent_and_more.py

@@ -0,0 +1,67 @@
+# Generated by Django 4.2.16 on 2024-12-08 03:03
+
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('v1', '0052_remove_dataset_is_closed'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='organization',
+            name='area',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='organizations', to='v1.area'),
+        ),
+        migrations.CreateModel(
+            name='RawDataSourceComponent',
+            fields=[
+                ('order', models.PositiveIntegerField(db_index=True, editable=False, verbose_name='order')),
+                ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
+                ('name', models.CharField(max_length=255)),
+                ('name_pt', models.CharField(max_length=255, null=True)),
+                ('name_en', models.CharField(max_length=255, null=True)),
+                ('name_es', models.CharField(max_length=255, null=True)),
+                ('description', models.TextField(blank=True, null=True)),
+                ('description_pt', models.TextField(blank=True, null=True)),
+                ('description_en', models.TextField(blank=True, null=True)),
+                ('description_es', models.TextField(blank=True, null=True)),
+                ('url', models.URLField(blank=True, max_length=500, null=True)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+                ('updated_at', models.DateTimeField(auto_now=True)),
+                ('version', models.IntegerField(blank=True, null=True)),
+                ('raw_data_source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='raw_data_source_components', to='v1.rawdatasource')),
+                ('status', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='raw_data_source_components', to='v1.status')),
+            ],
+            options={
+                'verbose_name': 'Raw Data Source Component',
+                'verbose_name_plural': 'Raw Data Source Components',
+                'db_table': 'raw_data_source_component',
+                'ordering': ['url'],
+            },
+        ),
+        migrations.AddField(
+            model_name='coverage',
+            name='raw_data_source_component',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='coverages', to='v1.rawdatasourcecomponent'),
+        ),
+        migrations.AddField(
+            model_name='observationlevel',
+            name='raw_data_source_component',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='observation_levels', to='v1.rawdatasourcecomponent'),
+        ),
+        migrations.AddField(
+            model_name='poll',
+            name='raw_data_source_component',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='polls', to='v1.rawdatasourcecomponent'),
+        ),
+        migrations.AddField(
+            model_name='update',
+            name='raw_data_source_component',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='updates', to='v1.rawdatasourcecomponent'),
+        ),
+    ]

backend/apps/api/v1/migrations/0054_rename_required_registration_rawdatasource_requires_registration.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.16 on 2024-12-08 03:10
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('v1', '0053_alter_organization_area_rawdatasourcecomponent_and_more'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='rawdatasource',
+            old_name='required_registration',
+            new_name='requires_registration',
+        ),
+    ]

backend/apps/api/v1/models.py

@@ -108,6 +108,13 @@ class Coverage(BaseModel):
         on_delete=models.CASCADE,
         related_name="coverages",
     )
+    raw_data_source_component = models.ForeignKey(
+        "RawDataSourceComponent",
+        blank=True,
+        null=True,
+        on_delete=models.CASCADE,
+        related_name="coverages",
+    )
     information_request = models.ForeignKey(
         "InformationRequest",
         blank=True,
@@ -784,6 +791,13 @@ class Update(BaseModel):
         on_delete=models.CASCADE,
         related_name="updates",
     )
+    raw_data_source_component = models.ForeignKey(
+        "RawDataSourceComponent",
+        blank=True,
+        null=True,
+        on_delete=models.CASCADE,
+        related_name="updates",
+    )
     information_request = models.ForeignKey(
         "InformationRequest",
         blank=True,
@@ -853,6 +867,13 @@ class Poll(BaseModel):
         on_delete=models.CASCADE,
         related_name="polls",
     )
+    raw_data_source_component = models.ForeignKey(
+        "RawDataSourceComponent",
+        blank=True,
+        null=True,
+        on_delete=models.CASCADE,
+        related_name="polls",
+    )
     information_request = models.ForeignKey(
         "InformationRequest",
         blank=True,
@@ -1675,7 +1696,7 @@ class RawDataSource(BaseModel, OrderedModel):
     contains_structured_data = models.BooleanField(default=False)
     contains_api = models.BooleanField(default=False)
     is_free = models.BooleanField(default=False)
-    required_registration = models.BooleanField(default=False)
+    requires_registration = models.BooleanField(default=False)
     version = models.IntegerField(null=True, blank=True)
     status = models.ForeignKey(
         "Status",
@@ -1701,15 +1722,70 @@ def __str__(self):
 
     @property
     def last_polled_at(self):
-        polls = [u.latest for u in self.polls.all() if u.latest]
+        polls = [
+            poll.latest 
+            for raw_data_source_component in self.raw_data_source_components.all()
+            for poll in raw_data_source_component.polls.all()
+            if poll.latest
+        ]
         return max(polls) if polls else None
 
     @property
     def last_updated_at(self):
-        updates = [u.latest for u in self.updates.all() if u.latest]
+        updates = [
+            update.latest 
+            for raw_data_source_component in self.raw_data_source_components.all()
+            for update in raw_data_source_component.updates.all()
+            if update.latest
+        ]
         return max(updates) if updates else None
 
 
+class RawDataSourceComponent(BaseModel, OrderedModel):
+    """Model definition for RawDataSourceComponent."""
+
+    id = models.UUIDField(primary_key=True, default=uuid4)
+    name = models.CharField(max_length=255)
+    description = models.TextField(blank=True, null=True)
+    url = models.URLField(max_length=500, blank=True, null=True)
+    raw_data_source = models.ForeignKey(
+        "RawDataSource", on_delete=models.CASCADE, related_name="raw_data_source_components"
+    )
+    created_at = models.DateTimeField(auto_now_add=True)
+    updated_at = models.DateTimeField(auto_now=True)
+    version = models.IntegerField(null=True, blank=True)
+    status = models.ForeignKey(
+        "Status",
+        on_delete=models.PROTECT,
+        related_name="raw_data_source_components",
+        null=True,
+        blank=True,
+    )
+    order_with_respect_to = ("raw_data_source",)
+
+    graphql_nested_filter_fields_whitelist = ["id"]
+
+    class Meta:
+        """Meta definition for RawDataSource."""
+
+        db_table = "raw_data_source_component"
+        verbose_name = "Raw Data Source Component"
+        verbose_name_plural = "Raw Data Source Components"
+        ordering = ["url"]
+
+    def __str__(self):
+        return f"{self.name} ({self.raw_data_source.name})"
+
+    @property
+    def last_polled_at(self):
+        polls = [poll.latest for poll in self.polls.all() if poll.latest]
+        return max(polls) if polls else None
+
+    @property
+    def last_updated_at(self):
+        updates = [update.latest for update in self.updates.all() if update.latest]
+        return max(updates) if updates else None
+
 class InformationRequest(BaseModel, OrderedModel):
     """Model definition for InformationRequest."""
 
@@ -1832,6 +1908,13 @@ class ObservationLevel(BaseModel, OrderedModel):
         on_delete=models.CASCADE,
         related_name="observation_levels",
     )
+    raw_data_source_component = models.ForeignKey(
+        "RawDataSourceComponent",
+        blank=True,
+        null=True,
+        on_delete=models.CASCADE,
+        related_name="observation_levels",
+    )
     information_request = models.ForeignKey(
         "InformationRequest",
         blank=True,

backend/apps/api/v1/translation.py

@@ -19,6 +19,7 @@
     Organization,
     QualityCheck,
     RawDataSource,
+    RawDataSourceComponent,
     Status,
     Table,
     Tag,
@@ -102,6 +103,11 @@ class RawDataSourceTranslationOptions(TranslationOptions):
         "description",
     )
 
+class RawDataSourceComponentTranslationOptions(TranslationOptions):
+    fields = (
+        "name",
+        "description",
+    )
 
 class StatusTranslationOptions(TranslationOptions):
     fields = ("name",)
@@ -139,6 +145,7 @@ class ThemeTranslationOptions(TranslationOptions):
 translator.register(Organization, OrganizationTranslationOptions)
 translator.register(QualityCheck, QualityCheckTranslationOptions)
 translator.register(RawDataSource, RawDataSourceTranslationOptions)
+translator.register(RawDataSourceComponent, RawDataSourceComponentTranslationOptions)
 translator.register(Status, StatusTranslationOptions)
 translator.register(Table, TableTranslationOptions)
 translator.register(Tag, TagTranslationOptions)