Merge pull request #706 from basedosdados/staging

Staging

backend/apps/api/v1/filters.py

@@ -2,7 +2,7 @@
 from django.contrib import admin
 from django.utils.translation import gettext_lazy
 
-from backend.apps.api.v1.models import Coverage, ObservationLevel, Organization
+from backend.apps.api.v1.models import Area, Coverage, ObservationLevel, Organization
 
 
 class OrganizationImageListFilter(admin.SimpleListFilter):
@@ -27,12 +27,13 @@ class DatasetOrganizationListFilter(admin.SimpleListFilter):
     parameter_name = "organization"
 
     def lookups(self, request, model_admin):
-        values = Organization.objects.order_by("name").distinct().values("name", "pk")
-        return [(v.get("pk"), v.get("name")) for v in values]
+        organizations = Organization.objects.all().order_by('slug')
+        return [(org.id, org.name) for org in organizations]
 
     def queryset(self, request, queryset):
         if self.value():
-            return queryset.filter(organization=self.value())
+            return queryset.filter(organizations__id=self.value())
+        return queryset
 
 
 class TableOrganizationListFilter(admin.SimpleListFilter):
@@ -99,3 +100,36 @@ def queryset(self, request, queryset):
             return queryset.filter(is_directory=True)
         if self.value() == "false":
             return queryset.filter(is_directory=False)
+
+
+class AreaAdministrativeLevelFilter(admin.SimpleListFilter):
+    title = "Administrative Level"
+    parameter_name = "administrative_level"
+
+    def lookups(self, request, model_admin):
+        return [
+            (0, '0'),
+            (1, '1'),
+            (2, '2'),
+            (3, '3'),
+            (4, '4'),
+            (5, '5'),
+        ]
+
+    def queryset(self, request, queryset):
+        if self.value() is not None:
+            return queryset.filter(administrative_level=self.value())
+
+
+class AreaParentFilter(admin.SimpleListFilter):
+    title = "Parent Area"
+    parameter_name = "parent"
+
+    def lookups(self, request, model_admin):
+        # Get all areas that have children, ordered by name
+        parents = Area.objects.filter(children__isnull=False).distinct().order_by('name')
+        return [(area.id, f"{area.name}") for area in parents]
+
+    def queryset(self, request, queryset):
+        if self.value():
+            return queryset.filter(parent_id=self.value())

backend/apps/api/v1/forms/__init__.py

@@ -4,9 +4,13 @@
     ColumnInlineForm,
     ColumnOriginalNameInlineForm,
     CoverageInlineForm,
+    MeasurementUnitInlineForm,
     ObservationLevelInlineForm,
+    PollInlineForm,
+    TableForm,
     TableInlineForm,
     UpdateInlineForm,
 )
 from backend.apps.api.v1.forms.reorder_columns_form import ReorderColumnsForm  # noqa: F401
 from backend.apps.api.v1.forms.reorder_tables_form import ReorderTablesForm  # noqa: F401
+from backend.apps.api.v1.forms.reorder_observation_levels_form import ReorderObservationLevelsForm  # noqa: F401

backend/apps/api/v1/forms/admin_form.py

@@ -6,7 +6,10 @@
     Column,
     ColumnOriginalName,
     Coverage,
+    MeasurementUnit,
     ObservationLevel,
+    Poll,
+    RawDataSource,
     Table,
     Update,
 )
@@ -23,6 +26,27 @@ class Meta:
         abstract = True
 
 
+class TableForm(forms.ModelForm):
+    class Meta:
+        model = Table
+        fields = '__all__'
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.instance:
+            # Check both the saved instance and current form data
+            dataset_id = self.instance.dataset_id
+            if not dataset_id and self.data:
+                dataset_id = self.data.get('dataset')
+
+            if dataset_id:
+                self.fields['raw_data_source'].queryset = RawDataSource.objects.filter(
+                    dataset_id=dataset_id
+                )
+            else:
+                self.fields['raw_data_source'].queryset = RawDataSource.objects.none()
+
+
 class TableInlineForm(UUIDHiddenIdForm):
     class Meta(UUIDHiddenIdForm):
         model = Table
@@ -40,14 +64,9 @@ class Meta(UUIDHiddenIdForm):
             "data_cleaned_by",
             "data_cleaning_description",
             "data_cleaning_code_url",
-            "raw_data_url",
             "auxiliary_files_url",
             "architecture_url",
             "source_bucket_name",
-            "uncompressed_file_size",
-            "compressed_file_size",
-            "number_rows",
-            "number_columns",
             "is_closed",
         ]
         readonly_fields = [
@@ -56,6 +75,12 @@ class Meta(UUIDHiddenIdForm):
         ]
 
 
+class MeasurementUnitInlineForm(UUIDHiddenIdForm):
+    class Meta(UUIDHiddenIdForm.Meta):
+        model = MeasurementUnit
+        fields = "__all__"
+
+
 class ColumnInlineForm(UUIDHiddenIdForm):
     class Meta(UUIDHiddenIdForm.Meta):
         model = Column
@@ -93,7 +118,18 @@ class Meta(UUIDHiddenIdForm.Meta):
 class ObservationLevelInlineForm(UUIDHiddenIdForm):
     class Meta(UUIDHiddenIdForm.Meta):
         model = ObservationLevel
-        fields = "__all__"
+        fields = [
+            "id",
+            "entity",
+            "table",
+            "raw_data_source",
+            "information_request",
+            "analysis",
+        ]
+        readonly_fields = [
+            "order",
+            "move_up_down_links",
+        ]
 
 
 class CoverageInlineForm(UUIDHiddenIdForm):
@@ -110,3 +146,12 @@ class UpdateInlineForm(UUIDHiddenIdForm):
     class Meta(UUIDHiddenIdForm.Meta):
         model = Update
         fields = "__all__"
+
+class PollInlineForm(forms.ModelForm):
+    class Meta:
+        model = Poll
+        fields = [
+            "entity",
+            "frequency",
+            "latest",
+        ]

backend/apps/api/v1/forms/reorder_observation_levels_form.py

@@ -0,0 +1,8 @@
+from django import forms
+
+class ReorderObservationLevelsForm(forms.Form):
+    ordered_entities = forms.CharField(
+        required=False,
+        widget=forms.Textarea(attrs={"rows": 10, "cols": 40}),
+        help_text="Enter entity names one per line in desired order",
+    )

backend/apps/api/v1/migrations/0037_area_entity_area_level_area_parent.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.2.16 on 2024-11-03 01:29
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('v1', '0036_datetimerange_units'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='area',
+            name='entity',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='areas', to='v1.entity'),
+        ),
+        migrations.AddField(
+            model_name='area',
+            name='level',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='area',
+            name='parent',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='children', to='v1.area'),
+        ),
+    ]

backend/apps/api/v1/migrations/0038_rename_level_area_administrative_level.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.16 on 2024-11-03 01:37
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('v1', '0037_area_entity_area_level_area_parent'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='area',
+            old_name='level',
+            new_name='administrative_level',
+        ),
+    ]

backend/apps/api/v1/migrations/0039_dataset_organizations.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 4.2.10 on 2024-05-10 15:30
+
+from django.db import migrations, models
+
+
+def migrate_organization_to_organizations(apps, schema_editor):
+    Dataset = apps.get_model('v1', 'Dataset')
+    for dataset in Dataset.objects.all():
+        if dataset.organization:
+            dataset.organizations.add(dataset.organization)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('v1', '0038_rename_level_area_administrative_level'),
+    ]
+
+    operations = [
+        # Add new ManyToMany field
+        migrations.AddField(
+            model_name='dataset',
+            name='organizations',
+            field=models.ManyToManyField(
+                related_name='datasets',
+                to='v1.organization',
+                verbose_name='Organizations',
+                help_text='Organizations associated with this dataset',
+            ),
+        ),
+        # Run data migration
+        migrations.RunPython(
+            migrate_organization_to_organizations,
+            reverse_code=migrations.RunPython.noop
+        ),
+        # Remove old ForeignKey field
+        migrations.RemoveField(
+            model_name='dataset',
+            name='organization',
+        ),
+    ]

backend/apps/api/v1/migrations/0040_table_publishers_data_cleaners.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 4.2.10 on 2024-05-10 16:00
+
+from django.conf import settings
+from django.db import migrations, models
+
+
+def migrate_publishers_and_cleaners(apps, schema_editor):
+    """Migrate existing ForeignKey relationships to ManyToMany"""
+    Table = apps.get_model('v1', 'Table')
+    for table in Table.objects.all():
+        # Store old ForeignKey values
+        old_publisher = getattr(table, 'published_by_old', None)
+        old_cleaner = getattr(table, 'data_cleaned_by_old', None)
+
+        # Add to new M2M fields if they existed
+        if old_publisher:
+            table.published_by.add(old_publisher)
+        if old_cleaner:
+            table.data_cleaned_by.add(old_cleaner)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('v1', '0039_dataset_organizations'),
+    ]
+
+    operations = [
+        # Rename old fields temporarily
+        migrations.RenameField(
+            model_name='table',
+            old_name='published_by',
+            new_name='published_by_old',
+        ),
+        migrations.RenameField(
+            model_name='table',
+            old_name='data_cleaned_by',
+            new_name='data_cleaned_by_old',
+        ),
+        # Add new M2M fields
+        migrations.AddField(
+            model_name='table',
+            name='published_by',
+            field=models.ManyToManyField(
+                blank=True,
+                related_name='tables_published',
+                to=settings.AUTH_USER_MODEL,
+                verbose_name='Published by',
+                help_text='People who published the table',
+            ),
+        ),
+        migrations.AddField(
+            model_name='table',
+            name='data_cleaned_by',
+            field=models.ManyToManyField(
+                blank=True,
+                related_name='tables_cleaned',
+                to=settings.AUTH_USER_MODEL,
+                verbose_name='Data cleaned by',
+                help_text='People who cleaned the data',
+            ),
+        ),
+        # Run data migration
+        migrations.RunPython(
+            migrate_publishers_and_cleaners,
+            reverse_code=migrations.RunPython.noop
+        ),
+        # Remove old fields
+        migrations.RemoveField(
+            model_name='table',
+            name='published_by_old',
+        ),
+        migrations.RemoveField(
+            model_name='table',
+            name='data_cleaned_by_old',
+        ),
+    ]