feat: M2M dataset-organization, table-published_by,data_cleaned_by

backend/apps/api/v1/admin.py

@@ -32,6 +32,7 @@
     PollInlineForm,
     ReorderColumnsForm,
     ReorderTablesForm,
+    TableForm,
     TableInlineForm,
     UpdateInlineForm,
 )
@@ -528,17 +529,22 @@ class DatasetAdmin(OrderedInlineModelAdminMixin, TabbedTranslationAdmin):
         "updated_at",
         "related_objects",
     ]
-    search_fields = ["name", "slug", "organization__name"]
+    search_fields = [
+        "name", 
+        "slug", 
+        "organizations__name"
+    ]
     filter_horizontal = [
         "tags",
         "themes",
+        "organizations",
     ]
     list_filter = [
         DatasetOrganizationListFilter,
     ]
     list_display = [
         "name",
-        "organization",
+        "get_organizations",
         "spatial_coverage",
         "temporal_coverage",
         "related_objects",
@@ -548,6 +554,11 @@ class DatasetAdmin(OrderedInlineModelAdminMixin, TabbedTranslationAdmin):
     ]
     ordering = ["-updated_at"]
 
+    def get_organizations(self, obj):
+        """Display all organizations for the dataset"""
+        return ", ".join([org.name for org in obj.organizations.all()])
+    get_organizations.short_description = "Organizations"
+
     def related_objects(self, obj):
         return format_html(
             "<a class='related-widget-wrapper-link add-related' "
@@ -556,11 +567,11 @@ def related_objects(self, obj):
             obj.tables.count(),
             "tables" if obj.tables.count() > 1 else "table",
         )
-
     related_objects.short_description = "Tables"
 
 
 class TableAdmin(OrderedInlineModelAdminMixin, TabbedTranslationAdmin):
+    form = TableForm
     actions = [
         reorder_columns,
         reset_column_order,
@@ -591,8 +602,11 @@ class TableAdmin(OrderedInlineModelAdminMixin, TabbedTranslationAdmin):
     autocomplete_fields = [
         "dataset",
         "partner_organization",
-        "published_by",
-        "data_cleaned_by",
+    ]
+    filter_horizontal = [
+        'published_by',
+        'data_cleaned_by',
+        'raw_data_source',
     ]
     list_display = [
         "name",
@@ -601,6 +615,8 @@ class TableAdmin(OrderedInlineModelAdminMixin, TabbedTranslationAdmin):
         "number_rows",
         "uncompressed_file_size",
         "page_views",
+        "get_publishers",
+        "get_data_cleaners",
         "created_at",
         "updated_at",
     ]
@@ -612,15 +628,15 @@ class TableAdmin(OrderedInlineModelAdminMixin, TabbedTranslationAdmin):
     ]
     ordering = ["-updated_at"]
 
-    def get_form(self, request, obj=None, **kwargs):
-        """Get form, and save the current object"""
-        self.current_obj = obj
-        return super().get_form(request, obj, **kwargs)
+    def get_publishers(self, obj):
+        """Display all publishers for the table"""
+        return ", ".join([f"{pub.first_name} {pub.last_name}" for pub in obj.published_by.all()])
+    get_publishers.short_description = "Publishers"
 
-    def formfield_for_manytomany(self, db_field, request, **kwargs):
-        if self.current_obj and db_field.name == "raw_data_source":
-            kwargs["queryset"] = RawDataSource.objects.filter(dataset=self.current_obj.dataset)
-        return super().formfield_for_manytomany(db_field, request, **kwargs)
+    def get_data_cleaners(self, obj):
+        """Display all data cleaners for the table"""
+        return ", ".join([f"{cleaner.first_name} {cleaner.last_name}" for cleaner in obj.data_cleaned_by.all()])
+    get_data_cleaners.short_description = "Data Cleaners"
 
 
 class TableNeighborAdmin(admin.ModelAdmin):
@@ -662,7 +678,7 @@ class ColumnAdmin(TabbedTranslationAdmin):
         "table",
     ]
     list_filter = [
-        "table__dataset__organization__name",
+        "table__dataset__organizations__name",
     ]
     autocomplete_fields = [
         "table",

backend/apps/api/v1/filters.py

@@ -27,12 +27,13 @@ class DatasetOrganizationListFilter(admin.SimpleListFilter):
     parameter_name = "organization"
 
     def lookups(self, request, model_admin):
-        values = Organization.objects.order_by("name").distinct().values("name", "pk")
-        return [(v.get("pk"), v.get("name")) for v in values]
+        organizations = Organization.objects.all().order_by('slug')
+        return [(org.id, org.name) for org in organizations]
 
     def queryset(self, request, queryset):
         if self.value():
-            return queryset.filter(organization=self.value())
+            return queryset.filter(organizations__id=self.value())
+        return queryset
 
 
 class TableOrganizationListFilter(admin.SimpleListFilter):

backend/apps/api/v1/forms/__init__.py

@@ -6,6 +6,7 @@
     CoverageInlineForm,
     ObservationLevelInlineForm,
     PollInlineForm,
+    TableForm,
     TableInlineForm,
     UpdateInlineForm,
 )

backend/apps/api/v1/forms/admin_form.py

@@ -8,6 +8,7 @@
     Coverage,
     ObservationLevel,
     Poll,
+    RawDataSource,
     Table,
     Update,
 )
@@ -24,6 +25,27 @@ class Meta:
         abstract = True
 
 
+class TableForm(forms.ModelForm):
+    class Meta:
+        model = Table
+        fields = '__all__'
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.instance:
+            # Check both the saved instance and current form data
+            dataset_id = self.instance.dataset_id
+            if not dataset_id and self.data:
+                dataset_id = self.data.get('dataset')
+
+            if dataset_id:
+                self.fields['raw_data_source'].queryset = RawDataSource.objects.filter(
+                    dataset_id=dataset_id
+                )
+            else:
+                self.fields['raw_data_source'].queryset = RawDataSource.objects.none()
+
+
 class TableInlineForm(UUIDHiddenIdForm):
     class Meta(UUIDHiddenIdForm):
         model = Table

backend/apps/api/v1/migrations/0039_dataset_organizations.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 4.2.10 on 2024-05-10 15:30
+
+from django.db import migrations, models
+
+
+def migrate_organization_to_organizations(apps, schema_editor):
+    Dataset = apps.get_model('v1', 'Dataset')
+    for dataset in Dataset.objects.all():
+        if dataset.organization:
+            dataset.organizations.add(dataset.organization)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('v1', '0038_rename_level_area_administrative_level'),
+    ]
+
+    operations = [
+        # Add new ManyToMany field
+        migrations.AddField(
+            model_name='dataset',
+            name='organizations',
+            field=models.ManyToManyField(
+                related_name='datasets',
+                to='v1.organization',
+                verbose_name='Organizations',
+                help_text='Organizations associated with this dataset',
+            ),
+        ),
+        # Run data migration
+        migrations.RunPython(
+            migrate_organization_to_organizations,
+            reverse_code=migrations.RunPython.noop
+        ),
+        # Remove old ForeignKey field
+        migrations.RemoveField(
+            model_name='dataset',
+            name='organization',
+        ),
+    ]

backend/apps/api/v1/migrations/0040_table_publishers_data_cleaners.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 4.2.10 on 2024-05-10 16:00
+
+from django.conf import settings
+from django.db import migrations, models
+
+
+def migrate_publishers_and_cleaners(apps, schema_editor):
+    """Migrate existing ForeignKey relationships to ManyToMany"""
+    Table = apps.get_model('v1', 'Table')
+    for table in Table.objects.all():
+        # Store old ForeignKey values
+        old_publisher = getattr(table, 'published_by_old', None)
+        old_cleaner = getattr(table, 'data_cleaned_by_old', None)
+
+        # Add to new M2M fields if they existed
+        if old_publisher:
+            table.published_by.add(old_publisher)
+        if old_cleaner:
+            table.data_cleaned_by.add(old_cleaner)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('v1', '0039_dataset_organizations'),
+    ]
+
+    operations = [
+        # Rename old fields temporarily
+        migrations.RenameField(
+            model_name='table',
+            old_name='published_by',
+            new_name='published_by_old',
+        ),
+        migrations.RenameField(
+            model_name='table',
+            old_name='data_cleaned_by',
+            new_name='data_cleaned_by_old',
+        ),
+        # Add new M2M fields
+        migrations.AddField(
+            model_name='table',
+            name='published_by',
+            field=models.ManyToManyField(
+                blank=True,
+                related_name='tables_published',
+                to=settings.AUTH_USER_MODEL,
+                verbose_name='Published by',
+                help_text='People who published the table',
+            ),
+        ),
+        migrations.AddField(
+            model_name='table',
+            name='data_cleaned_by',
+            field=models.ManyToManyField(
+                blank=True,
+                related_name='tables_cleaned',
+                to=settings.AUTH_USER_MODEL,
+                verbose_name='Data cleaned by',
+                help_text='People who cleaned the data',
+            ),
+        ),
+        # Run data migration
+        migrations.RunPython(
+            migrate_publishers_and_cleaners,
+            reverse_code=migrations.RunPython.noop
+        ),
+        # Remove old fields
+        migrations.RemoveField(
+            model_name='table',
+            name='published_by_old',
+        ),
+        migrations.RemoveField(
+            model_name='table',
+            name='data_cleaned_by_old',
+        ),
+    ]