diff --git a/backend/apps/api/v1/admin.py b/backend/apps/api/v1/admin.py index f35445c4..693fd77c 100644 --- a/backend/apps/api/v1/admin.py +++ b/backend/apps/api/v1/admin.py @@ -9,8 +9,11 @@ from django.http import HttpRequest from django.shortcuts import render from django.utils.html import format_html +from django.urls import reverse from modeltranslation.admin import TabbedTranslationAdmin, TranslationStackedInline from ordered_model.admin import OrderedInlineModelAdminMixin, OrderedStackedInline +from django.contrib.auth.admin import UserAdmin +from django.contrib.auth.models import User from backend.apps.api.v1.filters import ( DatasetOrganizationListFilter, @@ -19,15 +22,21 @@ TableDirectoryListFilter, TableObservationListFilter, TableOrganizationListFilter, + AreaAdministrativeLevelFilter, + AreaParentFilter, ) from backend.apps.api.v1.forms import ( CloudTableInlineForm, ColumnInlineForm, ColumnOriginalNameInlineForm, CoverageInlineForm, + MeasurementUnitInlineForm, ObservationLevelInlineForm, + PollInlineForm, ReorderColumnsForm, + ReorderObservationLevelsForm, ReorderTablesForm, + TableForm, TableInlineForm, UpdateInlineForm, ) @@ -50,6 +59,8 @@ Key, Language, License, + MeasurementUnit, + MeasurementUnitCategory, ObservationLevel, Organization, Pipeline, @@ -61,6 +72,7 @@ Tag, Theme, Update, + Poll, ) from backend.apps.api.v1.tasks import ( rebuild_search_index_task, @@ -80,6 +92,12 @@ class OrderedTranslatedInline(OrderedStackedInline, TranslationStackedInline): pass +class MeasurementUnitInline(OrderedTranslatedInline): + model = MeasurementUnit + form = MeasurementUnitInlineForm + extra = 0 + show_change_link = True + class ColumnInline(OrderedTranslatedInline): model = Column form = ColumnInlineForm @@ -123,30 +141,61 @@ class ColumnOriginalNameInline(TranslationStackedInline): ] -class CloudTableInline(admin.StackedInline): +class CloudTableInline(admin.TabularInline): model = CloudTable - form = CloudTableInlineForm extra = 0 - fields = [ - "id", + can_delete = False + show_change_link = True + readonly_fields = [ "gcp_project_id", "gcp_dataset_id", "gcp_table_id", ] + fields = readonly_fields + template = 'admin/cloud_table_inline.html' + + def has_add_permission(self, request, obj=None): + return False + + def has_change_permission(self, request, obj=None): + return False -class ObservationLevelInline(admin.StackedInline): +class ObservationLevelInline(OrderedStackedInline): model = ObservationLevel form = ObservationLevelInlineForm extra = 0 - fields = [ - "id", - "entity", - ] - autocomplete_fields = [ + show_change_link = True + readonly_fields = [ "entity", + "order", + "move_up_down_links", ] + fields = readonly_fields + template = 'admin/observation_level_inline.html' + ordering = ["order"] + + def get_formset(self, request, obj=None, **kwargs): + self.parent_obj = obj + return super().get_formset(request, obj, **kwargs) + def get_ordering_prefix(self): + """Return the appropriate ordering prefix based on parent model""" + if isinstance(self.parent_obj, Table): + return 'table' + elif isinstance(self.parent_obj, RawDataSource): + return 'rawdatasource' + elif isinstance(self.parent_obj, InformationRequest): + return 'informationrequest' + elif isinstance(self.parent_obj, Analysis): + return 'analysis' + return super().get_ordering_prefix() + + def has_add_permission(self, request, obj=None): + return False + + def has_change_permission(self, request, obj=None): + return False class TableInline(OrderedTranslatedInline): model = Table @@ -246,6 +295,23 @@ class UpdateInline(admin.StackedInline): ] +class PollInline(admin.StackedInline): + model = Poll + form = PollInlineForm + extra = 0 + fields = [ + "id", + "entity", + "frequency", + "latest", + "pipeline", + ] + autocomplete_fields = [ + "entity", + "pipeline", + ] + + ################################################################################ # Model Admins Actions ################################################################################ @@ -419,6 +485,48 @@ def reset_column_order(modeladmin, request, queryset): reset_column_order.short_description = "Reiniciar ordem das colunas" +def reorder_observation_levels(modeladmin, request, queryset): + """Reorder observation levels in respect to parent""" + if "do_action" in request.POST: + form = ReorderObservationLevelsForm(request.POST) + if form.is_valid(): + if queryset.count() != 1: + messages.error( + request, + "To pass the names manually you must select only one parent.", + ) + return + + parent = queryset.first() + ordered_entities = form.cleaned_data["ordered_entities"].split() + + # Get observation levels for this parent + if hasattr(parent, 'observation_levels'): + obs_levels = parent.observation_levels.all() + + # Create a mapping of entity names to observation levels + obs_by_entity = {ol.entity.name: ol for ol in obs_levels} + + # Update order based on provided entity names + for i, entity_name in enumerate(ordered_entities): + if entity_name in obs_by_entity: + obs_by_entity[entity_name].order = i + obs_by_entity[entity_name].save() + + messages.success(request, "Observation levels reordered successfully") + else: + messages.error(request, "Selected object has no observation levels") + else: + form = ReorderObservationLevelsForm() + return render( + request, + "admin/reorder_observation_levels.html", + {"title": "Reorder observation levels", "parents": queryset, "form": form}, + ) + +reorder_observation_levels.short_description = "Alterar ordem dos níveis de observação" + + ################################################################################ # Model Admins ################################################################################ @@ -431,11 +539,21 @@ class AreaAdmin(TabbedTranslationAdmin): list_display = [ "name", "slug", + "administrative_level", + "parent", ] search_fields = [ "name", "slug", ] + list_filter = [ + AreaAdministrativeLevelFilter, + AreaParentFilter, + ] + autocomplete_fields = [ + "parent", + "entity", + ] class OrganizationAdmin(TabbedTranslationAdmin): @@ -489,33 +607,45 @@ class DatasetAdmin(OrderedInlineModelAdminMixin, TabbedTranslationAdmin): readonly_fields = [ "id", "full_slug", - "coverage", + "spatial_coverage", + "temporal_coverage", "contains_tables", "contains_raw_data_sources", "contains_information_requests", + "page_views", "created_at", "updated_at", "related_objects", ] - search_fields = ["name", "slug", "organization__name"] + search_fields = [ + "name", + "slug", + "organizations__name" + ] filter_horizontal = [ "tags", "themes", + "organizations", ] list_filter = [ DatasetOrganizationListFilter, ] list_display = [ "name", - "organization", - "coverage", + "get_organizations", + "spatial_coverage", + "temporal_coverage", "related_objects", - "page_views", "created_at", "updated_at", ] ordering = ["-updated_at"] + def get_organizations(self, obj): + """Display all organizations for the dataset""" + return ", ".join([org.name for org in obj.organizations.all()]) + get_organizations.short_description = "Organizations" + def related_objects(self, obj): return format_html( " 1 else "table", ) - related_objects.short_description = "Tables" +class CustomUserAdmin(UserAdmin): + search_fields = ['username', 'first_name', 'last_name', 'email'] + +if User in admin.site._registry: + admin.site.unregister(User) +admin.site.register(User, CustomUserAdmin) + class TableAdmin(OrderedInlineModelAdminMixin, TabbedTranslationAdmin): + form = TableForm actions = [ reorder_columns, reset_column_order, + reorder_observation_levels, update_table_metadata, update_table_neighbors, update_page_views, @@ -548,25 +686,33 @@ class TableAdmin(OrderedInlineModelAdminMixin, TabbedTranslationAdmin): "partitions", "created_at", "updated_at", + "spatial_coverage", + "full_temporal_coverage", "coverage_datetime_units", + "number_rows", + "number_columns", + "uncompressed_file_size", + "compressed_file_size", + "page_views", ] search_fields = [ "name", "dataset__name", ] autocomplete_fields = [ - "dataset", - "partner_organization", - "published_by", - "data_cleaned_by", + 'dataset', + 'partner_organization', + 'published_by', + 'data_cleaned_by', + ] + filter_horizontal = [ + 'raw_data_source', ] list_display = [ "name", "dataset", - "number_columns", - "number_rows", - "uncompressed_file_size", - "page_views", + "get_publishers", + "get_data_cleaners", "created_at", "updated_at", ] @@ -578,15 +724,26 @@ class TableAdmin(OrderedInlineModelAdminMixin, TabbedTranslationAdmin): ] ordering = ["-updated_at"] - def get_form(self, request, obj=None, **kwargs): - """Get form, and save the current object""" - self.current_obj = obj - return super().get_form(request, obj, **kwargs) + def get_queryset(self, request): + """Optimize queryset by prefetching related objects""" + return super().get_queryset(request).prefetch_related( + 'published_by', + 'data_cleaned_by' + ) + + def get_publishers(self, obj): + """Display all publishers for the table""" + # Convert to list to avoid multiple DB hits + publishers = list(obj.published_by.all()) + return ", ".join(f"{pub.first_name} {pub.last_name}" for pub in publishers) + get_publishers.short_description = "Publishers" - def formfield_for_manytomany(self, db_field, request, **kwargs): - if self.current_obj and db_field.name == "raw_data_source": - kwargs["queryset"] = RawDataSource.objects.filter(dataset=self.current_obj.dataset) - return super().formfield_for_manytomany(db_field, request, **kwargs) + def get_data_cleaners(self, obj): + """Display all data cleaners for the table""" + # Convert to list to avoid multiple DB hits + cleaners = list(obj.data_cleaned_by.all()) + return ", ".join(f"{cleaner.first_name} {cleaner.last_name}" for cleaner in cleaners) + get_data_cleaners.short_description = "Data Cleaners" class TableNeighborAdmin(admin.ModelAdmin): @@ -609,6 +766,34 @@ class TableNeighborAdmin(admin.ModelAdmin): ordering = ["table_a", "table_b"] +class MeasurementUnitCategoryAdmin(TabbedTranslationAdmin): + list_display = [ + "slug", + "name", + ] + search_fields = [ + "slug", + "name", + ] + +class MeasurementUnitAdmin(TabbedTranslationAdmin): + list_display = [ + "slug", + "name", + "tex", + "category", + ] + search_fields = [ + "slug", + "name", + "tex", + "category__name", + ] + list_filter = [ + "category", + ] + + class ColumnForm(forms.ModelForm): class Meta: model = Column @@ -628,7 +813,7 @@ class ColumnAdmin(TabbedTranslationAdmin): "table", ] list_filter = [ - "table__dataset__organization__name", + "table__dataset__organizations__name", ] autocomplete_fields = [ "table", @@ -637,6 +822,8 @@ class ColumnAdmin(TabbedTranslationAdmin): readonly_fields = [ "id", "order", + "spatial_coverage", + "temporal_coverage", ] search_fields = ["name", "table__name"] inlines = [ @@ -660,7 +847,37 @@ class ColumnOriginalNameAdmin(TabbedTranslationAdmin): inlines = [CoverageInline] +def reset_observation_level_order(modeladmin, request, queryset): + """Reset observation level order in respect to parent""" + # Group observation levels by their parent + by_table = {} + by_raw_data_source = {} + by_information_request = {} + by_analysis = {} + + for obs in queryset: + if obs.table_id: + by_table.setdefault(obs.table_id, []).append(obs) + elif obs.raw_data_source_id: + by_raw_data_source.setdefault(obs.raw_data_source_id, []).append(obs) + elif obs.information_request_id: + by_information_request.setdefault(obs.information_request_id, []).append(obs) + elif obs.analysis_id: + by_analysis.setdefault(obs.analysis_id, []).append(obs) + + # Reset order within each parent group + for parent_levels in [by_table, by_raw_data_source, by_information_request, by_analysis]: + for levels in parent_levels.values(): + sorted_levels = sorted(levels, key=lambda x: x.entity.name) + for i, obs_level in enumerate(sorted_levels): + obs_level.order = i + obs_level.save() + +reset_observation_level_order.short_description = "Reiniciar ordem dos níveis de observação" + + class ObservationLevelAdmin(admin.ModelAdmin): + actions = [reset_observation_level_order] readonly_fields = [ "id", ] @@ -678,6 +895,9 @@ class ObservationLevelAdmin(admin.ModelAdmin): ] list_filter = [ "entity__category__name", + "table", + "raw_data_source", + "information_request", ] list_display = [ "__str__", @@ -687,7 +907,10 @@ class ObservationLevelAdmin(admin.ModelAdmin): ] -class RawDataSourceAdmin(TabbedTranslationAdmin): +class RawDataSourceAdmin(OrderedInlineModelAdminMixin, TabbedTranslationAdmin): + actions = [ + reorder_observation_levels, + ] list_display = ["name", "dataset", "created_at", "updated_at"] search_fields = ["name", "dataset__name"] readonly_fields = ["id", "created_at", "updated_at"] @@ -699,15 +922,26 @@ class RawDataSourceAdmin(TabbedTranslationAdmin): "languages", "area_ip_address_required", ] - inlines = [CoverageInline] + inlines = [ + CoverageInline, + ObservationLevelInline, + PollInline, + ] -class InformationRequestAdmin(TabbedTranslationAdmin): +class InformationRequestAdmin(OrderedInlineModelAdminMixin, TabbedTranslationAdmin): + actions = [ + reorder_observation_levels, + ] list_display = ["__str__", "dataset", "created_at", "updated_at"] search_fields = ["__str__", "dataset__name"] readonly_fields = ["id", "created_at", "updated_at"] autocomplete_fields = ["dataset"] - inlines = [CoverageInline, ObservationLevelInline] + inlines = [ + CoverageInline, + ObservationLevelInline, + PollInline, + ] class CoverageTypeAdminFilter(admin.SimpleListFilter): @@ -770,7 +1004,7 @@ class DateTimeRangeAdmin(admin.ModelAdmin): class CoverageAdmin(admin.ModelAdmin): - readonly_fields = ["id"] + readonly_fields = ["id", "datetime_ranges_display"] list_display = [ "area", "coverage_type", @@ -794,9 +1028,44 @@ class CoverageAdmin(admin.ModelAdmin): "information_request__dataset__name", "column__name", ] - inlines = [ - DateTimeRangeInline, - ] + + def datetime_ranges_display(self, obj): + """Display datetime ranges with links to their admin pages""" + ranges = obj.datetime_ranges.all() + links = [] + for dt_range in ranges: + url = reverse('admin:v1_datetimerange_change', args=[dt_range.id]) + links.append( + format_html('{}', url, str(dt_range)) + ) + + # Add link to add new datetime range + add_url = reverse('admin:v1_datetimerange_add') + f'?coverage={obj.id}' + links.append( + format_html( + 'Add DateTime Range', + add_url + ) + ) + + return format_html('
'.join(links)) + + datetime_ranges_display.short_description = "DateTime Ranges" + + def get_queryset(self, request): + """Optimize queryset by prefetching related objects""" + qs = super().get_queryset(request).select_related( + 'table', + 'column', + 'raw_data_source', + 'information_request', + 'area' + ) + # Add prefetch for datetime_ranges and their units + return qs.prefetch_related( + 'datetime_ranges', + 'datetime_ranges__units' + ) class EntityCategoryAdmin(TabbedTranslationAdmin): @@ -951,6 +1220,9 @@ class AnalysisTypeAdmin(TabbedTranslationAdmin): class AnalysisAdmin(TabbedTranslationAdmin): + actions = [ + reorder_observation_levels, + ] readonly_fields = [ "id", ] @@ -1006,6 +1278,44 @@ class QualityCheckAdmin(TabbedTranslationAdmin): ] +class PollAdmin(admin.ModelAdmin): + readonly_fields = [ + "id", + ] + search_fields = [ + "entity__name", + "raw_data_source__name", + "information_request__dataset__name", + ] + autocomplete_fields = [ + "entity", + "pipeline", + "raw_data_source", + "information_request", + ] + list_filter = [ + "entity__category__name", + ] + list_display = [ + "__str__", + "raw_data_source", + "information_request", + ] + +class PipelineAdmin(admin.ModelAdmin): + readonly_fields = [ + "id", + ] + search_fields = [ + "id", + "github_url", + ] + list_display = [ + "id", + "github_url", + ] + + admin.site.register(Analysis, AnalysisAdmin) admin.site.register(AnalysisType, AnalysisTypeAdmin) admin.site.register(Area, AreaAdmin) @@ -1024,9 +1334,11 @@ class QualityCheckAdmin(TabbedTranslationAdmin): admin.site.register(Key, KeyAdmin) admin.site.register(Language, LanguageAdmin) admin.site.register(License, LicenseAdmin) +admin.site.register(MeasurementUnit, MeasurementUnitAdmin) +admin.site.register(MeasurementUnitCategory, MeasurementUnitCategoryAdmin) admin.site.register(ObservationLevel, ObservationLevelAdmin) admin.site.register(Organization, OrganizationAdmin) -admin.site.register(Pipeline) +admin.site.register(Pipeline, PipelineAdmin) admin.site.register(RawDataSource, RawDataSourceAdmin) admin.site.register(Status, StatusAdmin) admin.site.register(Table, TableAdmin) @@ -1035,3 +1347,4 @@ class QualityCheckAdmin(TabbedTranslationAdmin): admin.site.register(Theme, ThemeAdmin) admin.site.register(Update, UpdateAdmin) admin.site.register(QualityCheck, QualityCheckAdmin) +admin.site.register(Poll, PollAdmin) diff --git a/backend/apps/api/v1/filters.py b/backend/apps/api/v1/filters.py index f262e8e5..caf69a36 100644 --- a/backend/apps/api/v1/filters.py +++ b/backend/apps/api/v1/filters.py @@ -2,7 +2,7 @@ from django.contrib import admin from django.utils.translation import gettext_lazy -from backend.apps.api.v1.models import Coverage, ObservationLevel, Organization +from backend.apps.api.v1.models import Area, Coverage, ObservationLevel, Organization class OrganizationImageListFilter(admin.SimpleListFilter): @@ -27,12 +27,13 @@ class DatasetOrganizationListFilter(admin.SimpleListFilter): parameter_name = "organization" def lookups(self, request, model_admin): - values = Organization.objects.order_by("name").distinct().values("name", "pk") - return [(v.get("pk"), v.get("name")) for v in values] + organizations = Organization.objects.all().order_by('slug') + return [(org.id, org.name) for org in organizations] def queryset(self, request, queryset): if self.value(): - return queryset.filter(organization=self.value()) + return queryset.filter(organizations__id=self.value()) + return queryset class TableOrganizationListFilter(admin.SimpleListFilter): @@ -99,3 +100,36 @@ def queryset(self, request, queryset): return queryset.filter(is_directory=True) if self.value() == "false": return queryset.filter(is_directory=False) + + +class AreaAdministrativeLevelFilter(admin.SimpleListFilter): + title = "Administrative Level" + parameter_name = "administrative_level" + + def lookups(self, request, model_admin): + return [ + (0, '0'), + (1, '1'), + (2, '2'), + (3, '3'), + (4, '4'), + (5, '5'), + ] + + def queryset(self, request, queryset): + if self.value() is not None: + return queryset.filter(administrative_level=self.value()) + + +class AreaParentFilter(admin.SimpleListFilter): + title = "Parent Area" + parameter_name = "parent" + + def lookups(self, request, model_admin): + # Get all areas that have children, ordered by name + parents = Area.objects.filter(children__isnull=False).distinct().order_by('name') + return [(area.id, f"{area.name}") for area in parents] + + def queryset(self, request, queryset): + if self.value(): + return queryset.filter(parent_id=self.value()) diff --git a/backend/apps/api/v1/forms/__init__.py b/backend/apps/api/v1/forms/__init__.py index c7e0c78e..9ead2290 100644 --- a/backend/apps/api/v1/forms/__init__.py +++ b/backend/apps/api/v1/forms/__init__.py @@ -4,9 +4,13 @@ ColumnInlineForm, ColumnOriginalNameInlineForm, CoverageInlineForm, + MeasurementUnitInlineForm, ObservationLevelInlineForm, + PollInlineForm, + TableForm, TableInlineForm, UpdateInlineForm, ) from backend.apps.api.v1.forms.reorder_columns_form import ReorderColumnsForm # noqa: F401 from backend.apps.api.v1.forms.reorder_tables_form import ReorderTablesForm # noqa: F401 +from backend.apps.api.v1.forms.reorder_observation_levels_form import ReorderObservationLevelsForm # noqa: F401 \ No newline at end of file diff --git a/backend/apps/api/v1/forms/admin_form.py b/backend/apps/api/v1/forms/admin_form.py index 1f2081bb..7dc470ed 100644 --- a/backend/apps/api/v1/forms/admin_form.py +++ b/backend/apps/api/v1/forms/admin_form.py @@ -6,7 +6,10 @@ Column, ColumnOriginalName, Coverage, + MeasurementUnit, ObservationLevel, + Poll, + RawDataSource, Table, Update, ) @@ -23,6 +26,27 @@ class Meta: abstract = True +class TableForm(forms.ModelForm): + class Meta: + model = Table + fields = '__all__' + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if self.instance: + # Check both the saved instance and current form data + dataset_id = self.instance.dataset_id + if not dataset_id and self.data: + dataset_id = self.data.get('dataset') + + if dataset_id: + self.fields['raw_data_source'].queryset = RawDataSource.objects.filter( + dataset_id=dataset_id + ) + else: + self.fields['raw_data_source'].queryset = RawDataSource.objects.none() + + class TableInlineForm(UUIDHiddenIdForm): class Meta(UUIDHiddenIdForm): model = Table @@ -40,14 +64,9 @@ class Meta(UUIDHiddenIdForm): "data_cleaned_by", "data_cleaning_description", "data_cleaning_code_url", - "raw_data_url", "auxiliary_files_url", "architecture_url", "source_bucket_name", - "uncompressed_file_size", - "compressed_file_size", - "number_rows", - "number_columns", "is_closed", ] readonly_fields = [ @@ -56,6 +75,12 @@ class Meta(UUIDHiddenIdForm): ] +class MeasurementUnitInlineForm(UUIDHiddenIdForm): + class Meta(UUIDHiddenIdForm.Meta): + model = MeasurementUnit + fields = "__all__" + + class ColumnInlineForm(UUIDHiddenIdForm): class Meta(UUIDHiddenIdForm.Meta): model = Column @@ -93,7 +118,18 @@ class Meta(UUIDHiddenIdForm.Meta): class ObservationLevelInlineForm(UUIDHiddenIdForm): class Meta(UUIDHiddenIdForm.Meta): model = ObservationLevel - fields = "__all__" + fields = [ + "id", + "entity", + "table", + "raw_data_source", + "information_request", + "analysis", + ] + readonly_fields = [ + "order", + "move_up_down_links", + ] class CoverageInlineForm(UUIDHiddenIdForm): @@ -110,3 +146,12 @@ class UpdateInlineForm(UUIDHiddenIdForm): class Meta(UUIDHiddenIdForm.Meta): model = Update fields = "__all__" + +class PollInlineForm(forms.ModelForm): + class Meta: + model = Poll + fields = [ + "entity", + "frequency", + "latest", + ] \ No newline at end of file diff --git a/backend/apps/api/v1/forms/reorder_observation_levels_form.py b/backend/apps/api/v1/forms/reorder_observation_levels_form.py new file mode 100644 index 00000000..46dd4ad9 --- /dev/null +++ b/backend/apps/api/v1/forms/reorder_observation_levels_form.py @@ -0,0 +1,8 @@ +from django import forms + +class ReorderObservationLevelsForm(forms.Form): + ordered_entities = forms.CharField( + required=False, + widget=forms.Textarea(attrs={"rows": 10, "cols": 40}), + help_text="Enter entity names one per line in desired order", + ) \ No newline at end of file diff --git a/backend/apps/api/v1/migrations/0037_area_entity_area_level_area_parent.py b/backend/apps/api/v1/migrations/0037_area_entity_area_level_area_parent.py new file mode 100644 index 00000000..025b6dd5 --- /dev/null +++ b/backend/apps/api/v1/migrations/0037_area_entity_area_level_area_parent.py @@ -0,0 +1,29 @@ +# Generated by Django 4.2.16 on 2024-11-03 01:29 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('v1', '0036_datetimerange_units'), + ] + + operations = [ + migrations.AddField( + model_name='area', + name='entity', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='areas', to='v1.entity'), + ), + migrations.AddField( + model_name='area', + name='level', + field=models.IntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name='area', + name='parent', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='children', to='v1.area'), + ), + ] diff --git a/backend/apps/api/v1/migrations/0038_rename_level_area_administrative_level.py b/backend/apps/api/v1/migrations/0038_rename_level_area_administrative_level.py new file mode 100644 index 00000000..e1511709 --- /dev/null +++ b/backend/apps/api/v1/migrations/0038_rename_level_area_administrative_level.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.16 on 2024-11-03 01:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('v1', '0037_area_entity_area_level_area_parent'), + ] + + operations = [ + migrations.RenameField( + model_name='area', + old_name='level', + new_name='administrative_level', + ), + ] diff --git a/backend/apps/api/v1/migrations/0039_dataset_organizations.py b/backend/apps/api/v1/migrations/0039_dataset_organizations.py new file mode 100644 index 00000000..82b9b79c --- /dev/null +++ b/backend/apps/api/v1/migrations/0039_dataset_organizations.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +# Generated by Django 4.2.10 on 2024-05-10 15:30 + +from django.db import migrations, models + + +def migrate_organization_to_organizations(apps, schema_editor): + Dataset = apps.get_model('v1', 'Dataset') + for dataset in Dataset.objects.all(): + if dataset.organization: + dataset.organizations.add(dataset.organization) + + +class Migration(migrations.Migration): + dependencies = [ + ('v1', '0038_rename_level_area_administrative_level'), + ] + + operations = [ + # Add new ManyToMany field + migrations.AddField( + model_name='dataset', + name='organizations', + field=models.ManyToManyField( + related_name='datasets', + to='v1.organization', + verbose_name='Organizations', + help_text='Organizations associated with this dataset', + ), + ), + # Run data migration + migrations.RunPython( + migrate_organization_to_organizations, + reverse_code=migrations.RunPython.noop + ), + # Remove old ForeignKey field + migrations.RemoveField( + model_name='dataset', + name='organization', + ), + ] \ No newline at end of file diff --git a/backend/apps/api/v1/migrations/0040_table_publishers_data_cleaners.py b/backend/apps/api/v1/migrations/0040_table_publishers_data_cleaners.py new file mode 100644 index 00000000..884109a6 --- /dev/null +++ b/backend/apps/api/v1/migrations/0040_table_publishers_data_cleaners.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +# Generated by Django 4.2.10 on 2024-05-10 16:00 + +from django.conf import settings +from django.db import migrations, models + + +def migrate_publishers_and_cleaners(apps, schema_editor): + """Migrate existing ForeignKey relationships to ManyToMany""" + Table = apps.get_model('v1', 'Table') + for table in Table.objects.all(): + # Store old ForeignKey values + old_publisher = getattr(table, 'published_by_old', None) + old_cleaner = getattr(table, 'data_cleaned_by_old', None) + + # Add to new M2M fields if they existed + if old_publisher: + table.published_by.add(old_publisher) + if old_cleaner: + table.data_cleaned_by.add(old_cleaner) + + +class Migration(migrations.Migration): + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('v1', '0039_dataset_organizations'), + ] + + operations = [ + # Rename old fields temporarily + migrations.RenameField( + model_name='table', + old_name='published_by', + new_name='published_by_old', + ), + migrations.RenameField( + model_name='table', + old_name='data_cleaned_by', + new_name='data_cleaned_by_old', + ), + # Add new M2M fields + migrations.AddField( + model_name='table', + name='published_by', + field=models.ManyToManyField( + blank=True, + related_name='tables_published', + to=settings.AUTH_USER_MODEL, + verbose_name='Published by', + help_text='People who published the table', + ), + ), + migrations.AddField( + model_name='table', + name='data_cleaned_by', + field=models.ManyToManyField( + blank=True, + related_name='tables_cleaned', + to=settings.AUTH_USER_MODEL, + verbose_name='Data cleaned by', + help_text='People who cleaned the data', + ), + ), + # Run data migration + migrations.RunPython( + migrate_publishers_and_cleaners, + reverse_code=migrations.RunPython.noop + ), + # Remove old fields + migrations.RemoveField( + model_name='table', + name='published_by_old', + ), + migrations.RemoveField( + model_name='table', + name='data_cleaned_by_old', + ), + ] \ No newline at end of file diff --git a/backend/apps/api/v1/migrations/0041_remove_table_raw_data_url_and_more.py b/backend/apps/api/v1/migrations/0041_remove_table_raw_data_url_and_more.py new file mode 100644 index 00000000..8c6e9035 --- /dev/null +++ b/backend/apps/api/v1/migrations/0041_remove_table_raw_data_url_and_more.py @@ -0,0 +1,123 @@ +# Generated by Django 4.2.16 on 2024-11-05 23:04 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('v1', '0040_table_publishers_data_cleaners'), + ] + + operations = [ + migrations.RemoveField( + model_name='table', + name='raw_data_url', + ), + migrations.AlterField( + model_name='analysis', + name='analysis_type', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='analyses', to='v1.analysistype'), + ), + migrations.AlterField( + model_name='area', + name='administrative_level', + field=models.IntegerField(blank=True, choices=[(0, '0'), (1, '1'), (2, '2'), (3, '3'), (4, '4'), (5, '5')], null=True), + ), + migrations.AlterField( + model_name='area', + name='entity', + field=models.ForeignKey(blank=True, limit_choices_to={'category__slug': 'spatial'}, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='areas', to='v1.entity'), + ), + migrations.AlterField( + model_name='area', + name='parent', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='children', to='v1.area'), + ), + migrations.AlterField( + model_name='column', + name='bigquery_type', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='columns', to='v1.bigquerytype'), + ), + migrations.AlterField( + model_name='column', + name='directory_primary_key', + field=models.ForeignKey(blank=True, limit_choices_to={'is_primary_key': True, 'table__is_directory': True}, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='columns', to='v1.column'), + ), + migrations.AlterField( + model_name='column', + name='observation_level', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='columns', to='v1.observationlevel'), + ), + migrations.AlterField( + model_name='column', + name='status', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='columns', to='v1.status'), + ), + migrations.AlterField( + model_name='coverage', + name='area', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='coverages', to='v1.area'), + ), + migrations.AlterField( + model_name='entity', + name='category', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='entities', to='v1.entitycategory'), + ), + migrations.AlterField( + model_name='informationrequest', + name='status', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='information_requests', to='v1.status'), + ), + migrations.AlterField( + model_name='observationlevel', + name='entity', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='observation_levels', to='v1.entity'), + ), + migrations.AlterField( + model_name='organization', + name='area', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='organizations', to='v1.area'), + ), + migrations.AlterField( + model_name='poll', + name='entity', + field=models.ForeignKey(blank=True, limit_choices_to={'category__slug': 'datetime'}, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='polls', to='v1.entity'), + ), + migrations.AlterField( + model_name='qualitycheck', + name='pipeline', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='quality_checks', to='v1.pipeline'), + ), + migrations.AlterField( + model_name='rawdatasource', + name='availability', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='raw_data_sources', to='v1.availability'), + ), + migrations.AlterField( + model_name='rawdatasource', + name='license', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='raw_data_sources', to='v1.license'), + ), + migrations.AlterField( + model_name='table', + name='license', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='tables', to='v1.license'), + ), + migrations.AlterField( + model_name='table', + name='partner_organization', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='partner_tables', to='v1.organization'), + ), + migrations.AlterField( + model_name='table', + name='pipeline', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='tables', to='v1.pipeline'), + ), + migrations.AlterField( + model_name='update', + name='entity', + field=models.ForeignKey(blank=True, limit_choices_to={'category__slug': 'datetime'}, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='updates', to='v1.entity'), + ), + ] diff --git a/backend/apps/api/v1/migrations/0042_measurementunit.py b/backend/apps/api/v1/migrations/0042_measurementunit.py new file mode 100644 index 00000000..f70c38f1 --- /dev/null +++ b/backend/apps/api/v1/migrations/0042_measurementunit.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.16 on 2024-11-05 23:20 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('v1', '0041_remove_table_raw_data_url_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='MeasurementUnit', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ('slug', models.SlugField(unique=True)), + ('name', models.CharField(max_length=255)), + ], + options={ + 'verbose_name': 'Measurement Unit', + 'verbose_name_plural': 'Measurement Units', + 'db_table': 'measurement_unit', + 'ordering': ['slug'], + }, + ), + ] diff --git a/backend/apps/api/v1/migrations/0043_add_measurement_unit_translations.py b/backend/apps/api/v1/migrations/0043_add_measurement_unit_translations.py new file mode 100644 index 00000000..e7eb73ba --- /dev/null +++ b/backend/apps/api/v1/migrations/0043_add_measurement_unit_translations.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("v1", "0042_measurementunit"), + ] + + operations = [ + migrations.AddField( + model_name="measurementunit", + name="name_pt", + field=models.CharField(max_length=255, null=True), + ), + migrations.AddField( + model_name="measurementunit", + name="name_en", + field=models.CharField(max_length=255, null=True), + ), + migrations.AddField( + model_name="measurementunit", + name="name_es", + field=models.CharField(max_length=255, null=True), + ), + ] \ No newline at end of file diff --git a/backend/apps/api/v1/migrations/0044_measurementunitcategory_measurementunit_tex_and_more.py b/backend/apps/api/v1/migrations/0044_measurementunitcategory_measurementunit_tex_and_more.py new file mode 100644 index 00000000..d79629e7 --- /dev/null +++ b/backend/apps/api/v1/migrations/0044_measurementunitcategory_measurementunit_tex_and_more.py @@ -0,0 +1,42 @@ +# Generated by Django 4.2.16 on 2024-11-06 00:03 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('v1', '0043_add_measurement_unit_translations'), + ] + + operations = [ + migrations.CreateModel( + name='MeasurementUnitCategory', + fields=[ + ('id', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)), + ('slug', models.SlugField(unique=True)), + ('name', models.CharField(max_length=255)), + ('name_pt', models.CharField(max_length=255, null=True)), + ('name_en', models.CharField(max_length=255, null=True)), + ('name_es', models.CharField(max_length=255, null=True)), + ], + options={ + 'verbose_name': 'Measurement Unit Category', + 'verbose_name_plural': 'Measurement Unit Categories', + 'db_table': 'measurement_unit_category', + 'ordering': ['slug'], + }, + ), + migrations.AddField( + model_name='measurementunit', + name='tex', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='measurementunit', + name='category', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='measurement_units', to='v1.measurementunitcategory'), + ), + ] diff --git a/backend/apps/api/v1/migrations/0045_add_measurement_categories_and_units.py b/backend/apps/api/v1/migrations/0045_add_measurement_categories_and_units.py new file mode 100644 index 00000000..fbff222b --- /dev/null +++ b/backend/apps/api/v1/migrations/0045_add_measurement_categories_and_units.py @@ -0,0 +1,198 @@ +# -*- coding: utf-8 -*- +from django.db import migrations + +def create_categories_and_units(apps, schema_editor): + MeasurementUnitCategory = apps.get_model('v1', 'MeasurementUnitCategory') + MeasurementUnit = apps.get_model('v1', 'MeasurementUnit') + + # Create categories + categories = { + 'distance': { + 'name': 'Distância', + 'name_pt': 'Distância', + 'name_en': 'Distance', + 'name_es': 'Distancia' + }, + 'area': { + 'name': 'Área', + 'name_pt': 'Área', + 'name_en': 'Area', + 'name_es': 'Área' + }, + 'mass': { + 'name': 'Massa', + 'name_pt': 'Massa', + 'name_en': 'Mass', + 'name_es': 'Masa' + }, + 'volume': { + 'name': 'Volume', + 'name_pt': 'Volume', + 'name_en': 'Volume', + 'name_es': 'Volumen' + }, + 'energy': { + 'name': 'Energia', + 'name_pt': 'Energia', + 'name_en': 'Energy', + 'name_es': 'Energía' + }, + 'people': { + 'name': 'Pessoas', + 'name_pt': 'Pessoas', + 'name_en': 'People', + 'name_es': 'Personas' + }, + 'currency': { + 'name': 'Moeda', + 'name_pt': 'Moeda', + 'name_en': 'Currency', + 'name_es': 'Moneda' + }, + 'economics': { + 'name': 'Economia', + 'name_pt': 'Economia', + 'name_en': 'Economics', + 'name_es': 'Economía' + }, + 'datetime': { + 'name': 'Data/Hora', + 'name_pt': 'Data/Hora', + 'name_en': 'Date/Time', + 'name_es': 'Fecha/Hora' + }, + 'percentage': { + 'name': 'Porcentagem', + 'name_pt': 'Porcentagem', + 'name_en': 'Percentage', + 'name_es': 'Porcentaje' + } + } + + category_objects = {} + for slug, names in categories.items(): + category = MeasurementUnitCategory.objects.create( + slug=slug, + name=names['name'], + name_pt=names['name_pt'], + name_en=names['name_en'], + name_es=names['name_es'] + ) + category_objects[slug] = category + + # Define units with their categories and translations + units = { + # Distance + 'kilometer': {'category': 'distance', 'name': 'Kilometer', 'name_pt': 'Quilômetro', 'name_en': 'Kilometer', 'name_es': 'Kilómetro', 'tex': 'km'}, + 'meter': {'category': 'distance', 'name': 'Meter', 'name_pt': 'Metro', 'name_en': 'Meter', 'name_es': 'Metro', 'tex': 'm'}, + 'centimeter': {'category': 'distance', 'name': 'Centimeter', 'name_pt': 'Centímetro', 'name_en': 'Centimeter', 'name_es': 'Centímetro', 'tex': 'cm'}, + 'mile': {'category': 'distance', 'name': 'Mile', 'name_pt': 'Milha', 'name_en': 'Mile', 'name_es': 'Milla', 'tex': 'mi'}, + 'foot': {'category': 'distance', 'name': 'Foot', 'name_pt': 'Pé', 'name_en': 'Foot', 'name_es': 'Pie', 'tex': 'pé'}, + 'inch': {'category': 'distance', 'name': 'Inch', 'name_pt': 'Polegada', 'name_en': 'Inch', 'name_es': 'Pulgada', 'tex': 'polegada'}, + + # Area + 'kilometer2': {'category': 'area', 'name': 'Square Kilometer', 'name_pt': 'Quilômetro Quadrado', 'name_en': 'Square Kilometer', 'name_es': 'Kilómetro Cuadrado', 'tex': 'km^2'}, + 'meter2': {'category': 'area', 'name': 'Square Meter', 'name_pt': 'Metro Quadrado', 'name_en': 'Square Meter', 'name_es': 'Metro Cuadrado', 'tex': 'm^2'}, + 'centimeter2': {'category': 'area', 'name': 'Square Centimeter', 'name_pt': 'Centímetro Quadrado', 'name_en': 'Square Centimeter', 'name_es': 'Centímetro Cuadrado', 'tex': 'cm^2'}, + 'hectare': {'category': 'area', 'name': 'Hectare', 'name_pt': 'Hectare', 'name_en': 'Hectare', 'name_es': 'Hectárea', 'tex': 'ha'}, + 'acre': {'category': 'area', 'name': 'Acre', 'name_pt': 'Acre', 'name_en': 'Acre', 'name_es': 'Acre', 'tex': 'ac'}, + 'mile2': {'category': 'area', 'name': 'Square Mile', 'name_pt': 'Milha Quadrada', 'name_en': 'Square Mile', 'name_es': 'Milla Cuadrada', 'tex': 'mi^2'}, + 'foot2': {'category': 'area', 'name': 'Square Foot', 'name_pt': 'Pé Quadrado', 'name_en': 'Square Foot', 'name_es': 'Pie Cuadrado', 'tex': 'ft^2'}, + 'inch2': {'category': 'area', 'name': 'Square Inch', 'name_pt': 'Polegada Quadrada', 'name_en': 'Square Inch', 'name_es': 'Pulgada Cuadrada', 'tex': 'in^2'}, + + # Mass + 'ton': {'category': 'mass', 'name': 'Ton', 'name_pt': 'Tonelada', 'name_en': 'Ton', 'name_es': 'Tonelada', 'tex': 'ton'}, + 'kilogram': {'category': 'mass', 'name': 'Kilogram', 'name_pt': 'Quilograma', 'name_en': 'Kilogram', 'name_es': 'Kilogramo', 'tex': 'kg'}, + 'gram': {'category': 'mass', 'name': 'Gram', 'name_pt': 'Grama', 'name_en': 'Gram', 'name_es': 'Gramo', 'tex': 'g'}, + 'miligram': {'category': 'mass', 'name': 'Milligram', 'name_pt': 'Miligrama', 'name_en': 'Milligram', 'name_es': 'Miligramo', 'tex': 'mg'}, + 'ounce': {'category': 'mass', 'name': 'Ounce', 'name_pt': 'Onça', 'name_en': 'Ounce', 'name_es': 'Onza', 'tex': 'oz'}, + + # Volume + 'gallon': {'category': 'volume', 'name': 'Gallon', 'name_pt': 'Galão', 'name_en': 'Gallon', 'name_es': 'Galón', 'tex': 'gal'}, + 'litre': {'category': 'volume', 'name': 'Litre', 'name_pt': 'Litro', 'name_en': 'Litre', 'name_es': 'Litro', 'tex': 'l'}, + 'militre': {'category': 'volume', 'name': 'Millilitre', 'name_pt': 'Mililitro', 'name_en': 'Millilitre', 'name_es': 'Mililitro', 'tex': 'ml'}, + 'meter3': {'category': 'volume', 'name': 'Cubic Meter', 'name_pt': 'Metro Cúbico', 'name_en': 'Cubic Meter', 'name_es': 'Metro Cúbico', 'tex': 'm^3'}, + 'mile3': {'category': 'volume', 'name': 'Cubic Mile', 'name_pt': 'Milha Cúbica', 'name_en': 'Cubic Mile', 'name_es': 'Milla Cúbica', 'tex': 'mi^3'}, + 'foot3': {'category': 'volume', 'name': 'Cubic Foot', 'name_pt': 'Pé Cúbico', 'name_en': 'Cubic Foot', 'name_es': 'Pie Cúbico', 'tex': 'ft^3'}, + 'inch3': {'category': 'volume', 'name': 'Cubic Inch', 'name_pt': 'Polegada Cúbica', 'name_en': 'Cubic Inch', 'name_es': 'Pulgada Cúbica', 'tex': 'in^3'}, + 'barrel': {'category': 'volume', 'name': 'Barrel', 'name_pt': 'Barril', 'name_en': 'Barrel', 'name_es': 'Barril', 'tex': 'barrel'}, + 'boe': {'category': 'volume', 'name': 'Barrel of Oil Equivalent', 'name_pt': 'Barril de Óleo Equivalente', 'name_en': 'Barrel of Oil Equivalent', 'name_es': 'Barril de Petróleo Equivalente', 'tex': 'barrel_e'}, + 'toe': {'category': 'volume', 'name': 'Tonne of Oil Equivalent', 'name_pt': 'Tonelada de Óleo Equivalente', 'name_en': 'Tonne of Oil Equivalent', 'name_es': 'Tonelada de Petróleo Equivalente', 'tex': 'ton_e'}, + + # Energy + 'watt': {'category': 'energy', 'name': 'Watt', 'name_pt': 'Watt', 'name_en': 'Watt', 'name_es': 'Vatio', 'tex': 'W'}, + 'kilowatt': {'category': 'energy', 'name': 'Kilowatt', 'name_pt': 'Kilowatt', 'name_en': 'Kilowatt', 'name_es': 'Kilovatio', 'tex': 'kW'}, + 'megawatt': {'category': 'energy', 'name': 'Megawatt', 'name_pt': 'Megawatt', 'name_en': 'Megawatt', 'name_es': 'Megavatio', 'tex': 'mW'}, + 'gigawatt': {'category': 'energy', 'name': 'Gigawatt', 'name_pt': 'Gigawatt', 'name_en': 'Gigawatt', 'name_es': 'Gigavatio', 'tex': 'gW'}, + 'terawatt': {'category': 'energy', 'name': 'Terawatt', 'name_pt': 'Terawatt', 'name_en': 'Terawatt', 'name_es': 'Teravatio', 'tex': 'tW'}, + 'volt': {'category': 'energy', 'name': 'Volt', 'name_pt': 'Volt', 'name_en': 'Volt', 'name_es': 'Voltio', 'tex': 'V'}, + 'kilovolt': {'category': 'energy', 'name': 'Kilovolt', 'name_pt': 'Kilovolt', 'name_en': 'Kilovolt', 'name_es': 'Kilovoltio', 'tex': 'kV'}, + 'megavolt': {'category': 'energy', 'name': 'Megavolt', 'name_pt': 'Megavolt', 'name_en': 'Megavolt', 'name_es': 'Megavoltio', 'tex': 'mV'}, + 'gigavolt': {'category': 'energy', 'name': 'Gigavolt', 'name_pt': 'Gigavolt', 'name_en': 'Gigavolt', 'name_es': 'Gigavoltio', 'tex': 'gV'}, + 'teravolt': {'category': 'energy', 'name': 'Teravolt', 'name_pt': 'Teravolt', 'name_en': 'Teravolt', 'name_es': 'Teravoltio', 'tex': 'tV'}, + + # People + 'person': {'category': 'people', 'name': 'Person', 'name_pt': 'Pessoa', 'name_en': 'Person', 'name_es': 'Persona', 'tex': 'per'}, + 'household': {'category': 'people', 'name': 'Household', 'name_pt': 'Domicílio', 'name_en': 'Household', 'name_es': 'Hogar', 'tex': 'dom'}, + + # Currency + 'ars': {'category': 'currency', 'name': 'Argentine Peso', 'name_pt': 'Peso Argentino', 'name_en': 'Argentine Peso', 'name_es': 'Peso Argentino', 'tex': 'ARS'}, + 'brl': {'category': 'currency', 'name': 'Brazilian Real', 'name_pt': 'Real', 'name_en': 'Brazilian Real', 'name_es': 'Real Brasileño', 'tex': 'BRL'}, + 'cad': {'category': 'currency', 'name': 'Canadian Dollar', 'name_pt': 'Dólar Canadense', 'name_en': 'Canadian Dollar', 'name_es': 'Dólar Canadiense', 'tex': 'CAD'}, + 'clp': {'category': 'currency', 'name': 'Chilean Peso', 'name_pt': 'Peso Chileno', 'name_en': 'Chilean Peso', 'name_es': 'Peso Chileno', 'tex': 'CLP'}, + 'usd': {'category': 'currency', 'name': 'US Dollar', 'name_pt': 'Dólar Americano', 'name_en': 'US Dollar', 'name_es': 'Dólar Estadounidense', 'tex': 'USD'}, + 'eur': {'category': 'currency', 'name': 'Euro', 'name_pt': 'Euro', 'name_en': 'Euro', 'name_es': 'Euro', 'tex': 'EUR'}, + 'gbp': {'category': 'currency', 'name': 'British Pound', 'name_pt': 'Libra Esterlina', 'name_en': 'British Pound', 'name_es': 'Libra Esterlina', 'tex': 'GBP'}, + 'cny': {'category': 'currency', 'name': 'Chinese Yuan', 'name_pt': 'Yuan Chinês', 'name_en': 'Chinese Yuan', 'name_es': 'Yuan Chino', 'tex': 'CNY'}, + 'inr': {'category': 'currency', 'name': 'Indian Rupee', 'name_pt': 'Rupia Indiana', 'name_en': 'Indian Rupee', 'name_es': 'Rupia India', 'tex': 'INR'}, + 'jpy': {'category': 'currency', 'name': 'Japanese Yen', 'name_pt': 'Iene Japonês', 'name_en': 'Japanese Yen', 'name_es': 'Yen Japonés', 'tex': 'JPY'}, + 'zar': {'category': 'currency', 'name': 'South African Rand', 'name_pt': 'Rand Sul-Africano', 'name_en': 'South African Rand', 'name_es': 'Rand Sudafricano', 'tex': 'ZAR'}, + + # Economics + 'minimum_wage': {'category': 'economics', 'name': 'Minimum Wage', 'name_pt': 'Salário Mínimo', 'name_en': 'Minimum Wage', 'name_es': 'Salario Mínimo', 'tex': 'sm'}, + + # Date-time + 'year': {'category': 'datetime', 'name': 'Year', 'name_pt': 'Ano', 'name_en': 'Year', 'name_es': 'Año', 'tex': 'y'}, + 'semester': {'category': 'datetime', 'name': 'Semester', 'name_pt': 'Semestre', 'name_en': 'Semester', 'name_es': 'Semestre', 'tex': 'sem'}, + 'quarter': {'category': 'datetime', 'name': 'Quarter', 'name_pt': 'Trimestre', 'name_en': 'Quarter', 'name_es': 'Trimestre', 'tex': 'q'}, + 'bimester': {'category': 'datetime', 'name': 'Bimester', 'name_pt': 'Bimestre', 'name_en': 'Bimester', 'name_es': 'Bimestre', 'tex': 'bim'}, + 'month': {'category': 'datetime', 'name': 'Month', 'name_pt': 'Mês', 'name_en': 'Month', 'name_es': 'Mes', 'tex': 'm'}, + 'week': {'category': 'datetime', 'name': 'Week', 'name_pt': 'Semana', 'name_en': 'Week', 'name_es': 'Semana', 'tex': 'w'}, + 'day': {'category': 'datetime', 'name': 'Day', 'name_pt': 'Dia', 'name_en': 'Day', 'name_es': 'Día', 'tex': 'd'}, + 'hour': {'category': 'datetime', 'name': 'Hour', 'name_pt': 'Hora', 'name_en': 'Hour', 'name_es': 'Hora', 'tex': 'h'}, + 'minute': {'category': 'datetime', 'name': 'Minute', 'name_pt': 'Minuto', 'name_en': 'Minute', 'name_es': 'Minuto', 'tex': 'min'}, + 'second': {'category': 'datetime', 'name': 'Second', 'name_pt': 'Segundo', 'name_en': 'Second', 'name_es': 'Segundo', 'tex': 's'}, + + # Percentage + 'percent': {'category': 'percentage', 'name': 'Percentage', 'name_pt': 'Porcentagem', 'name_en': 'Percentage', 'name_es': 'Porcentaje', 'tex': '%'}, + } + + for slug, unit_data in units.items(): + MeasurementUnit.objects.create( + slug=slug, + name=unit_data['name'], + name_pt=unit_data['name_pt'], + name_en=unit_data['name_en'], + name_es=unit_data['name_es'], + tex=unit_data['tex'], + category=category_objects[unit_data['category']] + ) + +def reverse_categories_and_units(apps, schema_editor): + MeasurementUnitCategory = apps.get_model('v1', 'MeasurementUnitCategory') + MeasurementUnit = apps.get_model('v1', 'MeasurementUnit') + + MeasurementUnit.objects.all().delete() + MeasurementUnitCategory.objects.all().delete() + +class Migration(migrations.Migration): + dependencies = [ + ('v1', '0044_measurementunitcategory_measurementunit_tex_and_more'), + ] + + operations = [ + migrations.RunPython( + create_categories_and_units, + reverse_categories_and_units + ), + ] \ No newline at end of file diff --git a/backend/apps/api/v1/migrations/0046_observationlevel_order.py b/backend/apps/api/v1/migrations/0046_observationlevel_order.py new file mode 100644 index 00000000..fbc194ef --- /dev/null +++ b/backend/apps/api/v1/migrations/0046_observationlevel_order.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +from django.db import migrations, models + +class Migration(migrations.Migration): + dependencies = [ + ("v1", "0045_add_measurement_categories_and_units"), + ] + + operations = [ + migrations.AddField( + model_name="observationlevel", + name="order", + field=models.PositiveIntegerField( + db_index=True, default=0, editable=False, verbose_name="order" + ), + preserve_default=False, + ), + ] \ No newline at end of file diff --git a/backend/apps/api/v1/migrations/0047_initialize_observation_level_order.py b/backend/apps/api/v1/migrations/0047_initialize_observation_level_order.py new file mode 100644 index 00000000..b8425838 --- /dev/null +++ b/backend/apps/api/v1/migrations/0047_initialize_observation_level_order.py @@ -0,0 +1,42 @@ +from django.db import migrations + +def initialize_observation_level_order(apps, schema_editor): + ObservationLevel = apps.get_model('v1', 'ObservationLevel') + + # Group by each possible parent type and set order + for table_id in ObservationLevel.objects.values_list('table_id', flat=True).distinct(): + if table_id: + for i, ol in enumerate(ObservationLevel.objects.filter(table_id=table_id)): + ol.order = i + ol.save() + + for rds_id in ObservationLevel.objects.values_list('raw_data_source_id', flat=True).distinct(): + if rds_id: + for i, ol in enumerate(ObservationLevel.objects.filter(raw_data_source_id=rds_id)): + ol.order = i + ol.save() + + for ir_id in ObservationLevel.objects.values_list('information_request_id', flat=True).distinct(): + if ir_id: + for i, ol in enumerate(ObservationLevel.objects.filter(information_request_id=ir_id)): + ol.order = i + ol.save() + + for analysis_id in ObservationLevel.objects.values_list('analysis_id', flat=True).distinct(): + if analysis_id: + for i, ol in enumerate(ObservationLevel.objects.filter(analysis_id=analysis_id)): + ol.order = i + ol.save() + +def reverse_migration(apps, schema_editor): + ObservationLevel = apps.get_model('v1', 'ObservationLevel') + ObservationLevel.objects.all().update(order=0) + +class Migration(migrations.Migration): + dependencies = [ + ('v1', '0046_observationlevel_order'), + ] + + operations = [ + migrations.RunPython(initialize_observation_level_order, reverse_migration), + ] \ No newline at end of file diff --git a/backend/apps/api/v1/migrations/0048_alter_observationlevel_options_and_more.py b/backend/apps/api/v1/migrations/0048_alter_observationlevel_options_and_more.py new file mode 100644 index 00000000..4ae3ed76 --- /dev/null +++ b/backend/apps/api/v1/migrations/0048_alter_observationlevel_options_and_more.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.16 on 2024-11-04 03:54 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('v1', '0047_initialize_observation_level_order'), + ] + + operations = [ + migrations.AlterModelOptions( + name='observationlevel', + options={'ordering': ['order'], 'verbose_name': 'Observation Level', 'verbose_name_plural': 'Observation Levels'}, + ), + ] diff --git a/backend/apps/api/v1/migrations/0049_poll_pipeline.py b/backend/apps/api/v1/migrations/0049_poll_pipeline.py new file mode 100644 index 00000000..2dc4b07d --- /dev/null +++ b/backend/apps/api/v1/migrations/0049_poll_pipeline.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.16 on 2024-11-06 04:14 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('v1', '0048_alter_observationlevel_options_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='poll', + name='pipeline', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='polls', to='v1.pipeline'), + ), + ] diff --git a/backend/apps/api/v1/migrations/0050_table_is_deprecated.py b/backend/apps/api/v1/migrations/0050_table_is_deprecated.py new file mode 100644 index 00000000..c703d6ac --- /dev/null +++ b/backend/apps/api/v1/migrations/0050_table_is_deprecated.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.16 on 2024-11-06 04:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('v1', '0049_poll_pipeline'), + ] + + operations = [ + migrations.AddField( + model_name='table', + name='is_deprecated', + field=models.BooleanField(default=False, help_text='We stopped maintaining this table for some reason. Examples: raw data deprecated, new version elsewhere, etc.'), + ), + ] diff --git a/backend/apps/api/v1/models.py b/backend/apps/api/v1/models.py index dc6bdcc8..c3233ddf 100644 --- a/backend/apps/api/v1/models.py +++ b/backend/apps/api/v1/models.py @@ -14,6 +14,9 @@ from backend.custom.storage import OverwriteStorage, upload_to, validate_image from backend.custom.utils import check_kebab_case, check_snake_case +import logging + +logger = logging.getLogger('django.request') class Area(BaseModel): """Area model""" @@ -21,6 +24,33 @@ class Area(BaseModel): id = models.UUIDField(primary_key=True, default=uuid4) slug = models.SlugField(unique=True) name = models.CharField(max_length=255, blank=False, null=False) + administrative_level = models.IntegerField( + null=True, + blank=True, + choices=[ + (0, '0'), + (1, '1'), + (2, '2'), + (3, '3'), + (4, '4'), + (5, '5'), + ] + ) + entity = models.ForeignKey( + "Entity", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="areas", + limit_choices_to={'category__slug': 'spatial'} + ) + parent = models.ForeignKey( + "Area", + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="children", + ) graphql_nested_filter_fields_whitelist = ["id"] @@ -35,6 +65,27 @@ class Meta: verbose_name_plural = "Areas" ordering = ["name"] + def clean(self): + """Validate the model fields.""" + errors = {} + if self.administrative_level is not None and self.administrative_level not in [0, 1, 2, 3]: + errors['administrative_level'] = 'Administrative level must be 0, 1, 2, or 3' + + if self.entity and self.entity.category.slug != 'spatial': + errors['entity'] = 'Entity must have category "spatial"' + + if self.parent and self.parent.slug != 'world': + if self.administrative_level is None: + errors['administrative_level'] = 'Administrative level is required when parent is set' + elif self.parent.administrative_level is None: + errors['parent'] = 'Parent must have an administrative level' + elif self.parent.administrative_level != self.administrative_level - 1: + errors['parent'] = 'Parent must have administrative level exactly one level above' + + if errors: + raise ValidationError(errors) + return super().clean() + class Coverage(BaseModel): """ @@ -96,7 +147,7 @@ class Coverage(BaseModel): "Area", blank=True, null=True, - on_delete=models.CASCADE, + on_delete=models.SET_NULL, related_name="coverages", ) is_closed = models.BooleanField("Is Closed", default=False) @@ -270,7 +321,10 @@ class Analysis(BaseModel): name = models.CharField(null=True, blank=True, max_length=255) description = models.TextField(null=True, blank=True) analysis_type = models.ForeignKey( - "AnalysisType", on_delete=models.CASCADE, related_name="analyses" + "AnalysisType", + on_delete=models.SET_NULL, + null=True, + related_name="analyses", ) datasets = models.ManyToManyField( "Dataset", @@ -384,7 +438,7 @@ class Organization(BaseModel): slug = models.SlugField(unique=False, max_length=255) name = models.CharField(max_length=255) description = models.TextField(blank=True, null=True) - area = models.ForeignKey("Area", on_delete=models.CASCADE, related_name="organizations") + area = models.ForeignKey("Area", on_delete=models.SET_NULL, null=True, related_name="organizations") created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) website = models.URLField(blank=True, null=True, max_length=255) @@ -455,8 +509,11 @@ class Dataset(BaseModel): slug = models.SlugField(unique=False, max_length=255) name = models.CharField(max_length=255) description = models.TextField(blank=True, null=True) - organization = models.ForeignKey( - "Organization", on_delete=models.CASCADE, related_name="datasets" + organizations = models.ManyToManyField( + "Organization", + related_name="datasets", + verbose_name="Organizations", + help_text="Organizations associated with this dataset" ) themes = models.ManyToManyField( "Theme", @@ -504,9 +561,9 @@ class Meta: @property def full_slug(self): - if self.organization.area.slug != "unknown": - return f"{self.organization.area.slug}_{self.organization.slug}_{self.slug}" - return f"{self.organization.slug}_{self.slug}" + if self.organizations.first().area.slug != "unknown": + return f"{self.organizations.first().area.slug}_{self.slug}" + return f"{self.slug}" @property def popularity(self): @@ -517,22 +574,62 @@ def popularity(self): return log10(self.page_views) @property - def coverage(self) -> dict: + def temporal_coverage(self) -> dict: """Temporal coverage of all related entities""" resources = [ *self.tables.all(), *self.raw_data_sources.all(), *self.information_requests.all(), ] - coverage = get_coverage(resources) - if coverage["start"] and coverage["end"]: - return f"{coverage['start']} - {coverage['end']}" - if coverage["start"]: - return f"{coverage['start']}" - if coverage["end"]: - return f"{coverage['end']}" + temporal_coverage = get_temporal_coverage(resources) + if temporal_coverage["start"] and temporal_coverage["end"]: + return f"{temporal_coverage['start']} - {temporal_coverage['end']}" + if temporal_coverage["start"]: + return f"{temporal_coverage['start']}" + if temporal_coverage["end"]: + return f"{temporal_coverage['end']}" return "" + @property + def spatial_coverage(self) -> list[str]: + """Union spatial coverage of all related resources""" + resources = [ + *self.tables.all(), + *self.raw_data_sources.all(), + *self.information_requests.all(), + ] + return sorted(list(get_spatial_coverage(resources))) + + @property + def spatial_coverage_name_pt(self) -> list[str]: + """Union spatial coverage of all related resources""" + resources = [ + *self.tables.all(), + *self.raw_data_sources.all(), + *self.information_requests.all(), + ] + return sorted(list(get_spatial_coverage_name(resources, locale = 'pt'))) + + @property + def spatial_coverage_name_en(self) -> list[str]: + """Union spatial coverage of all related resources""" + resources = [ + *self.tables.all(), + *self.raw_data_sources.all(), + *self.information_requests.all(), + ] + return sorted(list(get_spatial_coverage_name(resources, locale = 'en'))) + + @property + def spatial_coverage_name_es(self) -> list[str]: + """Union spatial coverage of all related resources""" + resources = [ + *self.tables.all(), + *self.raw_data_sources.all(), + *self.information_requests.all(), + ] + return sorted(list(get_spatial_coverage_name(resources, locale = 'es'))) + @property def entities(self) -> list[dict]: """Entity of all related resources""" @@ -671,8 +768,9 @@ class Update(BaseModel): "Entity", blank=True, null=True, - on_delete=models.CASCADE, - related_name="updates" + on_delete=models.SET_NULL, + related_name="updates", + limit_choices_to={'category__slug': 'datetime'} ) frequency = models.IntegerField(blank=True, null=True) lag = models.IntegerField(blank=True, null=True) @@ -714,6 +812,7 @@ class Meta: def clean(self) -> None: """Assert that only one of "table", "raw_data_source", "information_request" is set""" + errors = {} count = 0 if self.table: count += 1 @@ -726,8 +825,10 @@ def clean(self) -> None: "One and only one of 'table', " "'raw_data_source', or 'information_request' must be set." ) - if self.entity.category.slug != "datetime": - raise ValidationError("Entity's category is not in category.slug = `datetime`.") + if self.entity and self.entity.category.slug != 'datetime': + errors['entity'] = 'Entity must have category "datetime"' + if errors: + raise ValidationError(errors) return super().clean() @@ -737,11 +838,19 @@ class Poll(BaseModel): "Entity", blank=True, null=True, - on_delete=models.CASCADE, - related_name="polls" + on_delete=models.SET_NULL, + related_name="polls", + limit_choices_to={'category__slug': 'datetime'} ) frequency = models.IntegerField(blank=True, null=True) latest = models.DateTimeField(blank=True, null=True) + pipeline = models.ForeignKey( + "Pipeline", + blank=True, + null=True, + on_delete=models.SET_NULL, + related_name="polls", + ) raw_data_source = models.ForeignKey( "RawDataSource", blank=True, @@ -772,12 +881,15 @@ class Meta: def clean(self) -> None: """Assert that only one of "raw_data_source", "information_request" is set""" + errors = {} if bool(self.raw_data_source) == bool(self.information_request): raise ValidationError( "One and only one of 'raw_data_source'," " or 'information_request' must be set." ) - if self.entity.category.slug != "datetime": - raise ValidationError("Entity's category is not in category.slug = `datetime`.") + if self.entity and self.entity.category.slug != 'datetime': + errors['entity'] = 'Entity must have category "datetime"' + if errors: + raise ValidationError(errors) return super().clean() @@ -798,23 +910,27 @@ class Table(BaseModel, OrderedModel): status = models.ForeignKey( "Status", on_delete=models.PROTECT, related_name="tables", null=True, blank=True ) + is_deprecated = models.BooleanField( + default=False, + help_text="We stopped maintaining this table for some reason. Examples: raw data deprecated, new version elsewhere, etc." + ) license = models.ForeignKey( "License", - on_delete=models.CASCADE, + on_delete=models.SET_NULL, related_name="tables", blank=True, null=True, ) partner_organization = models.ForeignKey( "Organization", - on_delete=models.CASCADE, + on_delete=models.SET_NULL, related_name="partner_tables", blank=True, null=True, ) pipeline = models.ForeignKey( "Pipeline", - on_delete=models.CASCADE, + on_delete=models.SET_NULL, related_name="tables", blank=True, null=True, @@ -822,23 +938,22 @@ class Table(BaseModel, OrderedModel): is_directory = models.BooleanField(default=False, blank=True, null=True) created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) - published_by = models.ForeignKey( + published_by = models.ManyToManyField( Account, - on_delete=models.CASCADE, related_name="tables_published", blank=True, - null=True, + verbose_name="Published by", + help_text="People who published the table", ) - data_cleaned_by = models.ForeignKey( + data_cleaned_by = models.ManyToManyField( Account, - on_delete=models.CASCADE, related_name="tables_cleaned", blank=True, - null=True, + verbose_name="Data cleaned by", + help_text="People who cleaned the data", ) data_cleaning_description = models.TextField(blank=True, null=True) data_cleaning_code_url = models.URLField(blank=True, null=True) - raw_data_url = models.URLField(blank=True, null=True, max_length=500) auxiliary_files_url = models.URLField(blank=True, null=True) architecture_url = models.URLField(blank=True, null=True) source_bucket_name = models.CharField( @@ -931,14 +1046,34 @@ def contains_closed_data(self): return False @property - def coverage(self) -> dict: + def temporal_coverage(self) -> dict: """Temporal coverage""" - return get_coverage([self]) + return get_temporal_coverage([self]) @property - def full_coverage(self) -> dict: + def full_temporal_coverage(self) -> dict: """Temporal coverage steps""" - return get_full_coverage([self]) + return get_full_temporal_coverage([self]) + + @property + def spatial_coverage(self) -> list[str]: + """Unique list of areas across all coverages""" + return sorted(list(get_spatial_coverage([self]))) + + @property + def spatial_coverage_name_pt(self) -> list[str]: + """Union spatial coverage of all related resources""" + return get_spatial_coverage_name([self], locale = 'pt') + + @property + def spatial_coverage_name_en(self) -> list[str]: + """Union spatial coverage of all related resources""" + return get_spatial_coverage_name([self], locale = 'en') + + @property + def spatial_coverage_name_es(self) -> list[str]: + """Union spatial coverage of all related resources""" + return get_spatial_coverage_name([self], locale = 'es') @property def neighbors(self) -> list[dict]: @@ -953,30 +1088,38 @@ def last_updated_at(self): return max(updates) if updates else None @property - def published_by_info(self) -> dict: - if not self.published_by: - return None - return { - "firstName": self.published_by.first_name, - "lastName": self.published_by.last_name, - "email": self.published_by.email, - "github": self.published_by.github, - "twitter": self.published_by.twitter, - "website": self.published_by.website, - } + def published_by_info(self) -> list[dict]: + """Return list of author information""" + if not self.published_by.exists(): + return [] + return [ + { + "firstName": author.first_name, + "lastName": author.last_name, + "email": author.email, + "github": author.github, + "twitter": author.twitter, + "website": author.website, + } + for author in self.published_by.all() + ] @property - def data_cleaned_by_info(self) -> dict: - if not self.data_cleaned_by: - return None - return { - "firstName": self.data_cleaned_by.first_name, - "lastName": self.data_cleaned_by.last_name, - "email": self.data_cleaned_by.email, - "github": self.data_cleaned_by.github, - "twitter": self.data_cleaned_by.twitter, - "website": self.data_cleaned_by.website, - } + def data_cleaned_by_info(self) -> list[dict]: + """Return list of data cleaner information""" + if not self.data_cleaned_by.exists(): + return [] + return [ + { + "firstName": cleaner.first_name, + "lastName": cleaner.last_name, + "email": cleaner.email, + "github": cleaner.github, + "twitter": cleaner.twitter, + "website": cleaner.website, + } + for cleaner in self.data_cleaned_by.all() + ] @property def coverage_datetime_units(self) -> str: @@ -1181,6 +1324,52 @@ class Meta: verbose_name_plural = "BigQuery Types" ordering = ["name"] +class MeasurementUnitCategory(BaseModel): + """Model definition for MeasurementUnitCategory.""" + + id = models.UUIDField(primary_key=True, default=uuid4) + slug = models.SlugField(unique=True) + name = models.CharField(max_length=255) + + graphql_nested_filter_fields_whitelist = ["id"] + + def __str__(self): + return str(self.slug) + + class Meta: + """Meta definition for Measurement Unit Category.""" + + db_table = "measurement_unit_category" + verbose_name = "Measurement Unit Category" + verbose_name_plural = "Measurement Unit Categories" + ordering = ["slug"] + +class MeasurementUnit(BaseModel): + """Model definition for MeasurementUnit.""" + + id = models.UUIDField(primary_key=True, default=uuid4) + slug = models.SlugField(unique=True) + name = models.CharField(max_length=255) + tex = models.CharField(max_length=255, blank=True, null=True) + category = models.ForeignKey( + "MeasurementUnitCategory", + on_delete=models.SET_NULL, + null=True, + related_name="measurement_units", + ) + + graphql_nested_filter_fields_whitelist = ["id"] + + def __str__(self): + return str(self.slug) + + class Meta: + """Meta definition for MeasurementUnit.""" + + db_table = "measurement_unit" + verbose_name = "Measurement Unit" + verbose_name_plural = "Measurement Units" + ordering = ["slug"] class Column(BaseModel, OrderedModel): """Model definition for Column.""" @@ -1190,17 +1379,18 @@ class Column(BaseModel, OrderedModel): name = models.CharField(max_length=255) name_staging = models.CharField(max_length=255, blank=True, null=True) bigquery_type = models.ForeignKey( - "BigQueryType", on_delete=models.CASCADE, related_name="columns" + "BigQueryType", on_delete=models.SET_NULL, null=True, related_name="columns" ) description = models.TextField(blank=True, null=True) covered_by_dictionary = models.BooleanField(default=False, blank=True, null=True) is_primary_key = models.BooleanField(default=False, blank=True, null=True) directory_primary_key = models.ForeignKey( "Column", - on_delete=models.PROTECT, + on_delete=models.SET_NULL, + null=True, related_name="columns", blank=True, - null=True, + limit_choices_to={'is_primary_key': True, 'table__is_directory': True} ) measurement_unit = models.CharField(max_length=255, blank=True, null=True) contains_sensitive_data = models.BooleanField(default=False, blank=True, null=True) @@ -1209,17 +1399,17 @@ class Column(BaseModel, OrderedModel): is_partition = models.BooleanField(default=False) observation_level = models.ForeignKey( "ObservationLevel", - on_delete=models.CASCADE, - related_name="columns", + on_delete=models.SET_NULL, null=True, + related_name="columns", blank=True, ) version = models.IntegerField(null=True, blank=True) status = models.ForeignKey( "Status", - on_delete=models.PROTECT, - related_name="columns", + on_delete=models.SET_NULL, null=True, + related_name="columns", blank=True, ) is_closed = models.BooleanField( @@ -1241,17 +1431,49 @@ class Meta: ordering = ["name"] @property - def coverage(self) -> dict: + def temporal_coverage(self) -> dict: """Temporal coverage of column if exists, if not table coverage""" - coverage = get_coverage([self]) + temporal_coverage = get_temporal_coverage([self]) fallback = defaultdict(lambda: None) - if not coverage["start"] or not coverage["end"]: - fallback = self.table.coverage + if not temporal_coverage["start"] or not temporal_coverage["end"]: + fallback = self.table.temporal_coverage return { - "start": coverage["start"] or fallback["start"], - "end": coverage["end"] or fallback["end"], + "start": temporal_coverage["start"] or fallback["start"], + "end": temporal_coverage["end"] or fallback["end"], } + @property + def spatial_coverage(self) -> list[str]: + """Unique list of areas across all coverages, falling back to table coverage if empty""" + coverage = get_spatial_coverage([self]) + if not coverage: + return get_spatial_coverage([self.table]) + return coverage + + @property + def spatial_coverage_name_pt(self) -> list[str]: + """Union spatial coverage of all related resources""" + coverage = get_spatial_coverage_name([self], locale = 'pt') + if not coverage: + coverage = get_spatial_coverage_name([self.table], locale = 'pt') + return coverage + + @property + def spatial_coverage_name_en(self) -> list[str]: + """Union spatial coverage of all related resources""" + coverage = get_spatial_coverage_name([self], locale = 'en') + if not coverage: + coverage = get_spatial_coverage_name([self.table], locale = 'en') + return coverage + + @property + def spatial_coverage_name_es(self) -> list[str]: + """Union spatial coverage of all related resources""" + coverage = get_spatial_coverage_name([self], locale = 'es') + if not coverage: + coverage = get_spatial_coverage_name([self.table], locale = 'es') + return coverage + @property def dir_column(self): """Column of directory table and column""" @@ -1435,15 +1657,15 @@ class RawDataSource(BaseModel, OrderedModel): "Dataset", on_delete=models.CASCADE, related_name="raw_data_sources" ) availability = models.ForeignKey( - "Availability", on_delete=models.CASCADE, related_name="raw_data_sources" + "Availability", on_delete=models.SET_NULL, null=True, related_name="raw_data_sources" ) languages = models.ManyToManyField("Language", related_name="raw_data_sources", blank=True) license = models.ForeignKey( "License", - on_delete=models.CASCADE, + on_delete=models.SET_NULL, + null=True, related_name="raw_data_sources", blank=True, - null=True, ) area_ip_address_required = models.ManyToManyField( "Area", related_name="raw_data_sources", blank=True @@ -1498,7 +1720,7 @@ class InformationRequest(BaseModel, OrderedModel): version = models.IntegerField(null=True, blank=True) status = models.ForeignKey( "Status", - on_delete=models.CASCADE, + on_delete=models.SET_NULL, related_name="information_requests", null=True, blank=True, @@ -1572,7 +1794,7 @@ class Entity(BaseModel): slug = models.SlugField(unique=True) name = models.CharField(max_length=255) category = models.ForeignKey( - "EntityCategory", on_delete=models.CASCADE, related_name="entities" + "EntityCategory", on_delete=models.SET_NULL, null=True, related_name="entities" ) graphql_nested_filter_fields_whitelist = ["id"] @@ -1589,12 +1811,12 @@ class Meta: ordering = ["slug"] -class ObservationLevel(BaseModel): +class ObservationLevel(BaseModel, OrderedModel): """Model definition for ObservationLevel.""" id = models.UUIDField(primary_key=True, default=uuid4) entity = models.ForeignKey( - "Entity", on_delete=models.CASCADE, related_name="observation_levels" + "Entity", on_delete=models.SET_NULL, null=True, related_name="observation_levels" ) table = models.ForeignKey( "Table", @@ -1625,6 +1847,8 @@ class ObservationLevel(BaseModel): related_name="observation_levels", ) + order_with_respect_to = ('table', 'raw_data_source', 'information_request', 'analysis') + graphql_nested_filter_fields_whitelist = ["id"] def __str__(self): @@ -1636,25 +1860,23 @@ class Meta: db_table = "observation_level" verbose_name = "Observation Level" verbose_name_plural = "Observation Levels" - ordering = ["id"] + ordering = ["order"] - def clean(self) -> None: - """Assert that only one of "table", "raw_data_source", "information_request" is set""" - count = 0 - if self.table: - count += 1 - if self.raw_data_source: - count += 1 - if self.information_request: - count += 1 - if self.analysis: - count += 1 - if count != 1: - raise ValidationError( - "One and only one of 'table', 'raw_data_source', " - "'information_request', 'analysis' must be set." - ) - return super().clean() + def get_ordering_queryset(self): + """Get queryset for ordering within the appropriate parent""" + qs = super().get_ordering_queryset() + + # Filter by the appropriate parent field + if self.table_id: + return qs.filter(table_id=self.table_id) + elif self.raw_data_source_id: + return qs.filter(raw_data_source_id=self.raw_data_source_id) + elif self.information_request_id: + return qs.filter(information_request_id=self.information_request_id) + elif self.analysis_id: + return qs.filter(analysis_id=self.analysis_id) + + return qs class DateTimeRange(BaseModel): @@ -1794,10 +2016,10 @@ class QualityCheck(BaseModel): updated_at = models.DateTimeField(auto_now=True) pipeline = models.ForeignKey( "Pipeline", - on_delete=models.CASCADE, - related_name="quality_checks", - blank=True, + on_delete=models.SET_NULL, null=True, + blank=True, + related_name="quality_checks", ) analysis = models.ForeignKey( "Analysis", @@ -1898,8 +2120,8 @@ def as_dict(self): return {"date": self.str, "type": self.type} -def get_coverage(resources: list) -> dict: - """Get maximum datetime coverage of resources +def get_temporal_coverage(resources: list) -> dict: + """Get maximum temporal coverage of resources Case: - Table A has data with dates between [X, Y] @@ -1918,8 +2140,8 @@ def get_coverage(resources: list) -> dict: return {"start": since.str, "end": until.str} -def get_full_coverage(resources: list) -> dict: - """Get datetime coverage steps of resources +def get_full_temporal_coverage(resources: list) -> dict: + """Get temporal coverage steps of resources Cases: - Table A has data with dates between [X, Y], where [X, Y] is open @@ -1957,3 +2179,105 @@ def get_full_coverage(resources: list) -> dict: return [open_since.as_dict, open_until.as_dict] if paid_since.str and paid_until.str: return [paid_since.as_dict, paid_until.as_dict] + +def get_spatial_coverage(resources: list) -> list: + """Get spatial coverage of resources by returning unique area slugs, keeping only the highest level in each branch + + For example: + - If areas = [br_mg_3100104, br_mg_3100104] -> returns [br_mg_3100104] + - If areas = [br_mg_3100104, br_sp_3500105] -> returns [br_mg_3100104, br_sp_3500105] + - If areas = [br_mg, us_ny, us] -> returns [br_mg, us] + - If areas = [br_mg, world, us] -> returns [world] + - If resources have no areas -> returns empty list + """ + # Collect all unique area slugs across resources + all_areas = set() + for resource in resources: + for coverage in resource.coverages.all(): + if coverage.area: + all_areas.add(coverage.area.slug) + + if not all_areas: + return [] + + # If 'world' is present, it encompasses everything + if 'world' in all_areas: + return ['world'] + + # Filter out areas that have a parent in the set + filtered_areas = set() + for area in all_areas: + parts = area.split('_') + is_parent_present = False + + # Check if any parent path exists in all_areas + for i in range(1, len(parts)): + parent = '_'.join(parts[:i]) + if parent in all_areas: + is_parent_present = True + break + + if not is_parent_present: + filtered_areas.add(area) + + return sorted(list(filtered_areas)) + +def get_spatial_coverage_name(resources: list, locale: str = 'pt') -> list: + """Get spatial coverage of resources by returning unique area names in the specified locale, + keeping only the highest level in each branch + + Args: + resources: List of resources to get coverage from + locale: Language code ('pt', 'en', etc). Defaults to 'pt' + + For example: + - If areas = [br_mg_3100104, br_mg_3100104] -> returns [Belo Horizonte] + - If areas = [br_mg_3100104, br_sp_3500105] -> returns [Belo Horizonte, São Paulo] + - If areas = [br_mg, us_ny, us] -> returns [Minas Gerais, United States] (en) + returns [Minas Gerais, Estados Unidos] (pt) + - If areas = [br_mg, world, us] -> returns [World] (en) + returns [Mundo] (pt) + - If resources have no areas -> returns empty list + """ + # Translation mapping for special cases + translations = { + 'world': { + 'pt': 'Mundo', + 'en': 'World', + 'es': 'Mundo', + } + } + + # Collect all unique areas (both slug and name) across resources + all_areas = {} + for resource in resources: + for coverage in resource.coverages.all(): + if coverage.area: + # Get localized name using getattr, fallback to default name if not found + localized_name = getattr(coverage.area, f'name_{locale}', None) or coverage.area.name + all_areas[coverage.area.slug] = localized_name + + if not all_areas: + return [] + + # If 'world' is present, it encompasses everything + if 'world' in all_areas: + return [translations['world'].get(locale, translations['world']['pt'])] + + # Filter out areas that have a parent in the set + filtered_areas = set() + for area_slug in all_areas: + parts = area_slug.split('_') + is_parent_present = False + + # Check if any parent path exists in all_areas + for i in range(1, len(parts)): + parent = '_'.join(parts[:i]) + if parent in all_areas: + is_parent_present = True + break + + if not is_parent_present: + filtered_areas.add(all_areas[area_slug]) + + return sorted(list(filtered_areas)) diff --git a/backend/apps/api/v1/schemas.py b/backend/apps/api/v1/schemas.py index 64ee7cac..eaa20a66 100644 --- a/backend/apps/api/v1/schemas.py +++ b/backend/apps/api/v1/schemas.py @@ -35,10 +35,24 @@ class Entity(BaseModel): name_es: str +class MeasurementUnit(BaseModel): + slug: str + name_pt: str + name_en: str + name_es: str + + class RawDataSource(BaseModel): id: str +class SpatialCoverage(BaseModel): + slug: str + name_pt: str + name_en: str + name_es: str + + class TemporalCoverage(BaseModel): start_date: str end_date: str @@ -67,11 +81,12 @@ class Dataset(BaseModel): contains_open_data: bool contains_closed_data: bool # - tags: List[Tag] themes: List[Theme] - entities: List[Entity] - temporal_coverage: List[str] organization: List[Organization] + temporal_coverage: List[str] + spatial_coverage: List[SpatialCoverage] + tags: List[Tag] + entities: List[Entity] class Facet(BaseModel): diff --git a/backend/apps/api/v1/search_indexes.py b/backend/apps/api/v1/search_indexes.py index 3488d837..1b46089b 100644 --- a/backend/apps/api/v1/search_indexes.py +++ b/backend/apps/api/v1/search_indexes.py @@ -47,6 +47,21 @@ class DatasetIndex(indexes.SearchIndex, indexes.Indexable): null=True, indexed=False, ) + + spatial_coverage = indexes.MultiValueField( + model_attr="spatial_coverage", + null=True, + faceted=True, + indexed=True, + ) + + temporal_coverage = indexes.MultiValueField( + model_attr="temporal_coverage", + null=True, + faceted=True, + indexed=True, + ) + table_id = indexes.MultiValueField( model_attr="tables__pk", @@ -88,55 +103,60 @@ class DatasetIndex(indexes.SearchIndex, indexes.Indexable): ) organization_id = indexes.MultiValueField( - model_attr="organization__pk", + model_attr="organizations__id", faceted=True, indexed=False, ) organization_slug = indexes.MultiValueField( - model_attr="organization__slug", + model_attr="organizations__slug", + faceted=True, + indexed=False, + ) + organization_name = indexes.MultiValueField( + model_attr="organizations__name", faceted=True, indexed=False, ) organization_name_pt = indexes.MultiValueField( - model_attr="organization__name_pt", + model_attr="organizations__name_pt", null=True, faceted=True, indexed=False, ) organization_name_en = indexes.MultiValueField( - model_attr="organization__name_en", + model_attr="organizations__name_en", null=True, faceted=True, indexed=False, ) organization_name_es = indexes.MultiValueField( - model_attr="organization__name_es", + model_attr="organizations__name_es", null=True, faceted=True, indexed=False, ) organization_picture = indexes.MultiValueField( - model_attr="organization__picture", + model_attr="organizations__picture", default="", indexed=False, ) organization_website = indexes.MultiValueField( - model_attr="organization__website", + model_attr="organizations__website", default="", indexed=False, ) organization_description_pt = indexes.MultiValueField( - model_attr="organization__description_pt", + model_attr="organizations__description_pt", null=True, indexed=False, ) organization_description_en = indexes.MultiValueField( - model_attr="organization__description_en", + model_attr="organizations__description_en", null=True, indexed=False, ) organization_description_es = indexes.MultiValueField( - model_attr="organization__description_es", + model_attr="organizations__description_es", null=True, indexed=False, ) @@ -213,12 +233,7 @@ class DatasetIndex(indexes.SearchIndex, indexes.Indexable): faceted=True, indexed=False, ) - temporal_coverage = indexes.MultiValueField( - default="", - model_attr="coverage", - indexed=False, - ) - + contains_open_data = indexes.BooleanField( model_attr="contains_open_data", indexed=False, @@ -293,4 +308,42 @@ def load_all_queryset(self, using=None): return self.get_model().objects.exclude(status__slug="under_review") def prepare_organization_picture(self, obj): - return getattr(obj.organization.picture, "name", None) + """ + Get pictures from all organizations associated with the dataset + """ + pictures = [] + for org in obj.organizations.all(): + pictures.append(getattr(org.picture, "name", None)) + return pictures + + def get_field_mapping(self): + mapping = super().get_field_mapping() + mapping['spatial_coverage'] = { + 'type': 'keyword', + 'store': True, + 'index': True, + } + return mapping + + def prepare(self, obj): + data = super().prepare(obj) + + organization_fields = [ + 'organization_id', + 'organization_slug', + 'organization_name', + 'organization_name_pt', + 'organization_name_en', + 'organization_name_es', + 'organization_picture', + 'organization_website', + 'organization_description_pt', + 'organization_description_en', + 'organization_description_es' + ] + + for field in organization_fields: + if field in data and not isinstance(data[field], (list, tuple)): + data[field] = [data[field]] if data[field] is not None else [] + + return data diff --git a/backend/apps/api/v1/search_views.py b/backend/apps/api/v1/search_views.py index b52a0a8e..069bbc45 100644 --- a/backend/apps/api/v1/search_views.py +++ b/backend/apps/api/v1/search_views.py @@ -7,19 +7,17 @@ from haystack.models import SearchResult from haystack.query import SearchQuerySet -from backend.apps.api.v1.models import Entity, Organization, Tag, Theme +from backend.apps.api.v1.models import Entity, Organization, Tag, Theme, Area - -import logging -logger = logging.getLogger(__name__) class DatasetSearchForm(FacetedSearchForm): load_all: bool = True def __init__(self, *args, **kwargs): self.contains = kwargs.pop("contains", None) or [] - self.tag = kwargs.pop("tag", None) or [] self.theme = kwargs.pop("theme", None) or [] self.organization = kwargs.pop("organization", None) or [] + self.spatial_coverage = kwargs.pop("spatial_coverage", None) + self.tag = kwargs.pop("tag", None) or [] self.observation_level = kwargs.pop("observation_level", None) or [] self.locale = kwargs.pop("locale", "pt") super().__init__(*args, **kwargs) @@ -28,19 +26,30 @@ def search(self): if not self.is_valid(): return self.no_query_found() + # Start with all results + sqs = self.searchqueryset.all() + + # Debug print to see all form data + print("DEBUG: Form data:", { + 'spatial_coverage': self.spatial_coverage, + 'theme': self.theme, + 'organization': self.organization, + 'tag': self.tag, + }) + + # Text search if provided if q := self.cleaned_data.get("q"): sqs = ( - self.searchqueryset - .auto_query(q) + sqs.auto_query(q) .filter_and(**{"text.edgengram": q}) .filter_or(**{f"text.snowball_{self.locale}": q}) ) - else: - sqs = self.no_query_found() + # Contains filters for qp_value in self.contains: sqs = sqs.narrow(f'contains_{qp_value}:"true"') + # Regular filters for qp_key, facet_key in [ ("tag", "tag_slug"), ("theme", "theme_slug"), @@ -50,6 +59,37 @@ def search(self): for qp_value in getattr(self, qp_key, []): sqs = sqs.narrow(f'{facet_key}:"{sqs.query.clean(qp_value)}"') + if self.spatial_coverage: + # Build queries for all coverage values + coverage_queries = [] + for coverage_list in self.spatial_coverage: + # Split the comma-separated values + coverages = coverage_list.split(',') + if 'world' in coverages: + # If world is in the list, only look for world coverage + coverage_queries = ['spatial_coverage_exact:"world"'] + break + else: + # Regular case: handle hierarchical patterns for each coverage + for coverage in coverages: + parts = coverage.split('_') + coverage_patterns = [ + '_'.join(parts[:i]) + for i in range(1, len(parts)) + ] + coverage_patterns.append(coverage) # Add the full coverage too + + # Build OR condition for all valid levels, including world + patterns = ' OR '.join( + f'spatial_coverage_exact:"{pattern}"' + for pattern in coverage_patterns + ['world'] + ) + coverage_queries.append(f'({patterns})') + + # Combine all coverage queries with AND + query = f'_exists_:spatial_coverage_exact AND {" AND ".join(coverage_queries)}' + sqs = sqs.raw_search(query) + return sqs def no_query_found(self): @@ -91,9 +131,10 @@ def locale(self): def get_form_kwargs(self): kwargs = super().get_form_kwargs() kwargs.update({"contains": self.request.GET.getlist("contains")}) - kwargs.update({"tag": self.request.GET.getlist("tag")}) kwargs.update({"theme": self.request.GET.getlist("theme")}) kwargs.update({"organization": self.request.GET.getlist("organization")}) + kwargs.update({"spatial_coverage": self.request.GET.getlist("spatial_coverage")}) + kwargs.update({"tag": self.request.GET.getlist("tag")}) kwargs.update({"observation_level": self.request.GET.getlist("observation_level")}) kwargs.update({"locale": self.locale}) return kwargs @@ -112,10 +153,11 @@ def get(self, request, *args, **kwargs): ) def get_facets(self, sqs: SearchQuerySet, facet_size=22): - sqs = sqs.facet("tag_slug", size=facet_size) sqs = sqs.facet("theme_slug", size=facet_size) - sqs = sqs.facet("entity_slug", size=facet_size) sqs = sqs.facet("organization_slug", size=facet_size) + sqs = sqs.facet("spatial_coverage", size=facet_size) + sqs = sqs.facet("tag_slug", size=facet_size) + sqs = sqs.facet("entity_slug", size=facet_size) facets = {} facet_counts = sqs.facet_counts() @@ -129,11 +171,12 @@ def get_facets(self, sqs: SearchQuerySet, facet_size=22): "count": value[1], } ) + for key_back, key_front, model in [ - ("tag_slug", "tags", Tag), ("theme_slug", "themes", Theme), - ("entity_slug", "observation_levels", Entity), ("organization_slug", "organizations", Organization), + ("tag_slug", "tags", Tag), + ("entity_slug", "observation_levels", Entity), ]: to_name = model.objects.values("slug", f"name_{self.locale}", "name") to_name = {e["slug"]: { @@ -145,6 +188,53 @@ def get_facets(self, sqs: SearchQuerySet, facet_size=22): translated_name = to_name.get(field["key"], {}) field["name"] = translated_name.get("name", field["key"]) field["fallback"] = translated_name.get("fallback", True) + + # Special handling for spatial coverage + if "spatial_coverage" in facets: + spatial_coverages = [] + coverage_counts = {} # Dictionary to track counts per slug + coverage_data = {} # Dictionary to store the full data per slug + + for field in facets.pop("spatial_coverage") or []: + coverage = field["key"] + areas = Area.objects.filter(slug=coverage, administrative_level=0) + + if coverage == "world": + field["name"] = "World" + field["fallback"] = False + + # Add all top-level areas (administrative_level = 0) + top_level_areas = Area.objects.filter(administrative_level=0) + for child_area in top_level_areas: + slug = child_area.slug + coverage_counts[slug] = coverage_counts.get(slug, 0) + field["count"] + coverage_data[slug] = { + "key": slug, + "name": getattr(child_area, f'name_{self.locale}') or child_area.name or slug, + "fallback": getattr(child_area, f'name_{self.locale}') is None + } + elif areas.exists(): + for area in areas: + slug = area.slug + coverage_counts[slug] = coverage_counts.get(slug, 0) + field["count"] + coverage_data[slug] = { + "key": slug, + "name": getattr(area, f'name_{self.locale}') or area.name or coverage, + "fallback": getattr(area, f'name_{self.locale}') is None + } + + # Create final list with collapsed counts and sort by count + spatial_coverages = [] + for slug, count in coverage_counts.items(): + entry = coverage_data[slug].copy() + entry["count"] = count + spatial_coverages.append(entry) + + # Sort by count in descending order + spatial_coverages.sort(key=lambda x: x["count"], reverse=True) + + facets["spatial_coverages"] = spatial_coverages + return facets def get_results(self, sqs: SearchQuerySet): @@ -160,15 +250,6 @@ def key(r): def as_search_result(result: SearchResult, locale='pt'): - tags = [] - for slug, name in zip(result.tag_slug or [], getattr(result, f"tag_name_{locale}") or []): - tags.append( - { - "slug": slug, - "name": name, - } - ) - themes = [] for slug, name in zip(result.theme_slug or [], getattr(result, f"theme_name_{locale}") or []): themes.append( @@ -178,23 +259,14 @@ def as_search_result(result: SearchResult, locale='pt'): } ) - entities = [] - for slug, name in zip(result.entity_slug or [], getattr(result, f"entity_name_{locale}") or []): - entities.append( - { - "slug": slug, - "name": name, - } - ) - organizations = [] for pk, slug, name, picture in zip( result.organization_id or [], result.organization_slug or [], - [(getattr(result, f"organization_name_{locale}") or []) or result.organization_name or result.organization_slug], + getattr(result, f"organization_name_{locale}") or result.organization_name or result.organization_slug or [], result.organization_picture or [], ): - picture = storage.url(picture) + picture = storage.url(picture) if picture else None organizations.append( { "id": pk, @@ -204,6 +276,39 @@ def as_search_result(result: SearchResult, locale='pt'): } ) + tags = [] + for slug, name in zip(result.tag_slug or [], getattr(result, f"tag_name_{locale}") or []): + tags.append( + { + "slug": slug, + "name": name, + } + ) + + entities = [] + for slug, name in zip(result.entity_slug or [], getattr(result, f"entity_name_{locale}") or []): + entities.append( + { + "slug": slug, + "name": name, + } + ) + + # Add spatial coverage translations + spatial_coverages = [] + for coverage in (result.spatial_coverage or []): + area = Area.objects.filter(slug=coverage).first() + if area: + spatial_coverages.append({ + 'slug': coverage, + 'name': getattr(area, f'name_{locale}') or area.name or coverage + }) + else: + spatial_coverages.append({ + 'slug': coverage, + 'name': coverage + }) + return { "updated_at": result.updated_at, "id": result.dataset_id, @@ -214,7 +319,8 @@ def as_search_result(result: SearchResult, locale='pt'): "themes": themes, "entities": entities, "organizations": organizations, - "temporal_coverages": result.temporal_coverage, + "temporal_coverage": result.temporal_coverage, + "spatial_coverage": spatial_coverages, "contains_open_data": result.contains_open_data, "contains_closed_data": result.contains_closed_data, "contains_tables": result.contains_tables, diff --git a/backend/apps/api/v1/templates/admin/cloud_table_inline.html b/backend/apps/api/v1/templates/admin/cloud_table_inline.html new file mode 100644 index 00000000..44ce34d8 --- /dev/null +++ b/backend/apps/api/v1/templates/admin/cloud_table_inline.html @@ -0,0 +1,15 @@ +{% load i18n admin_urls %} + +{# Include the standard stacked inline template #} +{% include "admin/edit_inline/tabular.html" %} + +{# Add the custom add button #} +{% if inline_admin_formset.formset.instance.pk %} +
+ + {% trans "Add Cloud Table" %} + +
+{% endif %} \ No newline at end of file diff --git a/backend/apps/api/v1/templates/admin/observation_level_inline.html b/backend/apps/api/v1/templates/admin/observation_level_inline.html new file mode 100644 index 00000000..f95ee531 --- /dev/null +++ b/backend/apps/api/v1/templates/admin/observation_level_inline.html @@ -0,0 +1,15 @@ +{% load i18n admin_urls %} + +{# Include the standard stacked inline template #} +{% include "admin/edit_inline/stacked.html" %} + +{# Add the custom add button #} +{% if inline_admin_formset.formset.instance.pk %} +
+ + {% trans "Add Observation Level" %} + +
+{% endif %} \ No newline at end of file diff --git a/backend/apps/api/v1/templates/admin/reorder_observation_levels.html b/backend/apps/api/v1/templates/admin/reorder_observation_levels.html new file mode 100644 index 00000000..00653b68 --- /dev/null +++ b/backend/apps/api/v1/templates/admin/reorder_observation_levels.html @@ -0,0 +1,32 @@ +{% extends "admin/base_site.html" %} + +{% block content %} +
+ {% csrf_token %} + + + +
+ {{ form.ordered_entities.label_tag}} +
+ {{ form.ordered_entities }} + {{ form.ordered_entities.help_text }} + {{ form.ordered_entities.errors }} +
+
+ +
+ +
+ +

Reorder observation levels for:

+ +
+{% endblock %} \ No newline at end of file diff --git a/backend/apps/api/v1/translation.py b/backend/apps/api/v1/translation.py index 1de97599..e279d6fc 100644 --- a/backend/apps/api/v1/translation.py +++ b/backend/apps/api/v1/translation.py @@ -14,6 +14,8 @@ InformationRequest, Language, License, + MeasurementUnit, + MeasurementUnitCategory, Organization, QualityCheck, RawDataSource, @@ -75,6 +77,14 @@ class LicenseTranslationOptions(TranslationOptions): fields = ("name",) +class MeasurementUnitTranslationOptions(TranslationOptions): + fields = ("name",) + + +class MeasurementUnitCategoryTranslationOptions(TranslationOptions): + fields = ("name",) + + class OrganizationTranslationOptions(TranslationOptions): fields = ("name", "description") @@ -124,6 +134,8 @@ class ThemeTranslationOptions(TranslationOptions): translator.register(InformationRequest, InformationRequestTranslationOptions) translator.register(Language, LanguageTranslationOptions) translator.register(License, LicenseTranslationOptions) +translator.register(MeasurementUnit, MeasurementUnitTranslationOptions) +translator.register(MeasurementUnitCategory, MeasurementUnitCategoryTranslationOptions) translator.register(Organization, OrganizationTranslationOptions) translator.register(QualityCheck, QualityCheckTranslationOptions) translator.register(RawDataSource, RawDataSourceTranslationOptions) diff --git a/backend/conftest.py b/backend/conftest.py index 40eb4cfc..0f9419e0 100644 --- a/backend/conftest.py +++ b/backend/conftest.py @@ -410,7 +410,6 @@ def fixture_tabela_bairros( is_directory=False, data_cleaning_description="Descrição da limpeza de dados", data_cleaning_code_url="http://cleaning.com/bairros", - raw_data_url="http://raw.com/bairros", auxiliary_files_url="http://aux.com/bairros", architecture_url="http://arch.com/bairros", source_bucket_name="basedosdados-dev",