From 96f3f333756765e74ec074d94c9f134d1ce80ab2 Mon Sep 17 00:00:00 2001 From: Ricardo Dahis Date: Thu, 7 Nov 2024 23:25:52 +1100 Subject: [PATCH 01/10] feat: closes #699 --- backend/apps/api/v1/admin.py | 1 + .../migrations/0051_remove_dataset_is_closed.py | 17 +++++++++++++++++ backend/apps/api/v1/models.py | 11 ++++++++--- 3 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 backend/apps/api/v1/migrations/0051_remove_dataset_is_closed.py diff --git a/backend/apps/api/v1/admin.py b/backend/apps/api/v1/admin.py index 693fd77c..a440a65a 100644 --- a/backend/apps/api/v1/admin.py +++ b/backend/apps/api/v1/admin.py @@ -612,6 +612,7 @@ class DatasetAdmin(OrderedInlineModelAdminMixin, TabbedTranslationAdmin): "contains_tables", "contains_raw_data_sources", "contains_information_requests", + "contains_closed_data", "page_views", "created_at", "updated_at", diff --git a/backend/apps/api/v1/migrations/0051_remove_dataset_is_closed.py b/backend/apps/api/v1/migrations/0051_remove_dataset_is_closed.py new file mode 100644 index 00000000..2012247a --- /dev/null +++ b/backend/apps/api/v1/migrations/0051_remove_dataset_is_closed.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.16 on 2024-11-07 12:24 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('v1', '0050_table_is_deprecated'), + ] + + operations = [ + migrations.RemoveField( + model_name='dataset', + name='is_closed', + ), + ] diff --git a/backend/apps/api/v1/models.py b/backend/apps/api/v1/models.py index c3233ddf..887e93b0 100644 --- a/backend/apps/api/v1/models.py +++ b/backend/apps/api/v1/models.py @@ -538,9 +538,6 @@ class Dataset(BaseModel): ) created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) - is_closed = models.BooleanField( - default=False, help_text="Dataset is for BD Pro subscribers only" - ) page_views = models.BigIntegerField( default=0, help_text="Number of page views by Google Analytics", @@ -662,14 +659,22 @@ def contains_closed_data(self): closed_data = False tables = self.tables.all() for table in tables: + # Check for closed coverages table_coverages = table.coverages.filter(is_closed=True) if table_coverages: closed_data = True break + + # Check for closed columns for column in table.columns.all(): if column.is_closed: # in the future it will be column.coverages closed_data = True break + + # Check if uncompressed file size is above 1 GB + if table.uncompressed_file_size and table.uncompressed_file_size > 1000000000: + closed_data = True + break return closed_data From b97a506012fc15901497f49b52b832c72d24a372 Mon Sep 17 00:00:00 2001 From: Ricardo Dahis Date: Mon, 11 Nov 2024 13:31:50 +1100 Subject: [PATCH 02/10] fix: property comment --- backend/apps/api/v1/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/apps/api/v1/models.py b/backend/apps/api/v1/models.py index 887e93b0..5b45c0b4 100644 --- a/backend/apps/api/v1/models.py +++ b/backend/apps/api/v1/models.py @@ -655,7 +655,7 @@ def contains_open_data(self): @property def contains_closed_data(self): - """Returns true if there are tables or columns with closed coverages""" + """Returns true if there are tables or columns with closed coverages, or if the uncompressed file size is above 1 GB""" closed_data = False tables = self.tables.all() for table in tables: From 103c41d394e7a29423894d90be71b2c8e729c80b Mon Sep 17 00:00:00 2001 From: isabelmeister Date: Wed, 13 Nov 2024 00:39:16 -0300 Subject: [PATCH 03/10] fix: create migration to add new url field in dataset --- .../0051_add_dataset_new_urlfield.py | 13 ++++++++++++ .../0051_add_textfield_all_models.py | 21 ------------------- 2 files changed, 13 insertions(+), 21 deletions(-) create mode 100644 backend/apps/api/v1/migrations/0051_add_dataset_new_urlfield.py delete mode 100644 backend/apps/api/v1/migrations/0051_add_textfield_all_models.py diff --git a/backend/apps/api/v1/migrations/0051_add_dataset_new_urlfield.py b/backend/apps/api/v1/migrations/0051_add_dataset_new_urlfield.py new file mode 100644 index 00000000..c8113fc0 --- /dev/null +++ b/backend/apps/api/v1/migrations/0051_add_dataset_new_urlfield.py @@ -0,0 +1,13 @@ +# Generated by Django 4.2.16 on 2024-11-13 01:13 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('v1', '0050_table_is_deprecated'), + ] + + operations = [ + ] diff --git a/backend/apps/api/v1/migrations/0051_add_textfield_all_models.py b/backend/apps/api/v1/migrations/0051_add_textfield_all_models.py deleted file mode 100644 index 68484ac9..00000000 --- a/backend/apps/api/v1/migrations/0051_add_textfield_all_models.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 4.2.16 on 2024-11-12 15:34 - -from django.db import migrations, models - -def add_new_field_to_all_tables(apps, schema_editor): - for model in apps.get_models(): - schema_editor.add_field( - model, - name='url_links', - field=models.TextField(max_length=255, default='', blank=True) - ) - -class Migration(migrations.Migration): - - dependencies = [ - ('v1', '0050_table_is_deprecated'), - ] - - operations = [ - migrations.RunPython(add_new_field_to_all_tables), - ] From 9e6ec9784767fac94f62f019e497cf8eb98ae894 Mon Sep 17 00:00:00 2001 From: isabelmeister Date: Wed, 13 Nov 2024 00:41:07 -0300 Subject: [PATCH 04/10] feat: add new url field to dataset --- .../api/v1/migrations/0051_add_dataset_new_urlfield.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/backend/apps/api/v1/migrations/0051_add_dataset_new_urlfield.py b/backend/apps/api/v1/migrations/0051_add_dataset_new_urlfield.py index c8113fc0..e303c87b 100644 --- a/backend/apps/api/v1/migrations/0051_add_dataset_new_urlfield.py +++ b/backend/apps/api/v1/migrations/0051_add_dataset_new_urlfield.py @@ -1,6 +1,6 @@ # Generated by Django 4.2.16 on 2024-11-13 01:13 -from django.db import migrations +from django.db import migrations, models class Migration(migrations.Migration): @@ -10,4 +10,9 @@ class Migration(migrations.Migration): ] operations = [ + migrations.AddField( + model_name='dataset', + name='link_url', + field=models.URLField(blank=True, null=True, verbose_name='Link') + ), ] From 6d9da9d8c3a9a8f0860e26c238c37f8b63ada237 Mon Sep 17 00:00:00 2001 From: isabelmeister Date: Wed, 13 Nov 2024 00:41:46 -0300 Subject: [PATCH 05/10] feat: display url field in dataset django --- backend/apps/api/v1/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/apps/api/v1/models.py b/backend/apps/api/v1/models.py index c3233ddf..d62f045e 100644 --- a/backend/apps/api/v1/models.py +++ b/backend/apps/api/v1/models.py @@ -545,6 +545,7 @@ class Dataset(BaseModel): default=0, help_text="Number of page views by Google Analytics", ) + link_url = models.URLField(blank=True, null=True) graphql_nested_filter_fields_whitelist = ["id", "slug"] From 47b815f8e08fb213f83fe163d8f922106a68c922 Mon Sep 17 00:00:00 2001 From: isabelmeister Date: Wed, 13 Nov 2024 00:45:13 -0300 Subject: [PATCH 06/10] fix(Model): add verbose_name in new urlField --- backend/apps/api/v1/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/apps/api/v1/models.py b/backend/apps/api/v1/models.py index d62f045e..2ab0d83b 100644 --- a/backend/apps/api/v1/models.py +++ b/backend/apps/api/v1/models.py @@ -545,7 +545,7 @@ class Dataset(BaseModel): default=0, help_text="Number of page views by Google Analytics", ) - link_url = models.URLField(blank=True, null=True) + link_url = models.URLField(blank=True, null=True, verbose_name='Link') graphql_nested_filter_fields_whitelist = ["id", "slug"] From 9ae3a25babce051af085916fa6d781b3a9ca0892 Mon Sep 17 00:00:00 2001 From: isabelmeister Date: Wed, 13 Nov 2024 22:18:45 -0300 Subject: [PATCH 07/10] feat: create migration --- .../0051_add_dataset_new_urlfield.py | 18 ------------------ .../migrations/0051_add_new_field_dataset.py | 13 +++++++++++++ 2 files changed, 13 insertions(+), 18 deletions(-) delete mode 100644 backend/apps/api/v1/migrations/0051_add_dataset_new_urlfield.py create mode 100644 backend/apps/api/v1/migrations/0051_add_new_field_dataset.py diff --git a/backend/apps/api/v1/migrations/0051_add_dataset_new_urlfield.py b/backend/apps/api/v1/migrations/0051_add_dataset_new_urlfield.py deleted file mode 100644 index e303c87b..00000000 --- a/backend/apps/api/v1/migrations/0051_add_dataset_new_urlfield.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 4.2.16 on 2024-11-13 01:13 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('v1', '0050_table_is_deprecated'), - ] - - operations = [ - migrations.AddField( - model_name='dataset', - name='link_url', - field=models.URLField(blank=True, null=True, verbose_name='Link') - ), - ] diff --git a/backend/apps/api/v1/migrations/0051_add_new_field_dataset.py b/backend/apps/api/v1/migrations/0051_add_new_field_dataset.py new file mode 100644 index 00000000..615c5de6 --- /dev/null +++ b/backend/apps/api/v1/migrations/0051_add_new_field_dataset.py @@ -0,0 +1,13 @@ +# Generated by Django 4.2.16 on 2024-11-13 16:59 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('v1', '0050_table_is_deprecated'), + ] + + operations = [ + ] From 6ab2f6c300c91a543aa0102c16241b480c681c29 Mon Sep 17 00:00:00 2001 From: isabelmeister Date: Wed, 13 Nov 2024 22:20:31 -0300 Subject: [PATCH 08/10] fix: change link field type and constraints in Dataset --- .../apps/api/v1/migrations/0051_add_new_field_dataset.py | 8 +++++++- backend/apps/api/v1/models.py | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/backend/apps/api/v1/migrations/0051_add_new_field_dataset.py b/backend/apps/api/v1/migrations/0051_add_new_field_dataset.py index 615c5de6..cb0dca8d 100644 --- a/backend/apps/api/v1/migrations/0051_add_new_field_dataset.py +++ b/backend/apps/api/v1/migrations/0051_add_new_field_dataset.py @@ -1,6 +1,6 @@ # Generated by Django 4.2.16 on 2024-11-13 16:59 -from django.db import migrations +from django.db import migrations, models class Migration(migrations.Migration): @@ -10,4 +10,10 @@ class Migration(migrations.Migration): ] operations = [ + migrations.AddField( + model_name='dataset', + name='link', + field=models.TextField(blank=True, null=True, default='', max_length=255 , verbose_name='Link') + ), ] + diff --git a/backend/apps/api/v1/models.py b/backend/apps/api/v1/models.py index 2ab0d83b..e382e832 100644 --- a/backend/apps/api/v1/models.py +++ b/backend/apps/api/v1/models.py @@ -545,7 +545,7 @@ class Dataset(BaseModel): default=0, help_text="Number of page views by Google Analytics", ) - link_url = models.URLField(blank=True, null=True, verbose_name='Link') + link_url = models.TextField(blank=True, null=True, default='', max_length=255 , verbose_name='Link') graphql_nested_filter_fields_whitelist = ["id", "slug"] From 2f14a925f47817ecd99811701c199fd49c6d7756 Mon Sep 17 00:00:00 2001 From: isabelmeister Date: Thu, 14 Nov 2024 10:27:40 -0300 Subject: [PATCH 09/10] fix: change field name --- backend/apps/api/v1/migrations/0051_add_new_field_dataset.py | 4 ++-- backend/apps/api/v1/models.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/apps/api/v1/migrations/0051_add_new_field_dataset.py b/backend/apps/api/v1/migrations/0051_add_new_field_dataset.py index cb0dca8d..0fcd2b45 100644 --- a/backend/apps/api/v1/migrations/0051_add_new_field_dataset.py +++ b/backend/apps/api/v1/migrations/0051_add_new_field_dataset.py @@ -12,8 +12,8 @@ class Migration(migrations.Migration): operations = [ migrations.AddField( model_name='dataset', - name='link', - field=models.TextField(blank=True, null=True, default='', max_length=255 , verbose_name='Link') + name='usage_guide', + field=models.TextField(blank=True, null=True, default='', max_length=255 , verbose_name='Guia de Uso') ), ] diff --git a/backend/apps/api/v1/models.py b/backend/apps/api/v1/models.py index e382e832..53cb249f 100644 --- a/backend/apps/api/v1/models.py +++ b/backend/apps/api/v1/models.py @@ -545,7 +545,7 @@ class Dataset(BaseModel): default=0, help_text="Number of page views by Google Analytics", ) - link_url = models.TextField(blank=True, null=True, default='', max_length=255 , verbose_name='Link') + usage_guide = models.TextField(blank=True, null=True, default='', max_length=255 , verbose_name='Guia de Uso') graphql_nested_filter_fields_whitelist = ["id", "slug"] From 6ab795bac64ce9cd79014be67a74c0c3e3c1d1c0 Mon Sep 17 00:00:00 2001 From: Ricardo Dahis Date: Mon, 25 Nov 2024 16:10:55 +1100 Subject: [PATCH 10/10] fix: migrations --- ...ve_dataset_is_closed.py => 0052_remove_dataset_is_closed.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename backend/apps/api/v1/migrations/{0051_remove_dataset_is_closed.py => 0052_remove_dataset_is_closed.py} (86%) diff --git a/backend/apps/api/v1/migrations/0051_remove_dataset_is_closed.py b/backend/apps/api/v1/migrations/0052_remove_dataset_is_closed.py similarity index 86% rename from backend/apps/api/v1/migrations/0051_remove_dataset_is_closed.py rename to backend/apps/api/v1/migrations/0052_remove_dataset_is_closed.py index 2012247a..e47b283f 100644 --- a/backend/apps/api/v1/migrations/0051_remove_dataset_is_closed.py +++ b/backend/apps/api/v1/migrations/0052_remove_dataset_is_closed.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ('v1', '0050_table_is_deprecated'), + ('v1', '0051_add_new_field_dataset'), ] operations = [