feat: add db id to dataset and table models

bd_api/apps/api/v1/migrations/0032_dataset_db_slug_table_db_slug.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 4.2.10 on 2024-05-13 02:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("v1", "0031_table_raw_data_source"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="dataset",
+            name="db_slug",
+            field=models.SlugField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name="table",
+            name="db_slug",
+            field=models.SlugField(blank=True, max_length=255, null=True),
+        ),
+    ]

bd_api/apps/api/v1/models.py

@@ -454,6 +454,7 @@ class Dataset(BaseModel):
 
     id = models.UUIDField(primary_key=True, default=uuid4)
     slug = models.SlugField(unique=False, max_length=255)
+    db_slug = models.SlugField(unique=False, max_length=255, blank=True, null=True)
     name = models.CharField(max_length=255)
     description = models.TextField(blank=True, null=True)
     organization = models.ForeignKey(
@@ -723,6 +724,7 @@ class Table(BaseModel, OrderedModel):
 
     id = models.UUIDField(primary_key=True, default=uuid4)
     slug = models.SlugField(unique=False, max_length=255)
+    db_slug = models.SlugField(unique=False, max_length=255, blank=True, null=True)
     name = models.CharField(max_length=255)
     description = models.TextField(blank=True, null=True)
     dataset = models.ForeignKey("Dataset", on_delete=models.CASCADE, related_name="tables")
@@ -810,6 +812,10 @@ class Meta:
             )
         ]
 
+    @property
+    def full_slug(self):
+        return self.slug
+
     @property
     def gbq_slug(self):
         """Get the slug used in Google Big Query"""

bd_api/apps/api/v1/tasks.py

@@ -6,7 +6,7 @@
 from google.api_core.exceptions import GoogleAPICallError
 from google.cloud.bigquery import Table as GBQTable
 from huey import crontab
-from huey.contrib.djhuey import periodic_task
+from huey.contrib.djhuey import periodic_task, task
 from loguru import logger
 from pandas import read_gbq
 from requests import get
@@ -30,6 +30,17 @@ def rebuild_search_index_task():
     call_command("rebuild_index", interactive=False, batchsize=100)
 
 
+@task()
+def update_db_id():
+    """Update db ids from datasets and tables, to ease filtering by the packages"""
+    for dataset in Dataset.objects.all():
+        dataset.db_slug = dataset.full_slug.replace("sa_br_", "br_")
+        dataset.save()
+    for table in Table.objects.all():
+        table.db_slug = f"{dataset.db_slug}.{table.slug}"
+        table.save()
+
+
 @periodic_task(crontab(day_of_week="1-5", hour="6", minute="0"))
 @production_task
 def update_table_metadata_task(table_pks: list[str] = None):