diff --git a/bd_api/apps/api/v1/admin.py b/bd_api/apps/api/v1/admin.py index 5873f4e4..22a8822d 100644 --- a/bd_api/apps/api/v1/admin.py +++ b/bd_api/apps/api/v1/admin.py @@ -263,7 +263,7 @@ def get_number_of_rows(table, bq_table): try: query = f""" SELECT COUNT(1) AS n_rows - FROM `{table.db_slug}` + FROM `{table.gbq_slug}` """ number_rows = read_gbq(query) number_rows = number_rows.loc[0, "n_rows"] @@ -285,18 +285,18 @@ def get_uncompressed_file_size(table, bq_table): if bq_table.table_type == "VIEW": try: file_size = 0 - folder_prefix = f"staging/{table.dataset.db_slug}/{table.db_slug}" - for blob in bucket.list_blobs(prefix=folder_prefix): + for blob in bucket.list_blobs(prefix=table.gcs_slug): file_size += blob.size except Exception as e: logger.warning(e) + return + creds = get_credentials() bq_client = GBQClient(credentials=creds) cs_client = GCSClient(credentials=creds) - bucket_name = "basedosdados-dev" - bucket = cs_client.get_bucket(bucket_name) + bucket = cs_client.get_bucket("basedosdados") tables: list[Table] = [] match str(modeladmin): @@ -312,7 +312,7 @@ def get_uncompressed_file_size(table, bq_table): for table in tables: try: - bq_table = bq_client.get_table(table.db_slug) + bq_table = bq_client.get_table(table.gbq_slug) table.number_rows = get_number_of_rows(table, bq_table) table.number_columns = get_number_of_columns(table, bq_table) table.uncompressed_file_size = get_uncompressed_file_size(table, bq_table) diff --git a/bd_api/apps/api/v1/models.py b/bd_api/apps/api/v1/models.py index 429384f0..fa23a88a 100644 --- a/bd_api/apps/api/v1/models.py +++ b/bd_api/apps/api/v1/models.py @@ -565,15 +565,6 @@ def get_success_url(self): """Get the success url for the dataset""" return reverse("datasetdetail", kwargs={"pk": self.object.pk}) - @property - def db_slug(self): - """Get the database slug""" - return self.full_slug.replace("sa_br", "br") - - @property - def get_graphql_db_slug(self): - return self.db_slug - @property def full_slug(self): """Get the full slug or Dataset""" @@ -980,16 +971,28 @@ class Meta: ] @property - def db_slug(self): - """Get the database slug""" - table_slug = self.slug - dataset_slug = self.dataset.db_slug - bucket_slug = self.source_bucket_name - return f"{bucket_slug}.{dataset_slug}.{table_slug}" + def gbq_slug(self): + """Get the slug used in Google Big Query""" + if cloud_table := self.cloud_tables.first(): + dataset = cloud_table.gcp_dataset_id + table = cloud_table.gcp_table_id + return f"basedosdados.{dataset}.{table}" + + @property + def gcs_slug(self): + """Get the slug used in Google Cloud Storage""" + if cloud_table := self.cloud_tables.first(): + dataset = cloud_table.gcp_dataset_id + table = cloud_table.gcp_table_id + return f"staging/{dataset}/{table}" + + @property + def get_graphql_gbq_slug(self): + return self.gbq_slug @property - def get_graphql_db_slug(self): - return self.db_slug + def get_graphql_gcs_slug(self): + return self.gcs_slug @property def partitions(self):