From 8714e76bd2eb92df5b26cef6254a1062ef9366c2 Mon Sep 17 00:00:00 2001 From: Ricardo Dahis Date: Tue, 21 Mar 2023 13:59:44 -0300 Subject: [PATCH 1/3] elements for update logic --- basedosdados_api/api/v1/models.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/basedosdados_api/api/v1/models.py b/basedosdados_api/api/v1/models.py index bf1e166f..0316b535 100644 --- a/basedosdados_api/api/v1/models.py +++ b/basedosdados_api/api/v1/models.py @@ -330,12 +330,14 @@ def get_success_url(self): return reverse("datasetdetail", kwargs={"pk": self.object.pk}) -class UpdateFrequency(BdmModel): +class Update(BdmModel): id = models.UUIDField(primary_key=True, default=uuid4) entity = models.ForeignKey( - "Entity", on_delete=models.CASCADE, related_name="update_frequencies" + "Entity", on_delete=models.CASCADE, related_name="updates" ) - number = models.IntegerField() + frequency = models.IntegerField() + lag = models.IntegerField() + last = models.DateTimeField(blank=True, null=True) graphql_nested_filter_fields_whitelist = ["id"] @@ -343,9 +345,9 @@ def __str__(self): return f"{str(self.number)} {str(self.entity)}" class Meta: - db_table = "update_frequency" - verbose_name = "Update Frequency" - verbose_name_plural = "Update Frequencies" + db_table = "update" + verbose_name = "Update" + verbose_name_plural = "Updates" ordering = ["number"] @@ -370,8 +372,8 @@ class Table(BdmModel): partner_organization = models.ForeignKey( "Organization", on_delete=models.CASCADE, related_name="partner_tables" ) - update_frequency = models.ForeignKey( - "UpdateFrequency", on_delete=models.CASCADE, related_name="tables" + update = models.ForeignKey( + "Update", on_delete=models.CASCADE, related_name="tables" ) pipeline = models.ForeignKey( "Pipeline", on_delete=models.CASCADE, related_name="tables" @@ -589,8 +591,8 @@ class RawDataSource(BdmModel): license = models.ForeignKey( "License", on_delete=models.CASCADE, related_name="raw_data_sources" ) - update_frequency = models.ForeignKey( - "UpdateFrequency", on_delete=models.CASCADE, related_name="raw_data_sources" + update = models.ForeignKey( + "Update", on_delete=models.CASCADE, related_name="raw_data_sources" ) area_ip_address_required = models.ManyToManyField( "Area", related_name="raw_data_sources", blank=True @@ -619,8 +621,8 @@ class InformationRequest(BdmModel): status = models.ForeignKey( "Status", on_delete=models.CASCADE, related_name="information_requests" ) - update_frequency = models.ForeignKey( - "UpdateFrequency", on_delete=models.CASCADE, related_name="information_requests" + update = models.ForeignKey( + "Update", on_delete=models.CASCADE, related_name="information_requests" ) created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) From 77baab737ec48f7e1f323a714183a6e7de475871 Mon Sep 17 00:00:00 2001 From: Ricardo Dahis Date: Wed, 22 Mar 2023 10:56:06 -0300 Subject: [PATCH 2/3] validation for datetime entity, makes fields optional --- basedosdados_api/api/v1/models.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/basedosdados_api/api/v1/models.py b/basedosdados_api/api/v1/models.py index 0316b535..07a14d05 100644 --- a/basedosdados_api/api/v1/models.py +++ b/basedosdados_api/api/v1/models.py @@ -336,7 +336,7 @@ class Update(BdmModel): "Entity", on_delete=models.CASCADE, related_name="updates" ) frequency = models.IntegerField() - lag = models.IntegerField() + lag = models.IntegerField(blank=True, null=True) last = models.DateTimeField(blank=True, null=True) graphql_nested_filter_fields_whitelist = ["id"] @@ -350,6 +350,12 @@ class Meta: verbose_name_plural = "Updates" ordering = ["number"] + def clean(self) -> None: + if self.entity.category.slug != "datetime": + raise ValidationError( + "Entity's category is not in category.slug = `datetime`." + ) + return super().clean() class Table(BdmModel): id = models.UUIDField(primary_key=True, default=uuid4) @@ -373,7 +379,11 @@ class Table(BdmModel): "Organization", on_delete=models.CASCADE, related_name="partner_tables" ) update = models.ForeignKey( - "Update", on_delete=models.CASCADE, related_name="tables" + "Update", + on_delete=models.CASCADE, + related_name="tables", + null=True, + blank=True, ) pipeline = models.ForeignKey( "Pipeline", on_delete=models.CASCADE, related_name="tables" @@ -592,7 +602,11 @@ class RawDataSource(BdmModel): "License", on_delete=models.CASCADE, related_name="raw_data_sources" ) update = models.ForeignKey( - "Update", on_delete=models.CASCADE, related_name="raw_data_sources" + "Update", + on_delete=models.CASCADE, + related_name="raw_data_sources", + null=True, + blank=True, ) area_ip_address_required = models.ManyToManyField( "Area", related_name="raw_data_sources", blank=True @@ -622,7 +636,11 @@ class InformationRequest(BdmModel): "Status", on_delete=models.CASCADE, related_name="information_requests" ) update = models.ForeignKey( - "Update", on_delete=models.CASCADE, related_name="information_requests" + "Update", + on_delete=models.CASCADE, + related_name="information_requests", + null=True, + blank=True, ) created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) From 43dc37c9ab31953175600849bbcd4629a2d9e894 Mon Sep 17 00:00:00 2001 From: Ricardo Dahis Date: Fri, 24 Mar 2023 10:34:28 -0300 Subject: [PATCH 3/3] fix: allow for null year --- basedosdados_api/api/v1/models.py | 79 ++++++++++++++++--------------- 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/basedosdados_api/api/v1/models.py b/basedosdados_api/api/v1/models.py index 07a14d05..5a02be7d 100644 --- a/basedosdados_api/api/v1/models.py +++ b/basedosdados_api/api/v1/models.py @@ -790,20 +790,22 @@ class DateTimeRange(BdmModel): graphql_nested_filter_fields_whitelist = ["id"] def __str__(self): - start_year = self.start_year or "" - start_month = f"-{self.start_month}" if self.start_month else "" - start_day = f"-{self.start_day}" if self.start_day else "" - start_hour = f" {self.start_hour}" if self.start_hour else "" - start_minute = f":{self.start_minute}" if self.start_minute else "" - start_second = f":{self.start_second}" if self.start_second else "" - end_year = self.end_year or "" - end_month = f"-{self.end_month}" if self.end_month else "" - end_day = f"-{self.end_day}" if self.end_day else "" - end_hour = f" {self.end_hour}" if self.end_hour else "" - end_minute = f":{self.end_minute}" if self.end_minute else "" - interval = f"({self.interval})" if self.interval else "()" - return f"{start_year}{start_month}{start_day}{start_hour}{start_minute}{start_second}{interval}\ - {end_year}{end_month}{end_day}{end_hour}{end_minute}" + #start_year = self.start_year or "" + #start_month = f"-{self.start_month}" if self.start_month else "" + #start_day = f"-{self.start_day}" if self.start_day else "" + #start_hour = f" {self.start_hour}" if self.start_hour else "" + #start_minute = f":{self.start_minute}" if self.start_minute else "" + #start_second = f":{self.start_second}" if self.start_second else "" + #end_year = self.end_year or "" + #end_month = f"-{self.end_month}" if self.end_month else "" + #end_day = f"-{self.end_day}" if self.end_day else "" + #end_hour = f" {self.end_hour}" if self.end_hour else "" + #end_minute = f":{self.end_minute}" if self.end_minute else "" + #end_second = f":{self.end_second}" if self.end_second else "" + #interval = f"({self.interval})" if self.interval else "()" + #return f"{start_year}-{start_month}-{start_day} {start_hour}:{start_minute}:{start_second}({interval})\ + # {end_year}-{end_month}-{end_day} {end_hour}:{end_minute}:{end_second}" + return self.id # TODO smarter string for cases when fields are null (year, month, etc) def clean(self) -> None: errors = {} @@ -811,30 +813,31 @@ def clean(self) -> None: if (self.start_year and self.end_year) and self.start_year > self.end_year: errors["start_year"] = ["Start year cannot be greater than end year"] - try: - start_datetime = datetime( - self.start_year, - self.start_month or 1, - self.start_day or 1, - self.start_hour or 0, - self.start_minute or 0, - self.start_second or 0, - ) - end_datetime = datetime( - self.end_year, - self.end_month or 1, - self.end_day or 1, - self.end_hour or 0, - self.end_minute or 0, - self.end_second or 0, - ) - if start_datetime > end_datetime: - errors["start_year"] = [ - "Start datetime cannot be greater than end datetime" - ] - - except TypeError: - errors["start_year"] = ["Start year or end year are invalid"] + if (self.start_year and self.end_year): + try: + start_datetime = datetime( + self.start_year, + self.start_month or 1, + self.start_day or 1, + self.start_hour or 0, + self.start_minute or 0, + self.start_second or 0, + ) + end_datetime = datetime( + self.end_year, + self.end_month or 1, + self.end_day or 1, + self.end_hour or 0, + self.end_minute or 0, + self.end_second or 0, + ) + if start_datetime > end_datetime: + errors["start_year"] = [ + "Start datetime cannot be greater than end datetime" + ] + + except TypeError: + errors["start_year"] = ["Start year or end year are invalid"] if self.start_day: max_day = calendar.monthrange(self.start_year, self.start_month)[1]