Merge pull request #112 from unb-mds/task/web-scraping

task(web-scraping): Definir intervalos de horário para cada data especial task

api/api/migrations/0004_alter_class_special_dates.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.5 on 2023-11-27 15:05
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0003_remove_class_workload_class_special_dates_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='class',
+            name='special_dates',
+            field=django.contrib.postgres.fields.ArrayField(base_field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=256), size=3), default=list, size=None),
+        ),
+    ]

api/api/models.py

@@ -42,7 +42,13 @@ class Class(models.Model):
     days = ArrayField(models.CharField(max_length=64))
     _class = models.CharField(max_length=64)
     discipline = models.ForeignKey(Discipline, on_delete=models.CASCADE, related_name='classes')
-    special_dates = ArrayField(models.CharField(max_length=256), default=list)
+    special_dates = ArrayField(
+        ArrayField(
+            models.CharField(max_length=256),
+            size=3,
+        ),
+        default=list
+    )
 
     def __str__(self):
         return self._class

api/utils/management/commands/updatemock.py

@@ -4,6 +4,7 @@
 from utils import sessions as sns, web_scraping as wbp
 from django.core.management.base import BaseCommand
 from pathlib import Path
+import re
 import json
 import os
 
@@ -26,12 +27,16 @@ def handle(self, *args: Any, **options: Any):
             current_year, current_period = sns.get_current_year_and_period()
             departments = wbp.get_list_of_departments()
             department = choice(departments)
-            
+
             with open(current_path / f"mock/sigaa.html", "a") as mock_file:
-                discipline_scraper = wbp.DisciplineWebScraper(department, current_year, current_period)
+                discipline_scraper = wbp.DisciplineWebScraper(
+                    department, current_year, current_period)
                 response = discipline_scraper.get_response_from_disciplines_post_request()
-                mock_file.write(self.response_decode(response))
-
+
+                striped_response = self.multiple_replace(
+                    self.response_decode(response))
+                mock_file.write(striped_response)
+
             with open(current_path / "mock/infos.json", "a") as info_file:
                 data = {
                     "year": current_year,
@@ -44,6 +49,15 @@ def handle(self, *args: Any, **options: Any):
             print('Não foi possível atualizar o mock!')
             print('Error:', error)
 
+    def multiple_replace(self, text):
+        replacement_dict = {
+            '\n': '',
+            '\t': '',
+            '\r': '',
+        }
+        pattern = re.compile('|'.join(map(re.escape, replacement_dict.keys())))
+        return pattern.sub(lambda match: replacement_dict[match.group(0)], text)
+
     def response_decode(self, response: Response) -> str:
         encoding = response.encoding if response.encoding else 'utf-8'
         return response.content.decode(encoding)

api/utils/mock/infos.json

@@ -1 +1 @@
-{"year": "2023", "period": "2", "department": "853"}
+{"year": "2023", "period": "2", "department": "1499"}