Merge branch 'feature/add_filter_data' into 'develop'

Feature/add filter data

See merge request gmv-bda/upm/inesdata-mov/data-generation!35

inesdata_mov_datasets/__init__.py

@@ -1 +1 @@
-__version__: str = "0.4.0.dev"
+__version__: str = "0.3.0.dev"

inesdata_mov_datasets/sources/extract/emt.py

@@ -246,7 +246,6 @@ async def token_control(config: Settings, date_slash: str, date_day: str) -> str
                 #Catching an error that ocurrs when the server doesnt provide the token expiration
                 try:
                     expiration_date_unix = data["data"][0]["tokenDteExpiration"]["$date"]
-                    print(expiration_date_unix)
                 except:
                     logger.error(f"Error saving time expiration from login. Solving the problem retrying the call.")
                     token = await login_emt(config, object_login_name, local_path=dir_path)
@@ -256,7 +255,6 @@ async def token_control(config: Settings, date_slash: str, date_day: str) -> str
                     expiration_date_unix / 1000
                 )  #  miliseconds to seconds
                 now = datetime.datetime.now()
-                print(now)
                 # Compare the time expiration of the token withthe actual date
                 if now >= expiration_date:  # reset token
                     token = await login_emt(config, object_login_name, local_path=dir_path)

inesdata_mov_datasets/utils.py

@@ -297,7 +297,6 @@ def check_local_file_exists(path_dir: Path, object_name: str) -> bool:
     """
     # Create a Path object for the file
     file_path = Path(path_dir) / object_name
-    print(file_path)
     # Check if the file exists
     if file_path.exists():
         return True

requirements/requirements.txt

@@ -4,7 +4,6 @@
 #
 #    pip-compile requirements/requirements.in
 #
-
 aioboto3==12.3.0
     # via -r requirements/requirements.in
 aiobotocore[boto3]==2.11.2
@@ -72,11 +71,11 @@ polars==0.20.22
     # via -r requirements/requirements.in
 pyarrow==15.0.0
     # via -r requirements/requirements.in
-pydantic==2.6.1
+pydantic==2.9.2
     # via
     #   -r requirements/requirements.in
     #   pydantic-settings
-pydantic-core==2.16.2
+pydantic-core==2.23.4
     # via pydantic
 pydantic-settings==2.1.0
     # via -r requirements/requirements.in

requirements/requirements_dev.in

@@ -1,5 +1,6 @@
 # Requirements for development
 -c requirements.txt
+aioresponses==0.7.6
 black~=23.3.0              # Code formater
 isort==5.13.2               # Sort imports
 pre-commit==3.2.2            # Pre-commit hooks

requirements/requirements_dev.txt

@@ -4,19 +4,30 @@
 #
 #    pip-compile requirements/requirements_dev.in
 #
---trusted-host pypi.org
---trusted-host files.pythonhosted.org
---trusted-host pypi.python.org
-
+aiohttp==3.9.3
+    # via
+    #   -c requirements/requirements.txt
+    #   aioresponses
+aioresponses==0.7.6
+    # via -r requirements/requirements_dev.in
+aiosignal==1.3.1
+    # via
+    #   -c requirements/requirements.txt
+    #   aiohttp
 annotated-types==0.6.0
     # via
     #   -c requirements/requirements.txt
     #   pydantic
 asttokens==2.4.1
     # via stack-data
+async-timeout==4.0.3
+    # via
+    #   -c requirements/requirements.txt
+    #   aiohttp
 attrs==23.2.0
     # via
     #   -c requirements/requirements.txt
+    #   aiohttp
     #   visions
 babel==2.14.0
     # via mkdocs-material
@@ -77,6 +88,11 @@ filelock==3.4.2
     # via virtualenv
 fonttools==4.50.0
     # via matplotlib
+frozenlist==1.4.1
+    # via
+    #   -c requirements/requirements.txt
+    #   aiohttp
+    #   aiosignal
 ghp-import==2.1.0
     # via mkdocs
 htmlmin==0.1.12
@@ -87,6 +103,7 @@ idna==3.6
     # via
     #   -c requirements/requirements.txt
     #   requests
+    #   yarl
 imagehash==4.3.1
     # via
     #   visions
@@ -177,6 +194,11 @@ mkdocs-material-extensions==1.3.1
     # via mkdocs-material
 more-itertools==10.2.0
     # via jaraco-classes
+multidict==6.0.5
+    # via
+    #   -c requirements/requirements.txt
+    #   aiohttp
+    #   yarl
 multimethod==1.11.2
     # via
     #   visions
@@ -268,11 +290,11 @@ pure-eval==0.2.2
     # via stack-data
 pycparser==2.21
     # via cffi
-pydantic==2.6.1
+pydantic==2.9.2
     # via
     #   -c requirements/requirements.txt
     #   ydata-profiling
-pydantic-core==2.16.2
+pydantic-core==2.23.4
     # via
     #   -c requirements/requirements.txt
     #   pydantic
@@ -429,6 +451,10 @@ widgetsnbextension==4.0.10
     # via ipywidgets
 wordcloud==1.9.3
     # via ydata-profiling
+yarl==1.9.4
+    # via
+    #   -c requirements/requirements.txt
+    #   aiohttp
 ydata-profiling==4.7.0
     # via -r requirements/requirements_dev.in
 zipp==3.17.0

test/test_create_aemet.py

@@ -130,7 +130,6 @@ def test_generate_day_df_no_files(mock_listdir, mock_logger):
 
     # Verificar que no se generó ningún archivo procesado
     processed_file_path = Path(storage_path) / Path("processed") / "aemet" / date / f"aemet_{date.replace('/', '')}.csv"
-    print(processed_file_path)
     assert not processed_file_path.is_file(), "No debería haberse creado un archivo procesado"
 
 @patch('inesdata_mov_datasets.sources.create.aemet.logger')

test/test_create_emt.py

@@ -221,7 +221,6 @@ def test_generate_line_df_from_file():
         "line",
     ]
     assert list(df.columns) == expected_columns
-    print(df)
     # Verificar el contenido del DataFrame
     assert df["dayType"].iloc[0] == "FESTIVO"  # Correspondiente al primer elemento de Direction1
     assert df["StartTime"].iloc[0] == "08:00"
@@ -676,7 +675,6 @@ def test_create_emt_success(mock_instantiate_logger, mock_to_csv, mock_mkdir, mo
 
     # Llamar a la función
     create_emt(mock_settings, "2024/10/08")
-    print(mock_to_csv)
     # Verificar que se llama a to_csv para exportar el DataFrame
     mock_to_csv.assert_called_once()
 

test/test_extract_emt.py

@@ -52,7 +52,6 @@ async def test_get_calendar_error():
 
             # Llamar a la función
             result = await get_calendar(session, start_date, end_date, headers)
-            print(result)
 
             # Verificar que el resultado sea un error manejado
             assert result == {"code": -1}
@@ -184,7 +183,6 @@ async def test_login_emt_success(mock_makedirs, mock_open, mock_requests_get, mo
 
     # Llama a la función
     token = await login_emt(mock_settings, object_login_name="login_response.json", local_path="/test/storage/")
-    print(token)
     # Verifica que el token retornado sea el esperado
     assert token == "mock_access_token"
 

test/test_extract_informo.py

@@ -159,7 +159,6 @@ async def test_save_informo_minio(mock_upload_objs, mock_check_s3_file_exists, m
     # Get the timezone from Madrid and formated the dates for the object_name of the files
     europe_timezone = pytz.timezone("Europe/Madrid")
     current_datetime = datetime.datetime.now(europe_timezone).replace(second=0)
-    print(current_datetime)
     formatted_date_slash = current_datetime.strftime(
         "%Y/%m/%d"
     ) 
@@ -194,7 +193,6 @@ async def test_save_informo_local(mock_open_func, mock_check_local_file_exists,
     # Get the timezone from Madrid and formated the dates for the object_name of the files
     europe_timezone = pytz.timezone("Europe/Madrid")
     current_datetime = datetime.datetime.now(europe_timezone).replace(second=0)
-    print(current_datetime)
     formatted_date_slash = current_datetime.strftime(
         "%Y/%m/%d"
     ) 

test/test_utils.py

@@ -363,7 +363,6 @@ def test_read_settings():
         with patch("yaml.safe_load", return_value=yaml.safe_load(yaml_content)):
             # Llamamos a la función para probarla
             settings = read_settings("path/to/config.yaml")
-            print(settings.sources.emt.credentials)
 
             # Verificamos que los valores cargados en settings sean correctos
             if settings.storage.default == "local" and settings.sources.aemet.credentials.api_key == "my_api_key":