From a1f241dd72c0b6372cb66d4c996e309a89765843 Mon Sep 17 00:00:00 2001 From: FWuellhorst Date: Thu, 18 Jan 2024 11:32:34 +0100 Subject: [PATCH] 38 logging instead of prints (#40) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add logging in all cases except print_df #38 * Add log-level to examples and tests * update logger stages --------- Co-authored-by: Martin Rätz --- .../to_epw_energyplus.py | 13 ++++++++---- .../to_mos_TMY3.py | 7 +++++-- .../unconverted_to_x.py | 10 +++++++--- aixweather/imports/DWD.py | 7 +++++-- .../transformation_functions/auxiliary.py | 14 ++++++++----- .../pass_through_handling.py | 8 ++++++-- .../time_observation_transformations.py | 7 ++++++- .../unit_conversions.py | 19 ++++++++++-------- .../variable_transformations.py | 20 ++++++++++++------- ...ll_DWD_historical_to_all_output_formats.py | 4 ++++ tests/utils_4_tests.py | 5 ++++- 11 files changed, 79 insertions(+), 35 deletions(-) diff --git a/aixweather/core_data_format_2_output_file/to_epw_energyplus.py b/aixweather/core_data_format_2_output_file/to_epw_energyplus.py index ca9f0f6..7c07b78 100644 --- a/aixweather/core_data_format_2_output_file/to_epw_energyplus.py +++ b/aixweather/core_data_format_2_output_file/to_epw_energyplus.py @@ -4,6 +4,8 @@ import csv import datetime as dt +import logging + import pandas as pd import numpy as np @@ -11,6 +13,9 @@ from aixweather.imports.utils_import import MetaData from aixweather.transformation_functions import auxiliary, time_observation_transformations, pass_through_handling +logger = logging.getLogger(__name__) + + """ format_epw information: for links see readme @@ -374,9 +379,9 @@ def line4_ground_temp(df): ) return ground_temp except KeyError as err: - print( - f"For adding the probably unnecessary ground temperature to the .epw file header, " - f"the following made it impossible: {err}" + logger.warn( + "Error while adding the probably unnecessary ground temperature to the .epw file " + "header. A placeholder will be used. Error: %s", err ) ground_temp = ground_temp + [0] # 0 ground layers @@ -622,6 +627,6 @@ def fill_full_last_day(df): df_as_list, df = format_data(df, start, stop) writer.writerows(df_as_list) - print(f"EPW file saved to {file_path}.") + logger.info("EPW file saved to %s.", file_path) return df diff --git a/aixweather/core_data_format_2_output_file/to_mos_TMY3.py b/aixweather/core_data_format_2_output_file/to_mos_TMY3.py index 0e1525c..4e4de8a 100644 --- a/aixweather/core_data_format_2_output_file/to_mos_TMY3.py +++ b/aixweather/core_data_format_2_output_file/to_mos_TMY3.py @@ -1,7 +1,7 @@ """ Converts core data to modelica TMY3Reader data """ - +import logging import calendar import datetime as dt import pandas as pd @@ -10,6 +10,9 @@ from aixweather.imports.utils_import import MetaData from aixweather.transformation_functions import auxiliary, time_observation_transformations, pass_through_handling + +logger = logging.getLogger(__name__) + """ format_modelica_TMY3 information: @@ -251,5 +254,5 @@ def to_mos( file.seek(0, 0) file.write(f"{header_of}\n{content}") - print(f"MOS file saved to {filepath}.") + logger.info("MOS file saved to %s.", filepath) return df diff --git a/aixweather/core_data_format_2_output_file/unconverted_to_x.py b/aixweather/core_data_format_2_output_file/unconverted_to_x.py index cc1288e..b084a09 100644 --- a/aixweather/core_data_format_2_output_file/unconverted_to_x.py +++ b/aixweather/core_data_format_2_output_file/unconverted_to_x.py @@ -2,6 +2,7 @@ converts core data to different simpler formats (currently without any transformation) """ +import logging import json import pickle import pandas as pd @@ -10,6 +11,9 @@ from aixweather.imports.utils_import import MetaData +logger = logging.getLogger(__name__) + + def to_pickle( core_df: pd.DataFrame, meta: MetaData, @@ -35,7 +39,7 @@ def to_pickle( with open(meta_file_path, "wb") as file: pickle.dump(meta, file) - print(f"Pickle saved to {file_path}, meta information saved to {meta_file_path}.") + logger.info("Pickle saved to %s, meta information saved to %s.", file_path, meta_file_path) return core_df @@ -70,7 +74,7 @@ def to_json( with open(meta_file_path, "w") as file: json.dump(meta_dict, file, indent=4) - print(f"JSON saved to {file_path}, meta information saved to {meta_file_path}.") + logger.info("JSON saved to %s, meta information saved to %s.", file_path, meta_file_path) return core_df @@ -104,6 +108,6 @@ def to_csv( with open(meta_file_path, "w") as file: json.dump(meta_dict, file, indent=4) - print(f"CSV saved to {file_path}, meta information saved to {meta_file_path}.") + logger.info("CSV saved to %s, meta information saved to %s.", file_path, meta_file_path) return core_df diff --git a/aixweather/imports/DWD.py b/aixweather/imports/DWD.py index cc33523..fc66186 100644 --- a/aixweather/imports/DWD.py +++ b/aixweather/imports/DWD.py @@ -1,7 +1,7 @@ """ imports weather data from the DWD """ - +import logging import zipfile import os import shutil @@ -13,6 +13,9 @@ from aixweather import definitions +logger = logging.getLogger(__name__) + + def import_DWD_historical(start: dt.datetime, station: str) -> pd.DataFrame: """ Pull historical data from DWD: @@ -348,7 +351,7 @@ def _download_DWD_file(url: str, zip_name: str): for i in range(4): # try retrieval 3 times try: urllib.request.urlretrieve(url + zip_name, total_zip_name) - print(f"Loaded: {total_zip_name}") + logger.debug("Loaded: %s", total_zip_name) # save unzipped files to folder_unzip extract_path = os.path.join(definitions.local_folder_temp, folder_unzip) diff --git a/aixweather/transformation_functions/auxiliary.py b/aixweather/transformation_functions/auxiliary.py index f332414..82afdfe 100644 --- a/aixweather/transformation_functions/auxiliary.py +++ b/aixweather/transformation_functions/auxiliary.py @@ -1,10 +1,13 @@ """ includes auxiliary functions for data handling and transformation """ - +import logging import pandas as pd import numpy as np +logger = logging.getLogger(__name__) + + def force_data_variable_convention( df: pd.DataFrame, format_desired: dict ) -> pd.DataFrame: @@ -104,14 +107,15 @@ def evaluate_transformations(core_format: dict, other_format: dict): ValueError: If a core variable in other_format doesn't match the core variable format. """ - print("\nEvaluate format.") + logger.debug("Evaluate format.") for key, value in other_format.items(): if value["core_name"] in core_format.keys(): # compare units if value["unit"] != core_format[value["core_name"]]["unit"]: - print( - f"Unit transformation required for {value['core_name']} from" - f" {value['unit']} to {core_format[value['core_name']]['unit']}." + logger.debug( + "Unit transformation required for %s from %s to %s.", + value['core_name'], value['unit'], + core_format[value['core_name']]['unit'] ) elif not value["core_name"]: pass diff --git a/aixweather/transformation_functions/pass_through_handling.py b/aixweather/transformation_functions/pass_through_handling.py index cf61a65..c894e50 100644 --- a/aixweather/transformation_functions/pass_through_handling.py +++ b/aixweather/transformation_functions/pass_through_handling.py @@ -1,6 +1,7 @@ """ This module contains auxiliary functions for data transformation, e.g. time shifts """ +import logging import pandas as pd @@ -8,6 +9,9 @@ from aixweather.imports.utils_import import MetaData +logger = logging.getLogger(__name__) + + def create_pass_through_variables( df_shifted: pd.DataFrame, df_no_shift: pd.DataFrame, @@ -33,7 +37,7 @@ def create_pass_through_variables( pd.DataFrame: The modified `df_shifted` DataFrame with added pass-through variables. """ - print("\nApply transformation for pass through variables.") + logger.debug("Apply transformation for pass through variables.") # perform same transformation df_no_shift, meta.executed_transformations_no_shift = transform_func(df_no_shift) @@ -93,7 +97,7 @@ def get_shifts_of_used_variables(used_variables, meta): if not is_identical: # dont add to df - print( + logger.debug( f"Calculation of the non-shifted {desired_variable} is " f"not valid due non consistent " f"time of measurement (shifting) of the required " diff --git a/aixweather/transformation_functions/time_observation_transformations.py b/aixweather/transformation_functions/time_observation_transformations.py index ede9b30..a4fb54f 100644 --- a/aixweather/transformation_functions/time_observation_transformations.py +++ b/aixweather/transformation_functions/time_observation_transformations.py @@ -4,9 +4,14 @@ """ import datetime +import logging + import pandas as pd +logger = logging.getLogger(__name__) + + def _shift_timestamps_and_interpolate(df: pd.DataFrame, backward: bool) -> pd.DataFrame: """ Shift and interpolate timestamps in a DataFrame by 30 minutes forward or backward. @@ -81,7 +86,7 @@ def shift_time_by_dict(format_dict: dict, df: pd.DataFrame) -> pd.DataFrame: # No measurement if not present, though avoid being triggered # when using this function in 2output (empty string) if value[core_name] not in df.columns and value[core_name]: - print(f"No measurements for {value[core_name]}.") + logger.debug("No measurements for %s.", value[core_name]) else: if value[meas_key] == "prec2ind": df.loc[:, value[core_name]] = avg_preceding_hour_2_indicated_time( diff --git a/aixweather/transformation_functions/unit_conversions.py b/aixweather/transformation_functions/unit_conversions.py index 5f63d1d..b180d3c 100644 --- a/aixweather/transformation_functions/unit_conversions.py +++ b/aixweather/transformation_functions/unit_conversions.py @@ -1,61 +1,64 @@ """ Includes functions to convert units in weather data. """ +import logging import pandas as pd +logger = logging.getLogger(__name__) + def Jcm2_to_Whm2(radiation: pd.Series): """convert radiance unit from J/cm^2 to Wh/m^2""" radiation = radiation / 0.36 - print(f"{radiation.name} transformed from from J/cm2 to Wh/m2") + logger.debug("%s transformed from from J/cm2 to Wh/m2", radiation.name) return radiation def Jm2_to_Whm2(radiation: pd.Series): """convert radiance unit from J/m^2 to Wh/m^2""" radiation = radiation / 3600 - print(f"{radiation.name} transformed from from J/m2 to Wh/m2") + logger.debug("%s transformed from from J/m2 to Wh/m2", radiation.name) return radiation def kJm2_to_Whm2(radiation: pd.Series): """convert radiance unit from kJ/m^2 to Wh/m^2""" radiation = radiation / 3.6 - print(f"{radiation.name} transformed from from kJ/m^2 to Wh/m^2") + logger.debug("%s transformed from from kJ/m^2 to Wh/m^2", radiation.name) return radiation def hPa_to_Pa(pressure: pd.Series): """convert pressure unit from hPa to Pa""" pressure = pressure * 100 - print(f"{pressure.name} transformed from from hPa to Pa") + logger.debug("%s transformed from from hPa to Pa", pressure.name) return pressure def eigth_to_tenth(cloudgrade: pd.Series): """convert cloudgrade from eighth to tenth""" cloudgrade = cloudgrade * 10 / 8 - print(f"{cloudgrade.name} transformed from from eighth to tenth") + logger.debug("%s transformed from from eighth to tenth", cloudgrade.name) return cloudgrade def percent_to_tenth(cloudgrade: pd.Series): """convert cloudgrade from percent to tenth""" cloudgrade = cloudgrade / 10 - print(f"{cloudgrade.name} transformed from from percent to tenth") + logger.debug("%s transformed from from percent to tenth", cloudgrade.name) return cloudgrade def kelvin_to_celcius(temperature: pd.Series): """convert temperature from kelvin to celcius""" temperature = temperature - 273.15 - print(f"{temperature.name} transformed from from kelvin to celcius") + logger.debug("%s transformed from from kelvin to celcius", temperature.name) return temperature def divide_by_1000(series: pd.Series): """divide by 1000""" series = series / 1000 - print(f"{series.name} transformed from x to x/1000") + logger.debug("%s transformed from x to x/1000", series.name) return series diff --git a/aixweather/transformation_functions/variable_transformations.py b/aixweather/transformation_functions/variable_transformations.py index 08dce77..6dc0f7b 100644 --- a/aixweather/transformation_functions/variable_transformations.py +++ b/aixweather/transformation_functions/variable_transformations.py @@ -2,6 +2,7 @@ This module includes variable transformations and functions to calculate variables from given values """ +import logging import pandas as pd import numpy as np @@ -11,6 +12,9 @@ from aixweather.imports.utils_import import MetaData +logger = logging.getLogger(__name__) + + def approximate_opaque_from_total_skycover(total_sky_cover): opaque_sky_cover = total_sky_cover return opaque_sky_cover @@ -205,17 +209,19 @@ def robust_transformation( if arg in df.columns: # if contains only NaN values if df[arg].isna().all(): - print( - f"The required variable {arg} has only nan, " - f"calculation of {desired_variable} aborted." + logger.debug( + "The required variable %s has only nan, " + "calculation of %s aborted.", + arg, desired_variable ) return df, calc_status new_args.append(df[arg]) args_of_columns.append(arg) else: - print( - f"The required variable {arg} is not in the dataframes " - f"columns, calculation of {desired_variable} aborted." + logger.debug( + "The required variable %s is not in the dataframes " + "columns, calculation of %s aborted.", + arg, desired_variable ) return df, calc_status else: @@ -224,7 +230,7 @@ def robust_transformation( # Apply transformation if variables are available df[desired_variable] = transformation_function(*new_args) - print(f"Calculated {desired_variable} from {args_of_columns}.") + logger.info("Calculated %s from %s.", desired_variable, args_of_columns) # Feed back whether calculation was done and with which variables calc_status = {desired_variable: args_of_columns} diff --git a/examples/e1_pull_DWD_historical_to_all_output_formats.py b/examples/e1_pull_DWD_historical_to_all_output_formats.py index b625a62..f989a78 100644 --- a/examples/e1_pull_DWD_historical_to_all_output_formats.py +++ b/examples/e1_pull_DWD_historical_to_all_output_formats.py @@ -12,6 +12,10 @@ def e1_pull_DWD_historical_to_all_output_formats(): 2. See examplary use of additional features and settings 3. Create weather data files """ + # Enable logging to see what is happening + import logging + logging.basicConfig(level="DEBUG") + # choose the project class according to the desired weather data origin from aixweather.project_class import ProjectClassDWDHistorical diff --git a/tests/utils_4_tests.py b/tests/utils_4_tests.py index 93cae08..17ac166 100644 --- a/tests/utils_4_tests.py +++ b/tests/utils_4_tests.py @@ -3,7 +3,7 @@ to load data, compare results and execute tests """ # pylint: disable=all - +import logging import re import json import shutil @@ -13,6 +13,9 @@ from aixweather import definitions from aixweather.imports.utils_import import MetaData +# Enable logging for all unit-tests to see what is happening +logging.basicConfig(level="DEBUG") + def load_mos(folder_tests, result_folder, file_name): with open(os.path.join(folder_tests, file_name), "r") as file: