38 logging instead of prints (#40)

* Add logging in all cases except print_df #38 * Add log-level to examples and tests * update logger stages --------- Co-authored-by: Martin Rätz <[email protected]>
RWTH-EBC · Jan 18, 2024 · a1f241d · a1f241d
1 parent 283fb93
commit a1f241d
Show file tree

Hide file tree

Showing 11 changed files with 79 additions and 35 deletions.
diff --git a/aixweather/core_data_format_2_output_file/to_epw_energyplus.py b/aixweather/core_data_format_2_output_file/to_epw_energyplus.py
@@ -4,13 +4,18 @@
 
 import csv
 import datetime as dt
+import logging
+
 import pandas as pd
 import numpy as np
 
 from aixweather import definitions
 from aixweather.imports.utils_import import MetaData
 from aixweather.transformation_functions import auxiliary, time_observation_transformations, pass_through_handling
 
+logger = logging.getLogger(__name__)
+
+
 """
 format_epw information:
 for links see readme
@@ -374,9 +379,9 @@ def line4_ground_temp(df):
             )
             return ground_temp
         except KeyError as err:
-            print(
-                f"For adding the probably unnecessary ground temperature to the .epw file header, "
-                f"the following made it impossible: {err}"
+            logger.warn(
+                "Error while adding the probably unnecessary ground temperature to the .epw file "
+                "header. A placeholder will be used. Error: %s", err
             )
             ground_temp = ground_temp + [0]  # 0 ground layers
 
@@ -622,6 +627,6 @@ def fill_full_last_day(df):
         df_as_list, df = format_data(df, start, stop)
         writer.writerows(df_as_list)
 
-    print(f"EPW file saved to {file_path}.")
+    logger.info("EPW file saved to %s.", file_path)
 
     return df
diff --git a/aixweather/core_data_format_2_output_file/to_mos_TMY3.py b/aixweather/core_data_format_2_output_file/to_mos_TMY3.py
@@ -1,7 +1,7 @@
 """
 Converts core data to modelica TMY3Reader data
 """
-
+import logging
 import calendar
 import datetime as dt
 import pandas as pd
@@ -10,6 +10,9 @@
 from aixweather.imports.utils_import import MetaData
 from aixweather.transformation_functions import auxiliary, time_observation_transformations, pass_through_handling
 
+
+logger = logging.getLogger(__name__)
+
 """
 format_modelica_TMY3 information:
 
@@ -251,5 +254,5 @@ def to_mos(
         file.seek(0, 0)
         file.write(f"{header_of}\n{content}")
 
-    print(f"MOS file saved to {filepath}.")
+    logger.info("MOS file saved to %s.", filepath)
     return df
diff --git a/aixweather/core_data_format_2_output_file/unconverted_to_x.py b/aixweather/core_data_format_2_output_file/unconverted_to_x.py
@@ -2,6 +2,7 @@
 converts core data to different simpler formats (currently without any transformation)
 """
 
+import logging
 import json
 import pickle
 import pandas as pd
@@ -10,6 +11,9 @@
 from aixweather.imports.utils_import import MetaData
 
 
+logger = logging.getLogger(__name__)
+
+
 def to_pickle(
     core_df: pd.DataFrame,
     meta: MetaData,
@@ -35,7 +39,7 @@ def to_pickle(
     with open(meta_file_path, "wb") as file:
         pickle.dump(meta, file)
 
-    print(f"Pickle saved to {file_path}, meta information saved to {meta_file_path}.")
+    logger.info("Pickle saved to %s, meta information saved to %s.", file_path, meta_file_path)
 
     return core_df
 
@@ -70,7 +74,7 @@ def to_json(
     with open(meta_file_path, "w") as file:
         json.dump(meta_dict, file, indent=4)
 
-    print(f"JSON saved to {file_path}, meta information saved to {meta_file_path}.")
+    logger.info("JSON saved to %s, meta information saved to %s.", file_path, meta_file_path)
 
     return core_df
 
@@ -104,6 +108,6 @@ def to_csv(
     with open(meta_file_path, "w") as file:
         json.dump(meta_dict, file, indent=4)
 
-    print(f"CSV saved to {file_path}, meta information saved to {meta_file_path}.")
+    logger.info("CSV saved to %s, meta information saved to %s.", file_path, meta_file_path)
 
     return core_df
diff --git a/aixweather/imports/DWD.py b/aixweather/imports/DWD.py
@@ -1,7 +1,7 @@
 """
 imports weather data from the DWD
 """
-
+import logging
 import zipfile
 import os
 import shutil
@@ -13,6 +13,9 @@
 from aixweather import definitions
 
 
+logger = logging.getLogger(__name__)
+
+
 def import_DWD_historical(start: dt.datetime, station: str) -> pd.DataFrame:
     """
     Pull historical data from DWD:
@@ -348,7 +351,7 @@ def _download_DWD_file(url: str, zip_name: str):
     for i in range(4):  # try retrieval 3 times
         try:
             urllib.request.urlretrieve(url + zip_name, total_zip_name)
-            print(f"Loaded: {total_zip_name}")
+            logger.debug("Loaded: %s", total_zip_name)
 
             # save unzipped files to folder_unzip
             extract_path = os.path.join(definitions.local_folder_temp, folder_unzip)

diff --git a/aixweather/transformation_functions/auxiliary.py b/aixweather/transformation_functions/auxiliary.py
@@ -1,10 +1,13 @@
 """
 includes auxiliary functions for data handling and transformation
 """
-
+import logging
 import pandas as pd
 import numpy as np
 
+logger = logging.getLogger(__name__)
+
+
 def force_data_variable_convention(
     df: pd.DataFrame, format_desired: dict
 ) -> pd.DataFrame:
@@ -104,14 +107,15 @@ def evaluate_transformations(core_format: dict, other_format: dict):
         ValueError: If a core variable in other_format doesn't match the core variable format.
     """
 
-    print("\nEvaluate format.")
+    logger.debug("Evaluate format.")
     for key, value in other_format.items():
         if value["core_name"] in core_format.keys():
             # compare units
             if value["unit"] != core_format[value["core_name"]]["unit"]:
-                print(
-                    f"Unit transformation required for {value['core_name']} from"
-                    f" {value['unit']} to {core_format[value['core_name']]['unit']}."
+                logger.debug(
+                    "Unit transformation required for %s from %s to %s.",
+                    value['core_name'], value['unit'],
+                    core_format[value['core_name']]['unit']
                 )
         elif not value["core_name"]:
             pass

diff --git a/aixweather/transformation_functions/pass_through_handling.py b/aixweather/transformation_functions/pass_through_handling.py
@@ -1,13 +1,17 @@
 """
 This module contains auxiliary functions for data transformation, e.g. time shifts
 """
+import logging
 
 import pandas as pd
 
 from aixweather.transformation_functions import auxiliary
 from aixweather.imports.utils_import import MetaData
 
 
+logger = logging.getLogger(__name__)
+
+
 def create_pass_through_variables(
     df_shifted: pd.DataFrame,
     df_no_shift: pd.DataFrame,
@@ -33,7 +37,7 @@ def create_pass_through_variables(
         pd.DataFrame: The modified `df_shifted` DataFrame with added pass-through variables.
     """
 
-    print("\nApply transformation for pass through variables.")
+    logger.debug("Apply transformation for pass through variables.")
     # perform same transformation
     df_no_shift, meta.executed_transformations_no_shift = transform_func(df_no_shift)
 
@@ -93,7 +97,7 @@ def get_shifts_of_used_variables(used_variables, meta):
 
         if not is_identical:
             # dont add to df
-            print(
+            logger.debug(
                 f"Calculation of the non-shifted {desired_variable} is "
                 f"not valid due non consistent "
                 f"time of measurement (shifting) of the required "

diff --git a/aixweather/transformation_functions/time_observation_transformations.py b/aixweather/transformation_functions/time_observation_transformations.py
@@ -4,9 +4,14 @@
 """
 
 import datetime
+import logging
+
 import pandas as pd
 
 
+logger = logging.getLogger(__name__)
+
+
 def _shift_timestamps_and_interpolate(df: pd.DataFrame, backward: bool) -> pd.DataFrame:
     """
     Shift and interpolate timestamps in a DataFrame by 30 minutes forward or backward.
@@ -81,7 +86,7 @@ def shift_time_by_dict(format_dict: dict, df: pd.DataFrame) -> pd.DataFrame:
         # No measurement if not present, though avoid being triggered
         # when using this function in 2output (empty string)
         if value[core_name] not in df.columns and value[core_name]:
-            print(f"No measurements for {value[core_name]}.")
+            logger.debug("No measurements for %s.", value[core_name])
         else:
             if value[meas_key] == "prec2ind":
                 df.loc[:, value[core_name]] = avg_preceding_hour_2_indicated_time(

diff --git a/aixweather/transformation_functions/unit_conversions.py b/aixweather/transformation_functions/unit_conversions.py
@@ -1,61 +1,64 @@
 """
 Includes functions to convert units in weather data.
 """
+import logging
 
 import pandas as pd
 
+logger = logging.getLogger(__name__)
+
 
 def Jcm2_to_Whm2(radiation: pd.Series):
     """convert radiance unit from J/cm^2 to Wh/m^2"""
     radiation = radiation / 0.36
-    print(f"{radiation.name} transformed from from J/cm2 to Wh/m2")
+    logger.debug("%s transformed from from J/cm2 to Wh/m2", radiation.name)
     return radiation
 
 
 def Jm2_to_Whm2(radiation: pd.Series):
     """convert radiance unit from J/m^2 to Wh/m^2"""
     radiation = radiation / 3600
-    print(f"{radiation.name} transformed from from J/m2 to Wh/m2")
+    logger.debug("%s transformed from from J/m2 to Wh/m2", radiation.name)
     return radiation
 
 
 def kJm2_to_Whm2(radiation: pd.Series):
     """convert radiance unit from kJ/m^2 to Wh/m^2"""
     radiation = radiation / 3.6
-    print(f"{radiation.name} transformed from from kJ/m^2 to Wh/m^2")
+    logger.debug("%s transformed from from kJ/m^2 to Wh/m^2", radiation.name)
     return radiation
 
 
 def hPa_to_Pa(pressure: pd.Series):
     """convert pressure unit from hPa to Pa"""
     pressure = pressure * 100
-    print(f"{pressure.name} transformed from from hPa to Pa")
+    logger.debug("%s transformed from from hPa to Pa", pressure.name)
     return pressure
 
 
 def eigth_to_tenth(cloudgrade: pd.Series):
     """convert cloudgrade from eighth to tenth"""
     cloudgrade = cloudgrade * 10 / 8
-    print(f"{cloudgrade.name} transformed from from eighth to tenth")
+    logger.debug("%s transformed from from eighth to tenth", cloudgrade.name)
     return cloudgrade
 
 
 def percent_to_tenth(cloudgrade: pd.Series):
     """convert cloudgrade from percent to tenth"""
     cloudgrade = cloudgrade / 10
-    print(f"{cloudgrade.name} transformed from from percent to tenth")
+    logger.debug("%s transformed from from percent to tenth", cloudgrade.name)
     return cloudgrade
 
 
 def kelvin_to_celcius(temperature: pd.Series):
     """convert temperature from kelvin to celcius"""
     temperature = temperature - 273.15
-    print(f"{temperature.name} transformed from from kelvin to celcius")
+    logger.debug("%s transformed from from kelvin to celcius", temperature.name)
     return temperature
 
 
 def divide_by_1000(series: pd.Series):
     """divide by 1000"""
     series = series / 1000
-    print(f"{series.name} transformed from x to x/1000")
+    logger.debug("%s transformed from x to x/1000", series.name)
     return series
diff --git a/aixweather/transformation_functions/variable_transformations.py b/aixweather/transformation_functions/variable_transformations.py
@@ -2,6 +2,7 @@
 This module includes variable transformations and functions
 to calculate variables from given values
 """
+import logging
 
 import pandas as pd
 import numpy as np
@@ -11,6 +12,9 @@
 from aixweather.imports.utils_import import MetaData
 
 
+logger = logging.getLogger(__name__)
+
+
 def approximate_opaque_from_total_skycover(total_sky_cover):
     opaque_sky_cover = total_sky_cover
     return opaque_sky_cover
@@ -205,17 +209,19 @@ def robust_transformation(
                 if arg in df.columns:
                     # if contains only NaN values
                     if df[arg].isna().all():
-                        print(
-                            f"The required variable {arg} has only nan, "
-                            f"calculation of {desired_variable} aborted."
+                        logger.debug(
+                            "The required variable %s has only nan, "
+                            "calculation of %s aborted.",
+                            arg, desired_variable
                         )
                         return df, calc_status
                     new_args.append(df[arg])
                     args_of_columns.append(arg)
                 else:
-                    print(
-                        f"The required variable {arg} is not in the dataframes "
-                        f"columns, calculation of {desired_variable} aborted."
+                    logger.debug(
+                        "The required variable %s is not in the dataframes "
+                        "columns, calculation of %s aborted.",
+                        arg, desired_variable
                     )
                     return df, calc_status
             else:
@@ -224,7 +230,7 @@ def robust_transformation(
 
         # Apply transformation if variables are available
         df[desired_variable] = transformation_function(*new_args)
-        print(f"Calculated {desired_variable} from {args_of_columns}.")
+        logger.info("Calculated %s from %s.", desired_variable, args_of_columns)
 
         # Feed back whether calculation was done and with which variables
         calc_status = {desired_variable: args_of_columns}

diff --git a/examples/e1_pull_DWD_historical_to_all_output_formats.py b/examples/e1_pull_DWD_historical_to_all_output_formats.py
@@ -12,6 +12,10 @@ def e1_pull_DWD_historical_to_all_output_formats():
     2. See examplary use of additional features and settings
     3. Create weather data files
     """
+    # Enable logging to see what is happening
+    import logging
+    logging.basicConfig(level="DEBUG")
+
     # choose the project class according to the desired weather data origin
     from aixweather.project_class import ProjectClassDWDHistorical
 

diff --git a/tests/utils_4_tests.py b/tests/utils_4_tests.py
@@ -3,7 +3,7 @@
 to load data, compare results and execute tests
 """
 # pylint: disable=all
-
+import logging
 import re
 import json
 import shutil
@@ -13,6 +13,9 @@
 from aixweather import definitions
 from aixweather.imports.utils_import import MetaData
 
+# Enable logging for all unit-tests to see what is happening
+logging.basicConfig(level="DEBUG")
+
 
 def load_mos(folder_tests, result_folder, file_name):
     with open(os.path.join(folder_tests, file_name), "r") as file: