Merge pull request #12 from chrisroadmap/utils

Add utilities

CHANGELOG.rst

@@ -12,6 +12,7 @@ Following the recommendations of https://keepachangelog.com/en/1.0.0/
 
 master
 ------
+- added: `utils` module (`#12 <https://github.com/chrisroadmap/climateforcing/pull/12>`_)
 - added: `geometry` module (`#10 <https://github.com/chrisroadmap/climateforcing/pull/10>`_)
 - removed: masked array keyword in `humidity` (`#10 <https://github.com/chrisroadmap/climateforcing/pull/10>`_)
 - fixed: wrong global mean latitude weighting in `aprp` (`#10 <https://github.com/chrisroadmap/climateforcing/pull/10>`_).

README.rst

@@ -42,10 +42,17 @@ Generates the components of shortwave effective radiative forcing (ERF) from cha
 - Zelinka, M. D., Andrews, T., Forster, P. M., and Taylor, K. E. (2014), Quantifying components of aerosol‐cloud‐radiation interactions in climate models, J. Geophys. Res. Atmos., 119, 7599–7615, https://doi.org/10.1002/2014JD021710.
 - Taylor, K. E., Crucifix, M., Braconnot, P., Hewitt, C. D., Doutriaux, C., Broccoli, A. J., Mitchell, J. F. B., & Webb, M. J. (2007). Estimating Shortwave Radiative Forcing and Response in Climate Models, Journal of Climate, 20(11), 2530–2543, https://doi.org/10.1175/JCLI4143.1
 
+
 atmos: general atmospheric physics tools
 ----------------------------------------
 humidity: Conversions for specific to relative humidity and vice versa. 
 
+
+geometry: quick and dirty area-weighted mean
+--------------------------------------------
+For when you relly want to know the global mean but don't want to think or download anything much. (Works nicely with `aprp`).
+
+
 twolayermodel: two-layer energy balance climate model
 -----------------------------------------------------
 Implementation of the Held et al (2010) and Geoffroy et al (2013a, 2013b) two-layer climate model. Thanks to `Glen Harris <https://www.metoffice.gov.uk/research/people/glen-harris/>`_ for the original code.
@@ -61,3 +68,8 @@ utci: Universal Climate Thermal Index
 Calculates a measure of heat stress based on meteorological data. The code provided is a Python translation of the original FORTRAN, used under kind permission of Peter Bröde. If you use this code please cite:
 
 - Bröde P, Fiala D, Blazejczyk K, Holmér I, Jendritzky G, Kampmann B, Tinz B, Havenith G, 2012. Deriving the operational procedure for the Universal Thermal Climate Index (UTCI). International Journal of Biometeorology 56, 481-494, https://doi.org/10.1007/s00484-011-0454-1
+
+
+utils: non-climate related utility functions
+--------------------------------------------
+downloads datasets from URLs and simple function to make nested directories from Python.

setup.py

@@ -1,5 +1,3 @@
-import os.path
-
 from setuptools import find_packages, setup
 
 import versioneer
@@ -15,7 +13,7 @@
 
 SOURCE_DIR = "src"
 
-REQUIREMENTS = ["numpy", "netCDF4", "pandas", "scipy"]
+REQUIREMENTS = ["numpy", "netCDF4", "pandas", "scipy", "tqdm"]
 REQUIREMENTS_TESTS = [
     "codecov",
     "coverage",
@@ -46,7 +44,6 @@
 # no tests/docs in `src` so don't need exclude
 PACKAGES = find_packages(SOURCE_DIR)
 PACKAGE_DIR = {"": SOURCE_DIR}
-PACKAGE_DATA = {"openscm_runner": [os.path.join("adapters", "fair_adapter", "*.csv")]}
 
 # Get the long description from the README file
 with open("README.rst", "r") as fh:
@@ -75,7 +72,6 @@
     keywords=["climate", "tools"],
     packages=PACKAGES,
     package_dir=PACKAGE_DIR,
-    package_data=PACKAGE_DATA,
     include_package_data=True,
     install_requires=REQUIREMENTS,
     extras_require=REQUIREMENTS_EXTRAS,

src/climateforcing/utils/__init__.py

@@ -0,0 +1,71 @@
+"""Utility functions."""
+
+import errno
+import os
+import urllib.request
+
+from tqdm import tqdm
+
+
+# This example is adapted from
+# https://stackoverflow.com/questions/15644964/python-progress-bar-and-downloads
+class _DownloadProgressBar(tqdm):
+    def update_to(self, blocks=1, blocksize=1, totalsize=None):
+        """Update download progress bar.
+
+        Inputs
+        ------
+            blocks : int, optional
+                Number of blocks transferred so far [default: 1].
+            blocksize : int, optional
+                Size of each block (in tqdm units) [default: 1].
+            totalsize : int, optional
+                Total size (in tqdm units). If [default: None] remains unchanged.
+        """
+        if totalsize is not None:
+            self.total = totalsize
+        self.update(blocks * blocksize - self.n)
+
+
+def check_and_download(url, filepath, clobber=False):
+    """Check prescence of a file and downloads if not present.
+
+    Inputs
+    ------
+        url :
+            url to download from
+        filepath : str
+            filename to download to
+        clobber : bool
+            False if download should not overwrite existing file, True if it should
+    """
+    if clobber or not os.path.isfile(filepath):
+        with _DownloadProgressBar(
+            unit="B", unit_scale=True, miniters=1, desc=url.split("/")[-1]
+        ) as progress:
+            urllib.request.urlretrieve(
+                url, filename=filepath, reporthook=progress.update_to
+            )
+
+
+# I got this from Stack Overflow, but can't find where now.
+def mkdir_p(path):
+    """Check to see if directory exists, and if not, create it.
+
+    Inputs
+    ------
+        path : str
+            directory to create
+
+    Raises
+    ------
+        OSError:
+            if directory cannot be created
+    """
+    try:
+        os.makedirs(path)
+    except OSError as exc:
+        if exc.errno == errno.EEXIST and os.path.isdir(path):
+            pass
+        else:
+            raise

tests/test_utils.py

@@ -0,0 +1,21 @@
+import os
+
+from climateforcing.utils import check_and_download, mkdir_p
+
+
+def test_mkdir_p():
+    mkdir_p("tests/testdata/dummy")
+    os.rmdir("tests/testdata/dummy")
+
+
+def test_check_and_download():
+    check_and_download(
+        "http://homepages.see.leeds.ac.uk/~mencsm/images/ta_q_kernel.png",
+        "tests/testdata/test_image.png",
+    )
+    check_and_download(
+        "http://homepages.see.leeds.ac.uk/~mencsm/images/ta_q_kernel.png",
+        "tests/testdata/test_image.png",
+        clobber=True,
+    )
+    os.remove("tests/testdata/test_image.png")