Add unit tests for gpm_api.io

gpm_api/io/checks.py

@@ -32,7 +32,7 @@
         Base directory where the GPM directory is located.
     """
     # Check base_dir does not end with /
-    if base_dir[-1] == "/":
+    if base_dir[-1] == os.path.sep:
         base_dir = base_dir[0:-1]
     # Retrieve last folder name
     dir_name = os.path.basename(base_dir)
@@ -157,15 +157,15 @@
     if isinstance(time, np.ndarray):
         if np.issubdtype(time.dtype, np.datetime64):
             if time.size == 1:
-                time = time.astype("datetime64[s]").tolist()
+                time = time.astype("datetime64[us]").tolist()
             else:
                 raise ValueError("Expecting a single timestep!")
         else:
             raise ValueError("The numpy array does not have a np.datetime64 dtype!")
 
-    # If np.datetime64, convert to datetime.datetime
+    # If np.datetime64, convert to datetime.datetime with microsecond precision
     if isinstance(time, np.datetime64):
-        time = time.astype("datetime64[s]").tolist()
+        time = time.astype("datetime64[us]").tolist()
     # If datetime.date, convert to datetime.datetime
     if not isinstance(time, (datetime.datetime, str)):
         time = datetime.datetime(time.year, time.month, time.day, 0, 0, 0)
@@ -178,11 +178,12 @@
 
 
 def check_date(date):
-    if not isinstance(date, (datetime.date, datetime.datetime)):
-        raise ValueError("date must be a datetime object")
-    if isinstance(date, datetime.datetime):
-        date = date.date()
-    return date
+    if date is None:
+        raise ValueError("date cannot be None")
+
+    # Use check_time to convert to datetime.datetime
+    datetime_obj = check_time(date)
+    return datetime_obj.date()
 
 
 def check_start_end_time(start_time, end_time):
@@ -225,25 +226,3 @@
 
     # -------------------------------------------------------------------------.
     return scan_mode
-
-
-def check_bbox(bbox):
-    """
-    Check correctness of bounding box.
-    bbox format: [lon_0, lon_1, lat_0, lat_1]
-    bbox should be provided with longitude between -180 and 180, and latitude
-    between -90 and 90.
-    """
-    if bbox is None:
-        return bbox
-    # If bbox provided
-    if not (isinstance(bbox, list) and len(bbox) == 4):
-        raise ValueError("Provide valid bbox [lon_0, lon_1, lat_0, lat_1]")
-    if bbox[2] > 90 or bbox[2] < -90 or bbox[3] > 90 or bbox[3] < -90:
-        raise ValueError("Latitude is defined between -90 and 90")
-    # Try to be sure that longitude is specified between -180 and 180
-    if bbox[0] > 180 or bbox[1] > 180:
-        print("bbox should be provided with longitude between -180 and 180")
-        bbox[0] = bbox[0] - 180
-        bbox[1] = bbox[1] - 180
-    return bbox

gpm_api/io/directories.py

@@ -44,8 +44,6 @@ def get_disk_dir_pattern(product, product_type, version):
         GPM product name. See: gpm_api.available_products()
     product_type : str, optional
         GPM product type. Either 'RS' (Research) or 'NRT' (Near-Real-Time).
-    date : datetime.date
-        Single date for which to retrieve the data.
     version : int, optional
         GPM version of the data to retrieve if product_type = 'RS'.
 

gpm_api/io/download.py

@@ -61,7 +61,7 @@
     # -------------------------------------------------------------------------.
     # Replace ftps with ftp to make curl work !!!
     # - curl expects ftp:// and not ftps://
-    server_path = server_path.replace("ftps", "ftp", 1)
+    server_path = server_path.replace("ftps://", "ftp://", 1)
     # -------------------------------------------------------------------------.
     ## Define command to run
     # Base command: curl -4 --ftp-ssl --user [user name]:[password] -n [url]
@@ -247,7 +247,6 @@
     progress_bar=True,
     verbose=False,
 ):
-
     if transfer_tool == "curl":
         list_cmd = [
             curl_cmd(src_path, dst_path, username, username)

gpm_api/tests/conftest.py

@@ -0,0 +1,137 @@
+import pytest
+import datetime
+from typing import Any, List, Dict
+from gpm_api.io.products import get_info_dict, available_products
+
+
+@pytest.fixture
+def product_types() -> List[str]:
+    """Return a list of all product types from the info dict"""
+    product_types = []
+    for product, props in get_info_dict().items():
+        product_types += props["product_types"]
+
+    product_types = list(set(product_types))  # Dedup list
+
+    return product_types
+
+
+@pytest.fixture
+def product_categories() -> List[str]:
+    """Return a list of product categories from the info dict"""
+
+    return list(set([props["product_category"] for props in get_info_dict().values()]))
+
+
+@pytest.fixture
+def product_levels() -> List[str]:
+    """Return a list of product levels from the info dict"""
+
+    # Available in gpm_api.io.checks.check_product_level()
+    return ["1A", "1B", "1C", "2A", "2B"]
+
+
+@pytest.fixture
+def versions() -> List[int]:
+    """Return a list of versions"""
+
+    # Available in gpm_api.io.checks.check_version()
+    return [4, 5, 6, 7]
+
+
+@pytest.fixture
+def products() -> List[str]:
+    """Return a list of all products regardless of type"""
+
+    return available_products()
+
+
+@pytest.fixture
+def username() -> str:
+    """Return a username
+
+    GPM uses an email address as username
+    """
+
+    return "[email protected]"
+
+
+@pytest.fixture
+def password() -> str:
+    """Return a password
+
+    GPM password is the username
+    """
+
+    return "[email protected]"
+
+
+@pytest.fixture
+def server_paths() -> Dict[str, Dict[str, Any]]:
+    """Return a list of probable GPM server paths"""
+
+    # Not validated to be real paths but follow the structure
+    return {
+        "ftps://arthurhouftps.pps.eosdis.nasa.gov/gpmdata/2020/07/05/radar/2A.GPM.DPR.V9-20211125.20200705-S170044-E183317.036092.V07A.HDF5": {
+            "year": 2020,
+            "month": 7,
+            "day": 5,
+            "product_category": "radar",
+            "product_type": "RS",
+            "start_time": datetime.datetime(2020, 7, 5, 17, 0, 44),
+            "end_time": datetime.datetime(2020, 7, 5, 18, 33, 17),
+            "version": 7,
+        },
+        "ftps://arthurhouftps.pps.eosdis.nasa.gov/gpmdata/2020/07/05/radar/2A.GPM.DPR.V9-20211125.20200705-S183318-E200550.036093.V07A.HDF5": {
+            "year": 2020,
+            "month": 7,
+            "day": 5,
+            "product_category": "radar",
+            "product_type": "RS",
+            "start_time": datetime.datetime(2020, 7, 5, 18, 33, 18),
+            "end_time": datetime.datetime(2020, 7, 5, 20, 5, 50),
+            "version": 7,
+        },
+        "ftps://arthurhouftps.pps.eosdis.nasa.gov/gpmdata/2020/07/05/radar/2A.GPM.DPR.V9-20211125.20200705-S200551-E213823.036094.V07A.HDF5": {
+            "year": 2020,
+            "month": 7,
+            "day": 5,
+            "product_category": "radar",
+            "product_type": "RS",
+            "start_time": datetime.datetime(2020, 7, 5, 20, 5, 51),
+            "end_time": datetime.datetime(2020, 7, 5, 21, 38, 23),
+            "version": 7,
+        },
+        # Include non-ftps folders
+        "ftp://arthurhouftps.pps.eosdis.nasa.gov/gpmdata/2020/07/05/radar/2A.GPM.DPR.V9-20211125.20200705-S213824-E231056.036095.V07A.HDF5": {
+            "year": 2020,
+            "month": 7,
+            "day": 5,
+            "product_category": "radar",
+            "product_type": "RS",
+            "start_time": datetime.datetime(2020, 7, 5, 21, 38, 24),
+            "end_time": datetime.datetime(2020, 7, 5, 23, 10, 56),
+            "version": 7,
+        },
+        "ftp://arthurhouftps.pps.eosdis.nasa.gov/gpmdata/2020/07/05/radar/2A.GPM.DPR.V9-20211125.20200705-S231057-E004329.036096.V07A.HDF5": {
+            "year": 2020,
+            "month": 7,
+            "day": 5,
+            "product_category": "radar",
+            "product_type": "RS",
+            "start_time": datetime.datetime(2020, 7, 5, 23, 10, 57),
+            "end_time": datetime.datetime(2020, 7, 6, 0, 43, 29),
+            "version": 7,
+        },
+        "ftp://arthurhouftps.pps.eosdis.nasa.gov/gpmdata/2020/07/05/radar/2A.GPM.DPR.V9-20211125.20200705-S004330-E021602.036097.V07A.HDF5": {
+            "year": 2020,
+            "month": 7,
+            "day": 5,
+            "product_category": "radar",
+            "product_type": "RS",
+            "start_time": datetime.datetime(2020, 7, 5, 0, 43, 30),
+            "end_time": datetime.datetime(2020, 7, 5, 2, 16, 2),
+            "version": 7,
+        },
+        # TODO: Add more products with varying attributes ...
+    }