Merge pull request #13 from kthyng/robust

Making package more robust

ci/environment-py3.10.yml

@@ -4,14 +4,17 @@ channels:
 dependencies:
   - python=3.10
   ############## These will have to be adjusted to your specific project
-  - intake
-  - intake-xarray
+  - cf_pandas
   - pandas
   - requests
+  - shapely
   ##############
   - pytest
   - pip:
-    - cf-pandas
     - codecov
     - pytest-cov
     - coverage[toml]
+    - git+https://github.com/intake/intake
+    - intake-parquet
+    - intake-xarray
+    # - "dask[complete]"

ci/environment-py3.8.yml

@@ -4,14 +4,17 @@ channels:
 dependencies:
   - python=3.8
   ############## These will have to be adjusted to your specific project
-  - intake
-  - intake-xarray
+  - cf_pandas
   - pandas
   - requests
+  - shapely
   ##############
   - pytest
   - pip:
-    - cf-pandas
     - codecov
     - pytest-cov
     - coverage[toml]
+    - git+https://github.com/intake/intake
+    - intake-parquet
+    - intake-xarray
+    # - "dask[complete]"

ci/environment-py3.9.yml

@@ -4,14 +4,17 @@ channels:
 dependencies:
   - python=3.9
   ############## These will have to be adjusted to your specific project
-  - intake
-  - intake-xarray
+  - cf_pandas
   - pandas
   - requests
+  - shapely
   ##############
   - pytest
   - pip:
-    - cf-pandas
     - codecov
     - pytest-cov
     - coverage[toml]
+    - git+https://github.com/intake/intake
+    - intake-parquet
+    - intake-xarray
+    # - "dask[complete]"

docs/api.rst

@@ -10,10 +10,20 @@
    :caption: Documentation
 
 
-Top-level API
-*************
+``intake-axds`` catalog
+***********************
 
-.. automodule:: intake_axds
+.. automodule:: intake_axds.axds_cat
+   :members:
+   :inherited-members:
+   :undoc-members:
+   :show-inheritance:
+
+
+``intake-axds`` utilities
+*************************
+
+.. automodule:: intake_axds.utils
    :members:
    :inherited-members:
    :undoc-members:

docs/conf.py

@@ -56,7 +56,6 @@
     "sphinx.ext.extlinks",
     "sphinx.ext.intersphinx",
     "numpydoc",
-    "nbsphinx",
     "IPython.sphinxext.ipython_directive",
     "IPython.sphinxext.ipython_console_highlighting",
     "sphinxcontrib.srclinks",

docs/demo_overview.md

@@ -6,7 +6,7 @@ jupytext:
     format_version: 0.13
     jupytext_version: 1.14.0
 kernelspec:
-  display_name: Python 3 (ipykernel)
+  display_name: Python 3.9.13 ('intake-axds_docs')
   language: python
   name: python3
 ---
@@ -17,17 +17,44 @@ kernelspec:
 import intake
 ```
 
-## First 10
+## Defaults
 
-The default page size is 10, so requesting platforms without any other input arguments will return the first 10 datasets.
+The default (and currently only) data type is "platform2".
+The default page size is 10, so requesting platforms without any other input arguments will return the first 10 datasets. The input argument `page_size` controls the maximum number of entries in the catalog.
 
 ```{code-cell} ipython3
-cat = intake.open_axds_cat(datatype='platform2')
+cat = intake.open_axds_cat()
 len(cat)
 ```
 
+```{code-cell} ipython3
+cat
+```
+
+```{code-cell} ipython3
+cat[list(cat)[0]]
+```
+
 ## Filter in time and space
 
+The longitude values `min_lon` and `max_lon` should be in the range -180 to 180.
+
+```{code-cell} ipython3
+kw = {
+    "min_lon": -180,
+    "max_lon": -158,
+    "min_lat": 50,
+    "max_lat": 66,
+    "min_time": '2015-1-1',
+    "max_time": '2015-1-2',
+}
+
+cat = intake.open_axds_cat(datatype='platform2', kwargs_search=kw, page_size=1000)
+len(cat)
+```
+
+## Additionally filter with keyword
+
 ```{code-cell} ipython3
 kw = {
     "min_lon": -180,
@@ -36,9 +63,10 @@ kw = {
     "max_lat": 66,
     "min_time": '2015-1-1',
     "max_time": '2020-1-1',
+    "search_for": "humpback"
 }
 
-cat = intake.open_axds_cat(datatype='platform2', kwargs_search=kw)
+cat = intake.open_axds_cat(datatype='platform2', kwargs_search=kw, page_size=1000)
 len(cat)
 ```
 
@@ -48,8 +76,10 @@ len(cat)
 
 ### Dataframe
 
+For dataframes, the data by default comes from csv files, but can be accessed by parquet files instead. The `dataframe_filetype` argument controls this. About half of the datasets are not available from parquet files.
+
 ```{code-cell} ipython3
-cat = intake.open_axds_cat(datatype='platform2', outtype='dataframe')
+cat = intake.open_axds_cat(datatype='platform2', outtype='dataframe', dataframe_filetype="csv")
 source_name = list(cat)[0]
 cat[source_name].read()
 ```
@@ -61,3 +91,30 @@ cat = intake.open_axds_cat(datatype='platform2', outtype='xarray')
 source_name = list(cat)[0]
 cat[source_name].read()
 ```
+
+## Catalog metadata and options
+
+Can provide metadata at the catalog level with input arguments `name`, `description`, and `metadata` to override the defaults.
+
+```{code-cell} ipython3
+cat = intake.open_axds_cat(name="Catalog name", description="This is the catalog.", page_size=1,
+                           metadata={"special entry": "platforms"})
+cat
+```
+
+The default `ttl` argument, or time before force-reloading the catalog, is `None`, but can be overridden by inputting a value:
+
+```{code-cell} ipython3
+cat.ttl is None
+```
+
+```{code-cell} ipython3
+cat = intake.open_axds_cat(page_size=1, ttl=60)
+cat.ttl
+```
+
+## Verbose
+
+```{code-cell} ipython3
+cat = intake.open_axds_cat(verbose=True)
+```

docs/environment.yml

@@ -5,10 +5,12 @@ channels:
 dependencies:
    - python=3.9
    # If your docs code examples depend on other packages add them here
-   - intake
-   - intake-xarray
+   - aiohttp
+   - cf_pandas
+   - h5netcdf
    - pandas
    - requests
+   - shapely
    # These are needed for the docs themselves
    - jupytext
    - numpydoc
@@ -23,7 +25,10 @@ dependencies:
    - pip
    - recommonmark
    - pip:
-     - cf-pandas
+     - git+https://github.com/intake/intake
+     - intake-parquet
+     - intake-xarray
+    #  - "dask[complete]"
      - docrep<=0.2.7
      - furo
      - nbsphinx>=0.8.7

environment.yml

@@ -6,11 +6,14 @@ dependencies:
   - pytest
   # Examples (remove and add as needed)
   - aiohttp
+  - cf_pandas
   - h5netcdf
-  - intake
-  - intake-xarray
   - pandas
   - pip
   - requests
+  - shapely
   - pip:
-    - cf-pandas
+    - git+https://github.com/intake/intake
+    - intake-parquet
+    - intake-xarray
+    # - "dask[complete]"