Skip to content

Commit

Permalink
linting
Browse files Browse the repository at this point in the history
  • Loading branch information
kthyng committed Sep 12, 2023
1 parent 94ac2e1 commit 095a4d9
Show file tree
Hide file tree
Showing 5 changed files with 107 additions and 44 deletions.
4 changes: 1 addition & 3 deletions intake_axds/axds.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,10 @@

from . import __version__
from .utils import (
check_station,
load_metadata,
make_data_url,
make_filter,
make_label,
make_metadata_url,
make_search_docs_url,
response_from_url,
)
Expand Down Expand Up @@ -120,7 +118,7 @@ def __init__(
self.uuid = metadata["uuid"]
self.search_docs_url = url

# not checking for now
# not checking for now
# # check station for if we want the output or not — for when source is used directly.
# _ = check_station(metadata, verbose=True)

Expand Down
10 changes: 6 additions & 4 deletions intake_axds/axds_cat.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@
from . import __version__
from .axds import AXDSSensorSource
from .utils import (
check_station,
load_metadata,
match_key_to_parameter,
match_std_names_to_parameter,
response_from_url,
check_station,
)


Expand Down Expand Up @@ -480,7 +480,7 @@ def _load(self):
description = f"AXDS dataset_id {uuid} of datatype {self.datatype}"

metadata = load_metadata(self.datatype, result)

keep_station = check_station(metadata, verbose=self.verbose)
if not keep_station:
continue
Expand Down Expand Up @@ -582,7 +582,9 @@ def _load(self):
entry._plugin = [plugin]

self._entries[uuid] = entry

# final tally
if self.verbose:
print(f"Final number of stations found after removing some: {len(self._entries)}.")
print(
f"Final number of stations found after removing some: {len(self._entries)}."
)
61 changes: 35 additions & 26 deletions intake_axds/utils.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
"""Utils to run."""

from importlib.metadata import PackageNotFoundError, version
from operator import itemgetter
from typing import Optional, Union
from nested_lookup import nested_lookup

import cf_pandas as cfp
import pandas as pd
import requests

from nested_lookup import nested_lookup
from shapely import wkt


Expand Down Expand Up @@ -159,20 +158,20 @@ def load_metadata(datatype: str, results: dict) -> dict: #: Dict[str, str]
found = [value for value in nested_lookup(key, results) if value is not None]
if len(found) > 0:
metadata[new_name] = found[0] # take first instance

new_name = "minTime"
found = nested_lookup('start', results, wild=True, with_keys=True)
for key, values in found.items():
found_dict = nested_lookup("start", results, wild=True, with_keys=True)
for key, values in found_dict.items():
if values == [None]:
continue
if len(values) == 1:
metadata[new_name] = values[0]
elif len(values) > 1:
metadata[new_name] = min(values)

new_name = "maxTime"
found = nested_lookup('end', results, wild=True, with_keys=True)
for key, values in found.items():
found_dict = nested_lookup("end", results, wild=True, with_keys=True)
for key, values in found_dict.items():
if values == [None]:
continue
if len(values) == 1:
Expand All @@ -187,36 +186,40 @@ def load_metadata(datatype: str, results: dict) -> dict: #: Dict[str, str]
p1 = wkt.loads(metadata["geospatial_bounds"])
keys = ["minLongitude", "minLatitude", "maxLongitude", "maxLatitude"]
metadata.update(dict(zip(keys, p1.bounds)))
metadata["variables_details"] = nested_lookup('variables', results)

metadata["variables_details"] = nested_lookup("variables", results)
metadata["variables"] = nested_lookup("standard_name", results)

elif datatype == "sensor_station":

# location is lon, lat, depth and type
# e.g. {'coordinates': [-123.711083, 38.914556, 0.0], 'type': 'Point'}
lon, lat, depth = nested_lookup('location', results)[0]["coordinates"]
lon, lat, depth = nested_lookup("location", results)[0]["coordinates"]
keys = ["minLongitude", "minLatitude", "maxLongitude", "maxLatitude"]
metadata.update(dict(zip(keys, [lon, lat, lon, lat])))

# e.g. 106793
metadata["internal_id"] = int([value for value in nested_lookup('id', results) if value is not None][0])

metadata["variables_details"] = nested_lookup('figures', results)[0]
metadata["variables"] = list(set(nested_lookup('datasetVariableId', results)))

metadata["datumConversions"] = nested_lookup('datumConversions', results)[0]
metadata["internal_id"] = int(
[value for value in nested_lookup("id", results) if value is not None][0]
)

metadata["variables_details"] = nested_lookup("figures", results)[0]
metadata["variables"] = list(set(nested_lookup("datasetVariableId", results)))

metadata["datumConversions"] = nested_lookup("datumConversions", results)[0]

filter = f"%7B%22stations%22:%5B%22{metadata['internal_id']}%22%5D%7D"
baseurl = "https://sensors.axds.co/api"
metadata_url = f"{baseurl}/metadata/filter/custom?filter={filter}"
metadata["metadata_url"] = metadata_url

# 1 or 2?
metadata["version"] = nested_lookup('version', results)[0]
metadata["version"] = nested_lookup("version", results)[0]

# name on other sites, esp for ERDDAP
metadata["foreignNames"] = list(set(nested_lookup('foreignName', results, wild=True)))
metadata["foreignNames"] = list(
set(nested_lookup("foreignName", results, wild=True))
)

return metadata

Expand All @@ -243,13 +246,15 @@ def check_station(metadata: dict, verbose: bool) -> bool:
keep = False
if verbose:
print(f"UUID {metadata['uuid']} is a webcam and should be skipped.")
# these are IOOS ERDDAP and were setup to be different stations so we can see which stations

# these are IOOS ERDDAP and were setup to be different stations so we can see which stations
# are successfully being served through IOOS RAs. It duplicates the data (purposely)
elif "ism-" in metadata["uuid"]:
keep = False
if verbose:
print(f"UUID {metadata['uuid']} is a duplicate station from IOOS and should be skipped.")
print(
f"UUID {metadata['uuid']} is a duplicate station from IOOS and should be skipped."
)

return keep

Expand Down Expand Up @@ -359,9 +364,11 @@ def make_metadata_url(filter: str) -> str:
return f"{baseurl}/metadata/filter/custom?filter={filter}"


def make_search_docs_url(internal_id: Optional[int] = None, uuid: Optional[str] = None) -> str:
def make_search_docs_url(
internal_id: Optional[int] = None, uuid: Optional[str] = None
) -> str:
"""Url for Axiom Search docs.
Uses whichever of internal_id and uuid is not None to formulate url.
Parameters
Expand All @@ -380,6 +387,8 @@ def make_search_docs_url(internal_id: Optional[int] = None, uuid: Optional[str]
return f"https://search.axds.co/v2/docs?verbose=false&id=sensor_station:{internal_id}"
elif uuid is not None:
return f"https://search.axds.co/v2/docs?verbose=false&id={uuid}"
else:
raise KeyError("Correct key was not input for return")


def response_from_url(url: str) -> Union[list, dict]:
Expand Down
12 changes: 3 additions & 9 deletions tests/test_axds_sensor.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,9 +133,7 @@ def test_intake_opener():


def test_binned():
source = AXDSSensorSource(
internal_id=123456, uuid="test", bin_interval="monthly"
)
source = AXDSSensorSource(internal_id=123456, uuid="test", bin_interval="monthly")
assert source.binned


Expand All @@ -158,13 +156,9 @@ def test_ids(mock_requests):

def test_times():
# doesn't need response because both internal_id and dataset_id are faked upon init
source = AXDSSensorSource(
internal_id=123456, uuid="fake", start_time="2000-1-1"
)
source = AXDSSensorSource(internal_id=123456, uuid="fake", start_time="2000-1-1")
assert source.end_time is None
source = AXDSSensorSource(
internal_id=123456, uuid="fake", end_time="2000-1-1"
)
source = AXDSSensorSource(internal_id=123456, uuid="fake", end_time="2000-1-1")
assert source.start_time is None


Expand Down
64 changes: 62 additions & 2 deletions tests/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,41 @@ def test_load_metadata():
},
}
metadata = utils.load_metadata(datatype, results)
test_results = {'uuid': 'uuid', 'title': 'label', 'summary': 'desc', 'minTime': '2000-1-1', 'maxTime': '2000-1-2', 'minLongitude': -123.711083, 'minLatitude': 38.914556, 'maxLongitude': -123.711083, 'maxLatitude': 38.914556, 'internal_id': 106793, 'variables_details': [{'label': 'label', 'parameterGroupId': 'parameterGroupId', 'plots': [{'subPlots': [{'datasetVariableId': 'datasetVariableId', 'parameterId': 'parameterId', 'label': 'label', 'deviceId': 'deviceId'}]}]}], 'variables': ['datasetVariableId'], 'datumConversions': [], 'metadata_url': 'https://sensors.axds.co/api/metadata/filter/custom?filter=%7B%22stations%22:%5B%22106793%22%5D%7D', 'version': 2, 'foreignNames': []}
test_results = {
"uuid": "uuid",
"title": "label",
"summary": "desc",
"minTime": "2000-1-1",
"maxTime": "2000-1-2",
"minLongitude": -123.711083,
"minLatitude": 38.914556,
"maxLongitude": -123.711083,
"maxLatitude": 38.914556,
"internal_id": 106793,
"variables_details": [
{
"label": "label",
"parameterGroupId": "parameterGroupId",
"plots": [
{
"subPlots": [
{
"datasetVariableId": "datasetVariableId",
"parameterId": "parameterId",
"label": "label",
"deviceId": "deviceId",
}
]
}
],
}
],
"variables": ["datasetVariableId"],
"datumConversions": [],
"metadata_url": "https://sensors.axds.co/api/metadata/filter/custom?filter=%7B%22stations%22:%5B%22106793%22%5D%7D",
"version": 2,
"foreignNames": [],
}
assert metadata == test_results

datatype = "platform2"
Expand Down Expand Up @@ -232,5 +266,31 @@ def test_load_metadata():
},
}
metadata = utils.load_metadata(datatype, results)
test_results = {'uuid': 'uuid', 'title': 'label', 'summary': 'desc', 'minTime': '2000-1-1', 'maxTime': '2000-1-2', 'institution': ['institution'], 'geospatial_bounds': 'POLYGON ((0 -80, 0 90, 359.9200439453125 90, 359.9200439453125 -80, 0 -80))', 'minLongitude': 0.0, 'minLatitude': -80.0, 'maxLongitude': 359.9200439453125, 'maxLatitude': 90.0, 'variables_details': [{'variable_name': {'attributes': {'standard_name': 'standard_name', 'units': 'units', 'unit_id': 'unit_id', 'long_name': 'long_name', 'parameter_id': 'parameter_id'}}}], 'variables': ['standard_name']}
test_results = {
"uuid": "uuid",
"title": "label",
"summary": "desc",
"minTime": "2000-1-1",
"maxTime": "2000-1-2",
"institution": ["institution"],
"geospatial_bounds": "POLYGON ((0 -80, 0 90, 359.9200439453125 90, 359.9200439453125 -80, 0 -80))",
"minLongitude": 0.0,
"minLatitude": -80.0,
"maxLongitude": 359.9200439453125,
"maxLatitude": 90.0,
"variables_details": [
{
"variable_name": {
"attributes": {
"standard_name": "standard_name",
"units": "units",
"unit_id": "unit_id",
"long_name": "long_name",
"parameter_id": "parameter_id",
}
}
}
],
"variables": ["standard_name"],
}
assert metadata == test_results

0 comments on commit 095a4d9

Please sign in to comment.