diff --git a/docker-compose.yml b/docker-compose.yml index dce6636a5..5077d4332 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,8 +44,6 @@ services: depends_on: elasticsearch: condition: service_healthy - volumes: - - api-config:/data/wis2box/config/pygeoapi/:rw healthcheck: test: ["CMD", "curl", "-f", "http://localhost/oapi/admin/resources"] interval: 5s @@ -157,5 +155,4 @@ volumes: es-data: minio-data: auth-data: - api-config: htpasswd: diff --git a/tests/data/downloads/.session-info.json b/tests/data/downloads/.session-info.json new file mode 100644 index 000000000..261fe5ee4 --- /dev/null +++ b/tests/data/downloads/.session-info.json @@ -0,0 +1,4 @@ +{ + "topics": {}, + "client_id": "wis2box-wis2downloader-9L2i4j9DSYltt7X2gUzfoaQpVhcS4wUb" +} diff --git a/wis2box-management/wis2box/api/__init__.py b/wis2box-management/wis2box/api/__init__.py index c96601d4f..4356173e8 100644 --- a/wis2box-management/wis2box/api/__init__.py +++ b/wis2box-management/wis2box/api/__init__.py @@ -25,6 +25,8 @@ from time import sleep +from owslib.ogcapi.records import Records + from wis2box import cli_helpers from wis2box.api.backend import load_backend from wis2box.api.config import load_config @@ -102,7 +104,6 @@ def setup_collection(meta: dict = {}) -> bool: backend = load_backend() if not backend.has_collection(data_name): - if not backend.add_collection(data_name): msg = f'Unable to setup backend for collection {data_name}' LOGGER.error(msg) @@ -241,13 +242,29 @@ def api(): @click.pass_context @cli_helpers.OPTION_VERBOSITY def setup(ctx, verbosity): - """Add collection items to API backend""" + """Add collection items to API using discovery-metadata""" api_config = load_config() - if not api_config.has_collection(''): - click.echo('API not ready') - else: - click.echo('API ready') + api_collections = api_config.list_collections() + # print resources in api_config + click.echo(f'Collections in api_config: {api_collections}') + backend = load_backend() + backend_collections = backend.list_collections() + click.echo(f'Collections in backend: {backend_collections}') + + oar = Records(DOCKER_API_URL) + try: + records = oar.collection_items('discovery-metadata') + except Exception as err: + click.echo(f'Issue loading discovery-metadata: {err}') + return False + for record in records['features']: + metadata_id = record['id'] + if metadata_id not in api_collections: + click.echo(f'Adding collection: {metadata_id}') + from wis2box.data import gcm + meta = gcm(record) + setup_collection(meta=meta) @click.command() diff --git a/wis2box-management/wis2box/api/backend/base.py b/wis2box-management/wis2box/api/backend/base.py index d085ef72c..70f5afd2e 100644 --- a/wis2box-management/wis2box/api/backend/base.py +++ b/wis2box-management/wis2box/api/backend/base.py @@ -40,6 +40,15 @@ def __init__(self, defs: dict) -> None: self.username = defs.get('username') self.password = defs.get('password') + def list_collections(self) -> list: + """ + List collections + + :returns: `list` of collection names + """ + + raise NotImplementedError() + def add_collection(self, name: str) -> bool: """ Add a collection diff --git a/wis2box-management/wis2box/api/backend/elastic.py b/wis2box-management/wis2box/api/backend/elastic.py index 65d176cf8..81220b346 100644 --- a/wis2box-management/wis2box/api/backend/elastic.py +++ b/wis2box-management/wis2box/api/backend/elastic.py @@ -196,6 +196,15 @@ def es_id(collection_id: str) -> Tuple[str]: """ return collection_id.lower().replace(':', '-') + def list_collections(self) -> list: + """ + List collections + + :returns: `list` of collection names + """ + + return [index for index in self.conn.indices.get_alias(index="*")] + def add_collection(self, collection_id: str) -> dict: """ Add a collection diff --git a/wis2box-management/wis2box/api/config/base.py b/wis2box-management/wis2box/api/config/base.py index 9133634be..a6c23ec56 100644 --- a/wis2box-management/wis2box/api/config/base.py +++ b/wis2box-management/wis2box/api/config/base.py @@ -33,6 +33,15 @@ def __init__(self, defs: dict) -> None: :param defs: `dict` of connection parameters """ + def list_collections(self) -> list: + """ + List collections + + :returns: `list` of collection names + """ + + raise NotImplementedError() + def get_collection(self, name: str) -> dict: """ Get a collection diff --git a/wis2box-management/wis2box/api/config/pygeoapi.py b/wis2box-management/wis2box/api/config/pygeoapi.py index ff61be533..dcb89c4e3 100644 --- a/wis2box-management/wis2box/api/config/pygeoapi.py +++ b/wis2box-management/wis2box/api/config/pygeoapi.py @@ -55,6 +55,23 @@ def __init__(self, defs: dict) -> None: self.http.mount('https://', adapter) self.http.mount('http://', adapter) + def list_collections(self) -> list: + """ + List collections + + :returns: `list` of collection names + """ + + collections = [] + + r = self.http.get(self.url) + r.raise_for_status() + resources = r.json() + for key in resources: + if resources[key]['type'] == 'collection': + collections.append(key) + return collections + def get_collection(self, name: str) -> dict: """ Get a collection