From a0ecb74dff1c445c852d118822788b8dff954bb8 Mon Sep 17 00:00:00 2001 From: Maaike Date: Fri, 8 Nov 2024 14:28:53 +0100 Subject: [PATCH 01/10] add daycli test --- .github/workflows/tests-docker.yml | 13 ++++ .../discovery/example-test-daycli.yml | 69 +++++++++++++++++++ .../metadata/station/example-daycli-test.csv | 2 + .../example-daycli-test/daycli.csv | 31 +++++++++ tests/integration/test_workflow.py | 3 +- 5 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 tests/data/metadata/discovery/example-test-daycli.yml create mode 100644 tests/data/metadata/station/example-daycli-test.csv create mode 100644 tests/data/observations/example-daycli-test/daycli.csv diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index dc73c6ab..75487d5e 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -184,6 +184,19 @@ jobs: curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA + - name: add example-test daycli + env: + TOPIC_HIERARCHY: origin/a/wis2/example-test/data/core/weather/surface-based-observations/daily + DISCOVERY_METADATA: /data/wis2box/metadata/discovery/example-test-daycli.yml + DISCOVERY_METADATA_ID: urn:wmo:md:example-test:surface-weather-observations:daily + STATION_METADATA: /data/wis2box/metadata/station/example-daycli-test.csv + TEST_DATA: /data/wis2box/observations/example-daycli-test + run: | + python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA + python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY + curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID + pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID + python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - name: add China GRIB2 data (universal pipeline) 🇨🇳 env: DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cn-grapes-geps-global.yml diff --git a/tests/data/metadata/discovery/example-test-daycli.yml b/tests/data/metadata/discovery/example-test-daycli.yml new file mode 100644 index 00000000..d31f0429 --- /dev/null +++ b/tests/data/metadata/discovery/example-test-daycli.yml @@ -0,0 +1,69 @@ +wis2box: + retention: P30D + topic_hierarchy: example-test/data/core/weather/surface-based-observations/daily + country: int + centre_id: example-test + data_mappings: + plugins: + csv: + - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR + template: daycli-template + notify: true + file-pattern: '^WIGOS_(\d-\d+-\d+-\w+)_.*\.csv$' + bufr4: + - plugin: wis2box.data.bufr2geojson.ObservationDataBUFR2GeoJSON + file-pattern: '^WIGOS_(\d-\d+-\d+-\w+)_.*\.bufr4$' + +mcf: + version: 1.0 + +metadata: + identifier: urn:wmo:md:example-test:surface-weather-observations:daily + hierarchylevel: dataset + +identification: + title: Daily values from land stations (DAYCLI) + abstract: Daily values from land stations (DAYCLI) + dates: + creation: 2024-03-04 + keywords: + default: + keywords: + - surface weather + - temperature + - observations + - daily + wmo: + keywords: + - weather + keywords_type: theme + vocabulary: + name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1. + url: https://codes.wmo.int/wis/topic-hierarchy/earth-system-discipline + extents: + spatial: + - bbox: [-180.0, -90.0, 180.0, 90.0] + crs: 4326 + temporal: + - begin: 2024-02-09 + end: null + resolution: P1H + rights: WMO Unified Policy for the International Exchange of Earth System Data + url: https://example.org/fixme + +contact: + host: &contact_poc + organization: World Meteorological Organisation + url: https://www.wmo.int/ + individualname: Firstname Lastname + positionname: Position Name + phone: null + fax: null + address: null + city: null + administrativearea: null + postalcode: null + country: null + email: you@example.org + hoursofservice: 0800h - 1600h UTC + contactinstructions: email diff --git a/tests/data/metadata/station/example-daycli-test.csv b/tests/data/metadata/station/example-daycli-test.csv new file mode 100644 index 00000000..6779c016 --- /dev/null +++ b/tests/data/metadata/station/example-daycli-test.csv @@ -0,0 +1,2 @@ +station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,barometer_height,territory_name,wmo_region +"DENVER INTERNATIONAL AIRPORT, CO.",0-20000-0-72565,72565,landFixed,39.8666666667,-104.6666666667,1656.0,1656.0,USA,northCentralAmericaCaribbean \ No newline at end of file diff --git a/tests/data/observations/example-daycli-test/daycli.csv b/tests/data/observations/example-daycli-test/daycli.csv new file mode 100644 index 00000000..c131f92a --- /dev/null +++ b/tests/data/observations/example-daycli-test/daycli.csv @@ -0,0 +1,31 @@ +wsi_series,wsi_issuer,wsi_issue_number,wsi_local,wmo_block_number,wmo_station_number,latitude,longitude,station_height_above_msl,temperature_siting_classification,precipitation_siting_classification,averaging_method,year,month,day,precipitation_day_offset,precipitation_hour,precipitation_minute,precipitation_second,precipitation,precipitation_flag,fresh_snow_day_offset,fresh_snow_hour,fresh_snow_minute,fresh_snow_second,fresh_snow_depth,fresh_snow_depth_flag,total_snow_day_offset,total_snow_hour,total_snow_minute,total_snow_second,total_snow_depth,total_snow_depth_flag,thermometer_height,maximum_temperature_day_offset,maximum_temperature_hour,maximum_temperature_minute,maximum_temperature_second,maximum_temperature,maximum_temperature_flag,minimum_temperature_day_offset,minimum_temperature_hour,minimum_temperature_minute,minimum_temperature_second,minimum_temperature,minimum_temperature_flag,average_temperature_day_offset,average_temperature_hour,average_temperature_minute,average_temperature_second,average_temperature,average_temperature_flag +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,1,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,278.15,0,0,7,0,1,273.75,0,0,0,0,1,276.25,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,2,0,7,0,1,1.5,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,279.25,0,0,7,0,1,273.75,0,0,0,0,1,275.75,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,3,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,288.75,0,0,7,0,1,274.25,0,0,0,0,1,280.05,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,4,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,293.75,0,0,7,0,1,274.85,0,0,0,0,1,281.75,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,5,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,296.45,0,0,7,0,1,279.85,0,0,0,0,1,287.25,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,6,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,299.85,0,0,7,0,1,277.55,0,0,0,0,1,286.95,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,7,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,298.15,0,0,7,0,1,280.95,0,0,0,0,1,287.5,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,8,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,283.75,0,0,7,0,1,271.55,0,0,0,0,1,281.05,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,9,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,287.05,0,0,7,0,1,272.15,0,0,0,0,1,278.05,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,10,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,287.55,0,0,7,0,1,273.75,0,0,0,0,1,281.55,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,11,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,286.45,0,0,7,0,1,268.85,0,0,0,0,1,277.85,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,12,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,285.35,0,0,7,0,1,266.55,0,0,0,0,1,276.55,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,13,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,292.55,0,0,7,0,1,274.25,0,0,0,0,1,284.05,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,14,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,293.15,0,0,7,0,1,268.85,0,0,0,0,1,281.25,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,15,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,296.45,0,0,7,0,1,280.95,0,0,0,0,1,290.25,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,16,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,294.85,0,0,7,0,1,274.85,0,0,0,0,1,288.55,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,17,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,275.95,0,0,7,0,1,266.05,0,0,0,0,1,276.25,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,18,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,287.55,0,0,7,0,1,262.15,0,0,0,0,1,271.75,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,19,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,292.05,0,0,7,0,1,275.35,0,0,0,0,1,280.05,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,20,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,283.75,0,0,7,0,1,274.85,0,0,0,0,1,281.45,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,21,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,285.95,0,0,7,0,1,269.35,0,0,0,0,1,277.65,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,22,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,293.15,0,0,7,0,1,272.65,0,0,0,0,1,281.95,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,23,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,294.85,0,0,7,0,1,274.25,0,0,0,0,1,283.95,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,24,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,285.95,0,0,7,0,1,264.95,0,0,0,0,1,281.05,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,25,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,288.15,0,0,7,0,1,264.35,0,0,0,0,1,273.75,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,26,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,293.15,0,0,7,0,1,278.15,0,0,0,0,1,283.15,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,27,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,285.35,0,0,7,0,1,271.05,0,0,0,0,1,281.85,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,28,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,292.55,0,0,7,0,1,271.05,0,0,0,0,1,278.75,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,29,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,295.35,0,0,7,0,1,277.05,0,0,0,0,1,286.15,0 +0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,30,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,289.25,0,0,7,0,1,274.85,0,0,0,0,1,281.65,0 diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py index c27ad856..4be8ceb4 100644 --- a/tests/integration/test_workflow.py +++ b/tests/integration/test_workflow.py @@ -58,7 +58,8 @@ def test_wis2downloader(): 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind-profile': 1, # noqa 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship': 5, # noqa 'origin/a/wis2/it-meteoam/data/core/weather/surface-based-observations/synop': 31, # noqa - 'origin/a/wis2/int-wmo-test/data/core/weather/advisories-warnings': 1 # noqa + 'origin/a/wis2/int-wmo-test/data/core/weather/advisories-warnings': 1, # noqa + 'origin/a/wis2/example-test/data/core/weather/surface-based-observations/daycli': 30 # noqa } topic_nfiles_dict_found = {} From 38c245cef8c99f6669abf83c08a774b28e373158 Mon Sep 17 00:00:00 2001 From: Maaike Date: Fri, 8 Nov 2024 15:35:30 +0100 Subject: [PATCH 02/10] fix test --- .github/workflows/tests-docker.yml | 4 ++-- tests/data/metadata/discovery/example-test-daycli.yml | 10 ++++++---- tests/integration/test_workflow.py | 8 +++++--- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index 75487d5e..add577f0 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -186,9 +186,9 @@ jobs: python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - name: add example-test daycli env: - TOPIC_HIERARCHY: origin/a/wis2/example-test/data/core/weather/surface-based-observations/daily + TOPIC_HIERARCHY: origin/a/wis2/example-test/data/core/climate/surface-based-observations/daily DISCOVERY_METADATA: /data/wis2box/metadata/discovery/example-test-daycli.yml - DISCOVERY_METADATA_ID: urn:wmo:md:example-test:surface-weather-observations:daily + DISCOVERY_METADATA_ID: urn:wmo:md:example-test:surface-climate-observations:daily STATION_METADATA: /data/wis2box/metadata/station/example-daycli-test.csv TEST_DATA: /data/wis2box/observations/example-daycli-test run: | diff --git a/tests/data/metadata/discovery/example-test-daycli.yml b/tests/data/metadata/discovery/example-test-daycli.yml index d31f0429..78825647 100644 --- a/tests/data/metadata/discovery/example-test-daycli.yml +++ b/tests/data/metadata/discovery/example-test-daycli.yml @@ -1,6 +1,6 @@ wis2box: retention: P30D - topic_hierarchy: example-test/data/core/weather/surface-based-observations/daily + topic_hierarchy: example-test/data/core/climate/surface-based-observations/daily country: int centre_id: example-test data_mappings: @@ -18,7 +18,7 @@ mcf: version: 1.0 metadata: - identifier: urn:wmo:md:example-test:surface-weather-observations:daily + identifier: urn:wmo:md:example-test:surface-climate-observations:daily hierarchylevel: dataset identification: @@ -29,10 +29,12 @@ identification: keywords: default: keywords: - - surface weather - temperature - - observations + - snow depth + - precipitation + - climate - daily + - daycli wmo: keywords: - weather diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py index 4be8ceb4..392fe487 100644 --- a/tests/integration/test_workflow.py +++ b/tests/integration/test_workflow.py @@ -59,7 +59,7 @@ def test_wis2downloader(): 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship': 5, # noqa 'origin/a/wis2/it-meteoam/data/core/weather/surface-based-observations/synop': 31, # noqa 'origin/a/wis2/int-wmo-test/data/core/weather/advisories-warnings': 1, # noqa - 'origin/a/wis2/example-test/data/core/weather/surface-based-observations/daycli': 30 # noqa + 'origin/a/wis2/example-test/data/core/climate/surface-based-observations/daycli': 30 # noqa } topic_nfiles_dict_found = {} @@ -171,7 +171,8 @@ def test_metadata_discovery_publish(): 'dz-meteoalgerie', 'ro-rnimh-test', 'cg-met', - 'int-wmo-test' + 'int-wmo-test', + 'example-test' ] for centre_id in centre_ids: @@ -305,7 +306,8 @@ def test_message_api(): 'ro-rnimh': 50, 'cg-met': 15, 'int-wmo': 13, - 'cn-cma': 11 + 'cn-cma': 11, + 'example-test': 30 } for key, value in counts.items(): url = f'{API_URL}/collections/messages/items?sortby=-datetime&q={key}&limit=1' # noqa From 3a23cc802dd971cf4c07f44cb2859218b668e1e0 Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 18 Nov 2024 15:40:13 +0100 Subject: [PATCH 03/10] fix file-pattern daycli input csv --- tests/data/metadata/discovery/example-test-daycli.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/metadata/discovery/example-test-daycli.yml b/tests/data/metadata/discovery/example-test-daycli.yml index 78825647..92799bb1 100644 --- a/tests/data/metadata/discovery/example-test-daycli.yml +++ b/tests/data/metadata/discovery/example-test-daycli.yml @@ -9,7 +9,7 @@ wis2box: - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR template: daycli-template notify: true - file-pattern: '^WIGOS_(\d-\d+-\d+-\w+)_.*\.csv$' + file-pattern: '^*\.csv$' bufr4: - plugin: wis2box.data.bufr2geojson.ObservationDataBUFR2GeoJSON file-pattern: '^WIGOS_(\d-\d+-\d+-\w+)_.*\.bufr4$' From 0b6083597c73f69598e4259908a1229365e1e94f Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 18 Nov 2024 15:48:50 +0100 Subject: [PATCH 04/10] fix file-pattern --- tests/data/metadata/discovery/example-test-daycli.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/metadata/discovery/example-test-daycli.yml b/tests/data/metadata/discovery/example-test-daycli.yml index 92799bb1..48a0e487 100644 --- a/tests/data/metadata/discovery/example-test-daycli.yml +++ b/tests/data/metadata/discovery/example-test-daycli.yml @@ -9,7 +9,7 @@ wis2box: - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR template: daycli-template notify: true - file-pattern: '^*\.csv$' + file-pattern: '^.*\.csv$' bufr4: - plugin: wis2box.data.bufr2geojson.ObservationDataBUFR2GeoJSON file-pattern: '^WIGOS_(\d-\d+-\d+-\w+)_.*\.bufr4$' From f6e1046c1dbd7394e3d013250bc7d0f6949c385b Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 18 Nov 2024 16:35:47 +0100 Subject: [PATCH 05/10] rename and adjust numbers --- .github/workflows/tests-docker.yml | 12 ++++++------ .../{example-test-daycli.yml => int-daycli-test.yml} | 6 +++--- tests/integration/test_workflow.py | 12 ++++++------ 3 files changed, 15 insertions(+), 15 deletions(-) rename tests/data/metadata/discovery/{example-test-daycli.yml => int-daycli-test.yml} (91%) diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index add577f0..8fd49f63 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -184,13 +184,13 @@ jobs: curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - - name: add example-test daycli + - name: add int-daycli-test daycli env: - TOPIC_HIERARCHY: origin/a/wis2/example-test/data/core/climate/surface-based-observations/daily - DISCOVERY_METADATA: /data/wis2box/metadata/discovery/example-test-daycli.yml - DISCOVERY_METADATA_ID: urn:wmo:md:example-test:surface-climate-observations:daily - STATION_METADATA: /data/wis2box/metadata/station/example-daycli-test.csv - TEST_DATA: /data/wis2box/observations/example-daycli-test + TOPIC_HIERARCHY: origin/a/wis2/int-daycli-test/data/core/climate/surface-based-observations/daily + DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-daycli-test-daycli.yml + DISCOVERY_METADATA_ID: urn:wmo:md:int-daycli-test:surface-climate-observations:daily + STATION_METADATA: /data/wis2box/metadata/station/example-int-daycli-test.csv + TEST_DATA: /data/wis2box/observations/example-int-daycli-test run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY diff --git a/tests/data/metadata/discovery/example-test-daycli.yml b/tests/data/metadata/discovery/int-daycli-test.yml similarity index 91% rename from tests/data/metadata/discovery/example-test-daycli.yml rename to tests/data/metadata/discovery/int-daycli-test.yml index 48a0e487..7414bd96 100644 --- a/tests/data/metadata/discovery/example-test-daycli.yml +++ b/tests/data/metadata/discovery/int-daycli-test.yml @@ -1,8 +1,8 @@ wis2box: retention: P30D - topic_hierarchy: example-test/data/core/climate/surface-based-observations/daily + topic_hierarchy: int-daycli-test/data/core/climate/surface-based-observations/daily country: int - centre_id: example-test + centre_id: int-daycli-test data_mappings: plugins: csv: @@ -18,7 +18,7 @@ mcf: version: 1.0 metadata: - identifier: urn:wmo:md:example-test:surface-climate-observations:daily + identifier: urn:wmo:md:int-daycli-test:surface-climate-observations:daily hierarchylevel: dataset identification: diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py index 392fe487..3a5ec371 100644 --- a/tests/integration/test_workflow.py +++ b/tests/integration/test_workflow.py @@ -59,7 +59,7 @@ def test_wis2downloader(): 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship': 5, # noqa 'origin/a/wis2/it-meteoam/data/core/weather/surface-based-observations/synop': 31, # noqa 'origin/a/wis2/int-wmo-test/data/core/weather/advisories-warnings': 1, # noqa - 'origin/a/wis2/example-test/data/core/climate/surface-based-observations/daycli': 30 # noqa + 'int-daycli-test/data/core/climate/surface-based-observations/daily': 30 # noqa } topic_nfiles_dict_found = {} @@ -119,15 +119,15 @@ def test_metadata_station_publish(): stations = r.json() - assert stations['numberReturned'] == 103 - assert stations['numberMatched'] == 103 + assert stations['numberReturned'] == 104 + assert stations['numberMatched'] == 104 def test_metadata_discovery_publish(): """Test discovery metadata publishing""" r = SESSION.get(f'{API_URL}/collections/discovery-metadata/items').json() - assert r['numberMatched'] == 10 + assert r['numberMatched'] == 11 r = SESSION.get(f'{API_URL}/collections/discovery-metadata/items/{ID}').json() # noqa @@ -172,7 +172,7 @@ def test_metadata_discovery_publish(): 'ro-rnimh-test', 'cg-met', 'int-wmo-test', - 'example-test' + 'int-daycli-test' ] for centre_id in centre_ids: @@ -307,7 +307,7 @@ def test_message_api(): 'cg-met': 15, 'int-wmo': 13, 'cn-cma': 11, - 'example-test': 30 + 'int-daycli-test': 30 } for key, value in counts.items(): url = f'{API_URL}/collections/messages/items?sortby=-datetime&q={key}&limit=1' # noqa From 69590186a20e38e6647fb4cc1158d7e76b3993f5 Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 18 Nov 2024 16:46:28 +0100 Subject: [PATCH 06/10] adjust names --- .github/workflows/tests-docker.yml | 8 ++++---- .../station/{example-daycli-test.csv => daycli-test.csv} | 0 .../{example-daycli-test => daycli-test}/daycli.csv | 0 3 files changed, 4 insertions(+), 4 deletions(-) rename tests/data/metadata/station/{example-daycli-test.csv => daycli-test.csv} (100%) rename tests/data/observations/{example-daycli-test => daycli-test}/daycli.csv (100%) diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index 8fd49f63..db6ab74a 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -184,13 +184,13 @@ jobs: curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - - name: add int-daycli-test daycli + - name: add int-daycli-test data (csv2bufr) env: TOPIC_HIERARCHY: origin/a/wis2/int-daycli-test/data/core/climate/surface-based-observations/daily - DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-daycli-test-daycli.yml + DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-daycli-test.yml DISCOVERY_METADATA_ID: urn:wmo:md:int-daycli-test:surface-climate-observations:daily - STATION_METADATA: /data/wis2box/metadata/station/example-int-daycli-test.csv - TEST_DATA: /data/wis2box/observations/example-int-daycli-test + STATION_METADATA: /data/wis2box/metadata/station/daycli-test.csv + TEST_DATA: /data/wis2box/observations/daycli-test run: | python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY diff --git a/tests/data/metadata/station/example-daycli-test.csv b/tests/data/metadata/station/daycli-test.csv similarity index 100% rename from tests/data/metadata/station/example-daycli-test.csv rename to tests/data/metadata/station/daycli-test.csv diff --git a/tests/data/observations/example-daycli-test/daycli.csv b/tests/data/observations/daycli-test/daycli.csv similarity index 100% rename from tests/data/observations/example-daycli-test/daycli.csv rename to tests/data/observations/daycli-test/daycli.csv From 8d25bf4037f495fce3dea24c7bf3fdecebd11b27 Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 18 Nov 2024 17:14:40 +0100 Subject: [PATCH 07/10] count based on data_id rather than q= --- tests/integration/test_workflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py index 3a5ec371..74fc6050 100644 --- a/tests/integration/test_workflow.py +++ b/tests/integration/test_workflow.py @@ -161,7 +161,7 @@ def test_metadata_discovery_publish(): r = SESSION.get(f'{API_URL}/collections/discovery-metadata/items', params=params).json() - assert r['numberMatched'] == 8 + assert r['numberMatched'] == 9 # test access of discovery metadata from notification message @@ -310,7 +310,7 @@ def test_message_api(): 'int-daycli-test': 30 } for key, value in counts.items(): - url = f'{API_URL}/collections/messages/items?sortby=-datetime&q={key}&limit=1' # noqa + url = f'{API_URL}/collections/messages/items?sortby=-datetime&data_id={key}&limit=1' # noqa r = SESSION.get(url).json() assert r['numberMatched'] == value From e01ca10835048d5bfd633eb3f59d52fa7b3a606c Mon Sep 17 00:00:00 2001 From: Maaike Date: Tue, 19 Nov 2024 10:32:19 +0100 Subject: [PATCH 08/10] different name to avoid match with into-wmo --- .github/workflows/tests-docker.yml | 8 ++++---- .../{int-daycli-test.yml => org-daycli-test.yml} | 6 +++--- tests/integration/test_workflow.py | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) rename tests/data/metadata/discovery/{int-daycli-test.yml => org-daycli-test.yml} (92%) diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml index db6ab74a..0b04ca0d 100644 --- a/.github/workflows/tests-docker.yml +++ b/.github/workflows/tests-docker.yml @@ -184,11 +184,11 @@ jobs: curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA - - name: add int-daycli-test data (csv2bufr) + - name: add org-daycli-test data (csv2bufr) env: - TOPIC_HIERARCHY: origin/a/wis2/int-daycli-test/data/core/climate/surface-based-observations/daily - DISCOVERY_METADATA: /data/wis2box/metadata/discovery/int-daycli-test.yml - DISCOVERY_METADATA_ID: urn:wmo:md:int-daycli-test:surface-climate-observations:daily + TOPIC_HIERARCHY: origin/a/wis2/org-daycli-test/data/core/climate/surface-based-observations/daily + DISCOVERY_METADATA: /data/wis2box/metadata/discovery/org-daycli-test.yml + DISCOVERY_METADATA_ID: urn:wmo:md:org-daycli-test:surface-climate-observations:daily STATION_METADATA: /data/wis2box/metadata/station/daycli-test.csv TEST_DATA: /data/wis2box/observations/daycli-test run: | diff --git a/tests/data/metadata/discovery/int-daycli-test.yml b/tests/data/metadata/discovery/org-daycli-test.yml similarity index 92% rename from tests/data/metadata/discovery/int-daycli-test.yml rename to tests/data/metadata/discovery/org-daycli-test.yml index 7414bd96..9ba6c21d 100644 --- a/tests/data/metadata/discovery/int-daycli-test.yml +++ b/tests/data/metadata/discovery/org-daycli-test.yml @@ -1,8 +1,8 @@ wis2box: retention: P30D - topic_hierarchy: int-daycli-test/data/core/climate/surface-based-observations/daily + topic_hierarchy: org-daycli-test/data/core/climate/surface-based-observations/daily country: int - centre_id: int-daycli-test + centre_id: org-daycli-test data_mappings: plugins: csv: @@ -18,7 +18,7 @@ mcf: version: 1.0 metadata: - identifier: urn:wmo:md:int-daycli-test:surface-climate-observations:daily + identifier: urn:wmo:md:org-daycli-test:surface-climate-observations:daily hierarchylevel: dataset identification: diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py index 74fc6050..b09c43ad 100644 --- a/tests/integration/test_workflow.py +++ b/tests/integration/test_workflow.py @@ -59,7 +59,7 @@ def test_wis2downloader(): 'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship': 5, # noqa 'origin/a/wis2/it-meteoam/data/core/weather/surface-based-observations/synop': 31, # noqa 'origin/a/wis2/int-wmo-test/data/core/weather/advisories-warnings': 1, # noqa - 'int-daycli-test/data/core/climate/surface-based-observations/daily': 30 # noqa + 'origin/a/wis2/org-daycli-test/data/core/climate/surface-based-observations/daily': 30 # noqa } topic_nfiles_dict_found = {} @@ -172,7 +172,7 @@ def test_metadata_discovery_publish(): 'ro-rnimh-test', 'cg-met', 'int-wmo-test', - 'int-daycli-test' + 'org-daycli-test' ] for centre_id in centre_ids: @@ -307,10 +307,10 @@ def test_message_api(): 'cg-met': 15, 'int-wmo': 13, 'cn-cma': 11, - 'int-daycli-test': 30 + 'org-daycli-test': 30 } for key, value in counts.items(): - url = f'{API_URL}/collections/messages/items?sortby=-datetime&data_id={key}&limit=1' # noqa + url = f'{API_URL}/collections/messages/items?sortby=-datetime&q={key}&limit=1' # noqa r = SESSION.get(url).json() assert r['numberMatched'] == value From 624cf42c773d5a91ef300033c6abd1b9795ee132 Mon Sep 17 00:00:00 2001 From: Maaike Date: Tue, 19 Nov 2024 10:41:13 +0100 Subject: [PATCH 09/10] can't get the count right ... --- tests/integration/test_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py index b09c43ad..f6a82317 100644 --- a/tests/integration/test_workflow.py +++ b/tests/integration/test_workflow.py @@ -307,7 +307,7 @@ def test_message_api(): 'cg-met': 15, 'int-wmo': 13, 'cn-cma': 11, - 'org-daycli-test': 30 + 'org-daycli': 37 } for key, value in counts.items(): url = f'{API_URL}/collections/messages/items?sortby=-datetime&q={key}&limit=1' # noqa From d230fd5767e09f7d2f45626d5af071aaa0546d52 Mon Sep 17 00:00:00 2001 From: Maaike Date: Tue, 19 Nov 2024 10:56:41 +0100 Subject: [PATCH 10/10] get the count right --- tests/integration/test_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_workflow.py b/tests/integration/test_workflow.py index f6a82317..4250ccee 100644 --- a/tests/integration/test_workflow.py +++ b/tests/integration/test_workflow.py @@ -307,7 +307,7 @@ def test_message_api(): 'cg-met': 15, 'int-wmo': 13, 'cn-cma': 11, - 'org-daycli': 37 + 'org-daycli': 31 } for key, value in counts.items(): url = f'{API_URL}/collections/messages/items?sortby=-datetime&q={key}&limit=1' # noqa