add daycli test

.github/workflows/tests-docker.yml

@@ -184,6 +184,19 @@ jobs:
         curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
         pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID
         python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
+    - name: add org-daycli-test data (csv2bufr)
+      env:
+        TOPIC_HIERARCHY: origin/a/wis2/org-daycli-test/data/core/climate/surface-based-observations/daily
+        DISCOVERY_METADATA: /data/wis2box/metadata/discovery/org-daycli-test.yml
+        DISCOVERY_METADATA_ID: urn:wmo:md:org-daycli-test:surface-climate-observations:daily
+        STATION_METADATA: /data/wis2box/metadata/station/daycli-test.csv
+        TEST_DATA: /data/wis2box/observations/daycli-test
+      run: |
+        python3 wis2box-ctl.py execute wis2box dataset publish $DISCOVERY_METADATA
+        python3 wis2box-ctl.py execute wis2box metadata station publish-collection --path $STATION_METADATA --topic-hierarchy $TOPIC_HIERARCHY
+        curl -s http://localhost/data/metadata/$DISCOVERY_METADATA_ID.json --output /tmp/$DISCOVERY_METADATA_ID
+        pywcmp ets validate /tmp/$DISCOVERY_METADATA_ID
+        python3 wis2box-ctl.py execute wis2box data ingest -mdi $DISCOVERY_METADATA_ID -p $TEST_DATA
     - name: add China GRIB2 data (universal pipeline) 🇨🇳
       env:
         DISCOVERY_METADATA: /data/wis2box/metadata/discovery/cn-grapes-geps-global.yml

tests/data/metadata/discovery/org-daycli-test.yml

@@ -0,0 +1,71 @@
+wis2box:
+    retention: P30D
+    topic_hierarchy: org-daycli-test/data/core/climate/surface-based-observations/daily
+    country: int
+    centre_id: org-daycli-test
+    data_mappings:
+        plugins:
+            csv:
+                - plugin: wis2box.data.csv2bufr.ObservationDataCSV2BUFR
+                  template: daycli-template
+                  notify: true
+                  file-pattern: '^.*\.csv$'
+            bufr4:
+                - plugin: wis2box.data.bufr2geojson.ObservationDataBUFR2GeoJSON
+                  file-pattern: '^WIGOS_(\d-\d+-\d+-\w+)_.*\.bufr4$'
+
+mcf:
+    version: 1.0
+
+metadata:
+    identifier: urn:wmo:md:org-daycli-test:surface-climate-observations:daily
+    hierarchylevel: dataset
+
+identification:
+    title: Daily values from land stations (DAYCLI) 
+    abstract: Daily values from land stations (DAYCLI)
+    dates:
+        creation: 2024-03-04
+    keywords:
+        default:
+            keywords:
+                - temperature
+                - snow depth
+                - precipitation
+                - climate
+                - daily
+                - daycli
+        wmo:
+            keywords:
+                - weather
+            keywords_type: theme
+            vocabulary:
+                name: Earth system disciplines as defined by the WMO Unified Data Policy, Resolution 1 (Cg-Ext(2021), Annex 1.
+                url: https://codes.wmo.int/wis/topic-hierarchy/earth-system-discipline
+    extents:
+        spatial:
+            - bbox: [-180.0, -90.0, 180.0, 90.0]
+              crs: 4326
+        temporal:
+            - begin: 2024-02-09
+              end: null
+              resolution: P1H
+    rights: WMO Unified Policy for the International Exchange of Earth System Data
+    url: https://example.org/fixme
+
+contact:
+    host: &contact_poc
+        organization: World Meteorological Organisation
+        url: https://www.wmo.int/
+        individualname: Firstname Lastname
+        positionname: Position Name
+        phone: null
+        fax: null
+        address: null
+        city: null
+        administrativearea: null
+        postalcode: null
+        country: null
+        email: [email protected]
+        hoursofservice: 0800h - 1600h UTC
+        contactinstructions: email

tests/data/metadata/station/daycli-test.csv

@@ -0,0 +1,2 @@
+station_name,wigos_station_identifier,traditional_station_identifier,facility_type,latitude,longitude,elevation,barometer_height,territory_name,wmo_region
+"DENVER INTERNATIONAL AIRPORT, CO.",0-20000-0-72565,72565,landFixed,39.8666666667,-104.6666666667,1656.0,1656.0,USA,northCentralAmericaCaribbean

tests/data/observations/daycli-test/daycli.csv

@@ -0,0 +1,31 @@
+wsi_series,wsi_issuer,wsi_issue_number,wsi_local,wmo_block_number,wmo_station_number,latitude,longitude,station_height_above_msl,temperature_siting_classification,precipitation_siting_classification,averaging_method,year,month,day,precipitation_day_offset,precipitation_hour,precipitation_minute,precipitation_second,precipitation,precipitation_flag,fresh_snow_day_offset,fresh_snow_hour,fresh_snow_minute,fresh_snow_second,fresh_snow_depth,fresh_snow_depth_flag,total_snow_day_offset,total_snow_hour,total_snow_minute,total_snow_second,total_snow_depth,total_snow_depth_flag,thermometer_height,maximum_temperature_day_offset,maximum_temperature_hour,maximum_temperature_minute,maximum_temperature_second,maximum_temperature,maximum_temperature_flag,minimum_temperature_day_offset,minimum_temperature_hour,minimum_temperature_minute,minimum_temperature_second,minimum_temperature,minimum_temperature_flag,average_temperature_day_offset,average_temperature_hour,average_temperature_minute,average_temperature_second,average_temperature,average_temperature_flag
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,1,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,278.15,0,0,7,0,1,273.75,0,0,0,0,1,276.25,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,2,0,7,0,1,1.5,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,279.25,0,0,7,0,1,273.75,0,0,0,0,1,275.75,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,3,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,288.75,0,0,7,0,1,274.25,0,0,0,0,1,280.05,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,4,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,293.75,0,0,7,0,1,274.85,0,0,0,0,1,281.75,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,5,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,296.45,0,0,7,0,1,279.85,0,0,0,0,1,287.25,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,6,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,299.85,0,0,7,0,1,277.55,0,0,0,0,1,286.95,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,7,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,298.15,0,0,7,0,1,280.95,0,0,0,0,1,287.5,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,8,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,283.75,0,0,7,0,1,271.55,0,0,0,0,1,281.05,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,9,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,287.05,0,0,7,0,1,272.15,0,0,0,0,1,278.05,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,10,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,287.55,0,0,7,0,1,273.75,0,0,0,0,1,281.55,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,11,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,286.45,0,0,7,0,1,268.85,0,0,0,0,1,277.85,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,12,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,285.35,0,0,7,0,1,266.55,0,0,0,0,1,276.55,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,13,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,292.55,0,0,7,0,1,274.25,0,0,0,0,1,284.05,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,14,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,293.15,0,0,7,0,1,268.85,0,0,0,0,1,281.25,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,15,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,296.45,0,0,7,0,1,280.95,0,0,0,0,1,290.25,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,16,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,294.85,0,0,7,0,1,274.85,0,0,0,0,1,288.55,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,17,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,275.95,0,0,7,0,1,266.05,0,0,0,0,1,276.25,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,18,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,287.55,0,0,7,0,1,262.15,0,0,0,0,1,271.75,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,19,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,292.05,0,0,7,0,1,275.35,0,0,0,0,1,280.05,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,20,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,283.75,0,0,7,0,1,274.85,0,0,0,0,1,281.45,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,21,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,285.95,0,0,7,0,1,269.35,0,0,0,0,1,277.65,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,22,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,293.15,0,0,7,0,1,272.65,0,0,0,0,1,281.95,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,23,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,294.85,0,0,7,0,1,274.25,0,0,0,0,1,283.95,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,24,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,285.95,0,0,7,0,1,264.95,0,0,0,0,1,281.05,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,25,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,288.15,0,0,7,0,1,264.35,0,0,0,0,1,273.75,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,26,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,293.15,0,0,7,0,1,278.15,0,0,0,0,1,283.15,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,27,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,285.35,0,0,7,0,1,271.05,0,0,0,0,1,281.85,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,28,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,292.55,0,0,7,0,1,271.05,0,0,0,0,1,278.75,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,29,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,295.35,0,0,7,0,1,277.05,0,0,0,0,1,286.15,0
+0,20000,0,72565,72,565,39.83,-104.663,1650,255,255,2,2021,11,30,0,7,0,1,0,0,0,7,0,1,0,0,0,12,0,0,0,0,2,0,7,0,1,289.25,0,0,7,0,1,274.85,0,0,0,0,1,281.65,0

tests/integration/test_workflow.py

@@ -58,7 +58,8 @@ def test_wis2downloader():
         'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/wind-profile': 1, # noqa
         'origin/a/wis2/int-wmo-test/data/core/weather/surface-based-observations/ship': 5, # noqa
         'origin/a/wis2/it-meteoam/data/core/weather/surface-based-observations/synop': 31, # noqa
-        'origin/a/wis2/int-wmo-test/data/core/weather/advisories-warnings': 1 # noqa	
+        'origin/a/wis2/int-wmo-test/data/core/weather/advisories-warnings': 1, # noqa
+        'origin/a/wis2/org-daycli-test/data/core/climate/surface-based-observations/daily': 30 # noqa
     }
 
     topic_nfiles_dict_found = {}
@@ -118,15 +119,15 @@ def test_metadata_station_publish():
 
     stations = r.json()
 
-    assert stations['numberReturned'] == 103
-    assert stations['numberMatched'] == 103
+    assert stations['numberReturned'] == 104
+    assert stations['numberMatched'] == 104
 
 
 def test_metadata_discovery_publish():
     """Test discovery metadata publishing"""
 
     r = SESSION.get(f'{API_URL}/collections/discovery-metadata/items').json()
-    assert r['numberMatched'] == 10
+    assert r['numberMatched'] == 11
 
     r = SESSION.get(f'{API_URL}/collections/discovery-metadata/items/{ID}').json()  # noqa
 
@@ -160,7 +161,7 @@ def test_metadata_discovery_publish():
     r = SESSION.get(f'{API_URL}/collections/discovery-metadata/items',
                     params=params).json()
 
-    assert r['numberMatched'] == 8
+    assert r['numberMatched'] == 9
 
     # test access of discovery metadata from notification message
 
@@ -170,7 +171,8 @@ def test_metadata_discovery_publish():
         'dz-meteoalgerie',
         'ro-rnimh-test',
         'cg-met',
-        'int-wmo-test'
+        'int-wmo-test',
+        'org-daycli-test'
     ]
 
     for centre_id in centre_ids:
@@ -304,7 +306,8 @@ def test_message_api():
         'ro-rnimh': 50,
         'cg-met': 15,
         'int-wmo': 13,
-        'cn-cma': 11
+        'cn-cma': 11,
+        'org-daycli': 31
     }
     for key, value in counts.items():
         url = f'{API_URL}/collections/messages/items?sortby=-datetime&q={key}&limit=1'  # noqa