From 3acb24e1051d4107c5e6cf6f20ae8be295874461 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Fri, 4 Oct 2024 09:19:48 -0400 Subject: [PATCH 1/8] add more configuration options to THREDDS catalog --- CHANGES.md | 24 ++++++++++++- .../components/thredds/catalog.xml.template | 14 ++------ birdhouse/components/thredds/default.env | 17 ++++++++++ birdhouse/env.local.example | 34 +++++++++++++++++++ 4 files changed, 76 insertions(+), 13 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a83d33ec9..174b71dbe 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,7 +15,29 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ -[//]: # (list changes here, using '-' for each new entry, remove this when items are added) +## Changes + +- THREDDS: add more options to configure catalog.xml + - Currently the default THREDDS configuration creates two default datasets, the Service Data dataset and the + Main dataset. The Service Data dataset is used internally and hosts WPS outputs. The Main dataset is the + place where users can access data served by THREDDS. Both of these are configured to serve files with the following + extensions: .nc .ncml .txt .md .rst .csv + + - In order to allow the THREDDS server to serve files with additional extensions, this introduces two new + variables: + - `THREDDS_SERVICE_DATA_EXTRA_FILE_FILTERS`: this allows users to specify additional [filter + elements](https://docs.unidata.ucar.edu/tds/current/userguide/tds_dataset_scan_ref.html#including-only-desired-files) to the Service Data dataset. This is especially useful if a WPS + outputs files with an extension other than the default (eg: .h5) to the `wps_outputs/` directory. + - `THREDDS_DATASET_DATASETSCAN_BODY`: this allows users to specify the whole body of the main dataset's + [``](https://docs.unidata.ucar.edu/tds/current/userguide/tds_dataset_scan_ref.html) element. + This allows users to fully customize how this dataset serves files. + + - We limit the configuration options for the Service Data dataset more than the main dataset because the Service + Data dataset requires a basic configuration in order to properly serve WPS outputs. Making significant changes + to this configuration could have unexpected negative impacts on WPS usage. + + - The defaults for these new variables are fully backwards compatible. Without changing these variables, the THREDDS + server should behave exactly the same as before. [2.5.3](https://github.com/bird-house/birdhouse-deploy/tree/2.5.3) (2024-09-11) ------------------------------------------------------------------------------------------------------------------ diff --git a/birdhouse/components/thredds/catalog.xml.template b/birdhouse/components/thredds/catalog.xml.template index 231b266c8..32553173a 100644 --- a/birdhouse/components/thredds/catalog.xml.template +++ b/birdhouse/components/thredds/catalog.xml.template @@ -27,24 +27,14 @@ + ${THREDDS_SERVICE_DATA_EXTRA_FILE_FILTERS} - - all - - - - - - - - - - + ${THREDDS_DATASET_DATASETSCAN_BODY} diff --git a/birdhouse/components/thredds/default.env b/birdhouse/components/thredds/default.env index e39a11007..74450f0f4 100644 --- a/birdhouse/components/thredds/default.env +++ b/birdhouse/components/thredds/default.env @@ -17,7 +17,22 @@ export THREDDS_SERVICE_DATA_LOCATION_NAME='Birdhouse' export THREDDS_DATASET_URL_PATH='datasets' export THREDDS_SERVICE_DATA_URL_PATH='birdhouse' +export THREDDS_SERVICE_DATA_EXTRA_FILE_FILTERS='' +export THREDDS_DATASET_DATASETSCAN_BODY=' + + all + + + + + + + + + + +' # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here VARS=" @@ -28,6 +43,7 @@ VARS=" \$THREDDS_DATASET_LOCATION_NAME \$THREDDS_DATASET_URL_PATH \$THREDDS_DATASET_LOCATION_ON_CONTAINER + \$THREDDS_DATASET_DATASETSCAN_BODY " OPTIONAL_VARS=" @@ -39,6 +55,7 @@ OPTIONAL_VARS=" \$THREDDS_IMAGE \$THREDDS_IMAGE_URI \$THREDDS_ADDITIONAL_CATALOG + \$THREDDS_SERVICE_DATA_EXTRA_FILE_FILTERS " export DELAYED_EVAL=" diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index 99be5ea12..716acc22c 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -476,6 +476,40 @@ export THREDDS_ADDITIONAL_CATALOG="" # #" +# Additional file filters to add for the Service Data THREDDS dataset. By default, the Service Data dataset will only +# serve files with the following extensions: .nc .ncml .txt .md .rst .csv +# If you need this dataset to serve other files you should update the THREDDS_SERVICE_DATA_EXTRA_FILE_FILTERS to add +# additional file filters. +# This may be useful to set if a WPS outputs files to the wps_outputs/ directory (hosted under the Service Data dataset) +# in a file format other than one of the defaults. +# See the example below which would also enable serving .png and .h5 files. +#export THREDDS_SERVICE_DATA_EXTRA_FILE_FILTERS=" +# +# +#" + +# Set this variable to customize the body of the XML element for the main THREDDS dataset. This is typically +# the dataset where you would store most of the data served by THREDDS (additional datasets can be configured by setting the +# THREDDS_ADDITIONAL_CATALOG variable). +# By default, the main dataset will only serve files with the following extensions: .nc .ncml .txt .md .rst .csv and will use +# the THREDDS service named "all" (see components/thredds/catalog.xml.template). However this can be customized if desired. +# See the example below which would change the configuration to also serve .h5 and .json files instead of .md and .rst files. +# See the THREDDS documentation for the element for all configuration options. +#export THREDDS_DATASET_DATASETSCAN_BODY=' +# +# all +# +# +# +# +# +# +# +# +# +# +#' + # Allow using Github as external AuthN/AuthZ provider with Magpie # To setup Github as login, goto under section [OAuth Apps] # and create a new Magpie application with configurations: From 5d4387c9438627217e0b92db24d8030c19c27f0a Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:20:09 -0400 Subject: [PATCH 2/8] change quotes in example [ci skip] --- birdhouse/env.local.example | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index 716acc22c..a130c2e8a 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -456,25 +456,25 @@ export GEOSERVER_ADMIN_PASSWORD="${__DEFAULT__GEOSERVER_ADMIN_PASSWORD}" # Additional catalogs for THREDDS. Add as many datasetScan XML blocks as needed to THREDDS_ADDITIONAL_CATALOG. # Each block defines a new top-level catalog. See birdhouse/components/thredds/catalog.xml.template for more information. -export THREDDS_ADDITIONAL_CATALOG="" -#export THREDDS_ADDITIONAL_CATALOG=" -# +export THREDDS_ADDITIONAL_CATALOG='' +#export THREDDS_ADDITIONAL_CATALOG=' +# # -# +# # all # # # -# -# -# -# -# -# +# +# +# +# +# +# # # # -#" +#' # Additional file filters to add for the Service Data THREDDS dataset. By default, the Service Data dataset will only # serve files with the following extensions: .nc .ncml .txt .md .rst .csv From 23d58acf6ec5f4b718ce53a3716123001d16ea33 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Fri, 11 Oct 2024 14:52:07 -0400 Subject: [PATCH 3/8] review feedback --- CHANGES.md | 18 +++++++-------- .../components/thredds/catalog.xml.template | 23 ++++++++----------- birdhouse/components/thredds/default.env | 18 ++++++++++----- 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 174b71dbe..03c1ca4bd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,27 +17,27 @@ ## Changes -- THREDDS: add more options to configure catalog.xml - - Currently the default THREDDS configuration creates two default datasets, the Service Data dataset and the - Main dataset. The Service Data dataset is used internally and hosts WPS outputs. The Main dataset is the +- THREDDS: add more options to configure `catalog.xml` + - The default THREDDS configuration creates two default datasets, the *Service Data* dataset and the + *Main* dataset. The *Service Data* dataset is used internally and hosts WPS outputs. The *Main* dataset is the place where users can access data served by THREDDS. Both of these are configured to serve files with the following extensions: .nc .ncml .txt .md .rst .csv - In order to allow the THREDDS server to serve files with additional extensions, this introduces two new - variables: + variables: - `THREDDS_SERVICE_DATA_EXTRA_FILE_FILTERS`: this allows users to specify additional [filter - elements](https://docs.unidata.ucar.edu/tds/current/userguide/tds_dataset_scan_ref.html#including-only-desired-files) to the Service Data dataset. This is especially useful if a WPS + elements](https://docs.unidata.ucar.edu/tds/current/userguide/tds_dataset_scan_ref.html#including-only-desired-files) to the *Service Data* dataset. This is especially useful if a WPS outputs files with an extension other than the default (eg: .h5) to the `wps_outputs/` directory. - - `THREDDS_DATASET_DATASETSCAN_BODY`: this allows users to specify the whole body of the main dataset's + - `THREDDS_DATASET_DATASETSCAN_BODY`: this allows users to specify the whole body of the *Main* dataset's [``](https://docs.unidata.ucar.edu/tds/current/userguide/tds_dataset_scan_ref.html) element. This allows users to fully customize how this dataset serves files. - - We limit the configuration options for the Service Data dataset more than the main dataset because the Service - Data dataset requires a basic configuration in order to properly serve WPS outputs. Making significant changes + - We limit the configuration options for the *Service Data* dataset more than the *Main* dataset because the *Service + Data* dataset requires a basic configuration in order to properly serve WPS outputs. Making significant changes to this configuration could have unexpected negative impacts on WPS usage. - The defaults for these new variables are fully backwards compatible. Without changing these variables, the THREDDS - server should behave exactly the same as before. + server should behave exactly the same as before. [2.5.3](https://github.com/bird-house/birdhouse-deploy/tree/2.5.3) (2024-09-11) ------------------------------------------------------------------------------------------------------------------ diff --git a/birdhouse/components/thredds/catalog.xml.template b/birdhouse/components/thredds/catalog.xml.template index 32553173a..17e3b2578 100644 --- a/birdhouse/components/thredds/catalog.xml.template +++ b/birdhouse/components/thredds/catalog.xml.template @@ -4,14 +4,14 @@ xmlns:xlink="http://www.w3.org/1999/xlink" > - - - - - - - - + + + + + + + + @@ -21,12 +21,7 @@ - - - - - - + ${THREDDS_DEFAULT_FILE_FILTERS} ${THREDDS_SERVICE_DATA_EXTRA_FILE_FILTERS} diff --git a/birdhouse/components/thredds/default.env b/birdhouse/components/thredds/default.env index 74450f0f4..d3b43e3fa 100644 --- a/birdhouse/components/thredds/default.env +++ b/birdhouse/components/thredds/default.env @@ -17,6 +17,15 @@ export THREDDS_SERVICE_DATA_LOCATION_NAME='Birdhouse' export THREDDS_DATASET_URL_PATH='datasets' export THREDDS_SERVICE_DATA_URL_PATH='birdhouse' +export THREDDS_DEFAULT_FILE_FILTERS=' + + + + + + +' + export THREDDS_SERVICE_DATA_EXTRA_FILE_FILTERS='' export THREDDS_DATASET_DATASETSCAN_BODY=' @@ -25,12 +34,7 @@ export THREDDS_DATASET_DATASETSCAN_BODY=' - - - - - - + ${THREDDS_DEFAULT_FILE_FILTERS} ' @@ -44,6 +48,7 @@ VARS=" \$THREDDS_DATASET_URL_PATH \$THREDDS_DATASET_LOCATION_ON_CONTAINER \$THREDDS_DATASET_DATASETSCAN_BODY + \$THREDDS_DEFAULT_FILE_FILTERS " OPTIONAL_VARS=" @@ -64,4 +69,5 @@ export DELAYED_EVAL=" THREDDS_SERVICE_DATA_LOCATION_ON_HOST THREDDS_IMAGE THREDDS_IMAGE_URI + THREDDS_DATASET_DATASETSCAN_BODY " From f57046fda206a1e52556f381d976c53a16f0925d Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:50:05 -0400 Subject: [PATCH 4/8] make magpie THREDDS permissions more configurable --- CHANGES.md | 10 ++++- .../config/magpie/providers.cfg.template | 37 ++++++++++--------- birdhouse/components/thredds/default.env | 5 +++ birdhouse/env.local.example | 35 +++++++++++++++--- 4 files changed, 64 insertions(+), 23 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 03c1ca4bd..9334470c6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -36,8 +36,16 @@ Data* dataset requires a basic configuration in order to properly serve WPS outputs. Making significant changes to this configuration could have unexpected negative impacts on WPS usage. + - In order to allow customization of the Magpie THREDDS configuration in case new file extensions are added we introduce + two additional variables: + - `THREDDS_MAGPIE_EXTRA_METADATA_PREFIXES`: additional file prefixes (ie. regular expression match patterns) that Magpie + should treat as metadata (accessible with "browse" permissions). + - `THREDDS_MAGPIE_EXTRA_DATA_PREFIXES`: additional file prefixes (ie. regular expression match patterns) that Magpie + should treat as data (accessible with "read" permissions). + - The defaults for these new variables are fully backwards compatible. Without changing these variables, the THREDDS - server should behave exactly the same as before. + server should behave exactly the same as before except that .md files and .rst files are now considered metadata + files according to the Magpie configuration, meaning that they can now be viewed with "browse" permissions. [2.5.3](https://github.com/bird-house/birdhouse-deploy/tree/2.5.3) (2024-09-11) ------------------------------------------------------------------------------------------------------------------ diff --git a/birdhouse/components/thredds/config/magpie/providers.cfg.template b/birdhouse/components/thredds/config/magpie/providers.cfg.template index 3ca5f9d84..499944ccb 100644 --- a/birdhouse/components/thredds/config/magpie/providers.cfg.template +++ b/birdhouse/components/thredds/config/magpie/providers.cfg.template @@ -15,21 +15,24 @@ providers: - ".+\\.ncml" # match longest extension first to avoid tuncating it by match of sorter '.nc' - ".+\\.nc" metadata_type: - prefixes: - - null # note: special YAML value evaluated as `no-prefix`, use quotes if literal value is needed - - "\\w+\\.gif" # threddsIcon, folder icon, etc. - - "\\w+\\.ico" # favicon - - "\\w+\\.txt" # licence - - "\\w+\\.css" # tds.css - - "catalog\\.\\w+" # note: special case for `THREDDS` top-level directory (root) accessed for `BROWSE` - - catalog - - ncml - - uddc - - iso + prefixes: [ + null, # note: special YAML value evaluated as `no-prefix`, use quotes if literal value is needed + "\\w+\\.gif", # threddsIcon, folder icon, etc. + "\\w+\\.ico", # favicon + "\\w+\\.css", # tds.css + "catalog\\.\\w+", # note: special case for `THREDDS` top-level directory (root) accessed for `BROWSE` + catalog, + ncml, + uddc, + iso, + ${THREDDS_MAGPIE_EXTRA_METADATA_PREFIXES} + ] data_type: - prefixes: - - fileServer - - dodsC - - wcs - - wms - - ncss + prefixes: [ + fileServer, + dodsC, + wcs, + wms, + ncss, + ${THREDDS_MAGPIE_EXTRA_DATA_PREFIXES} + ] diff --git a/birdhouse/components/thredds/default.env b/birdhouse/components/thredds/default.env index d3b43e3fa..7ce986d92 100644 --- a/birdhouse/components/thredds/default.env +++ b/birdhouse/components/thredds/default.env @@ -17,6 +17,9 @@ export THREDDS_SERVICE_DATA_LOCATION_NAME='Birdhouse' export THREDDS_DATASET_URL_PATH='datasets' export THREDDS_SERVICE_DATA_URL_PATH='birdhouse' +export THREDDS_MAGPIE_EXTRA_METADATA_PREFIXES='".+\\.txt", ".+\\.md", ".+\\.rst"' +export THREDDS_MAGPIE_EXTRA_DATA_PREFIXES='' + export THREDDS_DEFAULT_FILE_FILTERS=' @@ -61,6 +64,8 @@ OPTIONAL_VARS=" \$THREDDS_IMAGE_URI \$THREDDS_ADDITIONAL_CATALOG \$THREDDS_SERVICE_DATA_EXTRA_FILE_FILTERS + \$THREDDS_MAGPIE_EXTRA_METADATA_PREFIXES + \$THREDDS_MAGPIE_EXTRA_DATA_PREFIXES " export DELAYED_EVAL=" diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index a130c2e8a..172b42dad 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -475,6 +475,12 @@ export THREDDS_ADDITIONAL_CATALOG='' # # #' +# It is possible to define additional compound services in the THREDDS_ADDITIONAL_CATALOG variable as well. +# This may be useful if you are creating a catalog that only provides a subset of the services defined in the +# compound service named "all" (see birdhouse/components/thredds/catalog.xml.template). +# DO NOT define any non-compound services in THREDDS_ADDITIONAL_CATALOG that is not an exact copy of one of the +# variables defined in "all"! Especially, do not change the "base" attribute of any existing service. +# Doing so may break the way that access permissions are enforced when accessing data through this service. # Additional file filters to add for the Service Data THREDDS dataset. By default, the Service Data dataset will only # serve files with the following extensions: .nc .ncml .txt .md .rst .csv @@ -493,7 +499,7 @@ export THREDDS_ADDITIONAL_CATALOG='' # THREDDS_ADDITIONAL_CATALOG variable). # By default, the main dataset will only serve files with the following extensions: .nc .ncml .txt .md .rst .csv and will use # the THREDDS service named "all" (see components/thredds/catalog.xml.template). However this can be customized if desired. -# See the example below which would change the configuration to also serve .h5 and .json files instead of .md and .rst files. +# See the example below which would change the configuration to also serve .h5 and .json files and exclude .md files. # See the THREDDS documentation for the element for all configuration options. #export THREDDS_DATASET_DATASETSCAN_BODY=' # @@ -501,15 +507,34 @@ export THREDDS_ADDITIONAL_CATALOG='' # # # -# -# -# +# ${THREDDS_DEFAULT_FILE_FILTERS} # # -# +# # #' +# Files served by THREDDS are considered to either contain data or metadata (or both). The THREDDS Magpie service allows +# us to handle access permissions different for metadata vs. data. Magpie let's users with "browse" permissions access +# metadata but only users with "read" permissions can access data. +# By accessing files through different THREDDS services (see THREDDS documentation), we can either read the metadata with +# "browse" permissions or the data itself with "read" permissions. For example, by default a NetCDF file can be accessed +# using the NCML service to get its metadata or through the NCSS service to access the data itself. +# +# If you have a file that you would like to be treated as metadata (Magpie will allow users with "browse" permissions to +# access it) no matter which THREDDS service is used to access it, add the file pattern to the `THREDDS_MAGPIE_EXTRA_METADATA_PREFIXES` +# variable. Similarly, if you have a file that you would like to be treated as data no matter which THREDDS service is used +# to access it, add the file pattern to the `THREDDS_MAGPIE_EXTRA_DATA_PREFIXES` variable. +# +# For example, if you want all files with a .h5 extension to be treated as data files in all cases, add '".+\\.h5"' to the +# `THREDDS_MAGPIE_EXTRA_DATA_PREFIXES` variable. Note that values are regular expressions (python) where slashes are double +# escaped. Expressions should be surrounded by double quotes and if multiple expressions are included they should be comma +# delimited. +# +# Current defaults are: +#export THREDDS_MAGPIE_EXTRA_METADATA_PREFIXES='".+\\.txt", ".+\\.md", ".+\\.rst"' +#export THREDDS_MAGPIE_EXTRA_DATA_PREFIXES='' + # Allow using Github as external AuthN/AuthZ provider with Magpie # To setup Github as login, goto under section [OAuth Apps] # and create a new Magpie application with configurations: From 10e69191d6c4c7abaebbcb619a1ca2130f877e9c Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Tue, 15 Oct 2024 10:42:24 -0400 Subject: [PATCH 5/8] fix stripped quotes --- birdhouse/components/thredds/default.env | 7 +++---- birdhouse/env.local.example | 12 ++++++------ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/birdhouse/components/thredds/default.env b/birdhouse/components/thredds/default.env index 7ce986d92..64778b3fd 100644 --- a/birdhouse/components/thredds/default.env +++ b/birdhouse/components/thredds/default.env @@ -31,15 +31,15 @@ export THREDDS_DEFAULT_FILE_FILTERS=' export THREDDS_SERVICE_DATA_EXTRA_FILE_FILTERS='' -export THREDDS_DATASET_DATASETSCAN_BODY=' - +export THREDDS_DATASET_DATASETSCAN_BODY=" + all ${THREDDS_DEFAULT_FILE_FILTERS} -' +" # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here VARS=" @@ -74,5 +74,4 @@ export DELAYED_EVAL=" THREDDS_SERVICE_DATA_LOCATION_ON_HOST THREDDS_IMAGE THREDDS_IMAGE_URI - THREDDS_DATASET_DATASETSCAN_BODY " diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index 172b42dad..4679acc8f 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -501,18 +501,18 @@ export THREDDS_ADDITIONAL_CATALOG='' # the THREDDS service named "all" (see components/thredds/catalog.xml.template). However this can be customized if desired. # See the example below which would change the configuration to also serve .h5 and .json files and exclude .md files. # See the THREDDS documentation for the element for all configuration options. -#export THREDDS_DATASET_DATASETSCAN_BODY=' -# +#export THREDDS_DATASET_DATASETSCAN_BODY=" +# # all # # # # ${THREDDS_DEFAULT_FILE_FILTERS} -# -# -# +# +# +# # -#' +#" # Files served by THREDDS are considered to either contain data or metadata (or both). The THREDDS Magpie service allows # us to handle access permissions different for metadata vs. data. Magpie let's users with "browse" permissions access From 51f66b37e5765098c3bf88e68152b796d49b30f4 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 16 Oct 2024 12:42:21 -0400 Subject: [PATCH 6/8] simplify defaults --- .../components/thredds/catalog.xml.template | 7 ++++++- birdhouse/components/thredds/default.env | 17 ++++++----------- birdhouse/env.local.example | 3 +-- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/birdhouse/components/thredds/catalog.xml.template b/birdhouse/components/thredds/catalog.xml.template index 17e3b2578..f7979d9b3 100644 --- a/birdhouse/components/thredds/catalog.xml.template +++ b/birdhouse/components/thredds/catalog.xml.template @@ -21,7 +21,12 @@ - ${THREDDS_DEFAULT_FILE_FILTERS} + + + + + + ${THREDDS_SERVICE_DATA_EXTRA_FILE_FILTERS} diff --git a/birdhouse/components/thredds/default.env b/birdhouse/components/thredds/default.env index 64778b3fd..fb741ded5 100644 --- a/birdhouse/components/thredds/default.env +++ b/birdhouse/components/thredds/default.env @@ -20,15 +20,6 @@ export THREDDS_SERVICE_DATA_URL_PATH='birdhouse' export THREDDS_MAGPIE_EXTRA_METADATA_PREFIXES='".+\\.txt", ".+\\.md", ".+\\.rst"' export THREDDS_MAGPIE_EXTRA_DATA_PREFIXES='' -export THREDDS_DEFAULT_FILE_FILTERS=' - - - - - - -' - export THREDDS_SERVICE_DATA_EXTRA_FILE_FILTERS='' export THREDDS_DATASET_DATASETSCAN_BODY=" @@ -37,7 +28,12 @@ export THREDDS_DATASET_DATASETSCAN_BODY=" - ${THREDDS_DEFAULT_FILE_FILTERS} + + + + + + " @@ -51,7 +47,6 @@ VARS=" \$THREDDS_DATASET_URL_PATH \$THREDDS_DATASET_LOCATION_ON_CONTAINER \$THREDDS_DATASET_DATASETSCAN_BODY - \$THREDDS_DEFAULT_FILE_FILTERS " OPTIONAL_VARS=" diff --git a/birdhouse/env.local.example b/birdhouse/env.local.example index 4679acc8f..8ca61f137 100644 --- a/birdhouse/env.local.example +++ b/birdhouse/env.local.example @@ -499,7 +499,7 @@ export THREDDS_ADDITIONAL_CATALOG='' # THREDDS_ADDITIONAL_CATALOG variable). # By default, the main dataset will only serve files with the following extensions: .nc .ncml .txt .md .rst .csv and will use # the THREDDS service named "all" (see components/thredds/catalog.xml.template). However this can be customized if desired. -# See the example below which would change the configuration to also serve .h5 and .json files and exclude .md files. +# See the example below which would change the configuration to serve .h5, .md, and .json files. # See the THREDDS documentation for the element for all configuration options. #export THREDDS_DATASET_DATASETSCAN_BODY=" # @@ -507,7 +507,6 @@ export THREDDS_ADDITIONAL_CATALOG='' # # # -# ${THREDDS_DEFAULT_FILE_FILTERS} # # # From d4661f1902e59bc0dc3520d704d0d4e83eeba102 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Wed, 16 Oct 2024 13:48:00 -0400 Subject: [PATCH 7/8] quote fix --- birdhouse/components/thredds/default.env | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/birdhouse/components/thredds/default.env b/birdhouse/components/thredds/default.env index fb741ded5..9af3f0204 100644 --- a/birdhouse/components/thredds/default.env +++ b/birdhouse/components/thredds/default.env @@ -22,8 +22,8 @@ export THREDDS_MAGPIE_EXTRA_DATA_PREFIXES='' export THREDDS_SERVICE_DATA_EXTRA_FILE_FILTERS='' -export THREDDS_DATASET_DATASETSCAN_BODY=" - +export THREDDS_DATASET_DATASETSCAN_BODY=' + all @@ -35,7 +35,7 @@ export THREDDS_DATASET_DATASETSCAN_BODY=" -" +' # add any new variables not already in 'VARS' or 'OPTIONAL_VARS' that must be replaced in templates here VARS=" From 15f1264028c29cb7a69b0d573bb10dfad3d52eb7 Mon Sep 17 00:00:00 2001 From: mishaschwartz <4380924+mishaschwartz@users.noreply.github.com> Date: Thu, 31 Oct 2024 11:06:05 -0400 Subject: [PATCH 8/8] =?UTF-8?q?Bump=20version:=202.5.3=20=E2=86=92=202.5.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 6 +++--- CHANGES.md | 5 +++++ Makefile | 2 +- README.rst | 8 ++++---- RELEASE.txt | 2 +- .../canarie-api/docker_configuration.py.template | 8 ++++---- docs/source/conf.py | 4 ++-- 7 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 16fd47ca3..e3e23dc1b 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.5.3 +current_version = 2.5.4 commit = True tag = False tag_name = {new_version} @@ -30,11 +30,11 @@ search = {current_version} replace = {new_version} [bumpversion:file:RELEASE.txt] -search = {current_version} 2024-09-11T22:57:09Z +search = {current_version} 2024-10-31T15:06:05Z replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ} [bumpversion:part:releaseTime] -values = 2024-09-11T22:57:09Z +values = 2024-10-31T15:06:05Z [bumpversion:file(version):birdhouse/components/canarie-api/docker_configuration.py.template] search = 'version': '{current_version}' diff --git a/CHANGES.md b/CHANGES.md index 9334470c6..2059da724 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -15,6 +15,11 @@ [Unreleased](https://github.com/bird-house/birdhouse-deploy/tree/master) (latest) ------------------------------------------------------------------------------------------------------------------ +[//]: # (list changes here, using '-' for each new entry, remove this when items are added) + +[2.5.4](https://github.com/bird-house/birdhouse-deploy/tree/2.5.4) (2024-10-31) +------------------------------------------------------------------------------------------------------------------ + ## Changes - THREDDS: add more options to configure `catalog.xml` diff --git a/Makefile b/Makefile index 190c95210..71e102f26 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Generic variables override SHELL := bash override APP_NAME := birdhouse-deploy -override APP_VERSION := 2.5.3 +override APP_VERSION := 2.5.4 # utility to remove comments after value of an option variable override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g") diff --git a/README.rst b/README.rst index e0924ff64..e6e54adb7 100644 --- a/README.rst +++ b/README.rst @@ -18,13 +18,13 @@ for a full-fledged production platform. * - citation - | |citation| -.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.5.3.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.5.4.svg :alt: Commits since latest release - :target: https://github.com/bird-house/birdhouse-deploy/compare/2.5.3...master + :target: https://github.com/bird-house/birdhouse-deploy/compare/2.5.4...master -.. |latest-version| image:: https://img.shields.io/badge/tag-2.5.3-blue.svg?style=flat +.. |latest-version| image:: https://img.shields.io/badge/tag-2.5.4-blue.svg?style=flat :alt: Latest Tag - :target: https://github.com/bird-house/birdhouse-deploy/tree/2.5.3 + :target: https://github.com/bird-house/birdhouse-deploy/tree/2.5.4 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest :alt: ReadTheDocs Build Status (latest version) diff --git a/RELEASE.txt b/RELEASE.txt index 17a7e48f6..850fab3bc 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -1 +1 @@ -2.5.3 2024-09-11T22:57:09Z +2.5.4 2024-10-31T15:06:05Z diff --git a/birdhouse/components/canarie-api/docker_configuration.py.template b/birdhouse/components/canarie-api/docker_configuration.py.template index b6eef6344..fe77d5a80 100644 --- a/birdhouse/components/canarie-api/docker_configuration.py.template +++ b/birdhouse/components/canarie-api/docker_configuration.py.template @@ -108,8 +108,8 @@ SERVICES = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '2.5.3', - 'releaseTime': '2024-09-11T22:57:09Z', + 'version': '2.5.4', + 'releaseTime': '2024-10-31T15:06:05Z', 'institution': '${BIRDHOUSE_INSTITUTION}', 'researchSubject': '${BIRDHOUSE_SUBJECT}', 'supportEmail': '${BIRDHOUSE_SUPPORT_EMAIL}', @@ -141,8 +141,8 @@ PLATFORMS = { # NOTE: # Below version and release time auto-managed by 'make VERSION=x.y.z bump'. # Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'. - 'version': '2.5.3', - 'releaseTime': '2024-09-11T22:57:09Z', + 'version': '2.5.4', + 'releaseTime': '2024-10-31T15:06:05Z', 'institution': '${BIRDHOUSE_INSTITUTION}', 'researchSubject': '${BIRDHOUSE_SUBJECT}', 'supportEmail': '${BIRDHOUSE_SUPPORT_EMAIL}', diff --git a/docs/source/conf.py b/docs/source/conf.py index 5010e4422..5e7577787 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -69,9 +69,9 @@ # built documents. # # The short X.Y version. -version = '2.5.3' +version = '2.5.4' # The full version, including alpha/beta/rc tags. -release = '2.5.3' +release = '2.5.4' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages.