make magpie THREDDS permissions more configurable

CHANGES.md

@@ -36,8 +36,16 @@
     Data* dataset requires a basic configuration in order to properly serve WPS outputs. Making significant changes
     to this configuration could have unexpected negative impacts on WPS usage.
 
+  - In order to allow customization of the Magpie THREDDS configuration in case new file extensions are added we introduce
+    two additional variables:
+    - `THREDDS_MAGPIE_EXTRA_METADATA_PREFIXES`: additional file prefixes (ie. regular expression match patterns) that Magpie
+      should treat as metadata (accessible with "browse" permissions).
+    - `THREDDS_MAGPIE_EXTRA_DATA_PREFIXES`: additional file prefixes (ie. regular expression match patterns) that Magpie
+      should treat as data (accessible with "read" permissions).
+
   - The defaults for these new variables are fully backwards compatible. Without changing these variables, the THREDDS
-    server should behave exactly the same as before.
+    server should behave exactly the same as before except that .md files and .rst files are now considered metadata
+    files according to the Magpie configuration, meaning that they can now be viewed with "browse" permissions.
 
 [2.5.3](https://github.com/bird-house/birdhouse-deploy/tree/2.5.3) (2024-09-11)
 ------------------------------------------------------------------------------------------------------------------

birdhouse/components/thredds/config/magpie/providers.cfg.template

@@ -15,21 +15,24 @@ providers:
         - ".+\\.ncml"  # match longest extension first to avoid tuncating it by match of sorter '.nc'
         - ".+\\.nc"
       metadata_type:
-        prefixes:
-          - null  # note: special YAML value evaluated as `no-prefix`, use quotes if literal value is needed
-          - "\\w+\\.gif"  # threddsIcon, folder icon, etc.
-          - "\\w+\\.ico"  # favicon
-          - "\\w+\\.txt"  # licence
-          - "\\w+\\.css"  # tds.css
-          - "catalog\\.\\w+"  # note: special case for `THREDDS` top-level directory (root) accessed for `BROWSE`
-          - catalog
-          - ncml
-          - uddc
-          - iso
+        prefixes: [
+          null,  # note: special YAML value evaluated as `no-prefix`, use quotes if literal value is needed
+          "\\w+\\.gif",  # threddsIcon, folder icon, etc.
+          "\\w+\\.ico",  # favicon
+          "\\w+\\.css",  # tds.css
+          "catalog\\.\\w+",  # note: special case for `THREDDS` top-level directory (root) accessed for `BROWSE`
+          catalog,
+          ncml,
+          uddc,
+          iso,
+          ${THREDDS_MAGPIE_EXTRA_METADATA_PREFIXES}
+        ]
       data_type:
-        prefixes:
-          - fileServer
-          - dodsC
-          - wcs
-          - wms
-          - ncss
+        prefixes: [
+          fileServer,
+          dodsC,
+          wcs,
+          wms,
+          ncss,
+          ${THREDDS_MAGPIE_EXTRA_DATA_PREFIXES}
+        ]

birdhouse/components/thredds/default.env

@@ -17,6 +17,9 @@ export THREDDS_SERVICE_DATA_LOCATION_NAME='Birdhouse'
 export THREDDS_DATASET_URL_PATH='datasets'
 export THREDDS_SERVICE_DATA_URL_PATH='birdhouse'
 
+export THREDDS_MAGPIE_EXTRA_METADATA_PREFIXES='".+\\.txt", ".+\\.md", ".+\\.rst"'
+export THREDDS_MAGPIE_EXTRA_DATA_PREFIXES=''
+
 export THREDDS_DEFAULT_FILE_FILTERS='
     <include wildcard="*.nc" />
     <include wildcard="*.ncml" />
@@ -61,6 +64,8 @@ OPTIONAL_VARS="
   \$THREDDS_IMAGE_URI
   \$THREDDS_ADDITIONAL_CATALOG
   \$THREDDS_SERVICE_DATA_EXTRA_FILE_FILTERS
+  \$THREDDS_MAGPIE_EXTRA_METADATA_PREFIXES
+  \$THREDDS_MAGPIE_EXTRA_DATA_PREFIXES
 "
 
 export DELAYED_EVAL="

birdhouse/env.local.example

@@ -475,6 +475,12 @@ export THREDDS_ADDITIONAL_CATALOG=''
 #
 #    </datasetScan>
 #'
+# It is possible to define additional compound services in the THREDDS_ADDITIONAL_CATALOG variable as well.
+# This may be useful if you are creating a catalog that only provides a subset of the services defined in the
+# compound service named "all" (see birdhouse/components/thredds/catalog.xml.template).
+# DO NOT define any non-compound services in THREDDS_ADDITIONAL_CATALOG that is not an exact copy of one of the
+# variables defined in "all"! Especially, do not change the "base" attribute of any existing service. 
+# Doing so may break the way that access permissions are enforced when accessing data through this service. 
 
 # Additional file filters to add for the Service Data THREDDS dataset. By default, the Service Data dataset will only
 # serve files with the following extensions: .nc .ncml .txt .md .rst .csv
@@ -493,23 +499,42 @@ export THREDDS_ADDITIONAL_CATALOG=''
 # THREDDS_ADDITIONAL_CATALOG variable).
 # By default, the main dataset will only serve files with the following extensions: .nc .ncml .txt .md .rst .csv and will use
 # the THREDDS service named "all" (see components/thredds/catalog.xml.template). However this can be customized if desired.
-# See the example below which would change the configuration to also serve .h5 and .json files instead of .md and .rst files.
+# See the example below which would change the configuration to also serve .h5 and .json files and exclude .md files. 
 # See the THREDDS documentation for the <datasetScan> element for all configuration options.
 #export THREDDS_DATASET_DATASETSCAN_BODY='
 #      <metadata inherited="true">
 #        <serviceName>all</serviceName>
 #      </metadata>
 #
 #      <filter>
-#        <include wildcard="*.nc" />
-#        <include wildcard="*.ncml" />
-#        <include wildcard="*.txt" />
+#        ${THREDDS_DEFAULT_FILE_FILTERS}
 #        <include wildcard="*.h5" />
 #        <include wildcard="*.json" />
-#        <include wildcard="*.csv" />
+#        <exclude wildcard="*.md" />
 #      </filter>
 #'
 
+# Files served by THREDDS are considered to either contain data or metadata (or both). The THREDDS Magpie service allows
+# us to handle access permissions different for metadata vs. data. Magpie let's users with "browse" permissions access
+# metadata but only users with "read" permissions can access data.
+# By accessing files through different THREDDS services (see THREDDS documentation), we can either read the metadata with
+# "browse" permissions or the data itself with "read" permissions. For example, by default a NetCDF file can be accessed
+# using the NCML service to get its metadata or through the NCSS service to access the data itself.
+# 
+# If you have a file that you would like to be treated as metadata (Magpie will allow users with "browse" permissions to
+# access it) no matter which THREDDS service is used to access it, add the file pattern to the `THREDDS_MAGPIE_EXTRA_METADATA_PREFIXES`
+# variable. Similarly, if you have a file that you would like to be treated as data no matter which THREDDS service is used
+# to access it, add the file pattern to the `THREDDS_MAGPIE_EXTRA_DATA_PREFIXES` variable.
+#
+# For example, if you want all files with a .h5 extension to be treated as data files in all cases, add '".+\\.h5"' to the 
+# `THREDDS_MAGPIE_EXTRA_DATA_PREFIXES` variable. Note that values are regular expressions (python) where slashes are double
+# escaped. Expressions should be surrounded by double quotes and if multiple expressions are included they should be comma
+# delimited.
+# 
+# Current defaults are:
+#export THREDDS_MAGPIE_EXTRA_METADATA_PREFIXES='".+\\.txt", ".+\\.md", ".+\\.rst"'
+#export THREDDS_MAGPIE_EXTRA_DATA_PREFIXES=''
+
 # Allow using Github as external AuthN/AuthZ provider with Magpie
 #   To setup Github as login, goto <https://github.com/settings/developers> under section [OAuth Apps]
 #   and create a new Magpie application with configurations: