Merge pull request #2550 from dtcenter/feature_2388_Update_TCI

Feature 2388 update tci

docs/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.py

@@ -5,50 +5,84 @@
 model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.conf
 
 """
+##############################################################################
+# .. contents::
+#    :depth: 1
+#    :local:
+#    :backlinks: none
+
 ##############################################################################
 # Scientific Objective
 # --------------------
-# This use case ingests two CESM (CAM and CLM) files and a new FLUX2015 dataset (NETCDF) that Paul Dirmeyer, GMU prepared from FLUXNET2015 ASCII dataset.
-# The use case will calculate Terrestrial Coupling Index (TCI) from CESM datasets.
-# Utilizing Python embedding, this use case taps into a new vital observation dataset and compares it to CESM simulations TCI forecast. 
-# Finally, it will generate plots of model TCI and observed TCI.
+# This use case ingests two CESM (CAM and CLM) files and raw FLUXNET2015 data.
+# The use case calculates the Terrestrial Coupling Index (TCI) from the CESM forecasts and FLUXNET observations.
+# Utilizing Python embedding, this use case taps into a new vital observation dataset and compares it to CESM forecasts of TCI. 
+# Finally, it will generate plots of model forecast TCI overlaid with TCI observations at FLUXNET sites.
+#
+# The reference for the Terrestrial Coupling Index calculation is as follows:
+#
+# Dirmeyer, P. A., 2011: The terrestrial segment of soil moisture-climate coupling. *Geophys. Res. Lett.*, **38**, L16702, doi: 10.1029/2011GL048268.
+#
 
 ##############################################################################
 # Datasets
 # ---------------------
 #
-# | **Forecast:** CESM 1979-1983 Simulations a. Community Land Model (CLM) and b. Community Atmosphere Model (CAM) 
-#
-# | **Observations:** FLUXNET2015 post processed and converted to NETCDF. This data includes data collected from multiple regional flux towers.
+# | **Forecast:** CESM 1979-1983 Simulations 
+# | * Community Land Model (CLM) file
+# | * Community Atmosphere Model (CAM) file
 #
+# | **Observations:** Raw FLUXNET2015 observations
 #
-# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases
+# | **Location:** All of the input data required for this use case can be found in the land_surface sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases
 # | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See `Running METplus`_ section for more information.
 #
-# | **Data Source:** CESM - CGD; FLUXNET2015 - Paul Dirmeyer
+# | **Data Source:** CESM - NSF NCAR Climate & Global Dynamics (CGD); FLUXNET2015 "SUBSET" Data Product: https://fluxnet.org/data/fluxnet2015-dataset/subset-data-product/
+#
+
+##############################################################################
+# Python Dependencies
+# ---------------------
+#
+# This use case requires the following Python dependencies::
+#
+# * Xarray
+# * Pandas
+# * METcalcpy 3.0.0+
+#
 
 ##############################################################################
 # METplus Components
 # ------------------
 #
 # This use case utilizes the METplus PyEmbedIngest to read the CESM files and calculate TCI using python embedding and a NETCDF file of the TCI is generated. 
-# The METplus PointStat processes the output of PyEmbedIngest and FLUXNET2015 dataset (using python embedding), and outputs the requested line types.
-# Then METplus PlotPointObs tool reads the output of PyEmbedIngest and FLUXNET2015 dataset and produce plots of TCI from CESM and point observations. 
+# The METplus PointStat processes the output of PyEmbedIngest and FLUXNET2015 dataset (using Python embedding), and outputs the requested line types.
+# Then the METplus PlotPointObs tool reads the output of PyEmbedIngest and FLUXNET2015 dataset and produce plots of TCI from CESM and point observations.
 # A custom loop runs through all the pre-defined seasons (DJF, MAM, JJA, SON) and runs PyEmbedIngest, PointStat, and PlotPointObs.
+#
 
 ##############################################################################
 # METplus Workflow
 # ----------------
 #
-# The PyEmbedIngest tool reads 2 CESM files containing Soil Moisture and Sensible Heat Flux, each composed of daily forecasts from
-# 1979 to 1983 and calculates TCI and generates a NETCDF file of the TCI. One FLUXNET2015 NETCDF file containing station observations 
-# of several variables including Coupling Index of Soil Moisture and Sensible Heat Flux is read by Python Embedding.
-#
+# The PyEmbedIngest tool reads 2 CESM files containing Soil Moisture (CLM file) and Sensible Heat Flux (CAM file), each composed of daily forecasts from
+# 1979 to 1983 and calculates TCI and generates a NETCDF file of the TCI. Raw CSV files containing FLUXNET station observations of latent heat flux (LE_F_MDS)
+# and soil water content at the shallowest level (SWC_F_MDS_1) are read using Python embedding, and TCI is computed.
+# 
 # | **Valid Beg:** 1979-01-01 at 00z
 # | **Valid End:** 1979-01-01 at 00z
-# 
+#
 # PointStat is used to compare the two new fields (TCI calculated from CESM dataset and FLUXNET2015).
-# Finally, PlotPointObs is run to plot the CESM TCI overlaying the FLUXNET2015 point observations. 
+# Finally, PlotPointObs is run to plot the CESM TCI overlaying the FLUXNET2015 point observations.
+#
+# .. note::
+# 
+#   The CESM forecasts cover a time period prior to the availability of FLUXNET observations. Thus,
+#   this use case should be considered a demonstration of the capability to read CESM forecast data, 
+#   raw FLUXNET observation data, and compute TCI, rather than a bonafide scientific application.
+#   The use case is designed to enforce seasonal alignment, but it is not designed to enforce date/time alignment. 
+#   In this case, the CESM data cover 1979-1983, whereas the sample FLUXNET observations cover varying time ranges depending on the site.
+#
 
 ##############################################################################
 # METplus Configuration
@@ -76,18 +110,107 @@
 # Python Embedding
 # ----------------
 #
-# This use case uses a Python embedding script to read input data
+# This use case uses a Python embedding script to read both the forecast and observation data, in order to compute TCI,
+# which is the diagnostic that is being verified by MET using PointStat. The CESM forecast data is read using:
 #
 # parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/cesm_tci.py
 #
 # .. highlight:: python
 # .. literalinclude:: ../../../../parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/cesm_tci.py
 #
+# The user can control all arguments to this script via the METplus use case configuration file using the following config entries:
+#
+# .. glossary::
+# 
+#    CESM_CAM_VAR
+#      The CESM Community Atmosphere Model variable to use for computing TCI
+#
+#      | *Default:* LHFLX
+#
+#    CESM_CLM_VAR
+#      The CESM Community Land Model variable to use for computing TCI
+#
+#      | *Default:* SOILWATER_10CM
+#
+#    CESM_CAM_FILE_PATH
+#      The absolute path to the CESM Community Atmosphere Model netcdf file
+#
+#      | *Default:*
+#
+#    CESM_CLM_FILE_PATH
+#      The absolute path to the CESM Community Land Model netcdf file
+#
+#      | *Default:*
+#
+# The raw FLUXNET2015 SUBSET data are read using:
+#
 # parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fluxnet2015_tci.py
 #
 # .. highlight:: python
 # .. literalinclude:: ../../../../parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fluxnet2015_tci.py
 #
+# The user can control all command line arguments to this script via METplus config entries:
+#
+# .. glossary::
+#
+#    FLUXNET_DATA_DIR
+#      The directory containing raw FLUXNET CSV files
+#
+#      | *Default:*
+#
+#    FLUXNET_LAT_HEAT_VAR
+#      The FLUXNET surface latent heat flux variable name to use for computing TCI
+#
+#      | *Default:* LE_F_MDS
+#
+#    FLUXNET_SOIL_MOIST_VAR
+#      The FLUXNET soil moisture variable name to use for computing TCI
+#
+#      | *Default:* SWC_F_MDS_1
+#
+#    FLUXNET_OBS_METADATA_FILE
+#      The absolute path to the fluxnetstations.csv metadata file included with the use case
+#
+#      | *Default:*
+#
+# and for data filtering options, via METplus config entries:
+#
+# .. glossary::
+#
+#    FLUXNET_SKIP_LEAP_DAYS
+#      Skip FLUXNET observations from 29 February days
+#
+#      | *Default:* True
+#
+#    FLUXNET_HIGHRES_QC_THRESH
+#      The fraction of higher temporal resolution FLUXNET data required to have
+#      passed quality control in order to use the daily data.
+#
+#      | *Default:* 0.8
+#
+#    FLUXNET_MIN_DAYS_PER_SEASON
+#      The minimum number of days to include in individual seasons at each site
+#
+#      | *Default:* 1
+#
+#    FLUXNET_MIN_DAYS_PER_SITE
+#      The minimum number of days for all seasons at each site
+#
+#      | *Default:* 10
+#
+#    FLUXNET_RAW_FILENAME_PATTERN
+#      A filename pattern matching the template of the raw FLUXNET CSV files
+#
+#      | *Default:* FLX_*_DD_*.csv
+#
+#    FLUXNET_DEBUG
+#      Turn on additional print statements from the Python embedding script
+#
+#      | *Default:* False
+#
+# Both of the above Python embedding scripts compute TCI using the ``calc_tci()`` function in METcalcpy. See the METcalcpy 
+# documentation for more information: https://metcalcpy.readthedocs.io/en/latest/index.html.
+#
 
 ##############################################################################
 # Running METplus
@@ -99,6 +222,7 @@
 #    run_metplus.py /path/to/METplus/parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.conf /path/to/user_system.conf
 #
 # See :ref:`running-metplus` for more information.
+#
 
 ##############################################################################
 # Expected Output

internal/tests/use_cases/all_use_cases.txt

@@ -88,7 +88,7 @@ Category: data_assimilation
 
 
 Category: land_surface
-0::PointStat_fcstCESM_obsFLUXNET2015_TCI:: model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.conf:: py_embed
+0::PointStat_fcstCESM_obsFLUXNET2015_TCI:: model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.conf:: metplotpy_env, py_embed
 
 
 Category: marine_and_cryosphere

parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.conf

@@ -40,107 +40,93 @@ CUSTOM_LOOP_LIST = DJF,MAM,JJA,SON
 LOG_LEVEL=DEBUG
 
 ############################################################
-# PyEmbedIngest Settings
+# PyEmbedIngest Settings for CESM forecast data
 # https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#pyembedingest
+# User can modify which flux and soil variable is extracted from the CESM files
 ############################################################
 
+CESM_CAM_VAR = LHFLX
+CESM_CLM_VAR = SOILWATER_10CM
+CESM_CAM_FILE_PATH = {INPUT_BASE}/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fcst/f.e21.FHIST.f09_f09_mg17.CESM2-CLM45physics.002.cam.h1.1979-83_CIvars.nc
+CESM_CLM_FILE_PATH = {INPUT_BASE}/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fcst/f.e21.FHIST.f09_f09_mg17.CESM2-CLM45physics.002.clm2.h1.1979-83_SoilWater10cm.nc
+
 PY_EMBED_INGEST_1_OUTPUT_DIR =
 PY_EMBED_INGEST_1_OUTPUT_TEMPLATE = {OUTPUT_BASE}/regrid_data_plane_{custom}.nc
 
-############################################################
-# The last argument is the Latent Heat Flux.
-# User can change it to use any other variable present in CESM files.
-############################################################
-
-PY_EMBED_INGEST_1_SCRIPT = {PARM_BASE}/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/cesm_tci.py {INPUT_BASE}/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/f.e21.FHIST.f09_f09_mg17.CESM2-CLM45physics.002.clm2.h1.1979-83_SoilWater10cm.nc {INPUT_BASE}/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/f.e21.FHIST.f09_f09_mg17.CESM2-CLM45physics.002.cam.h1.1979-83_CIvars.nc {custom} LHFLX 
+PY_EMBED_INGEST_1_SCRIPT = {PARM_BASE}/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/cesm_tci.py {CESM_CAM_FILE_PATH} {CESM_CAM_VAR} {CESM_CLM_FILE_PATH} {CESM_CLM_VAR} {custom}
 
 PY_EMBED_INGEST_1_TYPE = NUMPY
-#PY_EMBED_INGEST_1_OUTPUT_GRID = G129
 PY_EMBED_INGEST_1_OUTPUT_GRID = "latlon 288 192 -90.0 -0.0 0.9375 1.25" 
 
 ############################################################
-# PointStat Settings 
-# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#pointstat
+# Python embedding settings for using FLUXNET2015 obs data with PointStat.
+# User can change the variables used from FLUXNET here to compute TCI.
+# The FLUXNET_* config items are also used for PlotPointObs below.
 ############################################################
+FLUXNET_DATA_DIR = {INPUT_BASE}/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/obs
+FLUXNET_LAT_HEAT_VAR = LE_F_MDS
+FLUXNET_SOIL_MOIST_VAR = SWC_F_MDS_1
+FLUXNET_OBS_METADATA_FILE={PARM_BASE}/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fluxnetstations.csv
 
-FCST_POINT_STAT_INPUT_TEMPLATE = PYTHON_NUMPY 
+OBS_POINT_STAT_INPUT_TEMPLATE = PYTHON_NUMPY={PARM_BASE}/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fluxnet2015_tci.py {FLUXNET_DATA_DIR} {FLUXNET_LAT_HEAT_VAR} {FLUXNET_SOIL_MOIST_VAR} {custom} {FLUXNET_OBS_METADATA_FILE}
 
 ############################################################
-# The default variables for the FLUXNET2015 observations id 10CM Soil Moisture
-# and Latent Heat Flux. User can change them in the python Embedding script
-# fluxnet_tci.py in the PointStat_fcstCESM_obsFLUXNET2015_TCI
-# directory.
+# PointStat Settings
+# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#pointstat
 ############################################################
-OBS_POINT_STAT_INPUT_TEMPLATE = PYTHON_NUMPY={PARM_BASE}/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fluxnet2015_tci.py {INPUT_BASE}/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/F2015_LoCo_AllChains_F2015.nc4 {custom}
-
-POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE}/PointStat
 
 ############################################################
 # Field Info
 # https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#field-info
 ############################################################
 
-POINT_STAT_ONCE_PER_FIELD = False
-
-FCST_POINT_STAT_INPUT_DIR = {OUTPUT_BASE}/
-FCST_POINT_STAT_INPUT_TEMPLATE = regrid_data_plane_{custom}.nc
-FCST_POINT_STAT_VAR1_LEVELS = 
 OBS_POINT_STAT_VAR1_NAME = TCI
-OBS_POINT_STAT_VAR1_LEVELS = L0 
+OBS_POINT_STAT_VAR1_LEVELS = L0
+OBS_POINT_STAT_WINDOW_BEGIN = -1000000000
+OBS_POINT_STAT_WINDOW_END = 2000000000
 
-FCST_POINT_STAT_VAR1_NAME = TCI_10cm_soil_depth 
+FCST_POINT_STAT_VAR1_NAME = TCI_10cm_soil_depth
 FCST_POINT_STAT_VAR1_LEVELS = Z10
-BOTH_POINT_STAT_VAR1_THRESH =
+FCST_POINT_STAT_INPUT_DIR = {OUTPUT_BASE}
+FCST_POINT_STAT_INPUT_TEMPLATE = regrid_data_plane_{custom}.nc
 
 ############################################################
-# PointStat Settings
-# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#pointstat
+# General PointStat Settings
 ############################################################
 
-LOG_POINT_STAT_VERBOSITY = 2
-
-POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped
-
+POINT_STAT_CONFIG_FILE = {PARM_BASE}/met_config/PointStatConfig_wrapped
+POINT_STAT_DESC = TCI
 POINT_STAT_INTERP_TYPE_METHOD = NEAREST
 POINT_STAT_INTERP_TYPE_WIDTH = 1
-
+POINT_STAT_MASK_GRID = FULL
+POINT_STAT_MASK_POLY = 
+POINT_STAT_MASK_SID =
+POINT_STAT_MESSAGE_TYPE = ADPSFC
+POINT_STAT_OBS_VALID_BEG = 19790101_000000 
+POINT_STAT_OBS_VALID_END = 20130101_000000 
+POINT_STAT_OFFSETS = 0
+POINT_STAT_ONCE_PER_FIELD = False
+POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE}/PointStat
 POINT_STAT_OUTPUT_FLAG_CTC = BOTH
 POINT_STAT_OUTPUT_FLAG_MPR = BOTH
 POINT_STAT_OUTPUT_FLAG_CNT = BOTH
-
-OBS_POINT_STAT_WINDOW_BEGIN = -1000000000
-OBS_POINT_STAT_WINDOW_END = 2000000000
-
-POINT_STAT_OFFSETS = 0
-
-MODEL = CESM
-
-POINT_STAT_DESC = TCI
-OBTYPE =
-
+POINT_STAT_OUTPUT_PREFIX = {custom}
 POINT_STAT_REGRID_TO_GRID = NONE
 POINT_STAT_REGRID_METHOD = BILIN
 POINT_STAT_REGRID_WIDTH = 2
 
-POINT_STAT_OBS_VALID_BEG = 19790101_000000 
-POINT_STAT_OBS_VALID_END = 20130101_000000 
-
-POINT_STAT_MASK_GRID = FULL
-POINT_STAT_MASK_POLY = 
-POINT_STAT_MASK_SID =
-
-POINT_STAT_MESSAGE_TYPE = ADPSFC
-
-POINT_STAT_OUTPUT_PREFIX = {custom}
+LOG_POINT_STAT_VERBOSITY = 2
+MODEL = CESM
+OBTYPE = FLUXNET
 
 ############################################################
 # PlotPointObs Settings
 # https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#plotpointobs
 ############################################################
 
-PLOT_POINT_OBS_INPUT_TEMPLATE =PYTHON_NUMPY={PARM_BASE}/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fluxnet2015_tci.py {INPUT_BASE}/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/F2015_LoCo_AllChains_F2015.nc4 {custom}
+PLOT_POINT_OBS_INPUT_TEMPLATE =PYTHON_NUMPY={PARM_BASE}/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fluxnet2015_tci.py {FLUXNET_DATA_DIR} {FLUXNET_LAT_HEAT_VAR} {FLUXNET_SOIL_MOIST_VAR} {custom} {FLUXNET_OBS_METADATA_FILE}
 
-PLOT_POINT_OBS_GRID_INPUT_DIR = {OUTPUT_BASE}/
+PLOT_POINT_OBS_GRID_INPUT_DIR = {OUTPUT_BASE}
 PLOT_POINT_OBS_GRID_INPUT_TEMPLATE = regrid_data_plane_{custom}.nc
 
 PLOT_POINT_OBS_OUTPUT_DIR = {OUTPUT_BASE}/plot_point_obs
@@ -170,3 +156,13 @@ PLOT_POINT_OBS_POINT_DATA =
     }
   }
 
+############################################################
+# FLUXNET Python Embedding Script options
+############################################################
+[user_env_vars]
+FLUXNET_SKIP_LEAP_DAYS=True
+FLUXNET_HIGHRES_QC_THRESH=0.8
+FLUXNET_MIN_DAYS_PER_SEASON=1
+FLUXNET_MIN_DAYS_PER_SITE_ALL_SEASONS=10
+FLUXNET_RAW_FILENAME_PATTERN=FLX_*_DD_*.csv
+FLUXNET_DEBUG=False