diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 0567a7db3e..0c927c2223 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -233,5 +233,10 @@ "category": "unstructured_grids", "index_list": "0", "run": false + }, + { + "category": "land_surface", + "index_list": "0", + "run": false } ] diff --git a/README.md b/README.md index 10f4a11a5f..d84305ba40 100644 --- a/README.md +++ b/README.md @@ -25,3 +25,5 @@ information, please read This infrastructure utilizes the NCEP produtil package, which is a platform-independent weather and ocean forecasting utility package developed at the National Oceanic and Atmospheric Administration (NOAA). + +For information about the support provided for releases, see our [Release Support Policy](https://metplus.readthedocs.io/en/develop/Release_Guide/index.html#release-support-policy). diff --git a/docs/Contributors_Guide/figure/1Issue-before-created.png b/docs/Contributors_Guide/figure/1Issue-before-created.png new file mode 100644 index 0000000000..6fd50eeae3 Binary files /dev/null and b/docs/Contributors_Guide/figure/1Issue-before-created.png differ diff --git a/docs/Contributors_Guide/figure/1PR-before-created.png b/docs/Contributors_Guide/figure/1PR-before-created.png new file mode 100644 index 0000000000..38f36e2764 Binary files /dev/null and b/docs/Contributors_Guide/figure/1PR-before-created.png differ diff --git a/docs/Contributors_Guide/figure/2Issue-after-created.png b/docs/Contributors_Guide/figure/2Issue-after-created.png new file mode 100644 index 0000000000..f86d6ee7c5 Binary files /dev/null and b/docs/Contributors_Guide/figure/2Issue-after-created.png differ diff --git a/docs/Contributors_Guide/figure/2PR-after-created.png b/docs/Contributors_Guide/figure/2PR-after-created.png new file mode 100644 index 0000000000..d9943e4534 Binary files /dev/null and b/docs/Contributors_Guide/figure/2PR-after-created.png differ diff --git a/docs/Contributors_Guide/figure/3Issue-set-status.png b/docs/Contributors_Guide/figure/3Issue-set-status.png new file mode 100644 index 0000000000..6a9629f80a Binary files /dev/null and b/docs/Contributors_Guide/figure/3Issue-set-status.png differ diff --git a/docs/Contributors_Guide/figure/3PR-set-status.png b/docs/Contributors_Guide/figure/3PR-set-status.png new file mode 100644 index 0000000000..03b7b5082f Binary files /dev/null and b/docs/Contributors_Guide/figure/3PR-set-status.png differ diff --git a/docs/Contributors_Guide/figure/4Issue-plus-one-set-cycle.png b/docs/Contributors_Guide/figure/4Issue-plus-one-set-cycle.png new file mode 100644 index 0000000000..a898b9f34d Binary files /dev/null and b/docs/Contributors_Guide/figure/4Issue-plus-one-set-cycle.png differ diff --git a/docs/Contributors_Guide/figure/4PR-plus-one-set-cycle.png b/docs/Contributors_Guide/figure/4PR-plus-one-set-cycle.png new file mode 100644 index 0000000000..248021ace0 Binary files /dev/null and b/docs/Contributors_Guide/figure/4PR-plus-one-set-cycle.png differ diff --git a/docs/Contributors_Guide/figure/5PR-link-issue.png b/docs/Contributors_Guide/figure/5PR-link-issue.png new file mode 100644 index 0000000000..9d251cf6bc Binary files /dev/null and b/docs/Contributors_Guide/figure/5PR-link-issue.png differ diff --git a/docs/Contributors_Guide/github_workflow.rst b/docs/Contributors_Guide/github_workflow.rst index 756923c237..161e1c44d9 100644 --- a/docs/Contributors_Guide/github_workflow.rst +++ b/docs/Contributors_Guide/github_workflow.rst @@ -324,8 +324,8 @@ Detailed instructions for each item can be found below or by clicking the link. .. _wo-find-issue: -Find the GitHub issue -^^^^^^^^^^^^^^^^^^^^^ +Find or Create a GitHub Issue +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * Go to the `METplus repository `_ and click on the `Issues tab `_. @@ -336,6 +336,46 @@ Find the GitHub issue `new Discussion `_ on the METplus GitHub Discussions page to ask if an issue should be created. +* If creating a new issue, select the + `"New Issue" `_ button + and review the categories of issues (e.g. Bug report, enhancement request, + New feature request, New use case, Sub-Issue, Task). Find an appropriate + categories and click on "Get Started" next to the category. + + Create a short, but descriptive title. In the 'write' window, follow the + directions and fill in the template. Add any additional comments/details. + When filling in the template, be sure to "Define the PR metadata, as + permissions allow. Select: **Assignee(s), Project(s)**, and **Milestone**". + + Before an issue is created, a "Project" can be selected, but there is no + option to select a cycle. + + .. image:: figure/1Issue-before-created.png + :width: 400 + + After the issue is created, more options appear under the "Project" section. + + .. image:: figure/2Issue-after-created.png + :width: 400 + + Click the "Status" drop down and select "Todo". + + .. image:: figure/3Issue-set-status.png + :width: 400 + + Click on "+1 more" then under "Cycle", click "Choose an iteration" and + select the current development cycle. + + .. image:: figure/4Issue-plus-one-set-cycle.png + :width: 400 + + After selecting the appropriate "Cycle", be sure to remove the + **alert: NEED CYCLE ASSIGNMENT** label, which is added by default. + + If the description of the issue is clear and does not need further + definition, be sure to remove the **alert: NEED MORE DEFINITION** + label, which is added by default. + .. _wo-fork-repo: Fork the dtcenter/METplus repository @@ -672,6 +712,29 @@ Open a pull request * When everything looks satisfactory, click on the green 'Create pull request' button. + * Before a pull request is created, a "Project" can be selected, but there + is no option to select a cycle. + + .. figure:: figure/1PR-before-created.png + + After the pull request is created, more options appear under the + "Project" section. + + .. figure:: figure/2PR-after-created.png + + Click the "Status" drop down and select "Review". + + .. figure:: figure/3PR-set-status.png + + Click on "+1 more" then under "Cycle", click "Choose an iteration" and + select the current development cycle. + + .. figure:: figure/4PR-plus-one-set-cycle.png + + To link the issue that correspondes to the pull request, click on the + gear next to "Development," type the issue number, then select the issue + that is displayed. + * An authorized METplus developer will accept the pull request (if everything meets acceptance criteria) and merge the code into the remote repository's develop branch. diff --git a/docs/Release_Guide/index.rst b/docs/Release_Guide/index.rst index 573eaa3ab5..6956f5311c 100644 --- a/docs/Release_Guide/index.rst +++ b/docs/Release_Guide/index.rst @@ -50,6 +50,21 @@ Bugfix Release A bugfix release introduces no new features, but fixes bugs in previous official releases and targets the most critical bugs affecting users. +Release Support Policy +====================== + +The METplus developers officially provide bug fix support for the latest +official release and for the developmental releases as described above. This +includes addressing critical bugs, security vulnerabilities, and functionality +issues. We acknowledge that certain exceptions may arise to cater to the +specific needs of funding institutions. These exceptions may include dedicating +resources, prioritizing bug fixes, or extending support beyond the standard +policy for funding-related requirements. Such exceptions will be evaluated on +a case-by-case basis, ensuring a mutually beneficial collaboration between our +software team and the respective funding institutions. For further inquiries or +to report any bugs, please contact our dedicated support team in the +`METplus GitHub Discussions Forum `_. + Instructions Summary ==================== diff --git a/docs/Release_Guide/release_steps/common/update_dtc_website.rst b/docs/Release_Guide/release_steps/common/update_dtc_website.rst index 2e430a5b2b..bbd39bc68d 100644 --- a/docs/Release_Guide/release_steps/common/update_dtc_website.rst +++ b/docs/Release_Guide/release_steps/common/update_dtc_website.rst @@ -16,7 +16,7 @@ Update DTC Website * For *Related Community Code* select both the METplus and the |projectName| options (For Macs, hold the Command key to select both). - * For *Version Label* type "|projectRepo| X.Y.Z betaN". + * For *Version Label* type "|projectRepo| X.Y.Z-betaN". * Select the release type (*Recommended* for official or bugfix releases or *Development* for development versions). @@ -48,8 +48,9 @@ Update DTC Website * Update the existing releases, as needed. - * For a development release, change any previous *Development* - versions to *Other*. + * For a development release, ensure the "Release Type" is set to + *Development* and change any previous *Development* versions to + *Other*. * For a bugfix or official release, change any previous *Recommended* versions to *Other*. @@ -59,22 +60,14 @@ Update DTC Website * Create or edit the "Coordinated METplus Version X.Y" software release. - * For a beta1 development release, create the "Coordinated METplus - Version X.Y" release entry if it doesn't already exist. Ensure the - "Release Type" is set to *Development*. Add links for the |projectRepo| - X.Y.Z beta1, the METplus "Documentation", the METplus "Existing Builds - and Docker" page, and the "Release Notes". Add to the Release Notes a link to the - |projectRepo| Release Notes. + * For an official release, create the "Coordinated METplus Version X.Y" + release entry if it doesn't already exist. Ensure the "Release Type" + is set to *Recommended*. Consider changing the "Release Type" of + previous coordinated releases from *Recommended* to *Other*. Add + links for the |projectRepo| X.Y.Z, the METplus "Documentation", + the METplus "Existing Builds and Docker" page, and the "Release Notes". + Make the Release Notes a link to the |projectRepo| Release Notes. - * For a beta2 or beyond development release, update the existing link and text in - the "Coordinated METplus Version X.Y" release section with the updated - development release information. - - * For an official release, update the existing link and text in - the "Coordinated METplus Version X.Y" release section with the X.Y.Z - information. Change the "Release Type" from *Development* to - *Recommended*. Consider changing the "Release Type" of previous coordinated releases from *Recommended* to *Other*. - * For a bugfix release, update the existing link and text in the "Coordinated METplus Version X.Y" release section with the X.Y.Z+1 information. diff --git a/docs/Users_Guide/overview.rst b/docs/Users_Guide/overview.rst index 0cea5af8c0..1d8e543abe 100644 --- a/docs/Users_Guide/overview.rst +++ b/docs/Users_Guide/overview.rst @@ -112,17 +112,17 @@ METplus Components Python Requirements - Description - Use Cases (only applicable for METplus wrappers component)(followed by python package name) - * - Python 3.8.6+ + * - Python 3.10.4+ - - METplus wrappers, :raw-html:`
` METcalcpy, :raw-html:`
` - METplotpy,:raw-html:`
` + METplotpy, :raw-html:`
` METdataio - - - * - cartopy - - >=0.18.0 + - >=0.21.1 - METplus wrappers, :raw-html:`
` METcalcpy, :raw-html:`
` METplotpy @@ -167,7 +167,7 @@ METplus Components Python Requirements oceanographic variables - * - dateutil - - >=2.8 + - >=2.8.2 - METplus wrappers - https://github.com/dateutil/dateutil/releases - provides powerful :raw-html:`
` @@ -208,7 +208,7 @@ METplus Components Python Requirements (h5py) <../generated/met_tool_wrapper/PCPCombine/PCPCombine_python_embedding.html>`_ * - imutils - - >=0.5.3 + - >=0.5.4 - METplotpy :raw-html:`
` - https://pypi.org/project/imutils/ - A series of convenience :raw-html:`
` @@ -236,7 +236,7 @@ METplus Components Python Requirements and scientific formats - * - lxml - - + - >=4.9.1 - METcalcpy, :raw-html:`
` METplotpy, :raw-html:`
` METdataio @@ -246,7 +246,7 @@ METplus Components Python Requirements libxml2 and libxslt - * - matplotlib - - >=3.3.4 + - >=3.6.3 - METplus wrappers, :raw-html:`
` METcalcpy, :raw-html:`
` METplotpy @@ -349,7 +349,7 @@ METplus Components Python Requirements (metplotpy) <../generated/model_applications/s2s/TCGen_fcstGFSO_obsBDECKS_GDF_TDF.html>`_ * - metpy - - + - >=1.4.0 - METplus wrappers - https://www.unidata.ucar.edu/software/metpy/ - a collection of tools :raw-html:`
` @@ -371,7 +371,7 @@ METplus Components Python Requirements \**REQUIRES Python 3.7 - * - netCDF4 - - >=1.5.4 + - >=1.6.2 - METplus wrappers, :raw-html:`
` METcalcpy, :raw-html:`
` METplotpy @@ -380,7 +380,7 @@ METplus Components Python Requirements the netCDF C library - For using MET Python embedding functionality in use cases * - numpy - - >=1.19.2 + - >=1.24.2 - METplus wrappers, :raw-html:`
` METcalcpy, :raw-html:`
` METplotpy, :raw-html:`
` @@ -394,7 +394,7 @@ METplus Components Python Requirements Fourier transforms, and more. - For using MET Python embedding functionality in use cases * - pandas - - >=1.0.5, <=1.2.3 (METdataio) + - >=1.5.2 - METplus wrappers, :raw-html:`
` METcalcpy, :raw-html:`
` METplotpy, :raw-html:`
` @@ -409,7 +409,7 @@ METplus Components Python Requirements language - For using MET Python embedding functionality in use cases * - pint - - >=0.18 + - >=0.20.1 - METcalcpy - https://github.com/hgrecco/pint - Python package to define, :raw-html:`
` @@ -417,7 +417,7 @@ METplus Components Python Requirements physical quantities - * - plotly - - >=4.9.0 + - >=5.13.0 - METcalcpy, :raw-html:`
` METplotpy - https://github.com/plotly/plotly.py @@ -458,7 +458,7 @@ METplus Components Python Requirements (pylab) <../generated/model_applications/s2s/UserScript_fcstGFS_obsERA_WeatherRegime.html>`_ * - pymysql - - + - >=1.0.2 - METcalcpy, :raw-html:`
` METplotpy, :raw-html:`
` METdataio @@ -468,7 +468,7 @@ METplus Components Python Requirements based on PEP 249 - * - pyproj - - 2.3.1 + - >=2.3.1 - METplus wrappers - https://github.com/pyproj4/pyproj/archive/v2.3.1rel.tar.gz - Python interface to PROJ :raw-html:`
` @@ -506,7 +506,7 @@ METplus Components Python Requirements (pyresample) <../generated/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMAP_climWOA_sss.html>`_ * - pytest - - >=5.2.1 + - >=7.2.1 - METcalcpy, :raw-html:`
` METplotpy, :raw-html:`
` METdataio @@ -528,7 +528,7 @@ METplus Components Python Requirements libraries like Plotly - * - pyyaml - - >=5.3.1 + - >=6.0 - METcalcpy, :raw-html:`
` METplotpy, :raw-html:`
` METdataio @@ -538,7 +538,7 @@ METplus Components Python Requirements programming language - * - scikit-image - - >=0.16.2 + - >=0.19.3 - METcalcpy, :raw-html:`
` METplotpy - https://scikit-image.org @@ -547,7 +547,7 @@ METplus Components Python Requirements processing - * - scikit-learn - - 0.23.2 + - >=1.2.2 - METplus wrappers, :raw-html:`
` METcalcpy, :raw-html:`
` METplotpy @@ -572,7 +572,7 @@ METplus Components Python Requirements (scikit-learn) <../generated/model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsSMAP_climWOA_sss.html>`_ * - scipy - - >=1.5.1 + - >=1.9.3 - METplus wrappers, :raw-html:`
` METcalcpy, :raw-html:`
` METplotpy @@ -635,7 +635,7 @@ METplus Components Python Requirements (sklearn) <../generated/model_applications/s2s/UserScript_fcstGFS_obsERA_WeatherRegime.html>`_ * - xarray - - >=0.17.0 + - >=2023.1.0 - METplus wrappers, :raw-html:`
` METcalcpy, :raw-html:`
` METplotpy diff --git a/docs/Users_Guide/quicksearch.rst b/docs/Users_Guide/quicksearch.rst index af6e2a46cb..7fca01c204 100644 --- a/docs/Users_Guide/quicksearch.rst +++ b/docs/Users_Guide/quicksearch.rst @@ -75,6 +75,7 @@ Use Cases by Application: | `Short Range <../search.html?q=ShortRangeAppUseCase&check_keywords=yes&area=default>`_ | `Data Assimilation <../search.html?q=DataAssimilationAppUseCase&check_keywords=yes&area=default>`_ | `Ensemble <../search.html?q=EnsembleAppUseCase&check_keywords=yes&area=default>`_ + | `Land Surface <../search.html?q=LandSurfaceAppUseCase&check_keywords=yes&area=default>`_ | `Marine and Cryosphere <../search.html?q=MarineAndCryosphereAppUseCase&check_keywords=yes&area=default>`_ | `Medium Range <../search.html?q=MediumRangeAppUseCase&check_keywords=yes&area=default>`_ | `Precipitation <../search.html?q=PrecipitationAppUseCase&check_keywords=yes&area=default>`_ @@ -91,6 +92,7 @@ Use Cases by Application: | **Short Range**: *ShortRangeAppUseCase* | **Data Assimilation**: *DataAssimilationAppUseCase* | **Ensemble**: *EnsembleAppUseCase* + | **LandSurface**: *LandSurfaceAppUseCase* | **Marine and Cryosphere**: *MarineAndCryosphereAppUseCase* | **Medium Range**: *MediumRangeAppUseCase* | **Precipitation**: *PrecipitationAppUseCase* diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index b569df9e8b..df1680d803 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -30,6 +30,49 @@ When applicable, release notes are followed by the `GitHub issue `__ number which describes the bugfix, enhancement, or new feature. +METplus Version 5.1.0-rc1 Release Notes (2023-06-07) +---------------------------------------------------- + + .. dropdown:: Enhancements + + .. dropdown:: Bugfix + + * Define the order of the forecast variables numerically rather than alphabetically + (`#2070 `_) + * Allow setting of convert, censor_thresh, and censor_val in regrid dictionary + (`#2082 `_) + * Skip-if-output-exists logic incorrectly skips files + (`#2096 `_) + * Prevent error if no commands were run because they were skipped + (`#2098 `_) + * Allow spaces for complex categorical thresholds + (`#2189 `_) + * PCPCombine - Extra field options not set in -subtract mode + (`#2161 `_) + * StatAnalysis time shifting failure + (`#2168 `_) + + .. dropdown:: New Use Cases + + * Read in Argo profile data netCDF files for use in METplus with python embedding + (`#1977 `_) + + .. dropdown:: Documentation + + * Update the METplus Components Python Requirements + (`#1978 `_, + `#2016 `_) + * Add documentation on support for releases to the Release Guide + (`#2106 `_) + + .. dropdown:: Internal + + * Update Contributor's Guide to use GH Action to update truth data + (`#2068 `_) + * Enhance GitHub Workflow documentation + (`#2147 `_) + * Update the development release guide instructions to remove references to a Coordinated release + (`#2159 `_) METplus Version 5.1.0-beta2 Release Notes (2023-04-26) ------------------------------------------------------ @@ -51,8 +94,8 @@ METplus Version 5.1.0-beta2 Release Notes (2023-04-26) (`#2082 `_) * METplus run errors if no commands were run (`#2098 `_) - * Fix the documentation PDF build - (`#2087 `_) + * TCPairs setting -diag option causes failure + (`#2179 `_) .. dropdown:: New Use Cases diff --git a/docs/_static/land_surface-PointStat_fcstCESM_obsFLUXNET2015_TCI.png b/docs/_static/land_surface-PointStat_fcstCESM_obsFLUXNET2015_TCI.png new file mode 100644 index 0000000000..95d91f7225 Binary files /dev/null and b/docs/_static/land_surface-PointStat_fcstCESM_obsFLUXNET2015_TCI.png differ diff --git a/docs/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.py b/docs/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.py new file mode 100644 index 0000000000..8fb804771c --- /dev/null +++ b/docs/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.py @@ -0,0 +1,173 @@ +""" +PointStat: CESM and FLUXNET2015 Terrestrial Coupling Index (TCI) +====================================================================== + +model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.conf + +""" +############################################################################## +# Scientific Objective +# -------------------- +# This use case ingests two CESM (CAM and CLM) files and a new FLUX2015 dataset (NETCDF) that Paul Dirmeyer, GMU prepared from FLUXNET2015 ASCII dataset. +# The use case will calculate Terrestrial Coupling Index (TCI) from CESM datasets. +# Utilizing Python embedding, this use case taps into a new vital observation dataset and compares it to CESM simulations TCI forecast. +# Finally, it will generate plots of model TCI and observed TCI. + +############################################################################## +# Datasets +# --------------------- +# +# | **Forecast:** CESM 1979-1983 Simulations a. Community Land Model (CLM) and b. Community Atmosphere Model (CAM) +# +# | **Observations:** FLUXNET2015 post processed and converted to NETCDF. This data includes data collected from multiple regional flux towers. +# +# +# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases +# | This tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See `Running METplus`_ section for more information. +# +# | **Data Source:** CESM - CGD; FLUXNET2015 - Paul Dirmeyer + +############################################################################## +# METplus Components +# ------------------ +# +# This use case utilizes the METplus PyEmbedIngest to read the CESM files and calculate TCI using python embedding and a NETCDF file of the TCI is generated. +# The METplus PointStat processes the output of PyEmbedIngest and FLUXNET2015 dataset (using python embedding), and outputs the requested line types. +# Then METplus PlotPointObs tool reads the output of PyEmbedIngest and FLUXNET2015 dataset and produce plots of TCI from CESM and point observations. +# A custom loop runs through all the pre-defined seasons (DJF, MAM, JJA, SON) and runs PyEmbedIngest, PointStat, and PlotPointObs. + +############################################################################## +# METplus Workflow +# ---------------- +# +# The PyEmbedIngest tool reads 2 CESM files containing Soil Moisture and Sensible Heat Flux, each composed of daily forecasts from +# 1979 to 1983 and calculates TCI and generates a NETCDF file of the TCI. One FLUXNET2015 NETCDF file containing station observations +# of several variables including Coupling Index of Soil Moisture and Sensible Heat Flux is read by Python Embedding. +# +# | **Valid Beg:** 1979-01-01 at 00z +# | **Valid End:** 1979-01-01 at 00z +# +# PointStat is used to compare the two new fields (TCI calculated from CESM dataset and FLUXNET2015). +# Finally, PlotPointObs is run to plot the CESM TCI overlaying the FLUXNET2015 point observations. + +############################################################################## +# METplus Configuration +# --------------------- +# +# METplus first loads all of the configuration files found in parm/metplus_config, +# then it loads any configuration files passed to METplus via the command line +# i.e. -c parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.conf +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.conf +# + +############################################################################## +# MET Configuration +# ----------------- +# +# METplus sets environment variables based on the values in the METplus configuration file. These variables are referenced in the MET configuration file. **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!** If there is a setting in the MET configuration file that is not controlled by an environment variable, you can add additional environment variables to be set only within the METplus environment using the [user_env_vars] section of the METplus configuration files. See the ‘User Defined Config’ section on the ‘System Configuration’ page of the METplus User’s Guide for more information. +# +# .. highlight:: bash +# .. literalinclude:: ../../../../parm/met_config/PointStatConfig_wrapped +# + +############################################################################## +# Python Embedding +# ---------------- +# +# This use case uses a Python embedding script to read input data +# +# parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/cesm_tci.py +# +# .. highlight:: python +# .. literalinclude:: ../../../../parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/cesm_tci.py +# +# parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fluxnet2015_tci.py +# +# .. highlight:: python +# .. literalinclude:: ../../../../parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fluxnet2015_tci.py +# + +############################################################################## +# Running METplus +# --------------- +# +# Pass the use case configuration file to the run_metplus.py script +# along with any user-specific system configuration files if desired:: +# +# run_metplus.py /path/to/METplus/parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.conf /path/to/user_system.conf +# +# See :ref:`running-metplus` for more information. + +############################################################################## +# Expected Output +# --------------- +# +# A successful run will output the following both to the screen and to the logfile:: +# +# INFO: METplus has successfully finished running. +# +# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated. +# Output for the use case will be found in 3 folders(relative to **OUTPUT_BASE**). +# Those folders are: +# +# * PyEmbedIngest +# +# The **OUTPUT_BASE** folder contains all of the TCI output calculated using CESM files in NETCDF format: +# +# * regrid_data_plane_DJF.nc +# * regrid_data_plane_JJA.nc +# * regrid_data_plane_MAM.nc +# * regrid_data_plane_SON.nc +# +# * PointStat +# +# The final folder, PointStat, contains all of the following output from the PointStat call: +# +# * point_stat_DJF_000000L_19790101_000000V_cnt.txt +# * point_stat_DJF_000000L_19790101_000000V_ctc.txt +# * point_stat_DJF_000000L_19790101_000000V_mpr.txt +# * point_stat_DJF_000000L_19790101_000000V.stat +# * point_stat_JJA_000000L_19790101_000000V_cnt.txt +# * point_stat_JJA_000000L_19790101_000000V_ctc.txt +# * point_stat_JJA_000000L_19790101_000000V_mpr.txt +# * point_stat_JJA_000000L_19790101_000000V.stat +# * point_stat_MAM_000000L_19790101_000000V_cnt.txt +# * point_stat_MAM_000000L_19790101_000000V_ctc.txt +# * point_stat_MAM_000000L_19790101_000000V_mpr.txt +# * point_stat_MAM_000000L_19790101_000000V.stat +# * point_stat_SON_000000L_19790101_000000V_cnt.txt +# * point_stat_SON_000000L_19790101_000000V_ctc.txt +# * point_stat_SON_000000L_19790101_000000V_mpr.txt +# * point_stat_SON_000000L_19790101_000000V.stat +# +# * PlotPointObs +# +# The final folder plot_point_obs, contains all of the plots from the PlotPointObs call: +# +# * cesm_fluxnet2015_DJF.ps +# * cesm_fluxnet2015_JJA.ps +# * cesm_fluxnet2015_MAM.ps +# * cesm_fluxnet2015_SON.ps +# + +############################################################################## +# Keywords +# -------- +# +# .. note:: +# +# * PyEmbedIngestToolUseCase +# * PointStatToolUseCase +# * PlotPointObsToolUseCase +# * PythonEmbeddingFileUseCase +# * NETCDFFileUseCase +# * LandSurfaceAppUseCase +# +# Navigate to the :ref:`quick-search` page to discover other similar use cases. +# +# +# +# sphinx_gallery_thumbnail_path = '_static/land_surface-PointStat_fcstCESM_obsFLUXNET2015_TCI.png' + diff --git a/docs/use_cases/model_applications/land_surface/README.rst b/docs/use_cases/model_applications/land_surface/README.rst new file mode 100644 index 0000000000..d382a42d8e --- /dev/null +++ b/docs/use_cases/model_applications/land_surface/README.rst @@ -0,0 +1,3 @@ +Land Surface +------------ +Land Model diagnostics and verification against observations diff --git a/internal/scripts/docker_env/README.md b/internal/scripts/docker_env/README.md index ff9136d24a..53a8eedbdc 100644 --- a/internal/scripts/docker_env/README.md +++ b/internal/scripts/docker_env/README.md @@ -403,6 +403,28 @@ cd /home/met_test/geovista /home/met_test/.conda/envs/geovista.${METPLUS_ENV_VERSION}/bin/pip3 install --no-deps --editable . ``` +## pandac.v5.1 (from py_embed_base.v5.1) + +### Docker + +``` +export METPLUS_ENV_VERSION=v5.1 +docker build -t dtcenter/metplus-envs:pandac.${METPLUS_ENV_VERSION} \ + --build-arg METPLUS_ENV_VERSION \ + --build-arg BASE_ENV=py_embed_base \ + --build-arg ENV_NAME=pandac . +docker push dtcenter/metplus-envs:pandac.${METPLUS_ENV_VERSION} +``` + +### Local + +``` +export METPLUS_ENV_VERSION=v5.1 +./scripts/pandac_env.sh ${METPLUS_ENV_VERSION} +``` + + + ## pytest.v5.1 (from metplus_base.v5.1) diff --git a/internal/scripts/docker_env/scripts/pandac_env.sh b/internal/scripts/docker_env/scripts/pandac_env.sh new file mode 100755 index 0000000000..70c81b5899 --- /dev/null +++ b/internal/scripts/docker_env/scripts/pandac_env.sh @@ -0,0 +1,29 @@ +#! /bin/sh + +################################################################################ +# Environment: pandac.v5.1 +# Last Updated: 2023-05-25 (mccabe@ucar.edu) +# Notes: Adds Python packages needed for PANDA-C use cases +# Python Packages: +# scipy==1.10.1 +# matplotlib==3.6.3 +# pygrib==2.1.4 +# +# Other Content: None +################################################################################ + +# version of METplus when the environment was updated, e.g. v5.1 +METPLUS_VERSION=$1 + +# Conda environment to create +ENV_NAME=pandac.${METPLUS_VERSION} + +# Conda environment to use as base for new environment +BASE_ENV=py_embed_base.${METPLUS_VERSION} + + +conda create -y --clone ${BASE_ENV} --name ${ENV_NAME} + +conda install -y --name ${ENV_NAME} -c conda-forge scipy==1.10.1 +conda install -y --name ${ENV_NAME} -c conda-forge matplotlib==3.6.3 +conda install -y --name ${ENV_NAME} -c conda-forge pygrib==2.1.4 diff --git a/internal/tests/pytests/util/config_metplus/test_config_metplus.py b/internal/tests/pytests/util/config_metplus/test_config_metplus.py index 5b374fe011..8dda66dea1 100644 --- a/internal/tests/pytests/util/config_metplus/test_config_metplus.py +++ b/internal/tests/pytests/util/config_metplus/test_config_metplus.py @@ -907,8 +907,7 @@ def test_getraw_instance_with_unset_var(metplus_config): ] ) @pytest.mark.util -def test_format_var_items_options_semicolon(config_value, - expected_result): +def test_format_var_items_options_semicolon(config_value, expected_result): time_info = {} field_configs = {'name': 'FNAME', diff --git a/internal/tests/pytests/util/string_manip/test_util_string_manip.py b/internal/tests/pytests/util/string_manip/test_util_string_manip.py index 22453a53fe..d53f19b469 100644 --- a/internal/tests/pytests/util/string_manip/test_util_string_manip.py +++ b/internal/tests/pytests/util/string_manip/test_util_string_manip.py @@ -128,6 +128,8 @@ def test_threshold(key, value): ("goSFP90", None), ("NA", [('NA', '')]), (",>=,==,!=,<,<=,gt,ge,eq,ne,lt,le and then a number Optionally can have multiple comparison/number pairs separated with && or ||. - Args: - @param thresh_string: String to examine, i.e. <=3.4 - Returns: - None if string does not match any valid comparison operators or does - not contain a number afterwards - regex match object with comparison operator in group 1 and - number in group 2 if valid + + @param thresh_string: String to examine, i.e. <=3.4 + @returns None if string does not match any valid comparison operators + or does not contain a number afterwards. Regex match object with + comparison operator in group 1 and number in group 2 if valid """ comparison_number_list = [] # split thresh string by || or && thresh_split = re.split(r'\|\||&&', thresh_string) # check each threshold for validity - for thresh in thresh_split: + for thresh in [item.strip() for item in thresh_split]: found_match = False for comp in list(VALID_COMPARISONS)+list(VALID_COMPARISONS.values()): # if valid, add to list of tuples @@ -359,14 +357,14 @@ def get_threshold_via_regex(thresh_string): return comparison_number_list -def validate_thresholds(thresh_list): +def validate_thresholds(thresh_list, logger=None): """ Checks list of thresholds to ensure all of them have the correct format Should be a comparison operator with number pair combined with || or && i.e. gt4 or >3&&<5 or gt3||lt1 - Args: - @param thresh_list list of strings to check - Returns: - True if all items in the list are valid format, False if not + + @param thresh_list list of strings to check + @param logger (optional) logging object to output error + @returns True if all items in the list are valid format, False if not """ valid = True for thresh in thresh_list: @@ -375,8 +373,12 @@ def validate_thresholds(thresh_list): valid = False if valid is False: - print("ERROR: Threshold values must use >,>=,==,!=,<,<=,gt,ge,eq,ne,lt, or le with a number, " - "optionally combined with && or ||") + err_str = ("Threshold values must use >,>=,==,!=,<,<=,gt,ge,eq,ne,lt, " + "or le with a number, optionally combined with && or ||") + if logger: + logger.error(err_str) + else: + print(f'ERROR: {err_str}') return False return True diff --git a/metplus/util/time_util.py b/metplus/util/time_util.py index cd53af2511..fed14ec35d 100755 --- a/metplus/util/time_util.py +++ b/metplus/util/time_util.py @@ -159,7 +159,7 @@ def ti_get_seconds_from_relativedelta(lead, valid_time=None): return None # if valid time is specified, use it to determine seconds - if valid_time: + if valid_time and isinstance(valid_time, datetime.datetime): return int((valid_time - (valid_time - lead)).total_seconds()) if lead.months != 0 or lead.years != 0: diff --git a/parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.conf b/parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.conf new file mode 100644 index 0000000000..4fe0905888 --- /dev/null +++ b/parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.conf @@ -0,0 +1,172 @@ +[config] + +# Documentation for this use case can be found at +# https://metplus.readthedocs.io/en/latest/generated/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.html + +# For additional information, please see the METplus Users Guide. +# https://metplus.readthedocs.io/en/latest/Users_Guide + +############################################################ +# Processes to run +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#process-list +############################################################ + +PROCESS_LIST = PyEmbedIngest,PointStat,PlotPointObs + +############################################################ +# Time Info +# LOOP_BY options are INIT, VALID, RETRO, and REALTIME +# If set to INIT or RETRO: +# INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set +# If set to VALID or REALTIME: +# VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set +# LEAD_SEQ is the list of forecast leads to process +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#timing-control +############################################################ + +LOOP_BY = VALID +VALID_TIME_FMT = %Y%m%d%H +VALID_BEG = 1979060100 +VALID_END = 1979060100 +VALID_INCREMENT = 24H + +LEAD_SEQ = 0 + +############################################################ +# Pre-determined seasons for the use case +############################################################ +CUSTOM_LOOP_LIST = DJF,MAM,JJA,SON + +LOG_LEVEL=DEBUG + +############################################################ +# PyEmbedIngest Settings +# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#pyembedingest +############################################################ + +PY_EMBED_INGEST_1_OUTPUT_DIR = +PY_EMBED_INGEST_1_OUTPUT_TEMPLATE = {OUTPUT_BASE}/regrid_data_plane_{custom}.nc + +############################################################ +# The last argument is the Latent Heat Flux. +# User can change it to use any other variable present in CESM files. +############################################################ + +PY_EMBED_INGEST_1_SCRIPT = {PARM_BASE}/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/cesm_tci.py {INPUT_BASE}/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/f.e21.FHIST.f09_f09_mg17.CESM2-CLM45physics.002.clm2.h1.1979-83_SoilWater10cm.nc {INPUT_BASE}/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/f.e21.FHIST.f09_f09_mg17.CESM2-CLM45physics.002.cam.h1.1979-83_CIvars.nc {custom} LHFLX + +PY_EMBED_INGEST_1_TYPE = NUMPY +#PY_EMBED_INGEST_1_OUTPUT_GRID = G129 +PY_EMBED_INGEST_1_OUTPUT_GRID = "latlon 288 192 -90.0 -0.0 0.9375 1.25" + +############################################################ +# PointStat Settings +# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#pointstat +############################################################ + +FCST_POINT_STAT_INPUT_TEMPLATE = PYTHON_NUMPY + +############################################################ +# The default variables for the FLUXNET2015 observations id 10CM Soil Moisture +# and Latent Heat Flux. User can change them in the python Embedding script +# fluxnet_tci.py in the PointStat_fcstCESM_obsFLUXNET2015_TCI +# directory. +############################################################ +OBS_POINT_STAT_INPUT_TEMPLATE = PYTHON_NUMPY={PARM_BASE}/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fluxnet2015_tci.py {INPUT_BASE}/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/F2015_LoCo_AllChains_F2015.nc4 {custom} + +POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE}/PointStat + +############################################################ +# Field Info +# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#field-info +############################################################ + +POINT_STAT_ONCE_PER_FIELD = False + +FCST_POINT_STAT_INPUT_DIR = {OUTPUT_BASE}/ +FCST_POINT_STAT_INPUT_TEMPLATE = regrid_data_plane_{custom}.nc +FCST_POINT_STAT_VAR1_LEVELS = +OBS_POINT_STAT_VAR1_NAME = TCI +OBS_POINT_STAT_VAR1_LEVELS = L0 + +FCST_POINT_STAT_VAR1_NAME = TCI_10cm_soil_depth +FCST_POINT_STAT_VAR1_LEVELS = Z10 +BOTH_POINT_STAT_VAR1_THRESH = + +############################################################ +# PointStat Settings +# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#pointstat +############################################################ + +LOG_POINT_STAT_VERBOSITY = 2 + +POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped + +POINT_STAT_INTERP_TYPE_METHOD = NEAREST +POINT_STAT_INTERP_TYPE_WIDTH = 1 + +POINT_STAT_OUTPUT_FLAG_CTC = BOTH +POINT_STAT_OUTPUT_FLAG_MPR = BOTH +POINT_STAT_OUTPUT_FLAG_CNT = BOTH + +OBS_POINT_STAT_WINDOW_BEGIN = -1000000000 +OBS_POINT_STAT_WINDOW_END = 2000000000 + +POINT_STAT_OFFSETS = 0 + +MODEL = CESM + +POINT_STAT_DESC = TCI +OBTYPE = + +POINT_STAT_REGRID_TO_GRID = NONE +POINT_STAT_REGRID_METHOD = BILIN +POINT_STAT_REGRID_WIDTH = 2 + +POINT_STAT_OBS_VALID_BEG = 19790101_000000 +POINT_STAT_OBS_VALID_END = 20130101_000000 + +POINT_STAT_MASK_GRID = FULL +POINT_STAT_MASK_POLY = +POINT_STAT_MASK_SID = + +POINT_STAT_MESSAGE_TYPE = ADPSFC + +POINT_STAT_OUTPUT_PREFIX = {custom} + +############################################################ +# PlotPointObs Settings +# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#plotpointobs +############################################################ + +PLOT_POINT_OBS_INPUT_TEMPLATE =PYTHON_NUMPY={PARM_BASE}/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fluxnet2015_tci.py {INPUT_BASE}/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/F2015_LoCo_AllChains_F2015.nc4 {custom} + +PLOT_POINT_OBS_GRID_INPUT_DIR = {OUTPUT_BASE}/ +PLOT_POINT_OBS_GRID_INPUT_TEMPLATE = regrid_data_plane_{custom}.nc + +PLOT_POINT_OBS_OUTPUT_DIR = {OUTPUT_BASE}/plot_point_obs +PLOT_POINT_OBS_OUTPUT_TEMPLATE = cesm_fluxnet2015_{custom}.ps + +PLOT_POINT_OBS_TITLE = {custom} CESM vs FLUXNET2015 TCI + +LOG_PLOT_POINT_OBS_VERBOSITY = 2 + +PLOT_POINT_OBS_GRID_DATA_FIELD = { name = "TCI_10cm_soil_depth"; level = "10cm_soil_depth"; } +PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_PLOT_MIN = -40.0 +PLOT_POINT_OBS_GRID_DATA_GRID_PLOT_INFO_PLOT_MAX = 55.0 + +PLOT_POINT_OBS_POINT_DATA = + { + msg_typ = "ADPSFC"; + obs_thresh = >-9999; + obs_var = "TCI"; + dotsize(x) = 3.5; + line_color = [0, 0, 0]; + fill_plot_info = { + flag = TRUE; + color_table = "MET_BASE/colortables/met_default.ctable"; + plot_min = -40.0; + plot_max = 55.0; + colorbar_flag = FALSE; + } + } + diff --git a/parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/cesm_tci.py b/parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/cesm_tci.py new file mode 100644 index 0000000000..b7a97bdc97 --- /dev/null +++ b/parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/cesm_tci.py @@ -0,0 +1,119 @@ +""" +Code adapted from Meg Fowler,CGD,NCAR +The code computes the Terrestrial Coupling Index (TCI) +from latent Heat Flux and 10 cm Soil Moisture +Designed to read Latent Heat Flux (from CAM) and Soil Temp (from CLM) +from two CESM files +User needs to provide the season (DJF, MAM, JJA, or SON) in the METplus conf file +User can change the variables to compute TCI +""" + + +import numpy as np +import xarray as xr +import pandas as pd +import datetime +import time +import sys +import os +import netCDF4 as nc + +if len(sys.argv) < 4: + print("Must specify the following elements: sfc_flux_file soil_file season (DJF, MAM, JJA, or SON)") + sys.exit(1) + +fileCLM = os.path.expandvars(sys.argv[1]) +fileCAM = os.path.expandvars(sys.argv[2]) +season = sys.argv[3] +var_y = sys.argv[4] + +print('Starting Terrestrial Coupling Index Calculation for: ',season) + +camDS_CLM45 = xr.open_dataset(fileCAM, decode_times=False) +print('Finished reading in CAM file') +clmDS_CLM45 = xr.open_dataset(fileCLM, decode_times=False) +print('Finished reading in CLM file') + +if season=="DJF": + ss = 0 +elif season=="MAM": + ss = 1 +elif season=="JJA": + ss = 2 +elif season=="SON": + ss = 3 +else: + sys.exit('ERROR : URECOGNIZED SEASON, PLEASE USE DJF, MAM, JJA, OR SON') + +units, reference_date = camDS_CLM45.time.attrs['units'].split('since') +camDS_CLM45['time'] = pd.date_range(start=reference_date, periods=camDS_CLM45.sizes['time'], freq='D') +camDS_time=camDS_CLM45.time[0] +dt_object = datetime.datetime.utcfromtimestamp(camDS_time.values.tolist()/1e9) +vDate=dt_object.strftime("%Y%m%d") + + +units, reference_date = clmDS_CLM45.time.attrs['units'].split('since') +clmDS_CLM45['time'] = pd.date_range(start=reference_date, periods=clmDS_CLM45.sizes['time'], freq='D') + +ds = camDS_CLM45 +ds['SOILWATER_10CM'] = (('time','lat','lon'), clmDS_CLM45.SOILWATER_10CM.values) + +xname = 'SOILWATER_10CM' # Controlling variable +#yname = 'LHFLX' # Responding variable +yname=str(var_y) + +xday = ds[xname].groupby('time.season') +yday = ds[yname].groupby('time.season') + +# Get the covariance of the two (numerator in coupling index) +covarTerm = ((xday - xday.mean()) * (yday - yday.mean())).groupby('time.season').sum() / xday.count() + +# Now compute the coupling index +couplingIndex = covarTerm/xday.std() +ci_season=couplingIndex[ss,:,:] + +ci = np.where(np.isnan(ci_season), -9999, ci_season) + +met_data = ci[:,:] +met_data = met_data[::-1].copy() + +#trim the lat/lon grids so they match the data fields +lat_met = camDS_CLM45.lat +lon_met = camDS_CLM45.lon +print(" Model Data shape: "+repr(met_data.shape)) +v_str = vDate +v_str = v_str + '_000000' +#print(" Valid date: "+v_str) +lat_ll = float(lat_met.min()) +lon_ll = float(lon_met.min()) +n_lat = lat_met.shape[0] +n_lon = lon_met.shape[0] +delta_lat = (float(lat_met.max()) - float(lat_met.min()))/float(n_lat) +delta_lon = (float(lon_met.max()) - float(lon_met.min()))/float(n_lon) + +print(f"variables:" + f"lat_ll: {lat_ll} lon_ll: {lon_ll} n_lat: {n_lat} n_lon: {n_lon} delta_lat: {delta_lat} delta_lon: {delta_lon}") + +attrs = { + 'valid': v_str, + 'init': v_str, + 'lead': "000000", + 'accum': "000000", + 'name': 'TCI', + 'standard_name': 'terrestrial_coupling_index', + 'long_name': 'terrestrial_coupling_index', + 'level': "10cm_soil_depth", + 'units': "W/m2", + + 'grid': { + 'type': "LatLon", + 'name': "CESM Grid", + 'lat_ll': lat_ll, + 'lon_ll': lon_ll, + 'delta_lat': delta_lat, + 'delta_lon': delta_lon, + 'Nlat': n_lat, + 'Nlon': n_lon, + } + } + diff --git a/parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fluxnet2015_tci.py b/parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fluxnet2015_tci.py new file mode 100644 index 0000000000..1a2209aab6 --- /dev/null +++ b/parm/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI/fluxnet2015_tci.py @@ -0,0 +1,77 @@ +""" +This python embedding code reads in the FLUXNET2015 NETCDF data +and passed to point_data. User need to pass on the season. +User can change the verifying variable +""" + + +import numpy +import sys +import os +import glob +import numpy as np +import xarray as xr +import datetime +from datetime import date, timedelta +import pandas as pd + + +if len(sys.argv) < 2: + print("Must specify the following elements: FLUXNET2015_file season") + sys.exit(1) + +obsfile = os.path.expandvars(sys.argv[1]) +season = sys.argv[2] + +f2015data = xr.open_dataset(obsfile, decode_times=False) + +if season=="DJF": + ss = 0 +elif season=="MAM": + ss = 1 +elif season=="JJA": + ss = 2 +elif season=="SON": + ss = 3 +else: + print('Unrecognized season, please use DJF, MAM, JJA, SON') + exit() + +start_year=f2015data['Start year'].values.tolist() +end_year=f2015data['End year'].values.tolist() +vld=pd.to_datetime(start_year,format='%Y') +vld=vld.strftime("%Y%m%d") +vld=vld+ '_000000' + +sid=f2015data['Station'].values.tolist() +print("Length :",len(sid)) +print("SID :",sid) + +lat=f2015data['Latitude'].values.tolist() +lon=f2015data['Longitude'].values.tolist() +# User can change the name of the variable below +obs=f2015data['CI Sfc_SM Latent_Heat'].values.tolist() +obs=np.array(obs) +obs=obs[:,ss] + + +#create dummy lists for the message type, elevation, variable name, level, height, and qc string +#numpy is more efficient at creating the lists, but need to convert to Pythonic lists +typ = np.full(len(sid),'ADPSFC').tolist() +elv = np.full(len(sid),10).tolist() +var = np.full(len(sid),'TCI').tolist() +lvl = np.full(len(sid),10).tolist() +hgt = np.zeros(len(sid),dtype=int).tolist() +qc = np.full(len(sid),'NA').tolist() +obs = np.where(np.isnan(obs), -9999, obs) +obs = np.full(len(sid),obs).tolist() +vld = np.full(len(sid),vld).tolist() + +l_tuple = list(zip(typ,sid,vld,lat,lon,elv,var,lvl,hgt,qc,obs)) +point_data = [list(ele) for ele in l_tuple] + +#print("Data Length:\t" + repr(len(point_data))) +#print("Data Type:\t" + repr(type(point_data))) + +#print(" Point Data Shape: ",np.shape(point_data)) +print(" Point Data: ",point_data)