From a12ecaccc00732654e4d14f57ede2e78d56304d8 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 7 Aug 2024 17:11:04 +0200 Subject: [PATCH 1/7] Use a time template in module documentation generation --- .../workflows/update_available_software.yml | 3 +++ mkdocs.yml | 3 +++ requirements.txt | 1 + .../available_software/available_software.py | 8 +++++- .../test_md_template_detailed_science_sol.md | 25 +++++++++++++++++++ scripts/available_software/tests/test_md.py | 12 +++++++++ scripts/update_generated_time.sh | 6 +++++ 7 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 scripts/available_software/tests/data/test_md_template_detailed_science_sol.md create mode 100755 scripts/update_generated_time.sh diff --git a/.github/workflows/update_available_software.yml b/.github/workflows/update_available_software.yml index 633769f29..85b666804 100644 --- a/.github/workflows/update_available_software.yml +++ b/.github/workflows/update_available_software.yml @@ -1,5 +1,6 @@ name: Update overview of available software in EESSI on: + pull_request: workflow_dispatch: schedule: # run every 4 hours @@ -34,6 +35,7 @@ jobs: cp docs/available_software/data/json_data.json docs/available_software/data/json_data.json.orig cp docs/available_software/data/json_data_detail.json docs/available_software/data/json_data_detail.json.orig + export TIME_GENERATED_TEMPLATE="{{ generated_time }}" python scripts/available_software/available_software.py git status @@ -47,6 +49,7 @@ jobs: else echo "JSON files in docs/available_software/data have been changed, PR should be opened" echo "json_data_changed=yes" >> $GITHUB_OUTPUT + ./scripts/update_generated_time.sh mkdocs.yml fi # remove original JSON files, or they'll end up in the PR being opened diff --git a/mkdocs.yml b/mkdocs.yml index 78b552aff..68f6c9a8b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -118,11 +118,14 @@ markdown_extensions: - pymdownx.emoji: emoji_index: !!python/name:material.extensions.emoji.twemoji emoji_generator: !!python/name:material.extensions.emoji.to_svg + # Enable macros so we can use global variables + - macros extra: # add links in bottom right social: - icon: fontawesome/brands/twitter link: https://twitter.com/eessi_hpc + generated_time: "Wed, 07 Aug 2024 at 17:08:54 CEST" extra_javascript: # mermaid diagram - https://cdn.jsdelivr.net/npm/mermaid@10/dist/mermaid.esm.min.mjs diff --git a/requirements.txt b/requirements.txt index cc56811d9..c4e40dd4e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ mkdocs-material mkdocs-redirects mkdocs-git-revision-date-localized-plugin mkdocs-toc-sidebar-plugin +mkdocs-macros-plugin diff --git a/scripts/available_software/available_software.py b/scripts/available_software/available_software.py index 273856b07..efd6b75bf 100644 --- a/scripts/available_software/available_software.py +++ b/scripts/available_software/available_software.py @@ -408,8 +408,14 @@ def generate_detail_pages(json_path, dest_path) -> None: data = json.load(json_data) all_targets = data["targets"] + + time_generated_template = os.environ.get('TIME_GENERATED_TEMPLATE') for software, content in data["software"].items(): - generate_software_detail_page(software, content, data["time_generated"], all_targets, dest_path) + if time_generated_template: + time_generated = time_generated_template + else: + time_generated = data["time_generated"] + generate_software_detail_page(software, content, time_generated, all_targets, dest_path) # -------------------------------------------------------------------------------------------------------- diff --git a/scripts/available_software/tests/data/test_md_template_detailed_science_sol.md b/scripts/available_software/tests/data/test_md_template_detailed_science_sol.md new file mode 100644 index 000000000..d879e0955 --- /dev/null +++ b/scripts/available_software/tests/data/test_md_template_detailed_science_sol.md @@ -0,0 +1,25 @@ +--- +hide: + - toc +--- + +science +======= + +# Available modules + + +The overview below shows which science installations are available per target architecture in EESSI, ordered based on software version (new to old). + +To start using science, load one of these modules using a `module load` command like: + +```shell +module load science/7.2.0 +``` + +*(This data was automatically generated on {{ generated_date }})* + +| |aarch64/generic|x86_64/amd/zen2| +| :---: | :---: | :---: | +|science/7.2.0|x|x| +|science/5.3.0|x|x| diff --git a/scripts/available_software/tests/test_md.py b/scripts/available_software/tests/test_md.py index 007824fbf..02fc16142 100644 --- a/scripts/available_software/tests/test_md.py +++ b/scripts/available_software/tests/test_md.py @@ -1,7 +1,9 @@ from mdutils.mdutils import MdUtils from available_software import get_unique_software_names, modules_eessi, generate_table_data, generate_module_table +from available_software import generate_detail_pages import os import filecmp +import shutil class TestMarkdown: @@ -22,6 +24,8 @@ def setup_class(cls): def teardown_class(cls): if os.path.exists("test_simple.md"): os.remove("test_simple.md") + if os.path.exists("detailed_md"): + shutil.rmtree("detailed_md") # --------------------------- # Markdown tests @@ -41,3 +45,11 @@ def test_md_simple(self): md_file.create_md_file() assert os.path.exists("test_simple.md") assert filecmp.cmp(self.path + "/data/test_md_simple_sol.md", "test_simple.md") + + def test_md_detailed_template(self): + os.environ["TIME_GENERATED_TEMPLATE"] = "{{ generated_date }}" + os.mkdir('detailed_md') + generate_detail_pages(self.path + "/data/test_json_simple_sol_detail.json", 'detailed_md') + del os.environ["TIME_GENERATED_TEMPLATE"] + assert os.path.exists("detailed_md/science.md") + assert filecmp.cmp(self.path + "/data/test_md_template_detailed_science_sol.md", "detailed_md/science.md") diff --git a/scripts/update_generated_time.sh b/scripts/update_generated_time.sh new file mode 100755 index 000000000..dde38e02d --- /dev/null +++ b/scripts/update_generated_time.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# Get the new date +NEW_DATE="$(date '+%a, %d %b %Y at %H:%M:%S %Z')" +# Inject it into the target file +sed -i 's/\(generated_time: "\)[^"]*\(".*\)/\1'"${NEW_DATE}"'\2/' $1 From 4db49682e1c351ea0755f18ac3fbe62f891a4a13 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 7 Aug 2024 17:23:07 +0200 Subject: [PATCH 2/7] Place plugin in right location --- mkdocs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index 68f6c9a8b..b2f629bd5 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -79,6 +79,8 @@ plugins: - git-revision-date-localized # necessary for search to work - search + # Enable macros so we can use global variables + - macros - redirects: redirect_maps: adding_software.md: adding_software/overview.md @@ -118,8 +120,6 @@ markdown_extensions: - pymdownx.emoji: emoji_index: !!python/name:material.extensions.emoji.twemoji emoji_generator: !!python/name:material.extensions.emoji.to_svg - # Enable macros so we can use global variables - - macros extra: # add links in bottom right social: From 0fd557d918fe70f4bb433e164533a233732bda6c Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 7 Aug 2024 17:24:09 +0200 Subject: [PATCH 3/7] Linting --- scripts/available_software/available_software.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/available_software/available_software.py b/scripts/available_software/available_software.py index efd6b75bf..61cdc76fc 100644 --- a/scripts/available_software/available_software.py +++ b/scripts/available_software/available_software.py @@ -412,9 +412,9 @@ def generate_detail_pages(json_path, dest_path) -> None: time_generated_template = os.environ.get('TIME_GENERATED_TEMPLATE') for software, content in data["software"].items(): if time_generated_template: - time_generated = time_generated_template + time_generated = time_generated_template else: - time_generated = data["time_generated"] + time_generated = data["time_generated"] generate_software_detail_page(software, content, time_generated, all_targets, dest_path) From 7e542a359ef48cd8d4e49efd6086421b98e46068 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 7 Aug 2024 17:29:07 +0200 Subject: [PATCH 4/7] Always make time update --- .github/workflows/update_available_software.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/update_available_software.yml b/.github/workflows/update_available_software.yml index 85b666804..0045d83a8 100644 --- a/.github/workflows/update_available_software.yml +++ b/.github/workflows/update_available_software.yml @@ -37,6 +37,8 @@ jobs: export TIME_GENERATED_TEMPLATE="{{ generated_time }}" python scripts/available_software/available_software.py + ./scripts/update_generated_time.sh mkdocs.yml + git status # determine whether pull request should be opened: @@ -49,7 +51,6 @@ jobs: else echo "JSON files in docs/available_software/data have been changed, PR should be opened" echo "json_data_changed=yes" >> $GITHUB_OUTPUT - ./scripts/update_generated_time.sh mkdocs.yml fi # remove original JSON files, or they'll end up in the PR being opened From b71628867a40f956bdacc317a0831cb9aff20b4c Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 7 Aug 2024 17:36:31 +0200 Subject: [PATCH 5/7] Add a quick test --- .github/workflows/update_available_software.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/update_available_software.yml b/.github/workflows/update_available_software.yml index 0045d83a8..46cf67e8c 100644 --- a/.github/workflows/update_available_software.yml +++ b/.github/workflows/update_available_software.yml @@ -40,6 +40,7 @@ jobs: ./scripts/update_generated_time.sh mkdocs.yml git status + git diff docs/available_software/detail/zstd.md # determine whether pull request should be opened: # if JSON files in docs/available_software/data have been updated, then a PR should be opened From 1d2dba7a4dceec8ee0495538266e2a706fdeea19 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 7 Aug 2024 17:42:21 +0200 Subject: [PATCH 6/7] Tidy up PR, ready for review --- .github/workflows/update_available_software.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/update_available_software.yml b/.github/workflows/update_available_software.yml index 46cf67e8c..d33dc16c8 100644 --- a/.github/workflows/update_available_software.yml +++ b/.github/workflows/update_available_software.yml @@ -1,6 +1,5 @@ name: Update overview of available software in EESSI on: - pull_request: workflow_dispatch: schedule: # run every 4 hours @@ -40,7 +39,6 @@ jobs: ./scripts/update_generated_time.sh mkdocs.yml git status - git diff docs/available_software/detail/zstd.md # determine whether pull request should be opened: # if JSON files in docs/available_software/data have been updated, then a PR should be opened From 458329410ed0e896af54cefb006aaf0918ea06c1 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 8 Aug 2024 09:39:00 +0200 Subject: [PATCH 7/7] Update mkdocs.yml Co-authored-by: Kenneth Hoste --- mkdocs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs.yml b/mkdocs.yml index b2f629bd5..e38ac6ffc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -125,6 +125,7 @@ extra: social: - icon: fontawesome/brands/twitter link: https://twitter.com/eessi_hpc + # this gets auto-updated via update_generated_time.sh script run in update_available_software.yml action generated_time: "Wed, 07 Aug 2024 at 17:08:54 CEST" extra_javascript: # mermaid diagram