Add binary build for linux S3 (#3936)

* Reorg and update options for doc and cmake config * Correct subfiling depends * Add linux S3 binaries to GH published files * Correct path * Try reducing parallel jobs in tests * Change examples jobs for test from 8 to 4
HDFGroup · Jan 9, 2024 · efe7cb7 · efe7cb7
1 parent c129328
commit efe7cb7
Show file tree

Hide file tree

Showing 7 changed files with 122 additions and 4 deletions.
diff --git a/.github/workflows/cmake-ctest.yml b/.github/workflows/cmake-ctest.yml
@@ -230,6 +230,70 @@ jobs:
               path: ${{ runner.workspace }}/build/${{ steps.set-file-base.outputs.FILE_BASE }}-osx12.tar.gz
               if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn`
 
+  build_and_test_S3_linux:
+  # Linux S3 (Ubuntu) w/ gcc + CMake
+  #
+    name: "Ubuntu gcc CMake S3"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install CMake Dependencies (Linux S3)
+        run: |
+           sudo apt-get install ninja-build doxygen graphviz
+           sudo apt install libssl3 libssl-dev libcurl4 libcurl4-openssl-dev
+
+      - name: Set file base name (Linux S3)
+        id: set-file-base
+        run: |
+          FILE_NAME_BASE=$(echo "${{ inputs.file_base }}")
+          echo "FILE_BASE=$FILE_NAME_BASE" >> $GITHUB_OUTPUT
+
+      # Get files created by release script
+      - name: Get tgz-tarball (Linux S3)
+        uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4.1.0
+        with:
+              name: tgz-tarball
+              path: ${{ github.workspace }}
+
+      - name: List files for the space (Linux S3)
+        run: |
+              ls -l ${{ github.workspace }}
+              ls ${{ runner.workspace }}
+
+      - name: Uncompress source (Linux S3)
+        run: tar -zxvf ${{ github.workspace }}/${{ steps.set-file-base.outputs.FILE_BASE }}.tar.gz
+
+      - name: Run ctest (Linux S3)
+        run: |
+          cd "${{ runner.workspace }}/hdf5/hdfsrc"
+          cmake --workflow --preset=ci-StdShar-GNUC-S3 --fresh
+        shell: bash
+
+      - name: Publish binary (Linux S3)
+        id: publish-ctest-binary
+        run: |
+          mkdir "${{ runner.workspace }}/build"
+          mkdir "${{ runner.workspace }}/build/hdf5"
+          cp ${{ runner.workspace }}/hdf5/hdfsrc/COPYING ${{ runner.workspace }}/build/hdf5
+          cp ${{ runner.workspace }}/hdf5/hdfsrc/COPYING_LBNL_HDF5 ${{ runner.workspace }}/build/hdf5
+          cp ${{ runner.workspace }}/hdf5/hdfsrc/README.md ${{ runner.workspace }}/build/hdf5
+          cp ${{ runner.workspace }}/hdf5/build/ci-StdShar-GNUC-S3/*.tar.gz ${{ runner.workspace }}/build/hdf5
+          cd "${{ runner.workspace }}/build"
+          tar -zcvf ${{ steps.set-file-base.outputs.FILE_BASE }}-ubuntu-2204_gcc_s3.tar.gz hdf5
+        shell: bash
+
+      - name: List files in the space (Linux S3)
+        run: |
+              ls ${{ github.workspace }}
+              ls -l ${{ runner.workspace }}
+
+      # Save files created by ctest script
+      - name: Save published binary (Linux S3)
+        uses: actions/upload-artifact@v4
+        with:
+              name: tgz-ubuntu-2204_gcc_s3-binary
+              path: ${{ runner.workspace }}/build/${{ steps.set-file-base.outputs.FILE_BASE }}-ubuntu-2204_gcc_s3.tar.gz
+              if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn`
+
 ####### intel builds
   build_and_test_win_intel:
   # Windows w/ OneAPI + CMake

diff --git a/.github/workflows/release-files.yml b/.github/workflows/release-files.yml
@@ -114,6 +114,12 @@ jobs:
               name: tgz-ubuntu-2204_gcc-binary
               path: ${{ github.workspace }}
 
+      - name: Get published binary (Linux S3)
+        uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4.1.0
+        with:
+              name: tgz-ubuntu-2204_gcc_s3-binary
+              path: ${{ github.workspace }}
+
       - name: Get published binary (Windows_intel)
         uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4.1.0
         with:
@@ -144,6 +150,7 @@ jobs:
               ${{ steps.get-file-base.outputs.FILE_BASE }}.zip
               ${{ steps.get-file-base.outputs.FILE_BASE }}-osx12.tar.gz
               ${{ steps.get-file-base.outputs.FILE_BASE }}-ubuntu-2204_gcc.tar.gz
+              ${{ steps.get-file-base.outputs.FILE_BASE }}-ubuntu-2204_gcc_s3.tar.gz
               ${{ steps.get-file-base.outputs.FILE_BASE }}-win-vs2022_cl.zip
               ${{ steps.get-file-base.outputs.FILE_BASE }}-ubuntu-2204_intel.tar.gz
               ${{ steps.get-file-base.outputs.FILE_BASE }}-win-vs2022_intel.zip
@@ -163,6 +170,7 @@ jobs:
               ${{ steps.get-file-base.outputs.FILE_BASE }}.zip
               ${{ steps.get-file-base.outputs.FILE_BASE }}-osx12.tar.gz
               ${{ steps.get-file-base.outputs.FILE_BASE }}-ubuntu-2204_gcc.tar.gz
+              ${{ steps.get-file-base.outputs.FILE_BASE }}-ubuntu-2204_gcc_s3.tar.gz
               ${{ steps.get-file-base.outputs.FILE_BASE }}-win-vs2022_cl.zip
               ${{ steps.get-file-base.outputs.FILE_BASE }}-ubuntu-2204_intel.tar.gz
               ${{ steps.get-file-base.outputs.FILE_BASE }}-win-vs2022_intel.zip

diff --git a/CMakePresets.json b/CMakePresets.json
@@ -93,6 +93,14 @@
         "EXAMPLES_DOWNLOAD": "ON"
       }
     },
+    {
+      "name": "ci-S3",
+      "hidden": true,
+      "cacheVariables": {
+        "HDF5_ENABLE_ROS3_VFD": "ON",
+        "HDF5_ENABLE_HDFS": "OFF"
+      }
+    },
     {
       "name": "ci-StdShar",
       "hidden": true,
@@ -149,6 +157,14 @@
         "ci-StdShar"
       ]
     },
+    {
+      "name": "ci-StdShar-GNUC-S3",
+      "description": "GNUC S3 Config for x64 (Release)",
+      "inherits": [
+        "ci-StdShar-GNUC",
+        "ci-S3"
+      ]
+    },
     {
       "name": "ci-StdShar-Intel",
       "description": "Intel Standard Config for x64 (Release)",
@@ -187,6 +203,15 @@
         "ci-x64-Release-GNUC"
       ]
     },
+    {
+      "name": "ci-StdShar-GNUC-S3",
+      "description": "GNUC S3 Build for x64 (Release)",
+      "configurePreset": "ci-StdShar-GNUC-S3",
+      "verbose": true,
+      "inherits": [
+        "ci-x64-Release-GNUC"
+      ]
+    },
     {
       "name": "ci-StdShar-Intel",
       "description": "Intel Standard Build for x64 (Release)",
@@ -224,6 +249,13 @@
         "ci-x64-Release-GNUC"
       ]
     },
+    {
+      "name": "ci-StdShar-GNUC-S3",
+      "configurePreset": "ci-StdShar-GNUC-S3",
+      "inherits": [
+        "ci-x64-Release-GNUC"
+      ]
+    },
     {
       "name": "ci-StdShar-Intel",
       "configurePreset": "ci-StdShar-Intel",
@@ -253,6 +285,11 @@
       "configurePreset": "ci-StdShar-GNUC",
       "inherits": "ci-x64-Release-GNUC"
     },
+    {
+      "name": "ci-StdShar-GNUC-S3",
+      "configurePreset": "ci-StdShar-GNUC-S3",
+      "inherits": "ci-x64-Release-GNUC"
+    },
     {
       "name": "ci-StdShar-Intel",
       "configurePreset": "ci-StdShar-Intel",
@@ -287,6 +324,15 @@
         {"type": "package", "name": "ci-StdShar-GNUC"}
       ]
     },
+    {
+      "name": "ci-StdShar-GNUC-S3",
+      "steps": [
+        {"type": "configure", "name": "ci-StdShar-GNUC-S3"},
+        {"type": "build", "name": "ci-StdShar-GNUC-S3"},
+        {"type": "test", "name": "ci-StdShar-GNUC-S3"},
+        {"type": "package", "name": "ci-StdShar-GNUC-S3"}
+      ]
+    },
     {
       "name": "ci-StdShar-Intel",
       "steps": [

diff --git a/HDF5Examples/C/CMakeLists.txt b/HDF5Examples/C/CMakeLists.txt
@@ -1,5 +1,5 @@
 cmake_minimum_required (VERSION 3.12)
-PROJECT (HDF5Examples_C)
+project (HDF5Examples_C)
 
 #-----------------------------------------------------------------------------
 # Build the C Examples

diff --git a/HDF5Examples/config/cmake-presets/hidden-presets.json b/HDF5Examples/config/cmake-presets/hidden-presets.json
@@ -274,7 +274,7 @@
       "execution": {
         "noTestsAction": "error",
         "timeout": 600,
-        "jobs": 8
+        "jobs": 4
       }
     },
     {

diff --git a/config/cmake-presets/hidden-presets.json b/config/cmake-presets/hidden-presets.json
@@ -400,7 +400,7 @@
       "execution": {
         "noTestsAction": "error",
         "timeout": 600,
-        "jobs": 8
+        "jobs": 4
       }
     },
     {

diff --git a/config/cmake/scripts/CTestScript.cmake b/config/cmake/scripts/CTestScript.cmake
@@ -81,7 +81,7 @@ if (CTEST_USE_TAR_SOURCE)
   ## Uncompress source if tar file provided
   ## --------------------------
   if (WIN32 AND NOT MINGW)
-    message (STATUS "extracting... [${CMAKE_EXECUTABLE_NAME} x ${CTEST_DASHBOARD_ROOT}\\${CTEST_USE_TAR_SOURCE}.zip]")
+    message (STATUS "extracting... [${CMAKE_EXECUTABLE_NAME} -E tar -xvf ${CTEST_DASHBOARD_ROOT}\\${CTEST_USE_TAR_SOURCE}.zip]")
     execute_process (COMMAND ${CMAKE_EXECUTABLE_NAME} -E tar -xvf ${CTEST_DASHBOARD_ROOT}\\${CTEST_USE_TAR_SOURCE}.zip RESULT_VARIABLE rv)
   else ()
     message (STATUS "extracting... [${CMAKE_EXECUTABLE_NAME} -E tar -xvf ${CTEST_DASHBOARD_ROOT}/${CTEST_USE_TAR_SOURCE}.tar]")