Merge branch '785-workaround-ci-artifacts-size-limitation' into 'deve…

…lop_stream'

Resolve "Workaround CI artifacts size limitation"

Closes #785

See merge request amd/libraries/rocPRIM!697

.gitlab-ci.yml

@@ -158,7 +158,7 @@ build:cmake-minimum-apt:
       -D CMAKE_CXX_COMPILER="$AMDCLANG"
       -D CMAKE_CXX_FLAGS="-Wall -Wextra -Werror"
       -D CMAKE_BUILD_TYPE="$BUILD_TYPE"
-      -D BUILD_$BUILD_TARGET=ON 
+      -D BUILD_$BUILD_TARGET=ON
       -D BUILD_EXAMPLE=ON
       -D GPU_TARGETS=$GPU_TARGETS
       -D AMDGPU_TEST_TARGETS=$GPU_TARGETS
@@ -241,15 +241,15 @@ build:windows:
         BUILD_TARGET: [BENCHMARK, TEST]
   script:
     - mkdir -p $CI_PROJECT_DIR/build
-    - cmake -G Ninja 
-      -S $CI_PROJECT_DIR 
-      -B $CI_PROJECT_DIR/build 
-      -D BUILD_$BUILD_TARGET=ON 
+    - cmake -G Ninja
+      -S $CI_PROJECT_DIR
+      -B $CI_PROJECT_DIR/build
+      -D BUILD_$BUILD_TARGET=ON
       -D GPU_TARGETS=$GPU_TARGET
-      -D CMAKE_CXX_COMPILER:PATH="${env:HIP_PATH}\bin\clang++.exe" 
-      -D CMAKE_PREFIX_PATH:PATH="${env:HIP_PATH}" 
-      -D CMAKE_BUILD_TYPE="$BUILD_TYPE" 
-    - cmake --build "$CI_PROJECT_DIR/build" 
+      -D CMAKE_CXX_COMPILER:PATH="${env:HIP_PATH}\bin\clang++.exe"
+      -D CMAKE_PREFIX_PATH:PATH="${env:HIP_PATH}"
+      -D CMAKE_BUILD_TYPE="$BUILD_TYPE"
+    - cmake --build "$CI_PROJECT_DIR/build"
   artifacts:
     paths:
       - $CI_PROJECT_DIR/build/test/test_*
@@ -271,6 +271,10 @@ autotune:build:
     - .cmake-minimum
     - .gpus:rocm-gpus
     - .rules:benchmark
+  before_script:
+    - !reference [".cmake-minimum", before_script]
+    - $SUDO_CMD apt-get update -qq
+    - $SUDO_CMD apt-get install -qq -y zstd
   variables:
     BENCHMARK_TARGETS: benchmark_config_tuning
   script:
@@ -291,22 +295,13 @@ autotune:build:
       -D GPU_TARGETS=$GPU_TARGETS
     - cmake --build . --target $BENCHMARK_TARGETS
     - 'rm -rf $BUILD_DIR/benchmark/benchmark*.parallel'
-    # remove benchmark executables if their size together is too large for gitlab ci to handle
-    - |
-      total_size_bytes=0
-      while read -r file_size; do
-        total_size_bytes=$((total_size_bytes + file_size))
-      done < <(stat --format="%s" benchmark/benchmark*)
-      total_size_gib="$(numfmt --round=down --to-unit=Gi "$total_size_bytes")"
-      if [ "$total_size_gib" -ge 3 ]; then
-        printf "Total size: %s (%d bytes) > 3GiB, skipping benchmark executables from the artifact.\n" \
-          "$(numfmt --to=iec-i "$total_size_bytes")" "$total_size_bytes"
-        rm benchmark/benchmark*
-      fi
+    # The autotune benchmarks get very large, above GitLabs upload limit. Fortunately they compress well.
+    # We'll put them all in a single archive to compress them to a few hundred MB.
+    - find benchmark -type f -executable -print0 | tar -I zstd -cvf benchmarks.tar.zstd --null -T -
 
   artifacts:
     paths:
-      - $BUILD_DIR/benchmark/benchmark*
+      - $BUILD_DIR/benchmarks.tar.zstd
       - $BUILD_DIR/.ninja_log
       - $BUILD_DIR/deps/googlebenchmark/
     expire_in: 1 week
@@ -350,7 +345,7 @@ test:
     - cd $CI_PROJECT_DIR/build
     - ctest --output-on-failure
 
-# Disabled due to extensive link times. 
+# Disabled due to extensive link times.
 # This is tracked in issue 679
 # test-windows-debug:
 #   extends:
@@ -525,9 +520,13 @@ autotune:execute-tuning:
   artifacts:
     paths:
       - ${AUTOTUNE_RESULT_DIR}/*.json
+  before_script:
+    - !reference [".cmake-minimum", before_script]
+    - $SUDO_CMD apt-get update -qq
+    - $SUDO_CMD apt-get install -qq -y zstd
   script:
-    - >
-      cd "${CI_PROJECT_DIR}"
+    - cd "${CI_PROJECT_DIR}"
+    - tar -I zstd -xvf "${BUILD_DIR}/benchmarks.tar.zstd" -C "${BUILD_DIR}/"
     - |
       if [ ! -d "${BUILD_DIR}/benchmark" ]; then
           echo "There are no benchmark executables. Run the build job with a BUILD_TARGET."