ci: parallelize CMake quickstart builds (#7648)

googleapis · Nov 22, 2021 · 785c3aa · 785c3aa
1 parent a5ebb4a
commit 785c3aa
Show file tree

Hide file tree

Showing 5 changed files with 111 additions and 16 deletions.
diff --git a/.cmake-format.py b/.cmake-format.py
@@ -42,6 +42,9 @@
             "URL": 1,
             "URL_HASH": 1,
             "BUILD_ALWAYS": 1,
+            "LOG_OUTPUT_ON_FAILURE": 1,
+            "SOURCE_DIR": 1,
+            "BINARY_DIR": 1,
         },
     }
 }
diff --git a/ci/cloudbuild/builds/cmake-install.sh b/ci/cloudbuild/builds/cmake-install.sh
@@ -160,8 +160,15 @@ for repo_root in "ci/verify_current_targets" "ci/verify_deprecated_targets"; do
 done
 
 # Tests the installed artifacts by building and running the quickstarts.
-quickstart::build_cmake_and_make "${INSTALL_PREFIX}"
-quickstart::run_cmake_and_make "${INSTALL_PREFIX}"
+# shellcheck disable=SC2046
+libraries="$(printf ";%s" $(quickstart::libraries))"
+libraries="${libraries:1}"
+cmake -G Ninja \
+  -S "${PROJECT_ROOT}/ci/verify_quickstart" \
+  -B "${PROJECT_ROOT}/cmake-out/quickstart" \
+  "-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX}" \
+  "-DLIBRARIES=${libraries}"
+cmake --build "${PROJECT_ROOT}/cmake-out/quickstart"
 
 # Deletes all the installed artifacts, and installs only the runtime components
 # to verify that we can still execute the compiled quickstart programs.

diff --git a/ci/cloudbuild/builds/lib/quickstart.sh b/ci/cloudbuild/builds/lib/quickstart.sh
@@ -53,13 +53,13 @@ function quickstart::build_gcs_grpc_quickstart() {
   local src_dir="${PROJECT_ROOT}/google/cloud/storage/quickstart"
 
   io::log "[ CMake ]"
-  local cmake_bin_dir="${PROJECT_ROOT}/cmake-out/quickstart-cmake-storage"
+  local cmake_bin_dir="${PROJECT_ROOT}/cmake-out/quickstart/cmake-storage"
   cmake -H"${src_dir}" -B"${cmake_bin_dir}" "-DCMAKE_PREFIX_PATH=${prefix}"
   cmake --build "${cmake_bin_dir}" --target quickstart_grpc
 
   echo
   io::log "[ Make ]"
-  local makefile_bin_dir="${PROJECT_ROOT}/cmake-out/quickstart-makefile-storage"
+  local makefile_bin_dir="${PROJECT_ROOT}/cmake-out/quickstart/makefile-storage"
   mkdir -p "${makefile_bin_dir}"
   PKG_CONFIG_PATH="${prefix}/lib64/pkgconfig:${prefix}/lib/pkgconfig:${PKG_CONFIG_PATH:-}" \
     make -C "${src_dir}" BIN="${makefile_bin_dir}" "${makefile_bin_dir}/quickstart_grpc"
@@ -71,13 +71,13 @@ function quickstart::build_one_quickstart() {
   local bin_dir_suffix="$3"
 
   io::log "[ CMake ]"
-  local cmake_bin_dir="${PROJECT_ROOT}/cmake-out/quickstart-cmake-${bin_dir_suffix}"
+  local cmake_bin_dir="${PROJECT_ROOT}/cmake-out/quickstart/cmake-${bin_dir_suffix}"
   cmake -H"${src_dir}" -B"${cmake_bin_dir}" "-DCMAKE_PREFIX_PATH=${prefix}"
   cmake --build "${cmake_bin_dir}"
 
   echo
   io::log "[ Make ]"
-  local makefile_bin_dir="${PROJECT_ROOT}/cmake-out/quickstart-makefile-${bin_dir_suffix}"
+  local makefile_bin_dir="${PROJECT_ROOT}/cmake-out/quickstart/makefile-${bin_dir_suffix}"
   mkdir -p "${makefile_bin_dir}"
   PKG_CONFIG_PATH="${prefix}/lib64/pkgconfig:${prefix}/lib/pkgconfig:${PKG_CONFIG_PATH:-}" \
     make -C "${src_dir}" BIN="${makefile_bin_dir}"
@@ -108,12 +108,12 @@ function quickstart::run_gcs_grpc_quickstart() {
   io::log_h2 "Running quickstart for GCS+gRPC"
 
   io::log "[ CMake ]"
-  local cmake_bin_dir="${PROJECT_ROOT}/cmake-out/quickstart-cmake-storage"
+  local cmake_bin_dir="${PROJECT_ROOT}/cmake-out/quickstart/cmake-storage"
   "${cmake_bin_dir}/quickstart_grpc" "${run_args[@]}"
 
   echo
   io::log "[ Make ]"
-  local makefile_bin_dir="${PROJECT_ROOT}/cmake-out/quickstart-makefile-storage"
+  local makefile_bin_dir="${PROJECT_ROOT}/cmake-out/quickstart/makefile-storage"
   LD_LIBRARY_PATH="${prefix}/lib64:${prefix}/lib:${LD_LIBRARY_PATH:-}" \
     "${makefile_bin_dir}/quickstart_grpc" "${run_args[@]}"
 }
@@ -125,12 +125,12 @@ function quickstart::run_one_quickstart() {
   local -a run_args=("$@")
 
   io::log "[ CMake ]"
-  local cmake_bin_dir="${PROJECT_ROOT}/cmake-out/quickstart-cmake-${bin_dir_suffix}"
+  local cmake_bin_dir="${PROJECT_ROOT}/cmake-out/quickstart/cmake-${bin_dir_suffix}"
   "${cmake_bin_dir}/quickstart" "${run_args[@]}"
 
   echo
   io::log "[ Make ]"
-  local makefile_bin_dir="${PROJECT_ROOT}/cmake-out/quickstart-makefile-${bin_dir_suffix}"
+  local makefile_bin_dir="${PROJECT_ROOT}/cmake-out/quickstart/makefile-${bin_dir_suffix}"
   LD_LIBRARY_PATH="${prefix}/lib64:${prefix}/lib:${LD_LIBRARY_PATH:-}" \
     "${makefile_bin_dir}/quickstart" "${run_args[@]}"
 }
diff --git a/ci/cloudbuild/builds/quickstart-cmake.sh b/ci/cloudbuild/builds/quickstart-cmake.sh
@@ -30,14 +30,20 @@ vcpkg_dir="$(vcpkg::root_dir)"
 env -C "${vcpkg_dir}" ./vcpkg remove --outdated --recurse
 env -C "${vcpkg_dir}" ./vcpkg install google-cloud-cpp
 
-# Compiles and runs all the quickstart CMake builds.
+# Compiles all the quickstart builds
+# shellcheck disable=SC2046
+libraries="$(printf ";%s" $(quickstart::libraries))"
+libraries="${libraries:1}"
+cmake -G Ninja \
+  -S "${PROJECT_ROOT}/ci/verify_quickstart" \
+  -B "${PROJECT_ROOT}/cmake-out/quickstart" \
+  -DCMAKE_TOOLCHAIN_FILE="${vcpkg_dir}/scripts/buildsystems/vcpkg.cmake" \
+  -DLIBRARIES="${libraries}"
+cmake --build "${PROJECT_ROOT}/cmake-out/quickstart" --target verify-quickstart-cmake
+
 for lib in $(quickstart::libraries); do
   io::log_h2 "Running CMake quickstart for ${lib}"
-  src_dir="${PROJECT_ROOT}/google/cloud/${lib}/quickstart"
-  bin_dir="${PROJECT_ROOT}/cmake-out/quickstart-${lib}"
-  flag="-DCMAKE_TOOLCHAIN_FILE=${vcpkg_dir}/scripts/buildsystems/vcpkg.cmake"
-  cmake -H"${src_dir}" -B"${bin_dir}" "${flag}"
-  cmake --build "${bin_dir}"
+  bin_dir="${PROJECT_ROOT}/cmake-out/quickstart/cmake-${lib}"
   mapfile -t run_args < <(quickstart::arguments "${lib}")
   "${bin_dir}/quickstart" "${run_args[@]}"
 done
diff --git a/ci/verify_quickstart/CMakeLists.txt b/ci/verify_quickstart/CMakeLists.txt
@@ -0,0 +1,79 @@
+# ~~~
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ~~~
+
+cmake_minimum_required(VERSION 3.14)
+project(verify-quickstart CXX)
+
+# The list of libraries must be provided in the command line
+if (NOT LIBRARIES)
+    message(FATAL_ERROR "Missing -DLIBRARIES argument, for example:"
+                        " -DLIBRARIES=\"storage;spanner\"")
+endif ()
+
+set(ARGS)
+if (CMAKE_PREFIX_PATH)
+    list(APPEND ARGS "-DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}")
+endif ()
+if (CMAKE_TOOLCHAIN_FILE)
+    list(APPEND ARGS "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}")
+endif ()
+
+add_custom_target(verify-quickstart-cmake ALL)
+add_custom_target(verify-quickstart-make ALL)
+
+include(ExternalProject)
+
+foreach (library IN LISTS LIBRARIES)
+    ExternalProject_Add(
+        verify-quickstart-cmake-${library}
+        EXCLUDE_FROM_ALL ON
+        PREFIX "${PROJECT_BINARY_DIR}/cmake-${library}-prefix"
+        SOURCE_DIR
+            "${PROJECT_SOURCE_DIR}/../../google/cloud/${library}/quickstart"
+        BINARY_DIR "${PROJECT_BINARY_DIR}/cmake-${library}"
+        CMAKE_ARGS ${ARGS}
+        INSTALL_COMMAND ""
+                        # Use LOG_* ON to keep builds quiet on success
+        LOG_DOWNLOAD ON
+        LOG_CONFIGURE ON
+        LOG_BUILD ON
+        LOG_OUTPUT_ON_FAILURE ON)
+    add_dependencies(verify-quickstart-cmake verify-quickstart-cmake-${library})
+
+    ExternalProject_Add(
+        verify-quickstart-makefile-${library}
+        EXCLUDE_FROM_ALL ON
+        PREFIX "${PROJECT_BINARY_DIR}/makefile-${library}-prefix"
+        SOURCE_DIR
+            "${PROJECT_SOURCE_DIR}/../../google/cloud/${library}/quickstart"
+        BINARY_DIR "${PROJECT_BINARY_DIR}/makefile-${library}"
+        CONFIGURE_COMMAND ""
+        BUILD_COMMAND
+            "env"
+            "PKG_CONFIG_PATH=${CMAKE_PREFIX_PATH}/lib64/pkgconfig:${CMAKE_PREFIX_PATH}/lib/pkgconfig:$ENV{PKG_CONFIG_PATH}"
+            "make"
+            "-C"
+            "<SOURCE_DIR>"
+            "BIN=<BINARY_DIR>"
+        INSTALL_COMMAND ""
+                        # Use LOG_* ON to keep builds quiet on success
+        LOG_DOWNLOAD ON
+        LOG_CONFIGURE ON
+        LOG_BUILD ON
+        LOG_OUTPUT_ON_FAILURE ON)
+    add_dependencies(verify-quickstart-make
+                     verify-quickstart-makefile-${library})
+endforeach ()