CI: Separate dependency build from plugin build

* Goal is to enable caching of dependency builds to speed up CI build times significantly * Should ease transition to new plugin build template version
WarmUpTill · Sep 9, 2023 · 129a1f7 · 129a1f7
1 parent 5324fa9
commit 129a1f7
Show file tree

Hide file tree

Showing 10 changed files with 588 additions and 204 deletions.
diff --git a/.github/actions/build-dependencies/action.yml b/.github/actions/build-dependencies/action.yml
@@ -0,0 +1,78 @@
+name: 'Setup plugin build dependencies'
+description: 'Builds the plugin build dependencies'
+inputs:
+  target:
+    description: 'Build target for dependencies'
+    required: true
+  config:
+    description: 'Build configuration'
+    required: false
+    default: 'Release'
+  visualStudio:
+    description: 'Visual Studio version (Windows only)'
+    required: false
+    default: 'Visual Studio 16 2019'
+  workingDirectory:
+    description: 'Working directory for packaging'
+    required: false
+    default: ${{ github.workspace }}
+runs:
+  using: 'composite'
+  steps:
+    - name: Setup cmake
+      uses: jwlawson/actions-setup-cmake@v1.13
+      with:
+        cmake-version: '3.24.x'
+    - name: Run macOS Build
+      if: ${{ runner.os == 'macOS' }}
+      shell: zsh {0}
+      run: |
+        build_args=(
+          -c ${{ inputs.config }}
+          -t macos-${{ inputs.target }}
+        )
+
+        if (( ${+CI} && ${+RUNNER_DEBUG} )) build_args+=(--debug)
+
+        ${{ inputs.workingDirectory }}/.github/scripts/build-deps-macos.zsh -o ${{ env.DEP_DIR }} ${build_args}
+
+    - name: Run Linux Build
+      if: ${{ runner.os == 'Linux' }}
+      shell: bash
+      run: |
+        build_args=(
+          -c ${{ inputs.config }}
+          -t linux-${{ inputs.target }}
+        )
+
+        if [[ -n "${CI}" && -n "${RUNNER_DEBUG}" ]]; then
+          build_args+=(--debug)
+        fi
+
+        ${{ inputs.workingDirectory }}/.github/scripts/build-deps-linux.sh -o ${{ env.DEP_DIR }} "${build_args[@]}"
+
+    - name: Restore cached dependencies
+      id: restore-cache
+      uses: actions/cache@v3
+      with:
+        path: ${{ env.DEP_DIR }}
+        key: ${{ env.DEP_DIR }}-${{ runner.os }}
+
+    - name: Run Windows Build
+      if: ${{ runner.os == 'Windows' && steps.restore-cache.outputs.cache-hit != 'true' }}
+      shell: pwsh
+      run: |
+        $BuildArgs = @{
+          Target = '${{ inputs.target }}'
+          Configuration = '${{ inputs.config }}'
+          CMakeGenerator = '${{ inputs.visualStudio }}'
+        }
+
+        if ( ( Test-Path env:CI ) -and ( Test-Path env:RUNNER_DEBUG ) ) {
+          $BuildArgs += @{
+            Debug = $true
+          }
+        }
+
+        ${{ inputs.workingDirectory }}/.github/scripts/Build-Deps-Windows.ps1 -OutDirName ${{ env.DEP_DIR }} @BuildArgs
+
diff --git a/.github/actions/build-plugin/action.yml b/.github/actions/build-plugin/action.yml
@@ -86,4 +86,4 @@ runs:
           }
         }
 
-        ${{ inputs.workingDirectory }}/.github/scripts/Build-Windows.ps1 @BuildArgs
+        ${{ inputs.workingDirectory }}/.github/scripts/Build-Windows.ps1 -ADVSSDepName ${{ env.DEP_DIR }} @BuildArgs
diff --git a/.github/scripts/.build-deps.zsh b/.github/scripts/.build-deps.zsh
@@ -0,0 +1,289 @@
+#!/usr/bin/env zsh
+
+builtin emulate -L zsh
+setopt EXTENDED_GLOB
+setopt PUSHD_SILENT
+setopt ERR_EXIT
+setopt ERR_RETURN
+setopt NO_UNSET
+setopt PIPE_FAIL
+setopt NO_AUTO_PUSHD
+setopt NO_PUSHD_IGNORE_DUPS
+setopt FUNCTION_ARGZERO
+
+## Enable for script debugging
+# setopt WARN_CREATE_GLOBAL
+# setopt WARN_NESTED_VAR
+# setopt XTRACE
+
+autoload -Uz is-at-least && if ! is-at-least 5.2; then
+  print -u2 -PR "%F{1}${funcstack[1]##*/}:%f Running on Zsh version %B${ZSH_VERSION}%b, but Zsh %B5.2%b is the minimum supported version. Upgrade Zsh to fix this issue."
+  exit 1
+fi
+
+_trap_error() {
+  print -u2 -PR '%F{1}    ✖︎ script execution error%f'
+  print -PR -e "
+    Callstack:
+    ${(j:\n     :)funcfiletrace}
+  "
+  exit 2
+}
+
+build() {
+  if (( ! ${+SCRIPT_HOME} )) typeset -g SCRIPT_HOME=${ZSH_ARGZERO:A:h}
+  local host_os=${${(s:-:)ZSH_ARGZERO:t:r}[3]}
+  local target="${host_os}-${CPUTYPE}"
+  local project_root=${SCRIPT_HOME:A:h:h}
+  local buildspec_file="${project_root}/buildspec.json"
+
+  trap '_trap_error' ZERR
+
+  fpath=("${SCRIPT_HOME}/utils.zsh" ${fpath})
+  autoload -Uz log_info log_error log_output set_loglevel check_${host_os} setup_${host_os} setup_obs setup_ccache
+
+  if [[ ! -r ${buildspec_file} ]] {
+    log_error \
+      'No buildspec.json found. Please create a build specification for your project.' \
+      'A buildspec.json.template file is provided in the repository to get you started.'
+    return 2
+  }
+
+  typeset -g -a skips=()
+  local -i _verbosity=1
+  local -r _version='1.0.0'
+  local -r -a _valid_targets=(
+    macos-x86_64
+    macos-arm64
+    macos-universal
+    linux-x86_64
+  )
+  local -r -a _valid_configs=(Debug RelWithDebInfo Release MinSizeRel)
+  if [[ ${host_os} == 'macos' ]] {
+    local -r -a _valid_generators=(Xcode Ninja 'Unix Makefiles')
+    local generator="${${CI:+Ninja}:-Xcode}"
+  } else {
+    local -r -a _valid_generators=(Ninja 'Unix Makefiles')
+    local generator='Ninja'
+  }
+  local -r _usage="
+Usage: %B${functrace[1]%:*}%b <option> [<options>]
+
+%BOptions%b:
+
+%F{yellow} Build configuration options%f
+ -----------------------------------------------------------------------------
+  %B-t | --target%b                     Specify target - default: %B%F{green}${host_os}-${CPUTYPE}%f%b
+  %B-c | --config%b                     Build configuration - default: %B%F{green}RelWithDebInfo%f%b
+  %B-o | --out%b                        Output directory - default: %B%F{green}RelWithDebInfo%f%b
+  %B--generator%b                       Specify build system to generate - default: %B%F{green}Ninja%f%b
+                                    Available generators:
+                                      - Ninja
+                                      - Unix Makefiles
+                                      - Xcode (macOS only)
+
+%F{yellow} Output options%f
+ -----------------------------------------------------------------------------
+  %B-q | --quiet%b                      Quiet (error output only)
+  %B-v | --verbose%b                    Verbose (more detailed output)
+  %B--skip-[all|build|deps|unpack]%b    Skip all|building OBS|checking for dependencies|unpacking dependencies
+  %B--debug%b                           Debug (very detailed and added output)
+
+%F{yellow} General options%f
+ -----------------------------------------------------------------------------
+  %B-h | --help%b                       Print this usage help
+  %B-V | --version%b                    Print script version information"
+
+  local -a args
+  while (( # )) {
+    case ${1} {
+      -t|--target|-c|--config|--generator)
+        if (( # == 1 )) || [[ ${2:0:1} == '-' ]] {
+          log_error "Missing value for option %B${1}%b"
+          log_output ${_usage}
+          exit 2
+        }
+        ;;
+    }
+    case ${1} {
+      --)
+        shift
+        args+=($@)
+        break
+        ;;
+      -t|--target)
+        if (( ! ${_valid_targets[(Ie)${2}]} )) {
+          log_error "Invalid value %B${2}%b for option %B${1}%b"
+          log_output ${_usage}
+          exit 2
+        }
+        target=${2}
+        shift 2
+        ;;
+      -c|--config)
+        if (( ! ${_valid_configs[(Ie)${2}]} )) {
+          log_error "Invalid value %B${2}%b for option %B${1}%b"
+          log_output ${_usage}
+          exit 2
+        }
+        BUILD_CONFIG=${2}
+        shift 2
+        ;;
+      -o|--out)
+        OUT_DIR="${2}"
+        shift 2
+        ;;
+      -q|--quiet) (( _verbosity -= 1 )) || true; shift ;;
+      -v|--verbose) (( _verbosity += 1 )); shift ;;
+      -h|--help) log_output ${_usage}; exit 0 ;;
+      -V|--version) print -Pr "${_version}"; exit 0 ;;
+      --debug) _verbosity=3; shift ;;
+      --generator)
+        if (( ! ${_valid_generators[(Ie)${2}]} )) {
+          log_error "Invalid value %B${2}%b for option %B${1}%b"
+          log_output ${_usage}
+          exit 2
+        }
+        generator=${2}
+        shift 2
+        ;;
+      --skip-*)
+        local _skip="${${(s:-:)1}[-1]}"
+        local _check=(all deps unpack build)
+        (( ${_check[(Ie)${_skip}]} )) || log_warning "Invalid skip mode %B${_skip}%b supplied"
+        typeset -g -a skips=(${skips} ${_skip})
+        shift
+        ;;
+      *) log_error "Unknown option: %B${1}%b"; log_output ${_usage}; exit 2 ;;
+    }
+  }
+
+  set -- ${(@)args}
+  set_loglevel ${_verbosity}
+
+  check_${host_os}
+  setup_ccache
+
+  typeset -g QT_VERSION
+  typeset -g DEPLOYMENT_TARGET
+  typeset -g OBS_DEPS_VERSION
+  setup_${host_os}
+
+  local product_name
+  local product_version
+  local git_tag="$(git describe --tags)"
+
+  read -r product_name product_version <<< \
+    "$(jq -r '. | {name, version} | join(" ")' ${buildspec_file})"
+
+  if [[ "${git_tag}" =~ '^([0-9]+\.){0,2}(\*|[0-9]+)$' ]] {
+    log_info "Using git tag as version identifier '${git_tag}'"
+    product_version="${git_tag}"
+  } else {
+    log_info "Using buildspec.json version identifier '${product_version}'"
+  }
+
+  if [[ -z "${OUT_DIR}" ]] {
+    OUT_DIR="advss-build-dependencies"
+  }
+  mkdir -p "${project_root}/../${OUT_DIR}"
+  local advss_dep_path="$(readlink -f ${project_root}/../${OUT_DIR})"
+
+  case ${host_os} {
+    macos)
+        local opencv_dir="${project_root}/deps/opencv"
+        local opencv_build_dir="${opencv_dir}/build_${target##*-}"
+
+        local -a opencv_cmake_args=(
+          -DCMAKE_BUILD_TYPE=Release
+          -DBUILD_LIST=core,imgproc,objdetect
+          -DCMAKE_OSX_ARCHITECTURES=${${target##*-}//universal/x86_64;arm64}
+          -DCMAKE_OSX_DEPLOYMENT_TARGET=${DEPLOYMENT_TARGET:-10.15}
+        )
+
+        if [ "${target}" != "macos-x86_64" ]; then
+          opencv_cmake_args+=(-DWITH_IPP=OFF)
+        fi
+
+        pushd ${opencv_dir}
+        log_info "Configure OpenCV ..."
+        cmake -S . -B ${opencv_build_dir} -G ${generator} ${opencv_cmake_args}
+
+        log_info "Building OpenCV ..."
+        cmake --build ${opencv_build_dir} --config Release
+
+        log_info "Installing OpenCV..."
+        cmake --install ${opencv_build_dir} --prefix "${advss_dep_path}" --config Release || true
+        popd
+
+        local leptonica_dir="${project_root}/deps/leptonica"
+        local leptonica_build_dir="${leptonica_dir}/build_${target##*-}"
+
+        local -a leptonica_cmake_args=(
+          -DCMAKE_BUILD_TYPE=Release
+          -DCMAKE_OSX_ARCHITECTURES=${${target##*-}//universal/x86_64;arm64}
+          -DCMAKE_OSX_DEPLOYMENT_TARGET=${DEPLOYMENT_TARGET:-10.15}
+          -DSW_BUILD=OFF
+          -DOPENJPEG_SUPPORT=OFF
+          -DLIBWEBP_SUPPORT=OFF
+          -DCMAKE_DISABLE_FIND_PACKAGE_GIF=TRUE
+          -DCMAKE_DISABLE_FIND_PACKAGE_JPEG=TRUE
+          -DCMAKE_DISABLE_FIND_PACKAGE_TIFF=TRUE
+          -DCMAKE_DISABLE_FIND_PACKAGE_PNG=TRUE
+        )
+
+        pushd ${leptonica_dir}
+        log_info "Configure Leptonica ..."
+        cmake -S . -B ${leptonica_build_dir} -G ${generator} ${leptonica_cmake_args}
+
+        log_info "Building Leptonica ..."
+        cmake --build ${leptonica_build_dir} --config Release
+
+        log_info "Installing Leptonica..."
+        # Workaround for "unknown file attribute: H" errors when running install
+        cmake --install ${leptonica_build_dir} --prefix "${advss_dep_path}" --config Release || :
+        popd
+
+        local tesseract_dir="${project_root}/deps/tesseract"
+        local tesseract_build_dir="${tesseract_dir}/build_${target##*-}"
+
+        local -a tesseract_cmake_args=(
+          -DCMAKE_BUILD_TYPE=Release
+          -DCMAKE_OSX_ARCHITECTURES=${${target##*-}//universal/x86_64;arm64}
+          -DCMAKE_OSX_DEPLOYMENT_TARGET=${DEPLOYMENT_TARGET:-10.15}
+          -DSW_BUILD=OFF
+          -DBUILD_TRAINING_TOOLS=OFF
+          -DCMAKE_PREFIX_PATH=${advss_dep_path}
+        )
+
+        if [ "${target}" != "macos-x86_64" ]; then
+          tesseract_cmake_args+=(
+            -DCMAKE_SYSTEM_PROCESSOR=aarch64
+            -DHAVE_AVX=FALSE
+            -DHAVE_AVX2=FALSE
+            -DHAVE_AVX512F=FALSE
+            -DHAVE_FMA=FALSE
+            -DHAVE_SSE4_1=FALSE
+            -DHAVE_NEON=TRUE
+          )
+          sed -i'.original' 's/HAVE_NEON FALSE/HAVE_NEON TRUE/g' "${tesseract_dir}/CMakeLists.txt"
+        fi
+
+        pushd ${tesseract_dir}
+        log_info "Configure Tesseract ..."
+        cmake -S . -B ${tesseract_build_dir} -G ${generator} ${tesseract_cmake_args}
+
+        log_info "Building Tesseract ..."
+        cmake --build ${tesseract_build_dir} --config Release
+
+        log_info "Installing Tesseract..."
+        cmake --install ${tesseract_build_dir} --prefix "${advss_dep_path}" --config Release
+        popd
+      ;;
+    linux)
+      # Nothing to do for now
+      ;;
+  }
+}
+
+build ${@}
-Original file line number
+Diff line change
@@ Expand Up / @@ -86,4 +86,4 @@ runs: @@
               }
             }
-            ${{ inputs.workingDirectory }}/.github/scripts/Build-Windows.ps1 @BuildArgs
+            ${{ inputs.workingDirectory }}/.github/scripts/Build-Windows.ps1 -ADVSSDepName ${{ env.DEP_DIR }} @BuildArgs