Skip to content
Build

Merge pull request #1021 from koordinates/merge-ancestor #6276

Sign in to view logs

Merge pull request #1021 from koordinates/merge-ancestor

Merge pull request #1021 from koordinates/merge-ancestor #6276

Workflow file for this run

.github/workflows/build.yml at 60b702d
---
name: Build
on: [push, pull_request]
env:
PY_VER: "3.11"
CMAKE_VERSION: "~3.25.0"
FORCE_COLOR: "YES"
KART_S3_TEST_DATA_POINT_CLOUD: "s3://kart-bring-your-own-data-poc/auckland-small-laz1.2/*.laz"
KART_S3_TEST_DATA_RASTER: "s3://kart-bring-your-own-data-poc/erorisk_si/*.tif"
AWS_NO_SIGN_REQUEST: "1"
jobs:
type-checking:
name: Run Type Checking
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: "3.11"
cache: 'pip' # caching pip dependencies
- run: pip install -r requirements/requirements.txt -r requirements/test.txt -r requirements/dev.txt
- name: Run mypy
run: mypy
# Increase rate limits for public registry
amazon-ecr-auth:
runs-on: ubuntu-latest
permissions:
id-token: write
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: 'arn:aws:iam::276514628126:role/ci-kart'
aws-region: us-east-1
mask-aws-account-id: 'false'
role-session-name: kart-ci-gha
- name: Login to Amazon ECR
id: login-ecr
uses: aws-actions/amazon-ecr-login@v2
with:
registry-type: public
mask-password: 'false'
outputs:
registry: ${{ steps.login-ecr.outputs.registry }}
docker_username: ${{ steps.login-ecr.outputs.docker_username_public_ecr_aws }}
docker_password: ${{ steps.login-ecr.outputs.docker_password_public_ecr_aws }}
#
# Linux development build on modern Ubuntu
#
Linux-dev:
name: "Linux dev"
runs-on: ubuntu-22.04
needs: amazon-ecr-auth
# We want to run on external PRs, but not on our own internal PRs as they'll be run
# by the push to the branch.
# https://git.luolix.topmunity/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012/7
if: >
(
github.event_name == 'push'
|| github.event.pull_request.head.repo.full_name != github.repository
)
env:
CCACHE_DIR: ${{ github.workspace }}/.cache/ccache
CCACHE_COMPRESS: "1"
KART_POSTGIS_URL: "postgresql://postgres:@localhost:5432/postgres"
KART_SQLSERVER_URL: "mssql://sa:PassWord1@localhost:1433/master"
KART_MYSQL_URL: "mysql://root:PassWord1@localhost:3306"
services:
postgis:
# https://github.com/postgis/docker-postgis/issues/216
image: ghcr.io/baosystems/postgis:15
options: >-
--health-cmd pg_isready
--health-interval 2s
--health-timeout 2s
--health-retries 5
-e POSTGRES_HOST_AUTH_METHOD=trust
ports:
- 5432:5432
sqlserver:
image: mcr.microsoft.com/mssql/server
options: >-
-e ACCEPT_EULA=Y
-e SA_PASSWORD=PassWord1
ports:
- 1433:1433
mysql:
image: public.ecr.aws/docker/library/mysql:latest
credentials:
username: ${{ needs.amazon-ecr-auth.outputs.docker_username }}
password: ${{ needs.amazon-ecr-auth.outputs.docker_password }}
options: >-
-e MYSQL_ROOT_PASSWORD=PassWord1
ports:
- 3306:3306
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
#
# setup
#
- name: "setup: cmake & ninja"
uses: lukka/get-cmake@v3.25.2 # workaround for node 20 issue
with:
cmakeVersion: "${{ env.CMAKE_VERSION }}"
- name: "setup: ccache"
uses: actions/cache@v3
with:
path: ${{ env.CCACHE_DIR }}
key: vendor-ccache-ubuntu-22.04-cmake-vcpkg
- name: "setup: misc"
run: |
sudo apt-get update -q -y
sudo apt-get install -q -y --no-install-recommends autoconf-archive ccache
mkdir -p ${{ env.CCACHE_DIR }}
echo "/usr/lib/ccache" >> $GITHUB_PATH
- name: "setup: vcpkg"
uses: lukka/run-vcpkg@v11.1
with:
vcpkgDirectory: "${{ github.workspace }}/vcpkg-vendor/vcpkg"
vcpkgJsonGlob: "**/vcpkg-vendor/vcpkg.json"
doNotCache: false # do-cache
#
# App Build
#
- name: "app: version"
id: version
run: |
if [[ '${{ github.ref_type }}' == 'tag' ]] && [[ '${{ github.ref_name }}' =~ ^v(.*) ]]; then
VER=$(echo ${{ github.ref_name }} | sed 's/^v//')
IS_RELEASE=1
else
VER=$(sed -E "s/(.*)/\1+ci.${{ github.run_number }}.git${GITHUB_SHA::8}/" kart/VERSION)
IS_RELEASE=0
fi
echo "App Version: $VER"
echo "Is Release? $IS_RELEASE"
echo "value=$VER" >> $GITHUB_OUTPUT
echo "KART_VERSION=$VER" >> $GITHUB_ENV
echo "is_release=$IS_RELEASE" >> $GITHUB_OUTPUT
- name: "app: configuration & vendor dependencies"
uses: lukka/run-cmake@v10.6
with:
configurePreset: ci-linux
- name: "app: build"
uses: lukka/run-cmake@v10.6
with:
configurePreset: ci-linux
buildPreset: ci-linux
#
# App tests & checks
#
- name: "test: smoke test"
run: |
./build/kart --version
- name: "test: create no_postgis environment for tests"
shell: bash
run: |
sudo apt-get -y install postgresql
psql postgresql://postgres:@localhost:5432/postgres -c 'CREATE DATABASE no_postgis;'
echo "KART_NO_POSTGIS_URL=postgresql://postgres:@localhost:5432/no_postgis" >> $GITHUB_ENV
- name: "test: install database drivers for tests"
run: |
curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add -
curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list > /dev/null
sudo apt-get update -q -y
sudo ACCEPT_EULA=Y apt-get install -q -y msodbcsql17
- name: "test: install fienode for reflink testing"
run: |
curl -L https://github.com/pwaller/fienode/releases/download/v1.0/fienode-linux-amd64 --output fienode
chmod +x fienode
mv fienode /usr/local/bin/
- name: "test: unit tests"
uses: lukka/run-cmake@v10.6
with:
configurePreset: ci-linux
testPreset: ci-linux
- name: "test: dependency license check"
run: |
cmake --build build --target py-license-check
#
# Packaging
#
- name: "bundle: assemble"
uses: lukka/run-cmake@v10.6
with:
configurePreset: ci-linux
buildPreset: ci-bundle-linux
- name: "bundle: smoke test"
shell: bash
run: |
./build/pyinstaller/dist/kart/kart --version
- name: "bundle: e2e tests"
uses: lukka/run-cmake@v10.6
with:
configurePreset: ci-linux
testPreset: ci-e2e-linux
#
# Install
#
- name: "install"
shell: bash
run: |
cmake --install ./build --verbose
/opt/kart/kart --version
command -v kart
kart --version
- name: "install: e2e tests"
shell: bash
run: |
time tests/scripts/e2e-1.sh
#
# Linux build for installers
# Runs on centos 7 for maximum compatibility
#
Linux-Package:
strategy:
fail-fast: false
matrix:
os:
- base: ubuntu-22.04
label: centos7/amd64
image: quay.io/pypa/manylinux2014_x86_64
arch: amd64
arch_triplet: x64-linux
build_parallel: ""
- base: buildjet-4vcpu-ubuntu-2204-arm
label: centos7/arm64
image: quay.io/pypa/manylinux2014_aarch64
arch: arm64
arch_triplet: arm64-linux
build_parallel: 1
name: Linux ${{ matrix.os.arch }}
runs-on: ${{ matrix.os.base }}
needs: amazon-ecr-auth
# We want to run on external PRs, but not on our own internal PRs as they'll be run
# by the push to the branch.
# https://git.luolix.topmunity/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012/7
if: >
(
github.event_name == 'push'
|| github.event.pull_request.head.repo.full_name != github.repository
)
services:
postgis:
# https://github.com/postgis/docker-postgis/issues/216
image: ghcr.io/baosystems/postgis:15
options: >-
--health-cmd pg_isready
--health-interval 2s
--health-timeout 2s
--health-retries 5
-e POSTGRES_HOST_AUTH_METHOD=trust
ports:
- 5432:5432
sqlserver:
image: mcr.microsoft.com/mssql/server
options: >-
-e ACCEPT_EULA=Y
-e SA_PASSWORD=PassWord1
ports:
- 1433:1433
mysql:
image: public.ecr.aws/docker/library/mysql:latest
credentials:
username: ${{ needs.amazon-ecr-auth.outputs.docker_username }}
password: ${{ needs.amazon-ecr-auth.outputs.docker_password }}
options: >-
-e MYSQL_ROOT_PASSWORD=PassWord1
ports:
- 3306:3306
env:
ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
KART_POSTGIS_URL: "postgresql://postgres:@postgis:5432/postgres"
SQLSERVER_URL: "mssql://sa:PassWord1@sqlserver:1433/master?TrustServerCertificate=yes"
KART_MYSQL_URL: "mysql://root:PassWord1@mysql:3306"
ARCH: "${{ matrix.os.arch }}"
ARCH_TRIPLET: "${{ matrix.os.arch_triplet }}"
GITHUB_WORKSPACE: "/__w/kart/kart" # FIXME?
PKG_CONFIG: "/__w/kart/kart/vcpkg-vendor/vcpkg/installed/${{ matrix.os.arch_triplet }}/tools/pkgconf/pkgconf"
PKG_CONFIG_PATH: "/__w/kart/kart/vcpkg-vendor/vcpkg/installed/${{ matrix.os.arch_triplet }}/lib/pkgconfig"
OPTIONS_FOR_CMAKE: >-
[
'-DPKG_CONFIG_EXECUTABLE=/__w/kart/kart/vcpkg-vendor/vcpkg/installed/${{ matrix.os.arch_triplet }}/tools/pkgconf/pkgconf',
'-DCPACK_DEBIAN_PACKAGE_ARCHITECTURE=${{ matrix.os.arch }}'
]
container:
image: ${{ matrix.os.image }}
env:
ACLOCAL_PATH: "/usr/local/share/aclocal:/usr/share/aclocal"
PATH: "/opt/python/cp311-cp311/bin:/opt/rh/devtoolset-10/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/github/home/.cargo/bin:/opt/mssql-tools/bin"
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
#
# setup
#
- name: "setup: cmake & ninja"
uses: lukka/get-cmake@v3.25.2 # workaround for node 20 issue
with:
cmakeVersion: "${{ env.CMAKE_VERSION }}"
- name: "setup: rust"
run: |
curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSf | sh -s -- -y
. "$HOME/.cargo/env"
rustc --version
- name: "setup: misc"
shell: bash
run: |
echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
git config --global --add safe.directory ${GITHUB_WORKSPACE}
yum install -y autoconf-archive perl-IPC-Cmd rpm-build unixODBC zip
echo "installing golang..."
curl -qL "https://go.dev/dl/go1.19.2.linux-${ARCH}.tar.gz" | tar xz -C /usr/local/
ln -sf /usr/local/go/bin/go /usr/local/go/bin/gofmt /usr/local/bin/
if [ "${ARCH}" == "arm64" ]; then
echo "VCPKG_FORCE_SYSTEM_BINARIES=1" >> $GITHUB_ENV
fi
- name: "setup: vcpkg"
uses: lukka/run-vcpkg@v11.1
with:
vcpkgDirectory: "${{ github.workspace }}/vcpkg-vendor/vcpkg"
vcpkgJsonGlob: "**/vcpkg-vendor/vcpkg.json"
doNotCache: false # do-cache
- name: "setup: pkgconf"
working-directory: /__w/_temp
shell: bash
run: |
${GITHUB_WORKSPACE}/vcpkg-vendor/vcpkg/vcpkg install pkgconf
#
# App Build
#
- name: "app: version"
id: version
shell: bash
run: |
if [[ '${{ github.ref_type }}' == 'tag' ]] && [[ '${{ github.ref_name }}' =~ ^v(.*) ]]; then
VER=$(echo ${{ github.ref_name }} | sed 's/^v//')
IS_RELEASE=1
else
VER=$(sed -E "s/(.*)/\1+ci.${{ github.run_number }}.git${GITHUB_SHA::8}/" kart/VERSION)
IS_RELEASE=0
fi
echo "App Version: $VER"
echo "Is Release? $IS_RELEASE"
echo "value=$VER" >> $GITHUB_OUTPUT
echo "KART_VERSION=$VER" >> $GITHUB_ENV
echo "is_release=$IS_RELEASE" >> $GITHUB_OUTPUT
- name: "app: configuration & vendor dependencies"
uses: lukka/run-cmake@v10.6
env:
LD_LIBRARY_PATH: "${{ env.LD_LIBRARY_PATH }}:${{ env.GITHUB_WORKSPACE }}/build/vcpkg_installed/${{ env.ARCH_TRIPLET }}/lib"
with:
configurePreset: ci-linux
configurePresetAdditionalArgs: ${{ env.OPTIONS_FOR_CMAKE }}
- name: "app: build"
shell: bash
run: |
cmake --build --preset=ci-linux --verbose --parallel ${{ matrix.os.build_parallel }}
- name: "vendor: save archive"
uses: actions/upload-artifact@v3
with:
name: vendor-linux-${{ env.ARCH }}-py${{ env.PY_VER }}
path: build/vcpkg-vendor/kart-vendor.*.tar.gz
if-no-files-found: error
#
# App tests & checks
#
- name: "test: smoke test"
shell: bash
run: |
./build/kart --version
- name: "test: create no_postgis environment for tests"
shell: bash
run: |
yum install -y postgresql
psql postgresql://postgres:@postgis:5432/postgres -c 'CREATE DATABASE no_postgis;'
echo "KART_NO_POSTGIS_URL=postgresql://postgres:@postgis:5432/no_postgis" >> $GITHUB_ENV
- name: "test: install MSSQL database drivers for tests"
if: matrix.os.arch != 'arm64'
shell: bash
run: |
curl https://packages.microsoft.com/config/rhel/7/prod.repo > /etc/yum.repos.d/mssql-release.repo
ACCEPT_EULA=Y yum install -y msodbcsql18 mssql-tools
echo "KART_SQLSERVER_URL=${SQLSERVER_URL}" >> $GITHUB_ENV
- name: "test: unit tests"
shell: bash
run: |
ctest --preset=ci-linux
#
# Packaging
#
- name: "bundle: assemble"
shell: bash
run: |
cmake --build --preset=ci-bundle-linux --verbose
- name: "bundle: smoke test"
shell: bash
run: |
./build/pyinstaller/dist/kart/kart --version
- name: "bundle: e2e tests"
shell: bash
run: |
ctest --preset=ci-e2e-linux
- name: "package: TGZ & DEB & RPM"
shell: bash
run: |
cpack --preset=ci-linux
- name: "package: save TGZ"
uses: actions/upload-artifact@v3
with:
name: Kart-linux-${{ matrix.os.arch }}-tgz
path: build/dist/Kart-*.tar.gz
if-no-files-found: error
- name: "package: save DEB"
uses: actions/upload-artifact@v3
with:
name: Kart-linux-${{ matrix.os.arch }}-deb
path: build/dist/kart_*.deb
if-no-files-found: error
- name: "package: save RPM"
uses: actions/upload-artifact@v3
with:
name: Kart-linux-${{ matrix.os.arch }}-rpm
path: build/dist/kart-*.rpm
if-no-files-found: error
- name: "release"
uses: softprops/action-gh-release@v1
if: "steps.version.outputs.is_release == 1"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
draft: true
files: |
build/dist/Kart-*.tar.gz
build/dist/kart_*.deb
build/dist/kart-*.rpm
#
# Linux Package tests
# Install on a wide variety of Linux distributions and run E2E tests
#
Linux-Package-Tests:
needs: [Linux-Package, amazon-ecr-auth]
strategy:
fail-fast: false
matrix:
arch:
- label: amd64
runner: ubuntu-22.04
- label: arm64
runner: buildjet-2vcpu-ubuntu-2204-arm
os:
- label: "tgz-ubuntu-22.04"
image: "public.ecr.aws/ubuntu/ubuntu:jammy"
type: "tgz"
- label: "ubuntu-22.04"
image: "public.ecr.aws/ubuntu/ubuntu:jammy"
type: "deb"
- label: "ubuntu-20.04"
image: "public.ecr.aws/ubuntu/ubuntu:focal"
type: "deb"
- label: "ubuntu-18.04"
image: "public.ecr.aws/ubuntu/ubuntu:bionic"
type: "deb"
- label: "debian-stable"
image: "public.ecr.aws/docker/library/debian:stable-slim"
type: "deb"
- label: "debian-oldstable"
image: "public.ecr.aws/docker/library/debian:oldstable-slim"
type: "deb"
- label: "amazonlinux-2023"
image: "public.ecr.aws/amazonlinux/amazonlinux:2023"
type: "rpm"
- label: "fedora-latest"
image: "public.ecr.aws/docker/library/fedora:latest"
type: "rpm"
- label: "almalinux-9"
image: "public.ecr.aws/docker/library/almalinux:9"
type: "rpm"
- label: "almalinux-8"
image: "public.ecr.aws/docker/library/almalinux:8"
type: "rpm"
runs-on: ${{ matrix.arch.runner }}
name: "Install-check: ${{ matrix.os.label }}/${{ matrix.arch.label }}"
continue-on-error: ${{ !!matrix.os.xfail }}
env:
ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
container:
image: ${{ matrix.os.image }}
credentials:
username: ${{ needs.amazon-ecr-auth.outputs.docker_username }}
password: ${{ needs.amazon-ecr-auth.outputs.docker_password }}
env:
DEBIAN_FRONTEND: noninteractive
steps:
- name: setup
shell: bash
run: |
case "${{ matrix.os.type }}" in
deb | tgz)
apt update -q
apt install -q -y --no-install-recommends sqlite3
;;
rpm)
yum install -y sqlite tar
;;
*)
exit 1
;;
esac
- name: get kart installer
uses: actions/download-artifact@v3
with:
name: Kart-linux-${{ matrix.arch.label }}-${{ matrix.os.type }}
- name: install kart
shell: bash
run: |
case "${{ matrix.os.type }}" in
deb)
apt install -y ./kart_*.deb
;;
rpm)
yum install -y ./kart-*.rpm
;;
tgz)
tar -xz -C /opt --strip-components=1 -f ./Kart-*.tar.gz
ln -sf /opt/kart/kart /usr/local/bin/kart
;;
*)
exit 1
;;
esac
- name: smoke test
shell: bash
run: |
/opt/kart/kart --version
command -v kart
kart --version
- uses: actions/checkout@v3
- name: e2e test (no helper)
env:
KART_USE_HELPER: "0"
shell: bash
run: |
time tests/scripts/e2e-1.sh
- name: e2e test (helper)
env:
KART_USE_HELPER: "1"
shell: bash
run: |
time tests/scripts/e2e-1.sh
#
# Linux development build using vendor archive & host python
#
Linux-dev-vendor:
name: "Linux dev+vendor"
needs: ["Linux-Package"]
runs-on: ubuntu-22.04
# We want to run on external PRs, but not on our own internal PRs as they'll be run
# by the push to the branch.
# https://git.luolix.topmunity/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012/7
if: >
(
github.event_name == 'push'
|| github.event.pull_request.head.repo.full_name != github.repository
)
env:
CCACHE_DIR: ${{ github.workspace }}/.cache/ccache
CCACHE_COMPRESS: "1"
PY_VER_ID: "cp311-cp311"
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
#
# setup
#
- name: "setup: python"
uses: actions/setup-python@v4
with:
python-version: ${{ env.PY_VER }}
cache: 'pip'
cache-dependency-path: 'requirements/*.txt'
- name: "setup: cmake & ninja"
uses: lukka/get-cmake@v3.25.2 # workaround for node 20 issue
with:
cmakeVersion: "${{ env.CMAKE_VERSION }}"
- name: "setup: ccache"
uses: actions/cache@v3
with:
path: ${{ env.CCACHE_DIR }}
key: vendor-ccache-ubuntu-22.04-cmake-vcpkg
- name: "setup: misc"
run: |
sudo apt-get update -q -y
sudo apt-get install -q -y --no-install-recommends autoconf-archive ccache unixodbc file sqlite3
mkdir -p ${{ env.CCACHE_DIR }}
echo "/usr/lib/ccache" >> $GITHUB_PATH
#
# App Build
#
- name: "app: version"
id: version
run: |
if [[ '${{ github.ref_type }}' == 'tag' ]] && [[ '${{ github.ref_name }}' =~ ^v(.*) ]]; then
VER=$(echo ${{ github.ref_name }} | sed 's/^v//')
IS_RELEASE=1
else
VER=$(sed -E "s/(.*)/\1+ci.${{ github.run_number }}.git${GITHUB_SHA::8}/" kart/VERSION)
IS_RELEASE=0
fi
echo "App Version: $VER"
echo "Is Release? $IS_RELEASE"
echo "value=$VER" >> $GITHUB_OUTPUT
echo "KART_VERSION=$VER" >> $GITHUB_ENV
echo "is_release=$IS_RELEASE" >> $GITHUB_OUTPUT
- name: "app: get vendor archive"
uses: actions/download-artifact@v3
with:
name: vendor-linux-amd64-py${{ env.PY_VER }}
- name: "app: configuration & vendor dependencies"
shell: bash
run: |
cmake -B ./build -S . \
-DUSE_VCPKG=NO \
-DVENDOR_ARCHIVE=./kart-vendor.${PY_VER_ID}-linux_x86_64.tar.gz \
-DBUILD_TESTING=YES
- name: "app: build"
shell: bash
run: |
cmake --build ./build --verbose
#
# App tests & checks
#
- name: "test: smoke test"
shell: bash
run: |
./build/kart --version
- name: "bundle: assemble"
shell: bash
run: |
cmake --build ./build --target bundle --verbose
- name: "bundle: smoke test"
shell: bash
run: |
./build/pyinstaller/dist/kart/kart --version
- name: "bundle: e2e tests"
shell: bash
working-directory: ./build
run: |
ctest -V -L e2e
#
# macOS
#
macOS:
strategy:
fail-fast: false
matrix:
os:
- id: macos-13
label: "13/x86_64"
arch_triplet: x64-osx
- id: "ghcr.io/cirruslabs/macos-ventura-xcode:14.3.1"
label: "13/arm64"
arch_triplet: arm64-osx
runs-on: ${{ matrix.os.id }}
name: macOS ${{ matrix.os.label }}
# We want to run on external PRs, but not on our own internal PRs as they'll be run
# by the push to the branch.
# https://git.luolix.topmunity/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012/7
if: >
(
github.event_name == 'push'
|| github.event.pull_request.head.repo.full_name != github.repository
)
env:
CCACHE_DIR: ${{ github.workspace }}/.cache/ccache
CCACHE_COMPRESS: "1"
ARCH_TRIPLET: ${{ matrix.os.arch_triplet }}
HOMEBREW_CACHE: ${{ github.workspace }}/.cache/brew
HOMEBREW_NO_INSTALL_CLEANUP: "1"
MACOS_CODESIGN_ID: ${{ secrets.MACOS_CODESIGN_ID }}
MACOS_PKGSIGN_ID: ${{ secrets.MACOS_PKGSIGN_ID }}
MACOS_NOTARIZE_KEYCHAIN_PROFILE: "NOTARIZE_AUTH"
MACOS_NOTARIZE_KEYCHAIN_FILENAME: "signing_temp"
MACOS_NOTARIZE_KEYCHAIN: "~/Library/Keychains/signing_temp.keychain-db"
# X.Y version needs to match PY_VER:
PY_VER_INSTALLER: "https://www.python.org/ftp/python/3.11.6/python-3.11.6-macos11.pkg"
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
#
# setup
#
- uses: maxim-lobanov/setup-xcode@v1
with:
xcode-version: latest-stable
# TODO: cache
- name: "setup: python"
run: |
curl -sSfL -o ${{runner.temp}}/py.pkg "${PY_VER_INSTALLER}"
sudo installer -pkg ${{runner.temp}}/py.pkg -target /
sudo ln -sf python3 /Library/Frameworks/Python.framework/Versions/${PY_VER}/bin/python
export PATH="/Library/Frameworks/Python.framework/Versions/${PY_VER}/bin:$PATH"
which python3
python3 -V --version
which python
python -V --version
echo /Library/Frameworks/Python.framework/Versions/${PY_VER}/bin >> $GITHUB_PATH
- name: "setup: golang"
uses: actions/setup-go@v4
with:
go-version: '^1.21.1'
- name: "setup: rust"
uses: dtolnay/rust-toolchain@stable
- name: "setup: homebrew cache"
id: cache-brew
uses: actions/cache@v3
with:
path: .cache/brew
key: brew-cmake-${{ matrix.os.id }}-vcpkg
- name: "setup: cmake & ninja"
uses: lukka/get-cmake@latest
with:
cmakeVersion: "${{ env.CMAKE_VERSION }}"
- name: "setup: ccache"
uses: actions/cache@v3
if: steps.cache-vendor-dist.outputs.cache-hit != 'true'
with:
path: ${{ env.CCACHE_DIR }}
key: vendor-ccache-${{ matrix.os.id }}-vcpkg
# The brew install command sometimes fails on the first try but succeeds on the second :-/
- name: "setup: misc"
run: |
PACKAGES="autoconf automake ccache libtool openssl pandoc pkg-config"
brew install -q $PACKAGES || brew install -q $PACKAGES
mkdir -p ${{ env.CCACHE_DIR }}
echo "$(brew --prefix)/opt/ccache/libexec" >> $GITHUB_PATH
- name: "setup: vcpkg"
uses: lukka/run-vcpkg@v11
with:
vcpkgDirectory: "${{ github.workspace }}/vcpkg-vendor/vcpkg"
vcpkgJsonGlob: "**/vcpkg-vendor/vcpkg.json"
doNotCache: false # do-cache
#
# App Build
#
- name: "app: version"
id: version
env:
MACOS_NOTARIZE_TEAMID: ${{ secrets.MACOS_NOTARIZE_TEAMID }}
run: |
if [[ '${{ github.ref_type }}' == 'tag' ]] && [[ '${{ github.ref_name }}' =~ ^v(.*) ]]; then
VER=$(echo ${{ github.ref_name }} | sed 's/^v//')
IS_RELEASE=1
else
VER=$(sed -E "s/(.*)/\1+ci.${{ github.run_number }}.git${GITHUB_SHA::8}/" kart/VERSION)
IS_RELEASE=0
fi
echo "App Version: $VER"
echo "Is Release? $IS_RELEASE"
echo "value=$VER" >> $GITHUB_OUTPUT
echo "KART_VERSION=$VER" >> $GITHUB_ENV
echo "is_release=$IS_RELEASE" >> $GITHUB_OUTPUT
if [[ -n "$MACOS_CODESIGN_ID" ]]; then
echo "MACOS_SIGN_BUNDLE=ON" >> $GITHUB_ENV
else
echo "MACOS_SIGN_BUNDLE=OFF" >> $GITHUB_ENV
fi
if (( $IS_RELEASE )) && [[ -n "$MACOS_PKGSIGN_ID" ]]; then
echo "MACOS_SIGN_PKG=ON" >> $GITHUB_ENV
else
echo "MACOS_SIGN_PKG=OFF" >> $GITHUB_ENV
fi
if (( $IS_RELEASE )) && [[ -n "$MACOS_NOTARIZE_TEAMID" ]]; then
echo "MACOS_NOTARIZE=ON" >> $GITHUB_ENV
else
echo "MACOS_NOTARIZE=OFF" >> $GITHUB_ENV
fi
- name: "app: configuration & vendor dependencies"
uses: lukka/run-cmake@v10.6
with:
configurePreset: ci-macos
- name: "app: build"
uses: lukka/run-cmake@v10.6
with:
configurePreset: ci-macos
buildPreset: ci-macos
- name: "vendor: save archive"
uses: actions/upload-artifact@v3
with:
name: vendor-macos-${{ runner.arch }}-py${{ env.PY_VER }}
path: build/vcpkg-vendor/kart-vendor.*.tar.gz
if-no-files-found: error
#
# App tests & checks
#
- name: "test: smoke test"
run: |
./build/kart --version
- name: "test: install clone_checker for reflink testing"
run: |
git clone https://github.com/dyorgio/apfs-clone-checker.git
cd apfs-clone-checker
gcc clone_checker.c -o clone_checker
chmod +x clone_checker
sudo mv clone_checker /usr/local/bin/
- name: "test: unit tests"
uses: lukka/run-cmake@v10.6
with:
configurePreset: ci-macos
testPreset: ci-macos
#
# Packaging
#
- name: "bundle: setup app signing certificate"
id: keychain
uses: apple-actions/import-codesign-certs@v1
if: env.MACOS_SIGN_BUNDLE == 'ON'
with:
# This action auto generates the keychain password (only works if the keychain name is "signing_temp").
keychain: ${{ env.MACOS_NOTARIZE_KEYCHAIN_FILENAME }}
p12-file-base64: ${{ secrets.MACOS_APP_CERT }}
p12-password: ${{ secrets.MACOS_CERT_PW }}
- name: "bundle: setup notarization auth"
if: env.MACOS_NOTARIZE == 'ON'
env:
MACOS_NOTARIZE_TEAMID: ${{ secrets.MACOS_NOTARIZE_TEAMID }}
MACOS_NOTARIZE_USER: ${{ secrets.MACOS_NOTARIZE_USER }}
MACOS_NOTARIZE_PW: ${{ secrets.MACOS_NOTARIZE_PW }}
run: |
xcrun notarytool store-credentials "$MACOS_NOTARIZE_KEYCHAIN_PROFILE" \
--apple-id "$MACOS_NOTARIZE_USER" \
--team-id "$MACOS_NOTARIZE_TEAMID" \
--password "$MACOS_NOTARIZE_PW" \
--keychain "$MACOS_NOTARIZE_KEYCHAIN"
- name: "bundle: assemble"
uses: lukka/run-cmake@v10.6
with:
configurePreset: ci-macos
buildPreset: ci-bundle-macos
- name: "bundle: smoke test"
shell: bash
run: |
./build/pyinstaller/dist/Kart.app/Contents/MacOS/kart --version
- name: "bundle: e2e tests"
uses: lukka/run-cmake@v10.6
with:
configurePreset: ci-macos
testPreset: ci-e2e-macos
- name: "package: setup installer signing certificate"
uses: apple-actions/import-codesign-certs@v1
if: env.MACOS_SIGN_PKG == 'ON'
with:
create-keychain: false
keychain-password: ${{ steps.keychain.outputs.keychain-password }}
p12-file-base64: ${{ secrets.MACOS_INSTALLER_CERT }}
p12-password: ${{ secrets.MACOS_CERT_PW }}
- name: "package: ZIP & PKG"
shell: bash
run: |
cpack --preset=ci-macos
- name: "package: save ZIP"
uses: actions/upload-artifact@v3
with:
name: Kart-macOS-${{ runner.arch }}-bundle
path: build/dist/Kart-*.zip
if-no-files-found: error
- name: "package: save PKG"
uses: actions/upload-artifact@v3
with:
name: Kart-macOS-${{ runner.arch }}-installer
path: build/dist/Kart-*.pkg
if-no-files-found: error
- name: "package: tests"
run: |
sudo installer -pkg build/dist/Kart-*.pkg -dumplog -target /
readlink $(which kart)
# make sure good sqlite is in path
export PATH="$(pwd)/build/vcpkg_installed/${{ env.ARCH_TRIPLET }}/tools:$PATH"
tests/scripts/e2e-1.sh
- name: "release"
uses: softprops/action-gh-release@v1
if: "steps.version.outputs.is_release == 1"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
draft: true
files: |
build/dist/Kart-*.zip
build/dist/Kart-*.pkg
#
# Windows
#
Windows:
strategy:
matrix:
os: [{id: windows-2019, label: "2019/x64"}]
runs-on: ${{ matrix.os.id }}
name: Windows ${{ matrix.os.label }}
env:
NINJA_VERSION: "~1.10.2" # workaround for python logging output buffering noise
SIGN_AZURE_CERTIFICATE: ${{ secrets.WIN_SIGN_AZURE_CERTIFICATE }}
# We want to run on external PRs, but not on our own internal PRs as they'll be run
# by the push to the branch.
# https://git.luolix.topmunity/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012/7
if: >
(
github.event_name == 'push'
|| github.event.pull_request.head.repo.full_name != github.repository
)
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
# Setup
- name: "msvc setup"
uses: ilammy/msvc-dev-cmd@v1
- name: "Harden vcvarsall.bat"
shell: pwsh
run: |
cd 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build'
mv vcvarsall.bat orig_vcvarsall.bat
echo @'
@if not "%VSCMD_DEBUG%" GEQ "3" echo off
FOR /F "tokens=*" %%g IN ('powershell -Command "($Env:PATH.split(';') | Select -Unique) -Join ';'"') do (SET PATH=%%g)
'@ > vcvarsall.bat
type orig_vcvarsall.bat >> vcvarsall.bat
- name: "setup: cmake"
uses: lukka/get-cmake@latest
with:
cmakeVersion: "${{ env.CMAKE_VERSION }}"
ninjaVersion: "${{ env.NINJA_VERSION }}"
- name: "setup: python"
uses: actions/setup-python@v4
with:
python-version: "${{ env.PY_VER }}"
architecture: x64
cache: 'pip'
cache-dependency-path: 'requirements/*.txt'
- name: "setup: vcpkg"
uses: lukka/run-vcpkg@v11
with:
vcpkgDirectory: "${{ github.workspace }}/vcpkg-vendor/vcpkg"
vcpkgJsonGlob: "**/vcpkg-vendor/vcpkg.json"
doNotCache: false # do-cache
- name: "setup: misc"
shell: pwsh
run: |
& dotnet tool install --global AzureSignTool --version 3.0.0
& choco install sqlite.shell
#
# App Build
#
- name: "app: version"
id: version
shell: bash
run: |
if [[ '${{ github.ref_type }}' == 'tag' ]] && [[ '${{ github.ref_name }}' =~ ^v(.*) ]]; then
VER=$(echo ${{ github.ref_name }} | sed 's/^v//')
IS_RELEASE=1
else
VER=$(sed -E "s/(.*)/\1+ci.${{ github.run_number }}.git${GITHUB_SHA::8}/" kart/VERSION)
IS_RELEASE=0
fi
echo "App Version: $VER"
echo "Is Release? $IS_RELEASE"
echo "value=$VER" >> $GITHUB_OUTPUT
echo "KART_VERSION=$VER" >> $GITHUB_ENV
echo "is_release=$IS_RELEASE" >> $GITHUB_OUTPUT
if (( $IS_RELEASE )) && [[ -n "$SIGN_AZURE_CERTIFICATE" ]]; then
echo "WIN_SIGN_BUNDLE=ON" >> $GITHUB_ENV
echo "WIN_SIGN_INSTALLER=ON" >> $GITHUB_ENV
else
echo "WIN_SIGN_BUNDLE=OFF" >> $GITHUB_ENV
echo "WIN_SIGN_INSTALLER=OFF" >> $GITHUB_ENV
fi
- name: "app: configuration & vendor dependencies"
uses: lukka/run-cmake@v10.6
with:
configurePreset: ci-windows
- name: "app: build"
uses: lukka/run-cmake@v10.6
with:
configurePreset: ci-windows
buildPreset: ci-windows
- name: "vendor: save archive"
uses: actions/upload-artifact@v3
with:
name: vendor-windows-${{ runner.arch }}-py${{ env.PY_VER }}
path: build/vcpkg-vendor/kart-vendor.*.zip
if-no-files-found: error
#
# App tests & checks
#
- name: "test: smoke test"
shell: bash
run: |
./build/kart.cmd --version
- name: "test: unit tests"
uses: lukka/run-cmake@v10.6
with:
configurePreset: ci-windows
testPreset: ci-windows
#
# Packaging
#
- name: "bundle: assemble"
uses: lukka/run-cmake@v10.6
env:
SIGN_AZURE_VAULT: ${{ secrets.WIN_SIGN_AZURE_VAULT }}
SIGN_AZURE_CLIENTID: ${{ secrets.WIN_SIGN_AZURE_CLIENTID }}
SIGN_AZURE_CLIENTSECRET: ${{ secrets.WIN_SIGN_AZURE_CLIENTSECRET }}
SIGN_AZURE_TENANTID: ${{ secrets.WIN_SIGN_AZURE_TENANTID }}
with:
configurePreset: ci-windows
buildPreset: ci-bundle-windows
- name: "bundle: smoke test"
shell: bash
run: |
./build/pyinstaller/dist/kart/kart.exe --version
- name: "bundle: e2e tests"
uses: lukka/run-cmake@v10.6
with:
configurePreset: ci-windows
testPreset: ci-e2e-windows
- name: "package: ZIP & MSI"
env:
SIGN_AZURE_VAULT: ${{ secrets.WIN_SIGN_AZURE_VAULT }}
SIGN_AZURE_CLIENTID: ${{ secrets.WIN_SIGN_AZURE_CLIENTID }}
SIGN_AZURE_CLIENTSECRET: ${{ secrets.WIN_SIGN_AZURE_CLIENTSECRET }}
SIGN_AZURE_TENANTID: ${{ secrets.WIN_SIGN_AZURE_TENANTID }}
shell: bash
run: |
if ! cpack --preset=ci-windows; then
cat build/dist/_CPack_Packages/win64/WIX/wix.log
exit 1
fi
- name: "package: save ZIP"
uses: actions/upload-artifact@v3
with:
name: Kart-windows-${{ runner.arch }}-bundle
path: build/dist/Kart-*.zip
if-no-files-found: error
- name: "package: save MSI"
uses: actions/upload-artifact@v3
with:
name: Kart-windows-${{ runner.arch }}-installer
path: build/dist/Kart-*.msi
if-no-files-found: error
- name: "package: tests"
shell: pwsh
run: |
$ErrorView = 'NormalView'
$msi = Resolve-Path -Path ".\build\dist\Kart-*.msi" | Select-Object -ExpandProperty Path
$sqlitePath = Resolve-Path -Path ".\build\vcpkg_installed\x64-windows\tools" | Select-Object -ExpandProperty Path
Write-Host "Installing $msi ..." -ForegroundColor Green
& "msiexec.exe" /I $msi /quiet /norestart /l* install.log | Out-Default
if (!$?) {
Get-Content install.log
throw "MSI install failed"
}
Write-Host "Refreshing system path..." -ForegroundColor Green
$env:Path = [System.Environment]::GetEnvironmentVariable("Path", "Machine") + ";" +[System.Environment]::GetEnvironmentVariable("Path", "User") + ";" + $sqlitePath
Write-Host "Checking Kart runs..." -ForegroundColor Green
& "kart" --version | Out-Default
if (!$?) {throw "Smoke test Failed"}
Write-Host "Running Kart E2E tests..." -ForegroundColor Green
& "tests\scripts\e2e-1.ps1" | Out-Default
if (!$?) {throw "E2E test Failed"}
- name: "release"
uses: softprops/action-gh-release@v1
if: "steps.version.outputs.is_release == 1"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
draft: true
files: |
build/dist/Kart-*.zip
build/dist/Kart-*.msi