Merge pull request #995 from koordinates/rc-vcpkg-2024.04.26

git 2.45.1 and vcpkg 2024.04.26

.github/workflows/build.yml

@@ -12,12 +12,37 @@ env:
   AWS_NO_SIGN_REQUEST: "1"
 
 jobs:
+  # Increase rate limits for public registry
+  amazon-ecr-auth:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: 'arn:aws:iam::276514628126:role/ci-kart'
+          aws-region: us-east-1
+          mask-aws-account-id: 'false'
+          role-session-name: kart-ci-gha
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+        with:
+          registry-type: public
+          mask-password: 'false'
+    outputs:
+      registry: ${{ steps.login-ecr.outputs.registry }}
+      docker_username: ${{ steps.login-ecr.outputs.docker_username_public_ecr_aws }}
+      docker_password: ${{ steps.login-ecr.outputs.docker_password_public_ecr_aws }}
+
   #
   # Linux development build on modern Ubuntu
   #
   Linux-dev:
     name: "Linux dev"
     runs-on: ubuntu-22.04
+    needs: amazon-ecr-auth
 
     # We want to run on external PRs, but not on our own internal PRs as they'll be run
     # by the push to the branch.
@@ -56,6 +81,9 @@ jobs:
           - 1433:1433
       mysql:
         image: public.ecr.aws/docker/library/mysql:latest
+        credentials:
+          username: ${{ needs.amazon-ecr-auth.outputs.docker_username }}
+          password: ${{ needs.amazon-ecr-auth.outputs.docker_password }}
         options: >-
           -e MYSQL_ROOT_PASSWORD=PassWord1
         ports:
@@ -216,15 +244,18 @@ jobs:
             image: quay.io/pypa/manylinux2014_x86_64
             arch: amd64
             arch_triplet: x64-linux
+            build_parallel: ""
 
           - base: buildjet-4vcpu-ubuntu-2204-arm
             label: centos7/arm64
             image: quay.io/pypa/manylinux2014_aarch64
             arch: arm64
             arch_triplet: arm64-linux
+            build_parallel: 1
 
     name: Linux ${{ matrix.os.arch }}
     runs-on: ${{ matrix.os.base }}
+    needs: amazon-ecr-auth
 
     # We want to run on external PRs, but not on our own internal PRs as they'll be run
     # by the push to the branch.
@@ -256,6 +287,9 @@ jobs:
           - 1433:1433
       mysql:
         image: public.ecr.aws/docker/library/mysql:latest
+        credentials:
+          username: ${{ needs.amazon-ecr-auth.outputs.docker_username }}
+          password: ${{ needs.amazon-ecr-auth.outputs.docker_password }}
         options: >-
           -e MYSQL_ROOT_PASSWORD=PassWord1
         ports:
@@ -364,7 +398,7 @@ jobs:
       - name: "app: build"
         shell: bash
         run: |
-          cmake --build --preset=ci-linux --verbose --parallel
+          cmake --build --preset=ci-linux --verbose --parallel ${{ matrix.os.build_parallel }}
 
       - name: "vendor: save archive"
         uses: actions/upload-artifact@v3
@@ -464,7 +498,7 @@ jobs:
   # Install on a wide variety of Linux distributions and run E2E tests
   #
   Linux-Package-Tests:
-    needs: ["Linux-Package"]
+    needs: [Linux-Package, amazon-ecr-auth]
     strategy:
       fail-fast: false
       matrix:
@@ -511,6 +545,9 @@ jobs:
 
     container:
       image: ${{ matrix.os.image }}
+      credentials:
+        username: ${{ needs.amazon-ecr-auth.outputs.docker_username }}
+        password: ${{ needs.amazon-ecr-auth.outputs.docker_password }}
       env:
         DEBIAN_FRONTEND: noninteractive
 

kart.spec

@@ -175,6 +175,8 @@ pyi_analysis = Analysis(
         'shellingham.posix',
         'shellingham.nt',
         *collect_submodules('sqlalchemy'),
+        # required for setuptools >=70.0.0 and pyinstaller <=6.7.0
+        'pkg_resources.extern',
     ],
     hookspath=[],
     runtime_hooks=[],

scripts/doc_gen.py

@@ -14,6 +14,8 @@
 import time
 import typing as t
 
+from kart import cli
+
 import click
 import rst2txt
 from docutils.core import publish_string
@@ -342,7 +344,6 @@ def main():
     input_dir = sys.argv[1]
     output_dir = Path(sys.argv[2])
     output_dir.mkdir(parents=True, exist_ok=True)
-    from kart import cli
 
     cli.load_all_commands()
     generate_help_pages(name, input_dir, cli.cli, target_dir=output_dir)

tests/point_cloud/test_workingcopy.py

@@ -1215,7 +1215,7 @@ def test_working_copy_progress_bar(
         progress_output = r.stderr.splitlines()
         progress_output = _remove_copy_on_write_warning(progress_output)
         assert progress_output[0] == "Writing tiles for dataset 1 of 1: auckland"
-        assert re.fullmatch(
+        assert re.match(
             r"auckland: 100%\|█+\| 16/16 \[[0-9:<]+, [0-9\.]+tile/s\]",
             progress_output[-1],
         )

tests/scripts/e2e-1.sh

@@ -25,7 +25,9 @@ function do_cleanup {
     if [ -f "$KART_HELPER_LOG" ]; then
         cat "$KART_HELPER_LOG"
     fi
-    rm -rf "$TMP_PATH"
+    if [ -z "${NO_CLEANUP-}" ]; then
+        rm -rf "$TMP_PATH"
+    fi
 }
 trap do_cleanup EXIT
 
@@ -35,10 +37,14 @@ if [ "$(uname -s)" == "Linux" ]; then
     cp -a /etc/skel/. "$HOME/"
 fi
 export SHELL=/bin/bash
+export PAGER=cat
 
 KART_PATH=$(dirname "$(realpath "$(command -v kart)")")
 echo "Kart is at: ${KART_PATH}"
 
+SQLITE3_PATH=$(dirname "$(realpath "$(command -v sqlite3)")")
+echo "sqlite3 is at: ${SQLITE3_PATH}"
+
 SPATIALITE_PATH=$(echo 'from kart import spatialite_path; print(spatialite_path)' | kart --post-mortem 2>/dev/null | grep spatialite | awk '{print $2}')
 echo "Spatialite is at: ${SPATIALITE_PATH}"
 

tests/test_working_copy_gpkg.py

@@ -1221,7 +1221,7 @@ def test_working_copy_progress_bar(cli_runner, data_working_copy, monkeypatch):
             progress_output[0]
             == "Writing features for dataset 1 of 1: nz_pa_points_topo_150k"
         )
-        assert re.fullmatch(
+        assert re.match(
             r"nz_pa_points_topo_150k: 100%\|█+\| 2143/2143 \[[0-9:<]+, [0-9\.]+F/s\]",
             progress_output[-1],
         )

vcpkg-vendor/CMakeLists.txt

@@ -493,8 +493,8 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
 
   ExternalProject_Add(
     git
-    URL https://github.com/koordinates/git/releases/download/kart-0.11.0-windows/MinGit-v2.34.0.windows.1.13-64-bit.zip
-    URL_HASH SHA256=7dcbf2826721bde668bdc809539bf8cdaf49c11d5a66dd3a003949e95e992687
+    URL https://github.com/koordinates/git/releases/download/kart-v0.15.2-windows-mingit-build/MinGit-2.45.1.7.g5be46a8262-64-bit.zip
+    URL_HASH SHA256=322a7b292c382f39d54ff4bd421f240d5ef16f10bffa485718076907750de351
     DOWNLOAD_NO_PROGRESS ON
     BUILD_IN_SOURCE 1 # avoid creation of a build directory
     INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/git
@@ -507,11 +507,19 @@ else()
   # Build Git with spatial-filter extension on macOS and Linux
   find_package(OpenSSL REQUIRED)
   pkg_check_modules(OPENSSL REQUIRED openssl)
-  pkg_check_modules(PCRE2 REQUIRED libpcre2-8)
   pkg_check_modules(LIBCURL REQUIRED libcurl)
+  find_package(pcre2 CONFIG REQUIRED)
 
   cmake_path(SET GIT_SPATIAL_FILTER_EXT contrib/filter-extensions/spatial/spatial.a)
 
+  if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "AppleClang")
+    set(CXX_STD_LIB_LDFLAGS "-lc++")
+  elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
+    set(CXX_STD_LIB_LDFLAGS "-L/lib -lstdc++")
+  else()
+    message(FATAL_ERROR "Unknown C++ compiler")
+  endif()
+
   set(gitBuildConfig
       "prefix=/"
       "V=1"
@@ -523,23 +531,24 @@ else()
       "NO_PERL=1"
       "NO_TCLTK=1"
       "RUNTIME_PREFIX=1"
-      "LIBPCREDIR=${PCRE2_PREFIX}"
+      "LIBPCREDIR=${CURRENT_PACKAGES_DIR}"
       "OPENSSLDIR=${OPENSSL_PREFIX}"
-      "CURL_CONFIG=${CURRENT_PACKAGES_DIR}/tools/curl/bin/curl-config"
+      "CURLDIR=${CURRENT_PACKAGES_DIR}"
+      "CURL_LDFLAGS=-lcurl"
       # stuff for building the spatial partial clone filter extension
       "CXX=${CMAKE_CXX_COMPILER}"
       "LINK=${CMAKE_CXX_COMPILER}"
+      "LDFLAGS=-L${CURRENT_PACKAGES_DIR}/lib"
       "FILTER_EXTENSIONS=${GIT_SPATIAL_FILTER_EXT}"
-      "CFLAGS=-I<SOURCE_DIR>"
-      "EXTENSION_LIBS=-L${CURRENT_PACKAGES_DIR}/lib -lsqlite3")
+      "EXTENSION_LIBS=-L${CURRENT_PACKAGES_DIR}/lib -lsqlite3 ${CXX_STD_LIB_LDFLAGS}")
 
   ExternalProject_Add(
     git
     GIT_REPOSITORY https://github.com/koordinates/git
-    GIT_TAG kart-v0.12.0-rc3
+    GIT_TAG kart-v0.15.2
     GIT_SHALLOW ON
     BUILD_IN_SOURCE ON
-    DEPENDS CURL::libcurl OpenSSL::SSL OpenSSL::Crypto unofficial::sqlite3::sqlite3
+    DEPENDS CURL::libcurl OpenSSL::SSL OpenSSL::Crypto unofficial::sqlite3::sqlite3 PCRE2::8BIT
     EXCLUDE_FROM_ALL ON
     CONFIGURE_COMMAND ""
     INSTALL_DIR env
@@ -619,7 +628,8 @@ add_custom_command(
   OUTPUT ${VENDOR_ARCHIVE}
   DEPENDS vendor_wheels git git-lfs ${VENDOR_CONFIG} fix_vendor_libs.py
   COMMAND ${CMAKE_COMMAND} -E rm -rf env/share/git-gui
-  COMMAND ${CMAKE_COMMAND} -E make_directory wheelhouse env env/lib env/tools env/share env/${EXE_DIR}
+  COMMAND ${CMAKE_COMMAND} -E make_directory wheelhouse env env/lib env/tools env/share
+          env/${EXE_DIR}
   COMMAND ${CMAKE_COMMAND} -E copy ${WHEEL_LIST} wheelhouse
   COMMAND ${CMAKE_COMMAND} -E copy ${LIB_LIST} env/lib
   COMMAND ${CMAKE_COMMAND} -E copy ${EXE_LIST} env/${EXE_DIR}

vcpkg-vendor/fix_vendor_libs.py

@@ -106,6 +106,7 @@
     SYSTEM_DEPS_ALLOW_LIST = [
         "advapi32.dll",
         "api-ms-win-core-path-l1-1-0.dll",
+        "api-ms-win-core-synch-l1-2-0.dll",
         "api-ms-win-crt-conio-l1-1-0.dll",
         "api-ms-win-crt-convert-l1-1-0.dll",
         "api-ms-win-crt-environment-l1-1-0.dll",
@@ -120,6 +121,7 @@
         "api-ms-win-crt-time-l1-1-0.dll",
         "api-ms-win-crt-utility-l1-1-0.dll",
         "bcrypt.dll",
+        "bcryptprimitives.dll",
         "crypt32.dll",
         "kernel32.dll",
         "kernel32.dll",

vcpkg-vendor/vcpkg-overlay-ports/gdal/cmake-project-include.cmake

@@ -1,3 +1,10 @@
+if(GDAL_USE_KEA)
+    find_package(Kealib CONFIG REQUIRED)
+    add_library(KEA::KEA ALIAS Kealib::Kealib)
+    set(GDAL_CHECK_PACKAGE_KEA_NAMES Kealib CACHE INTERNAL "vcpkg")
+    set(GDAL_CHECK_PACKAGE_KEA_TARGETS Kealib::Kealib CACHE INTERNAL "vcpkg")
+endif()
+
 if(GDAL_USE_WEBP)
     find_package(WebP CONFIG REQUIRED)
     add_library(WEBP::WebP ALIAS WebP::webp)

vcpkg-vendor/vcpkg-overlay-ports/gdal/portfile.cmake

@@ -2,12 +2,13 @@ vcpkg_from_github(
     OUT_SOURCE_PATH SOURCE_PATH
     REPO OSGeo/gdal
     REF "v${VERSION}"
-    SHA512 ae0238b2741c673927f80b3d0765fbb0eb2ddb8699441564e67dd7bb6a429b45e4eac57f9c4e93ecea7a4525f28c05e6f3097828c40d5af6a66dda83d2245242
+    SHA512 5b155229225e904b3619628ec27efdd273d9f083c1ee4f6d94041897d5bc9c3133590b70885ad61fc8864da2f334a75cf32bafe7f776c40bbbc3673fe842c986
     HEAD_REF master
     PATCHES
         find-link-libraries.patch
         fix-gdal-target-interfaces.patch
         libkml.patch
+        target-is-valid.patch
 )
 # `vcpkg clean` stumbles over one subdir
 file(REMOVE_RECURSE "${SOURCE_PATH}/autotest")
@@ -32,6 +33,7 @@ vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS
         iconv            GDAL_USE_ICONV
         jpeg             GDAL_USE_JPEG
         core             GDAL_USE_JSONC
+        kea              GDAL_USE_KEA
         lerc             GDAL_USE_LERC
         libkml           GDAL_USE_LIBKML
         lzma             GDAL_USE_LIBLZMA
@@ -86,6 +88,7 @@ vcpkg_cmake_configure(
         -DGDAL_USE_EXTERNAL_LIBS=OFF
         -DGDAL_BUILD_OPTIONAL_DRIVERS=ON
         -DOGR_BUILD_OPTIONAL_DRIVERS=ON
+        -DFIND_PACKAGE2_KEA_ENABLED=OFF
         -DGDAL_CHECK_PACKAGE_MySQL_NAMES=unofficial-libmariadb
         -DGDAL_CHECK_PACKAGE_MySQL_TARGETS=unofficial::libmariadb
         -DMYSQL_LIBRARIES=unofficial::libmariadb
@@ -115,34 +118,33 @@ list(APPEND CMAKE_PROGRAM_PATH \"\${vcpkg_host_prefix}/tools/pkgconf\")"
 if (BUILD_APPS)
     vcpkg_copy_tools(
         TOOL_NAMES
-            gdalinfo
-            gdalbuildvrt
-            gdaladdo
+            gdal_contour
+            gdal_create
+            gdal_footprint
             gdal_grid
-            gdal_translate
             gdal_rasterize
-            gdalsrsinfo
-            gdalenhance
-            gdalmanage
-            gdaltransform
-            gdaltindex
-            gdaldem
-            gdal_create
+            gdal_translate
             gdal_viewshed
-            nearblack
-            ogrlineref
-            ogrtindex
-            gdalwarp
-            gdal_contour
+            gdaladdo
+            gdalbuildvrt
+            gdaldem
+            gdalenhance
+            gdalinfo
             gdallocationinfo
-            ogrinfo
-            ogr2ogr
-            ogrlineref
-            nearblack
+            gdalmanage
             gdalmdiminfo
             gdalmdimtranslate
+            gdalsrsinfo
+            gdaltindex
+            gdaltransform
+            gdalwarp
             gnmanalyse
             gnmmanage
+            nearblack
+            ogr2ogr
+            ogrinfo
+            ogrlineref
+            ogrtindex
             sozip
         AUTO_CLEAN
     )

vcpkg-vendor/vcpkg-overlay-ports/gdal/target-is-valid.patch

@@ -0,0 +1,21 @@
+diff --git a/cmake/helpers/CheckDependentLibraries.cmake b/cmake/helpers/CheckDependentLibraries.cmake
+index f5c920a..53494c9 100644
+--- a/cmake/helpers/CheckDependentLibraries.cmake
++++ b/cmake/helpers/CheckDependentLibraries.cmake
+@@ -41,6 +41,7 @@ endif()
+ # https://github.com/OSGeo/gdal/issues/5324
+ function (gdal_check_target_is_valid target res_var)
+   get_target_property(_interface_include_directories ${target} "INTERFACE_INCLUDE_DIRECTORIES")
++  get_target_property(_type ${target} "TYPE")
+   if(_interface_include_directories)
+     foreach(_dir IN LISTS _interface_include_directories)
+       if(NOT EXISTS "${_dir}")
+@@ -54,7 +55,7 @@ function (gdal_check_target_is_valid target res_var)
+     # property, but a GeoTIFF_INCLUDE_DIRS variable.
+     set_target_properties(${target} PROPERTIES
+                           INTERFACE_INCLUDE_DIRECTORIES "${GeoTIFF_INCLUDE_DIRS}")
+-  else()
++  elseif(NOT _type STREQUAL "INTERFACE" AND _type STREQUAL "ALIAS")
+      message(WARNING "Target ${target} has no INTERFACE_INCLUDE_DIRECTORIES property. Ignoring that target.")
+      set(${res_var} FALSE PARENT_SCOPE)
+      return()