From 610dc17376c4609ecf75cdf6b42870b2c70ba7fb Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Wed, 22 Nov 2023 13:51:53 -0500 Subject: [PATCH 1/4] rework build and Dockerfile to be parameterized --- .github/workflows/ci.yml | 29 ++- dockerfiles/Dockerfile | 420 +++++++++++++++++++++++++++++++++++++++ scripts/build.sh | 3 +- 3 files changed, 442 insertions(+), 10 deletions(-) create mode 100644 dockerfiles/Dockerfile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a71760..29a9ede 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: if: "!contains(github.event.head_commit.message, '[skip ci]')" strategy: matrix: - gdal-version: [3.6] + gdal-version: ["3.6.4", "3.7.3", "3.8.0"] steps: - uses: actions/checkout@v4 @@ -35,22 +35,31 @@ jobs: python -m pip install boto3 click - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to Github - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: Build - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 with: platforms: linux/amd64 context: . load: true - file: dockerfiles/Dockerfile.gdal${{ matrix.gdal-version }} + file: dockerfiles/Dockerfile + build-args: | + GDAL_VERSION=${{ matrix.gdal-version }} tags: ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ matrix.gdal-version }} cache-from: type=gha cache-to: type=gha,mode=max @@ -74,12 +83,14 @@ jobs: aws-region: us-east-1 - name: Publish docker image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v5 if: github.ref == 'refs/heads/master' with: platforms: linux/amd64 context: . - file: dockerfiles/Dockerfile.gdal${{ matrix.gdal-version }} + file: dockerfiles/Dockerfile + build-args: | + GDAL_VERSION=${{ matrix.gdal-version }} push: true cache-from: type=gha tags: ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ matrix.gdal-version }} @@ -124,7 +135,7 @@ jobs: - uses: actions/setup-node@v4 - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 + uses: aws-actions/configure-aws-credentials@v4 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -136,7 +147,7 @@ jobs: python scripts/list.py | jq > layer.json - name: Update resources - uses: test-room-7/action-update-file@v1 + uses: test-room-7/action-update-file@v1.8.0 with: file-path: layer.json commit-msg: Update layer [skip ci] diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile new file mode 100644 index 0000000..5df1162 --- /dev/null +++ b/dockerfiles/Dockerfile @@ -0,0 +1,420 @@ +FROM public.ecr.aws/lambda/provided:al2 as builder + +LABEL maintainer="lambgeo " + +RUN yum update -y && \ + yum install -y git autoconf libtool flex bison cmake make \ + tar gzip gcc gcc-c++ automake16 libpng-devel nasm \ + libxml2-devel readline-devel openssl-devel curl-devel \ + cmake3 && \ + yum clean all && \ + rm -rf /var/cache/yum /var/lib/yum/history + +ENV PREFIX /opt + +WORKDIR /opt + +ENV LD_LIBRARY_PATH $PREFIX/lib:$LD_LIBRARY_PATH + +# pkg-config +ENV PKGCONFIG_VERSION=0.29.2 +RUN mkdir /tmp/pkg-config \ + && curl -sfL https://pkg-config.freedesktop.org/releases/pkg-config-${PKGCONFIG_VERSION}.tar.gz | tar zxf - -C /tmp/pkg-config --strip-components=1 \ + && cd /tmp/pkg-config \ + && CFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --with-internal-glib \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/pkg-config + +ENV PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig/ + +# sqlite +RUN mkdir /tmp/sqlite \ + && curl -sfL https://www.sqlite.org/2020/sqlite-autoconf-3330000.tar.gz | tar zxf - -C /tmp/sqlite --strip-components=1 \ + && cd /tmp/sqlite \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --disable-static \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/sqlite + +# libexpat +RUN mkdir /tmp/libexpat \ + && curl -sfL https://github.com/libexpat/libexpat/releases/download/R_2_2_10/expat-2.2.10.tar.gz | tar zxf - -C /tmp/libexpat --strip-components=1 \ + && cd /tmp/libexpat \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --disable-static \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/libexpat + +ENV \ + SQLITE3_LIBS="-L${PREFIX}/lib -lsqlite3" \ + SQLITE3_INCLUDE_DIR="${PREFIX}/include" \ + SQLITE3_CFLAGS="$CFLAGS -I${PREFIX}/include" \ + PATH=${PREFIX}/bin/:$PATH + +# libxml2 +RUN mkdir /tmp/libxml2 \ + && curl -sfL https://gitlab.gnome.org/GNOME/libxml2/-/archive/v2.9.10/libxml2-v2.9.10.tar.gz | tar zxf - -C /tmp/libxml2 --strip-components=1 \ + && cd /tmp/libxml2 \ + && ./autogen.sh --prefix=$PREFIX --with-ftp=no --with-http=no --with-python=no \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/libxml2 + +# nghttp2 +ENV NGHTTP2_VERSION=1.42.0 +RUN mkdir /tmp/nghttp2 \ + && curl -sfL https://github.com/nghttp2/nghttp2/releases/download/v${NGHTTP2_VERSION}/nghttp2-${NGHTTP2_VERSION}.tar.gz | tar zxf - -C /tmp/nghttp2 --strip-components=1 \ + && cd /tmp/nghttp2 \ + && ./configure --enable-lib-only --prefix=$PREFIX \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/nghttp2 + +# libcurl +ENV CURL_VERSION=7.73.0 +RUN mkdir /tmp/libcurl \ + && curl -sfL https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz | tar zxf - -C /tmp/libcurl --strip-components=1 \ + && cd /tmp/libcurl \ + && ./configure --disable-manual --disable-cookies --with-nghttp2=$PREFIX --prefix=$PREFIX \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/libcurl + +# libdeflate +RUN mkdir /tmp/libdeflate \ + && curl -sfL https://github.com/ebiggers/libdeflate/archive/v1.10.tar.gz | tar zxf - -C /tmp/libdeflate --strip-components=1 \ + && cd /tmp/libdeflate \ + && make -j $(nproc) --silent PREFIX=$PREFIX && make install \ + && rm -rf /tmp/libdeflate + +# libpng +ENV LIBPNG_VERSION=1.6.37 +RUN mkdir /tmp/png \ + && curl -sfL https://github.com/glennrp/libpng/archive/v${LIBPNG_VERSION}.tar.gz | tar zxf - -C /tmp/png --strip-components=1 \ + && cd /tmp/png \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/png + +# libjpeg_turbo +ENV LIBJPEG_TURBO_VERSION=2.1.3 +RUN mkdir /tmp/jpeg \ + && curl -sfL https://github.com/libjpeg-turbo/libjpeg-turbo/archive/${LIBJPEG_TURBO_VERSION}.tar.gz | tar zxf - -C /tmp/jpeg --strip-components=1 \ + && cd /tmp/jpeg \ + && mkdir build && cd build \ + && cmake3 .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/jpeg + +# webp +ENV WEBP_VERSION=1.1.0 +RUN mkdir /tmp/webp \ + && curl -sfL https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-${WEBP_VERSION}.tar.gz | tar zxf - -C /tmp/webp --strip-components=1 \ + && cd /tmp/webp \ + && CFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/webp + +# zstd +ENV ZSTD_VERSION=1.4.5 +RUN mkdir /tmp/zstd \ + && curl -sfL https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz | tar zxf - -C /tmp/zstd --strip-components=1 \ + && cd /tmp/zstd \ + && make -j $(nproc) PREFIX=$PREFIX ZSTD_LEGACY_SUPPORT=0 CFLAGS=-O1 --silent && make install PREFIX=$PREFIX ZSTD_LEGACY_SUPPORT=0 CFLAGS=-O1 \ + && rm -rf /tmp/zstd + +# lerc +ENV LERC_VERSION=2.2.1 +RUN mkdir /tmp/lerc \ + && curl -sfL https://github.com/Esri/lerc/archive/v${LERC_VERSION}.tar.gz | tar zxf - -C /tmp/lerc --strip-components=1 \ + && cd /tmp/lerc \ + && cmake3 . \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/lerc + +# libtiff +ENV LIBTIFF_VERSION=4.5.0 +RUN mkdir /tmp/libtiff \ + && curl -sfL https://download.osgeo.org/libtiff/tiff-${LIBTIFF_VERSION}.tar.gz | tar zxf - -C /tmp/libtiff --strip-components=1 \ + && cd /tmp/libtiff \ + && LDFLAGS="-Wl,-rpath,'\$\$ORIGIN'" CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure \ + --prefix=$PREFIX \ + --disable-static \ + --enable-rpath \ + --with-lerc-lib-dir=$PREFIX/lib \ + --with-lerc-include-dir=$PREFIX/include \ + --with-libdeflate-lib-dir=$PREFIX/lib \ + --with-libdeflate-include-dir=$PREFIX/include \ + --with-jpeg-lib-dir=$PREFIX/lib \ + --with-jpeg-include-dir=$PREFIX/include \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/libtiff + +# postgres +ENV PG_VERSION=15.2 +RUN mkdir /tmp/postgres \ + && curl -sfL https://ftp.postgresql.org/pub/source/v${PG_VERSION}/postgresql-${PG_VERSION}.tar.gz | tar zxf - -C /tmp/postgres --strip-components=1 \ + && cd /tmp/postgres \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --with-openssl \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/postgres + +# openjpeg +ENV OPENJPEG_VERSION=2.5.0 +RUN mkdir /tmp/openjpeg \ + && curl -sfL https://github.com/uclouvain/openjpeg/archive/v${OPENJPEG_VERSION}.tar.gz | tar zxf - -C /tmp/openjpeg --strip-components=1 \ + && cd /tmp/openjpeg \ + && mkdir build && cd build \ + && cmake3 .. \ + -DBUILD_THIRDPARTY:BOOL=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/openjpeg + +# geos +ENV GEOS_VERSION=3.11.2 +RUN mkdir /tmp/geos \ + && curl -sfL https://github.com/libgeos/geos/archive/refs/tags/${GEOS_VERSION}.tar.gz | tar zxf - -C /tmp/geos --strip-components=1 \ + && cd /tmp/geos \ + && mkdir build && cd build \ + && cmake3 .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/geos + +# proj +# We still provide proj-datumgrid but the user can choose to set `PROJ_NETWORK=ON` to use network grids +# ENV PROJ_VERSION=9.2.0 +# RUN mkdir /tmp/proj && mkdir /tmp/proj/data \ +# && curl -sfL https://github.com/OSGeo/proj/archive/${PROJ_VERSION}.tar.gz | tar zxf - -C /tmp/proj --strip-components=1 \ +# && curl -sfL http://download.osgeo.org/proj/proj-datumgrid-latest.tar.gz | tar zxf - -C /tmp/proj/data \ +# && cd /tmp/proj \ +# && aclocal && autoheader && libtoolize --force --copy && automake --add-missing --copy && autoconf \ +# && LDFLAGS="-Wl,-rpath,'\$\$ORIGIN'" CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --disable-static --enable-lto \ +# && make -j $(nproc) --silent && make install \ +# && rm -rf /tmp/proj + +ENV PROJ_VERSION=9.2.0 +RUN mkdir /tmp/proj && mkdir /tmp/proj/data \ + && curl -sfL https://github.com/OSGeo/proj/archive/${PROJ_VERSION}.tar.gz | tar zxf - -C /tmp/proj --strip-components=1 \ + && cd /tmp/proj \ + && mkdir build && cd build \ + && cmake3 .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_INSTALL_INCLUDEDIR:PATH=include \ + -DBUILD_TESTING=OFF \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/proj + +# GeoTIFF +ENV LIBGEOTIFF_VERSION=1.7.1 +RUN mkdir /tmp/libgeotiff \ + && curl -sfL https://github.com/OSGeo/libgeotiff/releases/download/${LIBGEOTIFF_VERSION}/libgeotiff-${LIBGEOTIFF_VERSION}.tar.gz | tar zxf - -C /tmp/libgeotiff --strip-components=1 \ + && cd /tmp/libgeotiff \ + && LDFLAGS="-Wl,-rpath,'\$\$ORIGIN'" CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure \ + --prefix=$PREFIX \ + --with-proj=$PREFIX \ + --with-jpeg=$PREFIX \ + --with-zip=yes \ + --with-zlib \ + --with-libtiff=$PREFIX \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/libgeotiff + +# Build blosc (needed for ZARR) +ENV BLOSC_VERSION=1.21.0 +RUN mkdir /tmp/blosc \ + && curl -sfL https://github.com/Blosc/c-blosc/archive/refs/tags/v${BLOSC_VERSION}.tar.gz | tar zxf - -C /tmp/blosc --strip-components=1 \ + && cd /tmp/blosc \ + && cmake3 -G"Unix Makefiles" \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_INSTALL_RPATH="$ORIGIN" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + -DBUILD_SHARED=ON \ + -DBUILD_STATIC=OFF \ + -DBUILD_BENCHMARKS=OFF \ + -DBUILD_FUZZERS=OFF \ + -DBUILD_TESTS=OFF \ + -DPREFER_EXTERNAL_LZ4=ON \ + -DPREFER_EXTERNAL_ZLIB=ON \ + -DPREFER_EXTERNAL_ZSTD=ON \ + -DDEACTIVATE_SNAPPY=OFF . \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/blosc + +# libcrypto (for GPC access) +RUN mkdir /tmp/cryptopp \ + && curl -sfL https://github.com/weidai11/cryptopp/archive/refs/tags/CRYPTOPP_8_6_0.tar.gz | tar zxf - -C /tmp/cryptopp --strip-components=1 \ + && cd /tmp/cryptopp \ + && make -j $(nproc) --silent && make install PREFIX=$PREFIX \ + && rm -rf /tmp/cryptopp + +# szip (for hdf) +ENV SZIP_VERSION=2.1.1 +RUN mkdir /tmp/szip \ + && curl -sfL https://support.hdfgroup.org/ftp/lib-external/szip/${SZIP_VERSION}/src/szip-${SZIP_VERSION}.tar.gz | tar zxf - -C /tmp/szip --strip-components=1 \ + && cd /tmp/szip \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --disable-static \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/szip + +# libhdf4 +ENV HDF4_VERSION=4.2.15 +RUN mkdir /tmp/hdf4 \ + && curl -sfL https://support.hdfgroup.org/ftp/HDF/releases/HDF${HDF4_VERSION}/src/hdf-${HDF4_VERSION}.tar | tar xvf - -C /tmp/hdf4 --strip-components=1 \ + && cd /tmp/hdf4 \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure \ + --prefix=$PREFIX \ + --with-szlib=$PREFIX \ + --with-jpeg=$PREFIX \ + --enable-shared \ + --disable-static \ + --disable-netcdf \ + --disable-fortran \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/hdf4 + +# libhdf5 +ENV HDF5_VERSION=1.12.0 +RUN mkdir /tmp/hdf5 \ + && curl -sfL https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${HDF5_VERSION%.*}/hdf5-${HDF5_VERSION}/src/hdf5-$HDF5_VERSION.tar.gz | tar zxf - -C /tmp/hdf5 --strip-components=1 \ + && cd /tmp/hdf5 \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure \ + --prefix=$PREFIX \ + --with-szlib=$PREFIX \ + --enable-cxx \ + --enable-thread-safe \ + --disable-static \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/hdf5 + +# NetCDF +ENV NETCDF_VERSION=4.7.4 +RUN mkdir /tmp/netcdf \ + && curl -sfL https://github.com/Unidata/netcdf-c/archive/v${NETCDF_VERSION}.tar.gz | tar zxf - -C /tmp/netcdf --strip-components=1 \ + && cd /tmp/netcdf \ + && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" CPPFLAGS="-I${PREFIX}/include" LDFLAGS="-L${PREFIX}/lib" ./configure \ + --with-default-chunk-size=67108864 \ + --with-chunk-cache-size=67108864 \ + --prefix=$PREFIX \ + --disable-static \ + --enable-netcdf4 \ + --enable-dap \ + --with-pic \ + && make -j $(nproc) --silent && make install && make clean \ + && rm -rf /tmp/netcdf + +# # JPEG-XL +# # libhwy +# RUN mkdir /tmp/libhwy \ +# && curl -sfL https://github.com/google/highway/archive/refs/tags/0.16.0.tar.gz | tar zxf - -C /tmp/libhwy --strip-components=1 \ +# && cd /tmp/libhwy \ +# && mkdir build && cd build \ +# && cmake3 .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_PREFIX=$PREFIX \ +# && make -j $(nproc) install && make clean \ +# && rm -rf /tmp/libhwy + +# # brotli +# ENV BROTLI_VERSION=1.0.9 +# RUN mkdir /tmp/brotli \ +# && curl -sfL https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz | tar zxf - -C /tmp/brotli --strip-components=1 \ +# && cd /tmp/brotli \ +# && mkdir build && cd build \ +# && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ../configure-cmake --prefix=$PREFIX \ +# && make -j $(nproc) --silent && make install && make clean \ +# && rm -rf /tmp/brotli + +# # # JpegXL +# ENV JXL_VERSION=0.8.1 +# RUN mkdir /tmp/jxl \ +# && curl -sfL https://github.com/libjxl/libjxl/archive/refs/tags/v${JXL_VERSION}.tar.gz | tar zxf - -C /tmp/jxl --strip-components=1 \ +# && cd /tmp/jxl \ +# && mkdir build && cd build \ +# && cmake3 .. -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_PREFIX=$PREFIX \ +# && make -j $(nproc) install && make clean \ +# && rm -rf /tmp/jxl + +# We use commit sha to make sure we are not using `cache` when building the docker image +# "7ca88116f5a46d429251361634eb24629f315076" is the latest commit on release/3.6 branch + +# GDAL +ARG GDAL_VERSION +RUN mkdir /tmp/gdal \ + && curl -sfL https://github.com/OSGeo/gdal/archive/refs/tags/v${GDAL_VERSION}.tar.gz | tar zxf - -C /tmp/gdal --strip-components=1 \ + && cd /tmp/gdal \ + && mkdir build && cd build \ + && cmake3 .. \ + -DGDAL_USE_EXTERNAL_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ + -DCMAKE_INSTALL_LIBDIR:PATH=lib \ + -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ + -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ + -DGDAL_SET_INSTALL_RELATIVE_RPATH=ON \ + -DGDAL_USE_TIFF_INTERNAL=OFF \ + -DGDAL_USE_GEOTIFF_INTERNAL=OFF \ + -DGDAL_USE_LERC_INTERNAL=OFF \ + -DLERC_INCLUDE_DIR=$PREFIX/include \ + -DLERC_LIBRARY=$PREFIX/lib/libLercLib.so \ + -DPNG_PNG_INCLUDE_DIR=$PREFIX/include \ + -DPNG_LIBRARY_RELEASE=$PREFIX/lib/libpng.so \ + -DBUILD_PYTHON_BINDINGS=OFF \ + && make -j $(nproc) --silent && make install \ + && rm -rf /tmp/gdal + +# patchelf +# from https://github.com/pypa/manylinux/blob/d8ef5d47433ba771fa4403fd48f352c586e06e43/docker/build_scripts/build.sh#L133-L138 +ENV PATCHELF_VERSION=0.10 +RUN mkdir /tmp/patchelf \ + && curl -sfL https://github.com/NixOS/patchelf/archive/${PATCHELF_VERSION}.tar.gz | tar zxf - -C /tmp/patchelf --strip-components=1 \ + && cd /tmp/patchelf && ./bootstrap.sh && ./configure \ + && make -j $(nproc) --silent && make install \ + && cd / && rm -rf /tmp/patchelf + +# # libturbojpeg.so is not used by GDAL. Only libjpeg.so* +RUN rm -f $PREFIX/lib/libturbojpeg.so* \ + # Only libwebp.so is used by GDAL + && rm -f $PREFIX/lib/libwebpmux.so* $PREFIX/lib/libwebpdemux.so* $PREFIX/lib/libwebpdecoder.so* + +# FIX +RUN for i in $PREFIX/bin/*; do patchelf --force-rpath --set-rpath '$ORIGIN/../lib' $i; done + +# Build final image +FROM public.ecr.aws/lambda/provided:al2 as runner + +ENV PREFIX /opt +COPY --from=builder /opt/lib/ $PREFIX/lib/ +COPY --from=builder /opt/include/ $PREFIX/include/ +COPY --from=builder /opt/share/ $PREFIX/share/ +COPY --from=builder /opt/bin/ $PREFIX/bin/ + +RUN export GDAL_VERSION=$(gdal-config --version) + +ENV \ + GDAL_DATA=$PREFIX/share/gdal \ + PROJ_LIB=$PREFIX/share/proj \ + GDAL_CONFIG=$PREFIX/bin/gdal-config \ + GEOS_CONFIG=$PREFIX/bin/geos-config \ + PATH=$PREFIX/bin:$PATH + +ENTRYPOINT bash diff --git a/scripts/build.sh b/scripts/build.sh index 61f9724..bebd156 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -8,7 +8,8 @@ echo "Building image for AWS Lambda | GDAL: ${GDAL_VERSION} | Runtime: ${RUNTIME docker buildx build \ --platform=linux/amd64 \ - -f dockerfiles/Dockerfile.gdal${GDAL_VERSION} \ + --build-arg GDAL_VERSION=${GDAL_VERSION} \ + -f dockerfiles/Dockerfile \ -t ghcr.io/lambgeo/lambda-gdal:${GDAL_VERSION} . docker buildx build \ From 952077460f04d92c60c5e769559be75984a60e02 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Wed, 22 Nov 2023 13:56:36 -0500 Subject: [PATCH 2/4] re-remote aws credes --- .github/workflows/ci.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29a9ede..795d7ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,13 +44,6 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - name: Build uses: docker/build-push-action@v5 with: From 84f93e12b6bba3bf595fda6f538e38bdfae8ab0a Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 28 Nov 2023 10:37:23 -0500 Subject: [PATCH 3/4] parameterize, but only with one version --- .github/workflows/ci.yml | 2 +- dockerfiles/Dockerfile | 3 +- dockerfiles/Dockerfile.gdal3.6 | 419 --------------------------------- scripts/build.sh | 5 +- 4 files changed, 6 insertions(+), 423 deletions(-) delete mode 100644 dockerfiles/Dockerfile.gdal3.6 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4415a9e..d947e0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: if: "!contains(github.event.head_commit.message, '[skip ci]')" strategy: matrix: - gdal-version: ["3.6.4", "3.7.3", "3.8.0"] + gdal-version: ["3.8.0"] steps: - uses: actions/checkout@v4 diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 5df1162..e4b9db6 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -357,6 +357,7 @@ RUN mkdir /tmp/netcdf \ # We use commit sha to make sure we are not using `cache` when building the docker image # "7ca88116f5a46d429251361634eb24629f315076" is the latest commit on release/3.6 branch + # GDAL ARG GDAL_VERSION RUN mkdir /tmp/gdal \ @@ -382,8 +383,8 @@ RUN mkdir /tmp/gdal \ && make -j $(nproc) --silent && make install \ && rm -rf /tmp/gdal -# patchelf # from https://github.com/pypa/manylinux/blob/d8ef5d47433ba771fa4403fd48f352c586e06e43/docker/build_scripts/build.sh#L133-L138 +# Install patchelf (latest with unreleased bug fixes) ENV PATCHELF_VERSION=0.10 RUN mkdir /tmp/patchelf \ && curl -sfL https://github.com/NixOS/patchelf/archive/${PATCHELF_VERSION}.tar.gz | tar zxf - -C /tmp/patchelf --strip-components=1 \ diff --git a/dockerfiles/Dockerfile.gdal3.6 b/dockerfiles/Dockerfile.gdal3.6 deleted file mode 100644 index ba3b55d..0000000 --- a/dockerfiles/Dockerfile.gdal3.6 +++ /dev/null @@ -1,419 +0,0 @@ -FROM public.ecr.aws/lambda/provided:al2 as builder - -LABEL maintainer="lambgeo " - -RUN yum update -y && \ - yum install -y git autoconf libtool flex bison cmake make \ - tar gzip gcc gcc-c++ automake16 libpng-devel nasm \ - libxml2-devel readline-devel openssl-devel curl-devel \ - cmake3 && \ - yum clean all && \ - rm -rf /var/cache/yum /var/lib/yum/history - -ENV PREFIX /opt - -WORKDIR /opt - -ENV LD_LIBRARY_PATH $PREFIX/lib:$LD_LIBRARY_PATH - -# pkg-config -ENV PKGCONFIG_VERSION=0.29.2 -RUN mkdir /tmp/pkg-config \ - && curl -sfL https://pkg-config.freedesktop.org/releases/pkg-config-${PKGCONFIG_VERSION}.tar.gz | tar zxf - -C /tmp/pkg-config --strip-components=1 \ - && cd /tmp/pkg-config \ - && CFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --with-internal-glib \ - && make -j $(nproc) --silent && make install && make clean \ - && rm -rf /tmp/pkg-config - -ENV PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig/ - -# sqlite -RUN mkdir /tmp/sqlite \ - && curl -sfL https://www.sqlite.org/2020/sqlite-autoconf-3330000.tar.gz | tar zxf - -C /tmp/sqlite --strip-components=1 \ - && cd /tmp/sqlite \ - && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --disable-static \ - && make -j $(nproc) --silent && make install && make clean \ - && rm -rf /tmp/sqlite - -# libexpat -RUN mkdir /tmp/libexpat \ - && curl -sfL https://github.com/libexpat/libexpat/releases/download/R_2_2_10/expat-2.2.10.tar.gz | tar zxf - -C /tmp/libexpat --strip-components=1 \ - && cd /tmp/libexpat \ - && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --disable-static \ - && make -j $(nproc) --silent && make install && make clean \ - && rm -rf /tmp/libexpat - -ENV \ - SQLITE3_LIBS="-L${PREFIX}/lib -lsqlite3" \ - SQLITE3_INCLUDE_DIR="${PREFIX}/include" \ - SQLITE3_CFLAGS="$CFLAGS -I${PREFIX}/include" \ - PATH=${PREFIX}/bin/:$PATH - -# libxml2 -RUN mkdir /tmp/libxml2 \ - && curl -sfL https://gitlab.gnome.org/GNOME/libxml2/-/archive/v2.9.10/libxml2-v2.9.10.tar.gz | tar zxf - -C /tmp/libxml2 --strip-components=1 \ - && cd /tmp/libxml2 \ - && ./autogen.sh --prefix=$PREFIX --with-ftp=no --with-http=no --with-python=no \ - && make -j $(nproc) --silent && make install \ - && rm -rf /tmp/libxml2 - -# nghttp2 -ENV NGHTTP2_VERSION=1.42.0 -RUN mkdir /tmp/nghttp2 \ - && curl -sfL https://github.com/nghttp2/nghttp2/releases/download/v${NGHTTP2_VERSION}/nghttp2-${NGHTTP2_VERSION}.tar.gz | tar zxf - -C /tmp/nghttp2 --strip-components=1 \ - && cd /tmp/nghttp2 \ - && ./configure --enable-lib-only --prefix=$PREFIX \ - && make -j $(nproc) --silent && make install \ - && rm -rf /tmp/nghttp2 - -# libcurl -ENV CURL_VERSION=7.73.0 -RUN mkdir /tmp/libcurl \ - && curl -sfL https://curl.haxx.se/download/curl-${CURL_VERSION}.tar.gz | tar zxf - -C /tmp/libcurl --strip-components=1 \ - && cd /tmp/libcurl \ - && ./configure --disable-manual --disable-cookies --with-nghttp2=$PREFIX --prefix=$PREFIX \ - && make -j $(nproc) --silent && make install \ - && rm -rf /tmp/libcurl - -# libdeflate -RUN mkdir /tmp/libdeflate \ - && curl -sfL https://github.com/ebiggers/libdeflate/archive/v1.10.tar.gz | tar zxf - -C /tmp/libdeflate --strip-components=1 \ - && cd /tmp/libdeflate \ - && make -j $(nproc) --silent PREFIX=$PREFIX && make install \ - && rm -rf /tmp/libdeflate - -# libpng -ENV LIBPNG_VERSION=1.6.37 -RUN mkdir /tmp/png \ - && curl -sfL https://github.com/glennrp/libpng/archive/v${LIBPNG_VERSION}.tar.gz | tar zxf - -C /tmp/png --strip-components=1 \ - && cd /tmp/png \ - && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX \ - && make -j $(nproc) --silent && make install \ - && rm -rf /tmp/png - -# libjpeg_turbo -ENV LIBJPEG_TURBO_VERSION=2.1.3 -RUN mkdir /tmp/jpeg \ - && curl -sfL https://github.com/libjpeg-turbo/libjpeg-turbo/archive/${LIBJPEG_TURBO_VERSION}.tar.gz | tar zxf - -C /tmp/jpeg --strip-components=1 \ - && cd /tmp/jpeg \ - && mkdir build && cd build \ - && cmake3 .. \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ - -DCMAKE_INSTALL_LIBDIR:PATH=lib \ - -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ - -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ - && make -j $(nproc) --silent && make install \ - && rm -rf /tmp/jpeg - -# webp -ENV WEBP_VERSION=1.1.0 -RUN mkdir /tmp/webp \ - && curl -sfL https://storage.googleapis.com/downloads.webmproject.org/releases/webp/libwebp-${WEBP_VERSION}.tar.gz | tar zxf - -C /tmp/webp --strip-components=1 \ - && cd /tmp/webp \ - && CFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX \ - && make -j $(nproc) --silent && make install \ - && rm -rf /tmp/webp - -# zstd -ENV ZSTD_VERSION=1.4.5 -RUN mkdir /tmp/zstd \ - && curl -sfL https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz | tar zxf - -C /tmp/zstd --strip-components=1 \ - && cd /tmp/zstd \ - && make -j $(nproc) PREFIX=$PREFIX ZSTD_LEGACY_SUPPORT=0 CFLAGS=-O1 --silent && make install PREFIX=$PREFIX ZSTD_LEGACY_SUPPORT=0 CFLAGS=-O1 \ - && rm -rf /tmp/zstd - -# lerc -ENV LERC_VERSION=2.2.1 -RUN mkdir /tmp/lerc \ - && curl -sfL https://github.com/Esri/lerc/archive/v${LERC_VERSION}.tar.gz | tar zxf - -C /tmp/lerc --strip-components=1 \ - && cd /tmp/lerc \ - && cmake3 . \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ - -DCMAKE_INSTALL_LIBDIR:PATH=lib \ - -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ - -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ - && make -j $(nproc) --silent && make install \ - && rm -rf /tmp/lerc - -# libtiff -ENV LIBTIFF_VERSION=4.5.0 -RUN mkdir /tmp/libtiff \ - && curl -sfL https://download.osgeo.org/libtiff/tiff-${LIBTIFF_VERSION}.tar.gz | tar zxf - -C /tmp/libtiff --strip-components=1 \ - && cd /tmp/libtiff \ - && LDFLAGS="-Wl,-rpath,'\$\$ORIGIN'" CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure \ - --prefix=$PREFIX \ - --disable-static \ - --enable-rpath \ - --with-lerc-lib-dir=$PREFIX/lib \ - --with-lerc-include-dir=$PREFIX/include \ - --with-libdeflate-lib-dir=$PREFIX/lib \ - --with-libdeflate-include-dir=$PREFIX/include \ - --with-jpeg-lib-dir=$PREFIX/lib \ - --with-jpeg-include-dir=$PREFIX/include \ - && make -j $(nproc) --silent && make install \ - && rm -rf /tmp/libtiff - -# postgres -ENV PG_VERSION=15.2 -RUN mkdir /tmp/postgres \ - && curl -sfL https://ftp.postgresql.org/pub/source/v${PG_VERSION}/postgresql-${PG_VERSION}.tar.gz | tar zxf - -C /tmp/postgres --strip-components=1 \ - && cd /tmp/postgres \ - && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --with-openssl \ - && make -j $(nproc) --silent && make install \ - && rm -rf /tmp/postgres - -# openjpeg -ENV OPENJPEG_VERSION=2.5.0 -RUN mkdir /tmp/openjpeg \ - && curl -sfL https://github.com/uclouvain/openjpeg/archive/v${OPENJPEG_VERSION}.tar.gz | tar zxf - -C /tmp/openjpeg --strip-components=1 \ - && cd /tmp/openjpeg \ - && mkdir build && cd build \ - && cmake3 .. \ - -DBUILD_THIRDPARTY:BOOL=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ - -DCMAKE_INSTALL_LIBDIR:PATH=lib \ - -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ - -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ - && make -j $(nproc) --silent && make install \ - && rm -rf /tmp/openjpeg - -# geos -ENV GEOS_VERSION=3.11.2 -RUN mkdir /tmp/geos \ - && curl -sfL https://github.com/libgeos/geos/archive/refs/tags/${GEOS_VERSION}.tar.gz | tar zxf - -C /tmp/geos --strip-components=1 \ - && cd /tmp/geos \ - && mkdir build && cd build \ - && cmake3 .. \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ - -DCMAKE_INSTALL_LIBDIR:PATH=lib \ - -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ - -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ - && make -j $(nproc) --silent && make install \ - && rm -rf /tmp/geos - -# proj -# We still provide proj-datumgrid but the user can choose to set `PROJ_NETWORK=ON` to use network grids -# ENV PROJ_VERSION=9.2.0 -# RUN mkdir /tmp/proj && mkdir /tmp/proj/data \ -# && curl -sfL https://github.com/OSGeo/proj/archive/${PROJ_VERSION}.tar.gz | tar zxf - -C /tmp/proj --strip-components=1 \ -# && curl -sfL http://download.osgeo.org/proj/proj-datumgrid-latest.tar.gz | tar zxf - -C /tmp/proj/data \ -# && cd /tmp/proj \ -# && aclocal && autoheader && libtoolize --force --copy && automake --add-missing --copy && autoconf \ -# && LDFLAGS="-Wl,-rpath,'\$\$ORIGIN'" CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --disable-static --enable-lto \ -# && make -j $(nproc) --silent && make install \ -# && rm -rf /tmp/proj - -ENV PROJ_VERSION=9.2.0 -RUN mkdir /tmp/proj && mkdir /tmp/proj/data \ - && curl -sfL https://github.com/OSGeo/proj/archive/${PROJ_VERSION}.tar.gz | tar zxf - -C /tmp/proj --strip-components=1 \ - && cd /tmp/proj \ - && mkdir build && cd build \ - && cmake3 .. \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ - -DCMAKE_INSTALL_LIBDIR:PATH=lib \ - -DCMAKE_INSTALL_INCLUDEDIR:PATH=include \ - -DBUILD_TESTING=OFF \ - -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ - -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ - && make -j $(nproc) --silent && make install \ - && rm -rf /tmp/proj - -# GeoTIFF -ENV LIBGEOTIFF_VERSION=1.7.1 -RUN mkdir /tmp/libgeotiff \ - && curl -sfL https://github.com/OSGeo/libgeotiff/releases/download/${LIBGEOTIFF_VERSION}/libgeotiff-${LIBGEOTIFF_VERSION}.tar.gz | tar zxf - -C /tmp/libgeotiff --strip-components=1 \ - && cd /tmp/libgeotiff \ - && LDFLAGS="-Wl,-rpath,'\$\$ORIGIN'" CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure \ - --prefix=$PREFIX \ - --with-proj=$PREFIX \ - --with-jpeg=$PREFIX \ - --with-zip=yes \ - --with-zlib \ - --with-libtiff=$PREFIX \ - && make -j $(nproc) --silent && make install \ - && rm -rf /tmp/libgeotiff - -# Build blosc (needed for ZARR) -ENV BLOSC_VERSION=1.21.0 -RUN mkdir /tmp/blosc \ - && curl -sfL https://github.com/Blosc/c-blosc/archive/refs/tags/v${BLOSC_VERSION}.tar.gz | tar zxf - -C /tmp/blosc --strip-components=1 \ - && cd /tmp/blosc \ - && cmake3 -G"Unix Makefiles" \ - -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ - -DCMAKE_INSTALL_LIBDIR:PATH=lib \ - -DCMAKE_INSTALL_RPATH="$ORIGIN" \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ - -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ - -DBUILD_SHARED=ON \ - -DBUILD_STATIC=OFF \ - -DBUILD_BENCHMARKS=OFF \ - -DBUILD_FUZZERS=OFF \ - -DBUILD_TESTS=OFF \ - -DPREFER_EXTERNAL_LZ4=ON \ - -DPREFER_EXTERNAL_ZLIB=ON \ - -DPREFER_EXTERNAL_ZSTD=ON \ - -DDEACTIVATE_SNAPPY=OFF . \ - && make -j $(nproc) --silent && make install && make clean \ - && rm -rf /tmp/blosc - -# libcrypto (for GPC access) -RUN mkdir /tmp/cryptopp \ - && curl -sfL https://github.com/weidai11/cryptopp/archive/refs/tags/CRYPTOPP_8_6_0.tar.gz | tar zxf - -C /tmp/cryptopp --strip-components=1 \ - && cd /tmp/cryptopp \ - && make -j $(nproc) --silent && make install PREFIX=$PREFIX \ - && rm -rf /tmp/cryptopp - -# szip (for hdf) -ENV SZIP_VERSION=2.1.1 -RUN mkdir /tmp/szip \ - && curl -sfL https://support.hdfgroup.org/ftp/lib-external/szip/${SZIP_VERSION}/src/szip-${SZIP_VERSION}.tar.gz | tar zxf - -C /tmp/szip --strip-components=1 \ - && cd /tmp/szip \ - && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure --prefix=$PREFIX --disable-static \ - && make -j $(nproc) --silent && make install && make clean \ - && rm -rf /tmp/szip - -# libhdf4 -ENV HDF4_VERSION=4.2.15 -RUN mkdir /tmp/hdf4 \ - && curl -sfL https://support.hdfgroup.org/ftp/HDF/releases/HDF${HDF4_VERSION}/src/hdf-${HDF4_VERSION}.tar | tar xvf - -C /tmp/hdf4 --strip-components=1 \ - && cd /tmp/hdf4 \ - && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure \ - --prefix=$PREFIX \ - --with-szlib=$PREFIX \ - --with-jpeg=$PREFIX \ - --enable-shared \ - --disable-static \ - --disable-netcdf \ - --disable-fortran \ - && make -j $(nproc) --silent && make install && make clean \ - && rm -rf /tmp/hdf4 - -# libhdf5 -ENV HDF5_VERSION=1.12.0 -RUN mkdir /tmp/hdf5 \ - && curl -sfL https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${HDF5_VERSION%.*}/hdf5-${HDF5_VERSION}/src/hdf5-$HDF5_VERSION.tar.gz | tar zxf - -C /tmp/hdf5 --strip-components=1 \ - && cd /tmp/hdf5 \ - && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ./configure \ - --prefix=$PREFIX \ - --with-szlib=$PREFIX \ - --enable-cxx \ - --enable-thread-safe \ - --disable-static \ - && make -j $(nproc) --silent && make install && make clean \ - && rm -rf /tmp/hdf5 - -# NetCDF -ENV NETCDF_VERSION=4.7.4 -RUN mkdir /tmp/netcdf \ - && curl -sfL https://github.com/Unidata/netcdf-c/archive/v${NETCDF_VERSION}.tar.gz | tar zxf - -C /tmp/netcdf --strip-components=1 \ - && cd /tmp/netcdf \ - && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" CPPFLAGS="-I${PREFIX}/include" LDFLAGS="-L${PREFIX}/lib" ./configure \ - --with-default-chunk-size=67108864 \ - --with-chunk-cache-size=67108864 \ - --prefix=$PREFIX \ - --disable-static \ - --enable-netcdf4 \ - --enable-dap \ - --with-pic \ - && make -j $(nproc) --silent && make install && make clean \ - && rm -rf /tmp/netcdf - -# # JPEG-XL -# # libhwy -# RUN mkdir /tmp/libhwy \ -# && curl -sfL https://github.com/google/highway/archive/refs/tags/0.16.0.tar.gz | tar zxf - -C /tmp/libhwy --strip-components=1 \ -# && cd /tmp/libhwy \ -# && mkdir build && cd build \ -# && cmake3 .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_PREFIX=$PREFIX \ -# && make -j $(nproc) install && make clean \ -# && rm -rf /tmp/libhwy - -# # brotli -# ENV BROTLI_VERSION=1.0.9 -# RUN mkdir /tmp/brotli \ -# && curl -sfL https://github.com/google/brotli/archive/refs/tags/v${BROTLI_VERSION}.tar.gz | tar zxf - -C /tmp/brotli --strip-components=1 \ -# && cd /tmp/brotli \ -# && mkdir build && cd build \ -# && CFLAGS="-O2 -Wl,-S" CXXFLAGS="-O2 -Wl,-S" ../configure-cmake --prefix=$PREFIX \ -# && make -j $(nproc) --silent && make install && make clean \ -# && rm -rf /tmp/brotli - -# # # JpegXL -# ENV JXL_VERSION=0.8.1 -# RUN mkdir /tmp/jxl \ -# && curl -sfL https://github.com/libjxl/libjxl/archive/refs/tags/v${JXL_VERSION}.tar.gz | tar zxf - -C /tmp/jxl --strip-components=1 \ -# && cd /tmp/jxl \ -# && mkdir build && cd build \ -# && cmake3 .. -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_PREFIX=$PREFIX \ -# && make -j $(nproc) install && make clean \ -# && rm -rf /tmp/jxl - -# We use commit sha to make sure we are not using `cache` when building the docker image -# "7ca88116f5a46d429251361634eb24629f315076" is the latest commit on release/3.6 branch - -# gdal -RUN mkdir /tmp/gdal \ - && curl -sfL https://github.com/OSGeo/gdal/archive/7ca88116f5a46d429251361634eb24629f315076.tar.gz | tar zxf - -C /tmp/gdal --strip-components=1 \ - && cd /tmp/gdal \ - && mkdir build && cd build \ - && cmake3 .. \ - -DGDAL_USE_EXTERNAL_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX \ - -DCMAKE_INSTALL_LIBDIR:PATH=lib \ - -DCMAKE_C_FLAGS="-O2 -Wl,-S" \ - -DCMAKE_CXX_FLAGS="-O2 -Wl,-S" \ - -DGDAL_SET_INSTALL_RELATIVE_RPATH=ON \ - -DGDAL_USE_TIFF_INTERNAL=OFF \ - -DGDAL_USE_GEOTIFF_INTERNAL=OFF \ - -DGDAL_USE_LERC_INTERNAL=OFF \ - -DLERC_INCLUDE_DIR=$PREFIX/include \ - -DLERC_LIBRARY=$PREFIX/lib/libLercLib.so \ - -DPNG_PNG_INCLUDE_DIR=$PREFIX/include \ - -DPNG_LIBRARY_RELEASE=$PREFIX/lib/libpng.so \ - -DBUILD_PYTHON_BINDINGS=OFF \ - && make -j $(nproc) --silent && make install \ - && rm -rf /tmp/gdal - -# from https://github.com/pypa/manylinux/blob/d8ef5d47433ba771fa4403fd48f352c586e06e43/docker/build_scripts/build.sh#L133-L138 -# Install patchelf (latest with unreleased bug fixes) -ENV PATCHELF_VERSION=0.10 -RUN mkdir /tmp/patchelf \ - && curl -sfL https://github.com/NixOS/patchelf/archive/${PATCHELF_VERSION}.tar.gz | tar zxf - -C /tmp/patchelf --strip-components=1 \ - && cd /tmp/patchelf && ./bootstrap.sh && ./configure \ - && make -j $(nproc) --silent && make install \ - && cd / && rm -rf /tmp/patchelf - -# # libturbojpeg.so is not used by GDAL. Only libjpeg.so* -RUN rm -f $PREFIX/lib/libturbojpeg.so* \ - # Only libwebp.so is used by GDAL - && rm -f $PREFIX/lib/libwebpmux.so* $PREFIX/lib/libwebpdemux.so* $PREFIX/lib/libwebpdecoder.so* - -# FIX -RUN for i in $PREFIX/bin/*; do patchelf --force-rpath --set-rpath '$ORIGIN/../lib' $i; done - -# Build final image -FROM public.ecr.aws/lambda/provided:al2 as runner - -ENV PREFIX /opt -COPY --from=builder /opt/lib/ $PREFIX/lib/ -COPY --from=builder /opt/include/ $PREFIX/include/ -COPY --from=builder /opt/share/ $PREFIX/share/ -COPY --from=builder /opt/bin/ $PREFIX/bin/ - -RUN export GDAL_VERSION=$(gdal-config --version) - -ENV \ - GDAL_DATA=$PREFIX/share/gdal \ - PROJ_LIB=$PREFIX/share/proj \ - GDAL_CONFIG=$PREFIX/bin/gdal-config \ - GEOS_CONFIG=$PREFIX/bin/geos-config \ - PATH=$PREFIX/bin:$PATH - -ENTRYPOINT bash diff --git a/scripts/build.sh b/scripts/build.sh index bebd156..d8bf951 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,6 +1,7 @@ #!/bin/bash GDAL_VERSION=$1 +GDAL_VERSION_TAG=${GDAL_VERSION%.*} RUNTIME=$2 RUNTIME_VERSION=$3 @@ -10,11 +11,11 @@ docker buildx build \ --platform=linux/amd64 \ --build-arg GDAL_VERSION=${GDAL_VERSION} \ -f dockerfiles/Dockerfile \ - -t ghcr.io/lambgeo/lambda-gdal:${GDAL_VERSION} . + -t ghcr.io/lambgeo/lambda-gdal:${GDAL_VERSION_TAG} . docker buildx build \ --platform=linux/amd64 \ --build-arg GDAL_VERSION=${GDAL_VERSION} \ --build-arg RUNTIME_VERSION=${RUNTIME_VERSION} \ -f dockerfiles/runtimes/${RUNTIME} \ - -t ghcr.io/lambgeo/lambda-gdal:${GDAL_VERSION}-${RUNTIME}${RUNTIME_VERSION} . + -t ghcr.io/lambgeo/lambda-gdal:${GDAL_VERSION_TAG}-${RUNTIME}${RUNTIME_VERSION} . From 8c4e282646d1148ef60801f08a39a307b00d6717 Mon Sep 17 00:00:00 2001 From: Phil Varner Date: Tue, 28 Nov 2023 11:08:05 -0500 Subject: [PATCH 4/4] replace matrix with env --- .github/workflows/ci.yml | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d947e0a..cd129e5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,13 +14,14 @@ on: - '*' pull_request: +env: + GDAL_VERSION: 3.8.0 + GDAL_VERSION_TAG: 3.8 + jobs: build: runs-on: ubuntu-latest if: "!contains(github.event.head_commit.message, '[skip ci]')" - strategy: - matrix: - gdal-version: ["3.8.0"] steps: - uses: actions/checkout@v4 @@ -52,8 +53,8 @@ jobs: load: true file: dockerfiles/Dockerfile build-args: | - GDAL_VERSION=${{ matrix.gdal-version }} - tags: ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ matrix.gdal-version }} + GDAL_VERSION=${{ env.GDAL_VERSION }} + tags: ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }} cache-from: type=gha cache-to: type=gha,mode=max @@ -63,7 +64,7 @@ jobs: --platform=linux/amd64 \ --entrypoint bash \ -v ${{ github.workspace }}:/local \ - --rm ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ matrix.gdal-version }} \ + --rm ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }} \ /local/tests/tests.sh deploy: @@ -97,10 +98,10 @@ jobs: context: . file: dockerfiles/Dockerfile build-args: | - GDAL_VERSION=${{ matrix.gdal-version }} + GDAL_VERSION=${{ env.GDAL_VERSION }} push: true cache-from: type=gha - tags: ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ matrix.gdal-version }} + tags: ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }} - name: Build and Deploy layers run: | @@ -108,9 +109,9 @@ jobs: --platform=linux/amd64 \ --entrypoint bash \ -v ${{ github.workspace }}:/local \ - --rm ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ matrix.gdal-version }} \ + --rm ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }} \ /local/scripts/create-layer.sh - python scripts/deploy.py ${{ matrix.gdal-version }} --deploy + python scripts/deploy.py ${{ env.GDAL_VERSION }} --deploy - name: Build Runtime and Push run: | @@ -118,11 +119,11 @@ jobs: for runtime in ${runtimes}; do docker build \ --platform=linux/amd64 \ - --build-arg GDAL_VERSION=${{ matrix.gdal-version }} \ + --build-arg GDAL_VERSION=${{ env.GDAL_VERSION }} \ --build-arg RUNTIME_VERSION=${runtime} \ -f dockerfiles/runtimes/python \ - -t ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ matrix.gdal-version }}-python${runtime} . - docker push ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ matrix.gdal-version }}-python${runtime} + -t ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }}-python${runtime} . + docker push ghcr.io/${{ github.repository_owner }}/lambda-gdal:${{ env.GDAL_VERSION_TAG }}-python${runtime} done update-layer: