From 5651283071a0736b58ed2c8a60c4f2e9b38ce04a Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 1 Nov 2019 09:58:22 +0100 Subject: [PATCH 1/2] Change Dockerfile to be a multistage build The existing Dockerfile produced a broken binary (static compile caused the binary to segfault). This patch changes the Dockerfile to - Build a non-static binary - Use a multi-stage build: separate stages are used for build and deploy; the build stage contains the tools needed for building the binary, after which the build artefacts are copied to the final stage. which produces the actual image. Using this approach, the build-tools can be kept in the Docker build-cache, which improves performance of repeated builds and does not require cleanup steps in the Dockerfile itself, which can simplify the steps. - The official Python image is used for the build stage, so that there is no need to install python manually. The version of python is set to the same version as the version used in CI, but can be overridden using at build-time using a build-arg: docker build --build-arg PYTHON_VERSION=x .. - Both build- and deploy stages use the debian "stretch" variants, which matches what's used in the current version of the Dockerfile, but could be updated to the current stable ("buster") variant if there is a need. - The final stage uses the "slim" variant to reduce the size. For comparison, below are sizes of the final image when using the regular, or "slim" version of the image: REPOSITORY TAG IMAGE ID CREATED SIZE jq stretch-slim 4b7f380970f0 About a minute ago 70.3MB jq stretch c5fa8b766cd4 7 minutes ago 119MB A .dockerignore file is also added to prevent "busting" the Docker build-cache if not needed, and steps in the Dockerfile are optimized for BuildKit, but remain compatible with the legacy builder. To build, and verify the image: DOCKER_BUILDKIT=1 docker build -t jq . And, test the image, us it (for example) to pretty-print its own "docker image inspect" output: docker image inspect jq | docker run -i --rm jq . [ { "Id": "sha256:4b7f380970f0d06d4ca9137fbbb3fa3f6554da996147d1d956cda6f44be25650", "RepoTags": [ "jq:latest", "jq:stretch-slim" ], Signed-off-by: Sebastiaan van Stijn --- .dockerignore | 60 +++++++++++++++++++++++++++++++++ Dockerfile | 92 +++++++++++++++++++++++++-------------------------- 2 files changed, 106 insertions(+), 46 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..18fdfebc85 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,60 @@ +*.o +*.a +*.lo +*.la +*.lai +*.so +*.so.* +*.gcda +*.gcno +*.gcov +*~ +.*.sw[a-p] +tags + +jq +!tests/modules/lib/jq/ +jq.1 + +# Generated source +src/builtin.inc +*.pc + +# Autotools junk +.libs +.deps +.dirstamp +libtool +*.log +stamp-h1 +config.log +config.status +autom4te.cache +INSTALL +Makefile +jq-*.tar.gz +configure +aclocal.m4 +Makefile.in +version.h +.remake-version-h +config.cache +*.rpm +m4/libtool.m4 +m4/ltoptions.m4 +m4/ltsugar.m4 +m4/ltversion.m4 +m4/lt~obsolete.m4 +tests/*.trs + +cscope.in.out +cscope.out +cscope.po.out +jq.dSYM + +# Don't bust the build-cache on Dockerfile/.dockerignore only changes +.dockerignore +.github +.travis.yml +appveyor.yml +build/* diff --git a/Dockerfile b/Dockerfile index f69f1f4df8..b07cdc619a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,54 +1,54 @@ -FROM debian:9 -ENV DEBIAN_FRONTEND=noninteractive \ - DEBCONF_NONINTERACTIVE_SEEN=true \ - LC_ALL=C.UTF-8 \ - LANG=C.UTF-8 +ARG PYTHON_VERSION=3.6.7 -COPY . /app - -# get dependencies, build, and remove anything we don't need for running jq. -# valgrind seems to have trouble with pthreads TLS so it's off. - -RUN apt-get update && \ - apt-get install -y \ - build-essential \ +FROM python:${PYTHON_VERSION}-stretch AS build +CMD ["bash"] +ENV LC_ALL=C.UTF-8 +ARG DEBIAN_FRONTEND=noninteractive +ARG DEBCONF_NONINTERACTIVE_SEEN=true +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ autoconf \ - libtool \ - git \ bison \ - flex \ - python3 \ - python3-pip \ - wget && \ - pip3 install pipenv && \ - (cd /app/docs && pipenv sync) && \ - (cd /app && \ - git submodule init && \ - git submodule update && \ - autoreconf -i && \ - ./configure --disable-valgrind --enable-all-static --prefix=/usr/local && \ - make -j8 && \ - make check && \ - make install ) && \ - (cd /app/modules/oniguruma && \ - make uninstall ) && \ - (cd /app && \ - make distclean ) && \ - apt-get purge -y \ build-essential \ - autoconf \ - libtool \ - bison \ - git \ flex \ - python3 \ - python3-pip && \ - apt-get autoremove -y && \ - rm -rf /app/modules/oniguruma/* && \ - rm -rf /app/modules/oniguruma/.git && \ - rm -rf /app/modules/oniguruma/.gitignore && \ - rm -rf /var/lib/apt/lists/* /var/lib/gems + git \ + libtool \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Install docs dependencies; we're only copying the Pipfile here, +# so that this is only run again if dependencies change +RUN pip3 install pipenv +COPY docs/Pipfile docs/Pipfile.lock ./docs/ +RUN cd docs && pipenv sync + +COPY .gitmodules . +COPY .git ./.git/ +RUN git submodule init +RUN git submodule update +RUN sed -i.bak '/^AM_INIT_AUTOMAKE(\[-Wno-portability 1\.14\])$/s/14/11/' modules/oniguruma/configure.ac + +COPY . . +RUN autoreconf -if +RUN ./configure --disable-valgrind --with-oniguruma=builtin YACC=/usr/bin/bison +RUN make BISON_PKGDATADIR=/usr/bin/bison src/parser.c || make src/parser.c +RUN make -j8 +RUN make check -j8 +RUN make DESTDIR=/build install +RUN /build/usr/local/bin/jq -V + +# Collect the files for the final image +FROM debian:stretch-slim AS deploy +ENV LC_ALL=C.UTF-8 +ARG DEBIAN_FRONTEND=noninteractive +ARG DEBCONF_NONINTERACTIVE_SEEN=true +RUN apt-get update \ + && apt-get install -y --no-install-recommends man \ + && rm -rf /var/lib/apt/lists/* +COPY --from=build /build/usr/local/. /usr/local/ +RUN jq -V ENTRYPOINT ["/usr/local/bin/jq"] -CMD [] +CMD ["--help"] From 0e98fdf7ddd8d668500126dc4bf679af21b9bc05 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 1 Nov 2019 10:37:40 +0100 Subject: [PATCH 2/2] Dockerfile: use Alpine image to build static binary This patch switches the Dockerfile to use an Alpine Linux base image to build the static binary. Given that the binary is now fully static, a minimal image can be produced that only contains the essential artefacts. The static version of the Docker image does not contain the man- pages, so the existing (debian based) Dockerfile is kept, but renamed to Dockerfile.debian, and can be built by specifying the alternative Dockerfile using the `-f` flag: DOCKER_BUILDKIT=1 docker build -t jq:stretch -f Dockerfile.debian . With this patch applied, the image size is reduced drastically: DOCKER_BUILDKIT=1 docker build -t jq . docker image ls REPOSITORY TAG IMAGE ID CREATED SIZE jq latest 7da2ea1c016d 5 seconds ago 800kB jq stretch-slim 4b7f380970f0 About a minute ago 70.3MB jq stretch c5fa8b766cd4 7 minutes ago 119MB The image can be tested, for example, by inspecting itself and pretty- printing the json: docker image inspect jq | docker run -i --rm jq . [ { "Id": "sha256:7da2ea1c016d97e7b52b09d5a323a1e7c0e4dbdbc77ce2715aedd3188721e4da", "RepoTags": [ "jq:latest", "jq:static" ], ... Signed-off-by: Sebastiaan van Stijn --- Dockerfile | 65 +++++++++++++++++------------------------------ Dockerfile.debian | 54 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 42 deletions(-) create mode 100644 Dockerfile.debian diff --git a/Dockerfile b/Dockerfile index b07cdc619a..39f2853e5f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,54 +1,35 @@ +FROM alpine:3.12 AS build -ARG PYTHON_VERSION=3.6.7 - -FROM python:${PYTHON_VERSION}-stretch AS build -CMD ["bash"] ENV LC_ALL=C.UTF-8 -ARG DEBIAN_FRONTEND=noninteractive -ARG DEBCONF_NONINTERACTIVE_SEEN=true -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ +ENV LANG=C.UTF-8 + +RUN apk add --no-cache \ + automake \ autoconf \ + build-base \ bison \ - build-essential \ flex \ git \ libtool \ - && rm -rf /var/lib/apt/lists/* + oniguruma-dev WORKDIR /app - -# Install docs dependencies; we're only copying the Pipfile here, -# so that this is only run again if dependencies change -RUN pip3 install pipenv -COPY docs/Pipfile docs/Pipfile.lock ./docs/ -RUN cd docs && pipenv sync - -COPY .gitmodules . -COPY .git ./.git/ -RUN git submodule init -RUN git submodule update -RUN sed -i.bak '/^AM_INIT_AUTOMAKE(\[-Wno-portability 1\.14\])$/s/14/11/' modules/oniguruma/configure.ac - COPY . . RUN autoreconf -if -RUN ./configure --disable-valgrind --with-oniguruma=builtin YACC=/usr/bin/bison -RUN make BISON_PKGDATADIR=/usr/bin/bison src/parser.c || make src/parser.c -RUN make -j8 -RUN make check -j8 -RUN make DESTDIR=/build install -RUN /build/usr/local/bin/jq -V - -# Collect the files for the final image -FROM debian:stretch-slim AS deploy -ENV LC_ALL=C.UTF-8 -ARG DEBIAN_FRONTEND=noninteractive -ARG DEBCONF_NONINTERACTIVE_SEEN=true -RUN apt-get update \ - && apt-get install -y --no-install-recommends man \ - && rm -rf /var/lib/apt/lists/* - -COPY --from=build /build/usr/local/. /usr/local/ -RUN jq -V -ENTRYPOINT ["/usr/local/bin/jq"] +RUN ./configure --disable-docs --enable-all-static CFLAGS='-Os -static -no-pie' CXXFLAGS='-Os -static -no-pie' +RUN make +RUN make check +RUN strip jq + +# Ensure that the built executable is really statically linked. +RUN file jq | grep -Fw 'statically linked' + +# The deploy stage is the final image, and only contains artefacts +# that should be published. +FROM scratch AS deploy +COPY --from=build /app/AUTHORS / +COPY --from=build /app/COPYING / +COPY --from=build /app/jq /jq +RUN ["/jq", "-V"] +ENTRYPOINT ["/jq"] CMD ["--help"] diff --git a/Dockerfile.debian b/Dockerfile.debian new file mode 100644 index 0000000000..5a3ada2424 --- /dev/null +++ b/Dockerfile.debian @@ -0,0 +1,54 @@ + +ARG PYTHON_VERSION=3.6.7 + +FROM python:${PYTHON_VERSION}-stretch AS build +CMD ["bash"] +ENV LC_ALL=C.UTF-8 +ARG DEBIAN_FRONTEND=noninteractive +ARG DEBCONF_NONINTERACTIVE_SEEN=true +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + autoconf \ + bison \ + build-essential \ + flex \ + git \ + libtool \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Install docs dependencies; we're only copying the Pipfile here, +# so that this is only run again if dependencies change +RUN pip3 install pipenv +COPY docs/Pipfile docs/Pipfile.lock ./docs/ +RUN cd docs && pipenv sync + +COPY .gitmodules . +COPY .git ./.git/ +RUN git submodule init +RUN git submodule update +RUN sed -i.bak '/^AM_INIT_AUTOMAKE(\[-Wno-portability 1\.14\])$/s/14/11/' modules/oniguruma/configure.ac + +COPY . . +RUN autoreconf -if +RUN ./configure --disable-valgrind --with-oniguruma=builtin YACC=/usr/bin/bison --prefix=/build/ +RUN make BISON_PKGDATADIR=/usr/bin/bison src/parser.c || make src/parser.c +RUN make -j8 +RUN make check -j8 +RUN make install +RUN /build/bin/jq -V + +# Collect the files for the final image +FROM debian:stretch-slim AS deploy +ENV LC_ALL=C.UTF-8 +ARG DEBIAN_FRONTEND=noninteractive +ARG DEBCONF_NONINTERACTIVE_SEEN=true +RUN apt-get update \ + && apt-get install -y --no-install-recommends man \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=build /build/. /usr/local/ +RUN jq -V +ENTRYPOINT ["/usr/local/bin/jq"] +CMD ["--help"]