From 85c62457040c584ff7742c5f6b5395858de29788 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Wed, 17 Feb 2021 11:02:11 -0800 Subject: [PATCH] Add support for rust 1.50 and start maintaining multiple rust containers (#11) * Maintain containers for multiple rust versions * update docs to reflect multi-rust --- README.md | 123 ++++++++++++++---- .../geo-ci.Dockerfile | 0 .../libproj-builder.Dockerfile | 0 .../proj-ci-without-system-proj.Dockerfile | 2 +- .../proj-ci.Dockerfile | 4 +- rust-1.50/geo-ci.Dockerfile | 31 +++++ rust-1.50/libproj-builder.Dockerfile | 42 ++++++ .../proj-ci-without-system-proj.Dockerfile | 11 ++ rust-1.50/proj-ci.Dockerfile | 10 ++ 9 files changed, 197 insertions(+), 26 deletions(-) rename geo-ci.Dockerfile => rust-1.49/geo-ci.Dockerfile (100%) rename libproj-builder.Dockerfile => rust-1.49/libproj-builder.Dockerfile (100%) rename proj-ci-without-system-proj.Dockerfile => rust-1.49/proj-ci-without-system-proj.Dockerfile (91%) rename proj-ci.Dockerfile => rust-1.49/proj-ci.Dockerfile (65%) create mode 100644 rust-1.50/geo-ci.Dockerfile create mode 100644 rust-1.50/libproj-builder.Dockerfile create mode 100644 rust-1.50/proj-ci-without-system-proj.Dockerfile create mode 100644 rust-1.50/proj-ci.Dockerfile diff --git a/README.md b/README.md index ed320d5..8821f17 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,100 @@ -# Georust Docker Images - -## Instructions for updating `proj` Docker images, the `proj` crate, and the `proj` version in use by the `geo` crate - -In this example, we'll be updating to `PROJ 7.2.1`, using the tag name `proj-7.2.1` - - -1. Edit `libproj-builder.Dockerfile` to download and build `PROJ` 7.2.1 -2. Build the image, tagging it with the new `PROJ` version: `docker build -t georust/libproj-builder:proj-7.2.1 -f libproj-builder.Dockerfile .` -3. Push the new tag to Docker Hub: `docker push georust/libproj-builder:proj-7.2.1` -4. You can now update the two child `proj` Dockerfiles in this repo to use the new `libproj-builder` tag. Then build and push them: -5. `docker build -t georust/proj-ci:proj-7.2.1 -f proj-ci.Dockerfile .` -6. `docker push georust/proj-ci:proj-7.2.1` -7. `docker build -t georust/proj-ci-without-system-proj:proj-7.2.1 -f proj-ci-without-system-proj.Dockerfile .` -8. `docker push georust/proj-ci-without-system-proj:proj-7.2.1` -9. The three images should now each have a new tag, `proj-7.2.1` on Docker Hub -10. You will now need to open a separate PR to update the `proj-sys` and `proj` tests. Note that this PR must be **manually merged** as bors cannot modify workfows. Edit https://github.com/georust/proj/.github/workflows/test.yml to use the Docker images with the new tag -11. When the PR in step 10 has merged, you can update the compressed `PROJ` version in [the proj repo](https://github.com/georust/proj/proj-sys/PROJSRC) -12. Edit the adjacent [https://github.com/georust/proj/proj-sys/build.rs](`build.rs`) to look for the updated PROJ version -13. Bump the `proj-sys` version and update `proj/Cargo.toml` to use it. You can now open a PR to update the `proj`crate -14. When the PR has merged and the `proj` crate has been published, you can update [geo](https://github.com/georust/geo) to use the new `proj` crate: -15. Modify `geo-ci.Dockerfile` to use the latest `libproj-builder` tag, then build `geo-ci` with a new tag and push it to Docker hub -16. You can now modify the `geo` test script to use the new `geo-ci` Docker tag, allowing the updated `proj` crate version in use by `geo` to be tested using the correct Docker images. +# GeoRust Docker Images + +Configuration for Docker containers used by georust projects, primarily for CI. + +## Supporting Multiple Versions with Docker Tags + +Not everyone is running the latest and greatest of everything at the same time, +so it makes sense to support a reasonable window of dependencies. + +Unlike a `git tag`, a `tag` in Docker is not conventionally treated as an +immutable alias to a precise single state. Rather, it's more like a named +"release track" or "semver constraint". + +Using the official Rust Docker containers as an example, the `rust:1.45` tag +should be read as "the latest container published for rust >=1.45.0 <1.46". The +`rust:1.45` tag initially referred to the 1.45.0 release, but was +later [updated to refer to the 1.45.1 release, and then again to the 1.45.2 +release](https://hub.docker.com/_/rust?tab=tags&page=1&ordering=last_updated&name=1.45). +This is by design - a docker tag does not have the same goals as a git tag. + +We lean into this strategy of "docker tag as semver". The current stable +version of Rust is 1.50.0, so if I published `geo-ci:rust-1.50` today based on +`rust:1.50`, it would include rustc 1.50.0. If Rust later publishes a +hypothetical 1.50.1 release, they would update their `rust:1.50` container. It +would be entirely normal and expected that I would rebuild at that point, and +clobber our `geo-ci:rust-1.50` tag with a new image based on rust 1.50.1. + +If we did want to use only a very specific rust version, rust publishes the 3 +number tag as well, e.g. `rust:1.45.1`. Similarly we could publish a new tag +for each minor patch (`geo-ci:rust-1.50.0`, `geo-ci:rust-1.50.1`, etc.), but +running CI against each patch seems like overkill at this point. + +## How to Update Rust + +### add a new set of Dockerfiles + +i.e. assume we're adding support for rust 1.50. + + cp -r rust-1.49 rust-1.50 + # edit all Dockerfiles and tags to refer to rust-1.50 instead of rust-1.49 + vim rust-1.50/* + + # optionally drop support for old unsupported versions + rm -fr rust-1.49 + +Push this to a branch, i.e. "mkirk/rust-1.50" and open a Draft PR, e.g. +https://github.com/georust/docker-images/pulls/11 + +### add builds to Dockerhub + +1. Add a new automated build to dockerhub for each container for the new rust + version. Note: The libproj-builder's container build must be done before + the others, since our other containers depend on libproj-builder. + 1. https://hub.docker.com/repository/docker/georust/libproj-builder/builds/edit + 2. Add new "Build Rule" with: + 1. Source Type: branch + 2. Source: mkirk/rust-1.50 (after our PR is merged, we'll have to replace this with `master`) + 3. Docker Tag: rust-1.50 + 4. Dockerfile location: rust-1.50/libproj-builder.Dockerfile + 3. Save and start that build. Once it's complete, you can repeat the process + for the other containers (geo-ci, proj-ci, proj-ci-without-system-proj) + +### Verify the new containers work + +Open a PR in the affected repositories referencing the new containers, e.g. https://github.com/georust/docker-images/pull/11. + +Make sure that CI successfully runs against the new containers. + +### Merge ahoy! + +If your new CI containers passed their tests, everything can be merged. + +The Dockerhub builds added should be updated to build from master, rather than +your PR branch. This is annoying, but I'm not sure of a better way. + +## How to Update Proj + +libproj (the cpp lib) is built using the [docker container builder +pattern](https://docs.docker.com/develop/develop-images/multistage-build/), and +then reused by multiple CI containers. + +In this example, we'll be updating to `PROJ 7.2.1` for rust-1.49. + +1. cd rust-1.49 +2. Edit `libproj-builder.Dockerfile` to download and build `PROJ` 7.2.1 +3. Build the image, tagging it with the corresponding rust version: `docker build -t georust/libproj-builder:rust-1.50 -f libproj-builder.Dockerfile .` +4. You can now update the two child `proj` Dockerfiles in this repo to use the new `libproj-builder` tag. +5. `docker build -t georust/proj-ci:rust-1.50 -f proj-ci.Dockerfile .` +6. `docker build -t georust/proj-ci-without-system-proj:rust-1.50 -f proj-ci-without-system-proj.Dockerfile .` +7. Clobber the existing `rust-1.50` tag for these three images on Docker Hub + - `docker push georust/libproj-builder:rust-1.50` + - `docker push georust/proj-ci:rust-1.50` + - `docker push georust/proj-ci-without-system-proj:rust-1.50` +8. Update the `proj` crate + - update the compressed `PROJ` version in [the proj repo](https://github.com/georust/proj/proj-sys/PROJSRC) + - Edit the adjacent [https://github.com/georust/proj/proj-sys/build.rs](`build.rs`) to look for the updated PROJ version + - Bump the `proj-sys` version and update `proj/Cargo.toml` to use it. +9. Update the `geo` crate + - When the PR has merged and the `proj` crate has been published, you can update [geo](https://github.com/georust/geo) to use the new `proj` crate: +10. Rebuild the `geo-ci.Dockerfile` which will pull in the updated `libproj-builder` tag, then push it to Docker hub. diff --git a/geo-ci.Dockerfile b/rust-1.49/geo-ci.Dockerfile similarity index 100% rename from geo-ci.Dockerfile rename to rust-1.49/geo-ci.Dockerfile diff --git a/libproj-builder.Dockerfile b/rust-1.49/libproj-builder.Dockerfile similarity index 100% rename from libproj-builder.Dockerfile rename to rust-1.49/libproj-builder.Dockerfile diff --git a/proj-ci-without-system-proj.Dockerfile b/rust-1.49/proj-ci-without-system-proj.Dockerfile similarity index 91% rename from proj-ci-without-system-proj.Dockerfile rename to rust-1.49/proj-ci-without-system-proj.Dockerfile index de6d7f0..3620c65 100644 --- a/proj-ci-without-system-proj.Dockerfile +++ b/rust-1.49/proj-ci-without-system-proj.Dockerfile @@ -3,7 +3,7 @@ # This container is based on the libproj-builder container, which has built # libproj, and thus has all the dependencies for building proj from source, but # we intentionally have not installed it to a system path. -FROM georust/libproj-builder:proj-7.2.1 +FROM georust/libproj-builder:rust-1.49 RUN apt-get update \ && DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \ diff --git a/proj-ci.Dockerfile b/rust-1.49/proj-ci.Dockerfile similarity index 65% rename from proj-ci.Dockerfile rename to rust-1.49/proj-ci.Dockerfile index b039d2b..1faadbf 100644 --- a/proj-ci.Dockerfile +++ b/rust-1.49/proj-ci.Dockerfile @@ -1,10 +1,10 @@ # https://hub.docker.com/orgs/georust/proj-ci -FROM georust/libproj-builder:proj-7.2.1 +FROM georust/libproj-builder:rust-1.49 RUN apt-get update \ && DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \ clang \ && rm -rf /var/lib/apt/lists/* -COPY --from=georust/libproj-builder:proj-7.2.1 /build/usr /usr +COPY --from=georust/libproj-builder:rust-1.49 /build/usr /usr diff --git a/rust-1.50/geo-ci.Dockerfile b/rust-1.50/geo-ci.Dockerfile new file mode 100644 index 0000000..5f80511 --- /dev/null +++ b/rust-1.50/geo-ci.Dockerfile @@ -0,0 +1,31 @@ +# https://hub.docker.com/orgs/georust/geo-ci + +# ------------------------------------------------------------------------------ +# tarpaulin build stage +# ------------------------------------------------------------------------------ + +FROM rust:1.50 as tarpaulin-builder +RUN cargo install cargo-tarpaulin --root /build + +# ------------------------------------------------------------------------------ +# Final stage +# ------------------------------------------------------------------------------ + +FROM rust:1.50 + +# clang and libtiff5 are needed to build geo with `--features use-proj` +# note: I think we can remove clang if we make bindgen optional, see https://github.com/georust/proj-sys/issues/24 +# curl is needed to run tarpaulin +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \ + ca-certificates \ + clang \ + curl \ + git \ + libtiff5 \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=georust/libproj-builder:rust-1.50 /build/usr /usr +COPY --from=tarpaulin-builder /build/bin /usr/bin + diff --git a/rust-1.50/libproj-builder.Dockerfile b/rust-1.50/libproj-builder.Dockerfile new file mode 100644 index 0000000..001d5a1 --- /dev/null +++ b/rust-1.50/libproj-builder.Dockerfile @@ -0,0 +1,42 @@ +# https://hub.docker.com/orgs/georust/libproj-builder + +# Builds libproj from source + +FROM rust:1.50 + +# Install dependencies +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get install -y \ + clang \ + libcurl4-gnutls-dev \ + libsqlite3-dev \ + libtiff5-dev \ + cmake \ + pkg-config \ + sqlite3 \ + wget \ + && rm -rf /var/lib/apt/lists/* + +# Compile and install to /build +# +# Note libproj is not installed to a system directory, so that a staged build +# can install from this container using something like: +# +# # PROJ stage +# FROM libproj-builder as libproj-builder +# ... +# # Maybe some additional stages +# ... +# # Output Container +# FROM my-base-image +# ... +# COPY --from=libproj-builder /build/usr /usr +# ... +RUN wget https://github.com/OSGeo/PROJ/releases/download/7.2.1/proj-7.2.1.tar.gz +RUN tar -xzvf proj-7.2.1.tar.gz +RUN mv proj-7.2.1 proj-src +WORKDIR /proj-src +RUN ./configure --prefix=/usr +RUN make -j$(nproc) +RUN make DESTDIR=/build install +RUN rm -fr /proj-src diff --git a/rust-1.50/proj-ci-without-system-proj.Dockerfile b/rust-1.50/proj-ci-without-system-proj.Dockerfile new file mode 100644 index 0000000..b303bf7 --- /dev/null +++ b/rust-1.50/proj-ci-without-system-proj.Dockerfile @@ -0,0 +1,11 @@ +# https://hub.docker.com/orgs/georust/proj-ci-without-system-proj + +# This container is based on the libproj-builder container, which has built +# libproj, and thus has all the dependencies for building proj from source, but +# we intentionally have not installed it to a system path. +FROM georust/libproj-builder:rust-1.50 + +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \ + clang \ + && rm -rf /var/lib/apt/lists/* diff --git a/rust-1.50/proj-ci.Dockerfile b/rust-1.50/proj-ci.Dockerfile new file mode 100644 index 0000000..e798a3f --- /dev/null +++ b/rust-1.50/proj-ci.Dockerfile @@ -0,0 +1,10 @@ +# https://hub.docker.com/orgs/georust/proj-ci + +FROM georust/libproj-builder:rust-1.50 + +RUN apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \ + clang \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=georust/libproj-builder:rust-1.50 /build/usr /usr