From b1905cf427f5fc998de92fe735b9d604d368d277 Mon Sep 17 00:00:00 2001 From: Dale Date: Wed, 4 Sep 2019 17:11:03 +1000 Subject: [PATCH 1/2] METRON-2246 - initial commit - change rpm-docker to mechanism so that input files are built into the docker iamge --- .../development/centos7/ansible.cfg | 1 + .../packaging/docker/rpm-docker/Dockerfile | 39 ++----------- .../packaging/docker/rpm-docker/build.sh | 5 ++ .../packaging/docker/rpm-docker/pom.xml | 45 +++++++------- .../rpm-docker/rpm-base-image/Dockerfile | 58 +++++++++++++++++++ 5 files changed, 89 insertions(+), 59 deletions(-) create mode 100644 metron-deployment/packaging/docker/rpm-docker/rpm-base-image/Dockerfile diff --git a/metron-deployment/development/centos7/ansible.cfg b/metron-deployment/development/centos7/ansible.cfg index 66f0afd3e5..70ac5de7e0 100644 --- a/metron-deployment/development/centos7/ansible.cfg +++ b/metron-deployment/development/centos7/ansible.cfg @@ -21,6 +21,7 @@ roles_path = ../../ansible/roles pipelining = True log_path = ./ansible.log callback_plugins = ../../ansible/callback_plugins +callback_whitelist = profile_tasks # fix for "ssh throws 'unix domain socket too long' " problem [ssh_connection] diff --git a/metron-deployment/packaging/docker/rpm-docker/Dockerfile b/metron-deployment/packaging/docker/rpm-docker/Dockerfile index f151678ea0..45c3a4802b 100644 --- a/metron-deployment/packaging/docker/rpm-docker/Dockerfile +++ b/metron-deployment/packaging/docker/rpm-docker/Dockerfile @@ -15,39 +15,8 @@ # limitations under the License. # -FROM centos:centos6 - -RUN yum install -y \ - tar \ - wget \ - java-1.8.0-openjdk \ - java-1.8.0-openjdk-devel \ - && cd /usr/src \ - && wget http://apache.cs.utah.edu/maven/maven-3/3.2.5/binaries/apache-maven-3.2.5-bin.tar.gz \ - && tar xzvf apache-maven-3.2.5-bin.tar.gz \ - && rm apache-maven-3.2.5-bin.tar.gz \ - && mv apache-maven-3.2.5 /opt/maven \ - && ln -s /opt/maven/bin/mvn /usr/bin/mvn \ - && yum install -y \ - asciidoc \ - rpm-build \ - rpm2cpio \ - tar \ - unzip \ - xmlto \ - zip \ - rpmlint \ - # install node so that the node dependencies can be packaged into the RPMs \ - && cd /root \ - && curl --silent --location https://rpm.nodesource.com/setup_6.x | bash - \ - && yum install -y \ - gcc-c++ \ - make \ - nodejs \ - # Remove packages just needed for builds \ - && yum remove -y \ - wget \ - # Clean up yum caches \ - && yum clean all - +FROM rpm-docker-base:latest +ADD build.sh /root/ +ADD SPECS /root/SPECS +ADD SOURCES /root/SOURCES WORKDIR /root diff --git a/metron-deployment/packaging/docker/rpm-docker/build.sh b/metron-deployment/packaging/docker/rpm-docker/build.sh index 68566b72e7..845128a4c0 100755 --- a/metron-deployment/packaging/docker/rpm-docker/build.sh +++ b/metron-deployment/packaging/docker/rpm-docker/build.sh @@ -46,6 +46,11 @@ fi rpmlint -i SPECS/metron.spec RPMS/*/metron* SRPMS/metron +echo "Copying RPMs from docker image to project target directory" +# Its faster to keep files in the docker image and copy them out at the end +cp -r RPMS target/ +cp -r SRPMS target/ + # Ensure original user permissions are maintained after build if [ $OWNER_UID -ne 0 ]; then chown -R $OWNER_UID * diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml index 63c0402227..941b27f9eb 100644 --- a/metron-deployment/packaging/docker/rpm-docker/pom.xml +++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml @@ -329,6 +329,24 @@ exec-maven-plugin 1.5.0 + + docker-build-base + package + + exec + + + docker + + build + -f + rpm-base-image/Dockerfile + -t + ${rpm.docker.tag}-base + rpm-base-image + + + docker-build package @@ -342,7 +360,7 @@ -f Dockerfile -t - ${rpm.docker.tag} + ${rpm.docker.tag}-metron . @@ -358,8 +376,8 @@ run -v - ${project.basedir}:/root - ${rpm.docker.tag}:latest + ${project.basedir}/target:/root/target:delegated + ${rpm.docker.tag}-metron:latest /bin/bash -c ./build.sh ${project.version} @@ -368,27 +386,6 @@ - - maven-resources-plugin - 3.0.1 - - - copy-rpms-target - package - - copy-resources - - - ${project.basedir}/target/RPMS - - - RPMS - - - - - - diff --git a/metron-deployment/packaging/docker/rpm-docker/rpm-base-image/Dockerfile b/metron-deployment/packaging/docker/rpm-docker/rpm-base-image/Dockerfile new file mode 100644 index 0000000000..1b896f3732 --- /dev/null +++ b/metron-deployment/packaging/docker/rpm-docker/rpm-base-image/Dockerfile @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM centos:centos6 + +RUN yum install -y \ + tar \ + wget \ + java-1.8.0-openjdk \ + java-1.8.0-openjdk-devel \ + && cd /usr/src \ + && wget http://apache.cs.utah.edu/maven/maven-3/3.2.5/binaries/apache-maven-3.2.5-bin.tar.gz \ + && tar xzvf apache-maven-3.2.5-bin.tar.gz \ + && rm apache-maven-3.2.5-bin.tar.gz \ + && mv apache-maven-3.2.5 /opt/maven \ + && ln -s /opt/maven/bin/mvn /usr/bin/mvn \ + && yum install -y \ + asciidoc \ + rpm-build \ + rpm2cpio \ + tar \ + unzip \ + xmlto \ + zip \ + rpmlint \ + # install node so that the node dependencies can be packaged into the RPMs \ + && cd /root \ + && curl --silent --location https://rpm.nodesource.com/setup_6.x | bash - \ + && yum install -y \ + gcc-c++ \ + make \ + nodejs \ + # Remove packages just needed for builds \ + && yum remove -y \ + wget \ + # Clean up yum caches \ + && yum clean all \ + && mkdir /root/SRPMS \ + && mkdir /root/RPMS \ + && mkdir /root/BUILD \ + && mkdir /root/BUILDROOT \ + && mkdir /root/target + +WORKDIR /root From 6e885b9413c1f6ebae3f330965f6d76e52410b7d Mon Sep 17 00:00:00 2001 From: Dale Date: Wed, 4 Sep 2019 18:02:12 +1000 Subject: [PATCH 2/2] METRON-2246 Updated readme to describe new docker build process --- .../packaging/docker/rpm-docker/README.md | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/metron-deployment/packaging/docker/rpm-docker/README.md b/metron-deployment/packaging/docker/rpm-docker/README.md index 4f2bc08f8f..dacfaa1bea 100644 --- a/metron-deployment/packaging/docker/rpm-docker/README.md +++ b/metron-deployment/packaging/docker/rpm-docker/README.md @@ -41,7 +41,7 @@ If you are installing Metron using Ambari, these packages are necessary prerequi If Metron has already been built, just the RPM packages can be built by executing the following commands. ``` cd metron-deployment - mvn clean package -Pbuild-rpms + mvn clean package -Pbuild-rpms -T 2C ``` ### How does this work? @@ -50,17 +50,23 @@ Using the `build-rpms` profile as shown above, effectively automates the followi 1. Copy the tarball for each Metron sub-project to the `target` working directory. -1. Build a Docker image of a CentOS host called `rpm-docker` that contains all of the tools needed to build the packages. +1. Build a Docker image of a CentOS host called `rpm-docker-base` that contains all of the tools needed to build the packages. ``` - docker build -t rpm-docker . + docker build -f rpm-base-image/Dockerfile -t rpm-docker-base rpm-base-image ``` - + +1. Build a customised version of rpm-docker-base that contains all source files that are required to be packaged. +This bypasses the performance penalty (on MacOS Docker at least) of accessing these source files via a Docker bind shared folder. + ``` + docker build -f Dockerfile -t rpm-docker-metron . + ``` + 1. Execute the `build.sh` script within the Docker container. The argument passed to the build script is the current version of Metron. ``` - docker run -v `pwd`:/root rpm-docker:latest /bin/bash -c ./build.sh + docker run -v `pwd`/target:/root/target:delegated rpm-docker-metron:latest /bin/bash -c ./build.sh ``` 1. This results in the RPMs being generated within the following directory. ``` - metron-deployment/packaging/docker/rpm-docker/RPMS/noarch + metron-deployment/packaging/docker/rpm-docker/target ```