METRON-2246 rpm-docker - minimise use of bind mounts due to performance

metron-deployment/development/centos7/ansible.cfg

@@ -21,6 +21,7 @@ roles_path = ../../ansible/roles
 pipelining = True
 log_path = ./ansible.log
 callback_plugins = ../../ansible/callback_plugins
+callback_whitelist = profile_tasks
 
 # fix for "ssh throws 'unix domain socket too long' " problem
 [ssh_connection]

metron-deployment/packaging/docker/rpm-docker/Dockerfile

@@ -15,39 +15,8 @@
 #  limitations under the License.
 #
 
-FROM centos:centos6
-
-RUN yum install -y \
-        tar \
-        wget \
-        java-1.8.0-openjdk \
-        java-1.8.0-openjdk-devel \
-    && cd /usr/src \
-    && wget http://apache.cs.utah.edu/maven/maven-3/3.2.5/binaries/apache-maven-3.2.5-bin.tar.gz \
-    && tar xzvf apache-maven-3.2.5-bin.tar.gz \
-    && rm apache-maven-3.2.5-bin.tar.gz \
-    && mv apache-maven-3.2.5 /opt/maven \
-    && ln -s /opt/maven/bin/mvn /usr/bin/mvn \
-    && yum install -y \
-        asciidoc \
-        rpm-build \
-        rpm2cpio \
-        tar \
-        unzip \
-        xmlto \
-        zip \
-        rpmlint \
-    # install node so that the node dependencies can be packaged into the RPMs \
-    && cd /root \
-    && curl --silent --location https://rpm.nodesource.com/setup_6.x | bash - \
-    && yum install -y \
-        gcc-c++ \
-        make \
-        nodejs \
-    # Remove packages just needed for builds \
-    && yum remove -y \
-        wget \
-    # Clean up yum caches \
-    && yum clean all
-
+FROM rpm-docker-base:latest
+ADD build.sh /root/
+ADD SPECS /root/SPECS
+ADD SOURCES /root/SOURCES
 WORKDIR /root

metron-deployment/packaging/docker/rpm-docker/README.md

@@ -41,7 +41,7 @@ If you are installing Metron using Ambari, these packages are necessary prerequi
 If Metron has already been built, just the RPM packages can be built by executing the following commands.
   ```
   cd metron-deployment
-  mvn clean package -Pbuild-rpms
+  mvn clean package -Pbuild-rpms -T 2C
   ```
 
 ### How does this work?
@@ -50,17 +50,23 @@ Using the `build-rpms` profile as shown above, effectively automates the followi
 
 1. Copy the tarball for each Metron sub-project to the `target` working directory.
 
-1. Build a Docker image of a CentOS host called `rpm-docker` that contains all of the tools needed to build the packages.
+1. Build a Docker image of a CentOS host called `rpm-docker-base` that contains all of the tools needed to build the packages.
     ```
-    docker build -t rpm-docker .
+    docker build -f rpm-base-image/Dockerfile -t rpm-docker-base rpm-base-image
     ```
-
+
+1. Build a customised version of rpm-docker-base that contains all source files that are required to be packaged. 
+This bypasses the performance penalty (on MacOS Docker at least) of accessing these source files via a Docker bind shared folder.
+    ```
+    docker build -f Dockerfile -t rpm-docker-metron .
+    ```
+
 1. Execute the `build.sh` script within the Docker container.  The argument passed to the build script is the current version of Metron.
     ```
-    docker run -v `pwd`:/root rpm-docker:latest /bin/bash -c ./build.sh <metron-version>
+    docker run -v `pwd`/target:/root/target:delegated rpm-docker-metron:latest /bin/bash -c ./build.sh <metron-version>
     ```
 
 1. This results in the RPMs being generated within the following directory.
     ```
-    metron-deployment/packaging/docker/rpm-docker/RPMS/noarch
+    metron-deployment/packaging/docker/rpm-docker/target
     ```

metron-deployment/packaging/docker/rpm-docker/build.sh

@@ -46,6 +46,11 @@ fi
 
 rpmlint -i SPECS/metron.spec RPMS/*/metron* SRPMS/metron
 
+echo "Copying RPMs from docker image to project target directory"
+# Its faster to keep files in the docker image and copy them out at the end
+cp -r RPMS target/
+cp -r SRPMS target/
+
 # Ensure original user permissions are maintained after build
 if [ $OWNER_UID -ne 0 ]; then
   chown -R $OWNER_UID *

metron-deployment/packaging/docker/rpm-docker/pom.xml

@@ -329,6 +329,24 @@
                         <artifactId>exec-maven-plugin</artifactId>
                         <version>1.5.0</version>
                         <executions>
+                            <execution>
+                                <id>docker-build-base</id>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <executable>docker</executable>
+                                    <arguments>
+                                        <argument>build</argument>
+                                        <argument>-f</argument>
+                                        <argument>rpm-base-image/Dockerfile</argument>
+                                        <argument>-t</argument>
+                                        <argument>${rpm.docker.tag}-base</argument>
+                                        <argument>rpm-base-image</argument>
+                                    </arguments>
+                                </configuration>
+                            </execution>
                             <execution>
                                 <id>docker-build</id>
                                 <phase>package</phase>
@@ -342,7 +360,7 @@
                                         <argument>-f</argument>
                                         <argument>Dockerfile</argument>
                                         <argument>-t</argument>
-                                        <argument>${rpm.docker.tag}</argument>
+                                        <argument>${rpm.docker.tag}-metron</argument>
                                         <argument>.</argument>
                                     </arguments>
                                 </configuration>
@@ -358,8 +376,8 @@
                                     <arguments>
                                         <argument>run</argument>
                                         <argument>-v</argument>
-                                        <argument>${project.basedir}:/root</argument>
-                                        <argument>${rpm.docker.tag}:latest</argument>
+                                        <argument>${project.basedir}/target:/root/target:delegated</argument>
+                                        <argument>${rpm.docker.tag}-metron:latest</argument>
                                         <argument>/bin/bash</argument>
                                         <argument>-c</argument>
                                         <argument>./build.sh ${project.version}</argument>
@@ -368,27 +386,6 @@
                             </execution>
                         </executions>
                     </plugin>
-                    <plugin>
-                        <artifactId>maven-resources-plugin</artifactId>
-                        <version>3.0.1</version>
-                        <executions>
-                            <execution>
-                                <id>copy-rpms-target</id>
-                                <phase>package</phase>
-                                <goals>
-                                    <goal>copy-resources</goal>
-                                </goals>
-                                <configuration>
-                                    <outputDirectory>${project.basedir}/target/RPMS</outputDirectory>
-                                    <resources>
-                                        <resource>
-                                            <directory>RPMS</directory>
-                                        </resource>
-                                    </resources>
-                                </configuration>
-                            </execution>
-                        </executions>
-                    </plugin>
                 </plugins>
             </build>
         </profile>

metron-deployment/packaging/docker/rpm-docker/rpm-base-image/Dockerfile

@@ -0,0 +1,58 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+FROM centos:centos6
+
+RUN yum install -y \
+        tar \
+        wget \
+        java-1.8.0-openjdk \
+        java-1.8.0-openjdk-devel \
+    && cd /usr/src \
+    && wget http://apache.cs.utah.edu/maven/maven-3/3.2.5/binaries/apache-maven-3.2.5-bin.tar.gz \
+    && tar xzvf apache-maven-3.2.5-bin.tar.gz \
+    && rm apache-maven-3.2.5-bin.tar.gz \
+    && mv apache-maven-3.2.5 /opt/maven \
+    && ln -s /opt/maven/bin/mvn /usr/bin/mvn \
+    && yum install -y \
+        asciidoc \
+        rpm-build \
+        rpm2cpio \
+        tar \
+        unzip \
+        xmlto \
+        zip \
+        rpmlint \
+    # install node so that the node dependencies can be packaged into the RPMs \
+    && cd /root \
+    && curl --silent --location https://rpm.nodesource.com/setup_6.x | bash - \
+    && yum install -y \
+        gcc-c++ \
+        make \
+        nodejs \
+    # Remove packages just needed for builds \
+    && yum remove -y \
+        wget \
+    # Clean up yum caches \
+    && yum clean all \
+    && mkdir /root/SRPMS \
+    && mkdir /root/RPMS \
+    && mkdir /root/BUILD \
+    && mkdir /root/BUILDROOT \
+    && mkdir /root/target
+
+WORKDIR /root