Merge pull request #146 from PagerDuty/updated-release-build

Updated to support new release build flow.

.gitignore

@@ -12,11 +12,9 @@
 /target/
 /tmp/
 
-/.vagrant/
-
-/target/
-
 /build-linux/data/
 /build-linux/gnupg/
+/build-linux/gpg-deb/
+/build-linux/gpg-rpm/
 /build-linux/target/
 

Dockerfile-centos-7

@@ -2,10 +2,10 @@ FROM centos/systemd
 
 ARG FPM_VERSION=1.11.0
 ARG PYTHON_VERSION=3
+ARG DOCKER_WORKDIR=/usr/share/pdagent
 ENV PYTHON_VERSION ${PYTHON_VERSION}
-
 ENV container docker
-ENV DOCKER_DEV_ENV true
+ENV DOCKER_WORKDIR ${DOCKER_WORKDIR}
 
 RUN yum install -y -q centos-release-scl
 RUN yum install -y -q createrepo
@@ -26,12 +26,7 @@ RUN source /opt/rh/rh-ruby23/enable && \
   /opt/rh/rh-ruby23/root/usr/bin/gem install -q --no-ri --no-rdoc -v $FPM_VERSION fpm
 RUN yum install -y python${PYTHON_VERSION}
 
-WORKDIR /pd-agent-install
-COPY . /pd-agent-install
-RUN  mkdir -p /pd-agent-install/build-linux/release
-RUN  cd /pd-agent-install/build-linux && ./make_rpm.sh /pd-agent-install/build-linux/gnupg /pd-agent-install/build-linux/release
-RUN  echo "Installing rpm package"
-
-RUN ./scripts/build.sh centos
+COPY . $DOCKER_WORKDIR
+WORKDIR $DOCKER_WORKDIR
 
 CMD ["/usr/sbin/init"]

Dockerfile-ubuntu

@@ -3,10 +3,11 @@ FROM ubuntu:${UBUNTU_VERSION}
 
 ARG FPM_VERSION=1.11.0
 ARG PYTHON_VERSION=3
+ARG DOCKER_WORKDIR=/usr/share/pdagent
 ENV PYTHON_VERSION ${PYTHON_VERSION}
 ENV container docker
 ENV DEBIAN_FRONTEND noninteractive
-ENV DOCKER_DEV_ENV true
+ENV DOCKER_WORKDIR ${DOCKER_WORKDIR}
 
 RUN apt-get update -y -qq
 RUN apt-get install -y -q apt-utils
@@ -41,13 +42,7 @@ RUN rm -f /lib/systemd/system/multi-user.target.wants/* \
 
 VOLUME [ "/sys/fs/cgroup" ]
 
-WORKDIR /pd-agent-install
-COPY . /pd-agent-install
-RUN  mkdir -p /pd-agent-install/build-linux/release
-RUN  cd /pd-agent-install/build-linux && ./make_deb.sh /pd-agent-install/build-linux/gnupg /pd-agent-install/build-linux/release
-RUN  echo "Installing debian package"
-
-# Installing the debian package will also enable pdagent in systemd
-RUN ./scripts/build.sh ubuntu
+COPY . $DOCKER_WORKDIR
+WORKDIR $DOCKER_WORKDIR
 
 CMD ["/lib/systemd/systemd"]

Makefile

@@ -1,52 +1,74 @@
 include build-linux/make_common.env
 export $(shell sed 's/=.*//' build-linux/make_common.env)
 
-main: build
-
+.PHONY: build
 build: ubuntu centos
 
+.PHONY: ubuntu
 ubuntu: target/deb target/tmp/GPG-KEY-pagerduty
 
-centos: target/rpm target/tmp/GPG-KEY-pagerduty
+.PHONY: centos
+centos: target/rpm target/tmp/GPG-KEY-RPM-pagerduty
 
+.PHONY: build-ubuntu
 build-ubuntu:
 	docker build . \
 		-t pdagent-ubuntu \
 		-f Dockerfile-ubuntu \
 		--build-arg FPM_VERSION="${FPM_VERSION}" \
 		--build-arg PYTHON_VERSION="${PYTHON_VERSION}" \
-		--build-arg UBUNTU_VERSION="${UBUNTU_VERSION}"
+		--build-arg UBUNTU_VERSION="${UBUNTU_VERSION}" \
+		--build-arg DOCKER_WORKDIR="${DOCKER_WORKDIR}"
 
+.PHONY: build-centos
 build-centos:
 	docker build . \
 		-t pdagent-centos \
 		-f Dockerfile-centos-7 \
 		--build-arg FPM_VERSION="${FPM_VERSION}" \
 		--build-arg PYTHON_VERSION="${PYTHON_VERSION}" \
-		--build-arg CENTOS_VERSION="${CENTOS_VERSION}"
+		--build-arg CENTOS_VERSION="${CENTOS_VERSION}" \
+		--build-arg DOCKER_WORKDIR="${DOCKER_WORKDIR}"
 
 target/deb: build-ubuntu
 	docker run \
-		-v `pwd`:/usr/share/pdagent \
+		-v `pwd`:${DOCKER_WORKDIR} \
 		-it pdagent-ubuntu \
-			/bin/sh -c "/bin/sh build-linux/make_deb.sh /usr/share/pdagent/build-linux/gnupg /usr/share/pdagent/target"
+			/bin/sh -c "/bin/sh build-linux/make_deb.sh ${DOCKER_WORKDIR}/build-linux/gpg-deb ${DOCKER_WORKDIR}/target"
 
 target/rpm: build-centos
 	docker run \
-		-v `pwd`:/usr/share/pdagent \
+		-v `pwd`:${DOCKER_WORKDIR} \
 		-it pdagent-centos \
-			/bin/sh -c "/bin/sh build-linux/make_rpm.sh /usr/share/pdagent/build-linux/gnupg /usr/share/pdagent/target"
+			/bin/sh -c "/bin/sh build-linux/make_rpm.sh ${DOCKER_WORKDIR}/build-linux/gpg-rpm ${DOCKER_WORKDIR}/target"
 
 target/tmp/GPG-KEY-pagerduty:
 	docker run \
-		-v `pwd`:/usr/share/pdagent \
+		-v `pwd`:${DOCKER_WORKDIR} \
+		-it pdagent-ubuntu \
+			/bin/sh -c "mkdir -p ${DOCKER_WORKDIR}/target/tmp; gpg --armor --export --homedir ${DOCKER_WORKDIR}/build-linux/gpg-deb > ${DOCKER_WORKDIR}/target/tmp/GPG-KEY-pagerduty"
+
+target/tmp/GPG-KEY-RPM-pagerduty:
+	docker run \
+		-v `pwd`:${DOCKER_WORKDIR} \
 		-it pdagent-ubuntu \
-			/bin/sh -c "mkdir -p /usr/share/pdagent/target/tmp; gpg --armor --export --homedir /usr/share/pdagent/build-linux/gnupg > /usr/share/pdagent/target/tmp/GPG-KEY-pagerduty"
+			/bin/sh -c "mkdir -p ${DOCKER_WORKDIR}/target/tmp; gpg --armor --export --homedir ${DOCKER_WORKDIR}/build-linux/gpg-rpm > ${DOCKER_WORKDIR}/target/tmp/GPG-KEY-RPM-pagerduty"
 
 .PHONY: test
 test:
 	find unit_tests -name "test_*.py" | xargs python run-tests.py
 
+.PHONY: test-integration
+test-integration: test-integration-ubuntu test-integration-centos
+
+.PHONY: test-integration-ubuntu
+test-integration-ubuntu: ubuntu
+	scripts/full-integration-test.sh ubuntu
+
+.PHONY: test-integration-centos
+test-integration-centos: centos
+	scripts/full-integration-test.sh centos
+
 .PHONY: clean
 clean:
 	rm -rf dist

README.md

@@ -63,15 +63,15 @@ For IDE setup instructions see [PyDev Setup](pydev-setup.md) or [IDEA Setup](ide
 
 ### Build Tools
 
-To perform a complete automated build, you'll need to install Vagrant and Scons.  See [Vagrant Setup](vagrant-setup.md) and [Scons Setup](scons-setup.md) for instructions.
+To perform a complete automated build, you'll need to install Docker and `make`.
 
 ### Running Unit Tests
 
 You can run the unit tests with the following command:
 
-`scons test-local`
+`make test`
 
-To run them without installing SCons, use the `run-tests.py` test runner, e.g.:
+To run them without using `make`, use the `run-tests.py` test runner, e.g.:
 
 `python run-tests.py unit_tests/test_*.py unit_tests/thirdparty/test_*.py`
 
@@ -98,41 +98,12 @@ make ubuntu
 make centos
 ```
 
-3. Run integration tests on the packages as follows:
-  * Edit the file `pdagenttestinteg/util.sh` and change the line `SVC_KEY=CHANGEME` to a real PagerDuty Service API Key in your pdt test account.
-  * Run the command:
-```
-scons test-integration
-```
-
-This will run the integration tests on the various VMs using the packagesbuilt in the previous step. Note that the tests will trigger and resolve some incidents when they run.
-
 If you want to build packages by hand, follow the instructions in the
 [Build Linux Howto](build-linux/howto.md).
 
 Similarly, you can check the SCons targets using `scons -h` for instructions on
 performing specific builds tasks and on specific VMs.
 
-#### Some helpful scripts
-
-```
-scripts/rev_pkgs.sh
-```
-
-This will remove the installed pdagent packages from the vagrant build machines, `agent-minimal-centos65` and `agent-minimal-ubuntu1204`, and run `scons local-repo gpg-home=build-linux/gnupg` to install them again.  Run this anytime you revise a package artifact like `build-linux/deb/postinst`.
-
-```
-scripts/kill_pids.sh
-```
-
-This will kill stray pdagent processes and cleanup the pidfile on all vagrant machines.  Run this
-if your changes are causing integration tests to fail due to improper process managment via `service` or `systemctl`.
-
-```
-scripts/setup_upgrade_test.sh
-```
-
-This will vagrant destroy, up and install the latest public repo pdagent package on machines for upgrade testing via `scons test-integration`.
 
 #License
 Copyright (c) 2013-2014, PagerDuty, Inc. <info@pagerduty.com>

build-linux/deb/postinst

@@ -40,11 +40,7 @@ install_init () {
 install_systemd () {
     cp /var/lib/pdagent/scripts/pdagent.service /lib/systemd/system
     systemctl enable pdagent
-    if [ -z $DOCKER_DEV_ENV ]; then
-        # Prevent starting pdagent in Docker environment because systemd 
-        # will do that after running the Docker systemd process
-        systemctl start pdagent
-    fi
+    systemctl start pdagent
 }
 
 if [ "$1" = "configure" ]; then

build-linux/howto.md

@@ -1,8 +1,8 @@
 # Linux Build Instructions
 
-These instructions assume you're running this on a Mac with Vagrant installed,
-and that the project directory ends up mounted in the VM at `/vagrant`. (this
-should happen automatically)
+These instructions assume you're running this on a Mac with Docker installed,
+and that the project directory ends up mounted in the VM at
+`/usr/share/pdagent`. (this should happen automatically)
 
 ## One-time setup of Development GPG keys:
 
@@ -26,17 +26,15 @@ the following instructions accordingly.
 Building the .deb:
 
 ```
-scons local-repo \
-  gpg-home=build-linux/gnupg \
-  virt=agent-minimal-ubuntu1204
+make ubuntu
 ```
 
 Install & test the .deb:
 ```
-vagrant ssh agent-minimal-ubuntu1204
+docker run -it pdagent-ubuntu /bin/bash
 
-sudo apt-key add /vagrant/target/tmp/GPG-KEY-pagerduty
-sudo sh -c 'echo "deb file:///vagrant/target deb/" \
+sudo apt-key add /usr/share/pdagent/target/tmp/GPG-KEY-pagerduty
+sudo sh -c 'echo "deb file:///usr/share/pdagent/target deb/" \
   >/etc/apt/sources.list.d/pdagent.list'
 sudo apt-get update
 sudo apt-get install pdagent
@@ -60,22 +58,20 @@ Rerun the test commands to ensure files are gone
 
 Building the .rpm:
 ```
-scons local-repo \
-  gpg-home=/gpg/path/used/earlier \
-  virt=agent-minimal-centos65
+make centos
 ```
 
 Install & test the .rpm:
 ```
-vagrant ssh agent-minimal-centos65
+docker run -it pdagent-centos /bin/bash
 
 sudo sh -c 'cat >/etc/yum.repos.d/pdagent.repo <<EOF
 [pdagent]
 name=PDAgent
-baseurl=file:///vagrant/target/rpm
+baseurl=file:///usr/share/pdagent/target/rpm
 enabled=1
 gpgcheck=1
-gpgkey=file:///vagrant/target/tmp/GPG-KEY-pagerduty
+gpgkey=file:///usr/share/pdagent/target/tmp/GPG-KEY-RPM-pagerduty
 EOF'
 
 sudo yum install -y pdagent

build-linux/make_common.env

@@ -4,3 +4,4 @@ CENTOS_VERSION=7
 FPM_VERSION=1.11.0
 PYTHON_VERSION=3
 UBUNTU_VERSION=16.04
+DOCKER_WORKDIR=/usr/share/pdagent

build-linux/make_package.sh

@@ -66,6 +66,8 @@ if [ -z "$_VERSION" ]; then
     exit 1
 fi
 
+echo = building version $_VERSION
+
 echo = cleaning build directories
 rm -fr data target
 mkdir data target
@@ -74,10 +76,10 @@ echo = /usr/bin/...
 mkdir -p data/usr/bin
 cp ../bin/pd-* data/usr/bin
 
-echo = /usr/share/pdagent/bin
-mkdir -p data/usr/share/pdagent/bin
-cp ../bin/pdagentd.py data/usr/share/pdagent/bin
-chmod a+rx data/usr/share/pdagent/bin/pdagentd.py
+echo = ${DOCKER_WORKDIR}/bin
+mkdir -p data${DOCKER_WORKDIR}/bin
+cp ../bin/pdagentd.py data${DOCKER_WORKDIR}/bin
+chmod a+rx data${DOCKER_WORKDIR}/bin/pdagentd.py
 
 echo = /var/...
 mkdir -p data/var/log/pdagent
@@ -131,7 +133,7 @@ else
 fi
 
 echo = FPM!
-_FPM_DEPENDS="--depends sudo --depends python"
+_FPM_DEPENDS="--depends sudo"
 
 _SIGN_OPTS=""
 if [ "$pkg_type" = "rpm" ]; then

build-linux/rpm/postinst

@@ -40,11 +40,7 @@ install_init () {
 install_systemd () {
     cp /var/lib/pdagent/scripts/pdagent.service /lib/systemd/system
     systemctl enable pdagent
-    if [ -z $DOCKER_DEV_ENV ]; then
-        # Prevent starting pdagent in Docker environment because systemd 
-        # will do that after running the Docker systemd process
-        systemctl start pdagent
-    fi
+    systemctl start pdagent
 }
 
 # Create pdagent user & group