armadaproject · jankaspar · Oct 17, 2019 · Oct 16, 2019 · Oct 16, 2019 · Oct 16, 2019
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -23,54 +23,42 @@ commands:
 
 jobs:
   build:
-    docker:
-      - image: circleci/golang:1.12.6
-        environment:
-          GO111MODULE: "on"
-
-      # Specify service dependencies here if necessary
-      # CircleCI maintains a library of pre-built images
-      # documented at https://circleci.com/docs/2.0/circleci-images/
-      # - image: circleci/postgres:9.4
-      - image: redis
-
-    #### TEMPLATE_NOTE: go expects specific checkout path representing url
-    #### expecting it in the form of
-    ####   /go/src/github.com/circleci/go-tool
-    ####   /go/src/bitbucket.org/circleci/go-tool
-    working_directory: /go/src/github.com/G-Research/k8s-batch
+    machine:
+      docker_layer_caching: true
+      image: ubuntu-1604:201903-01
+    environment:
+      GO111MODULE: "on"
+      GOPATH: "/home/circleci/go"
+      GOCACHE: "/home/circleci/go/cache"
+    working_directory: ~/go/src/github.com/G-Research/k8s-batch
     steps:
       - checkout
-      - setup_remote_docker
+      - run: e2e/setup/setup_environment.sh
+      - run:
+          name: Start cluster for e2e tests
+          command: ./e2e/setup/setup_cluster_ci.sh
 
       - restore_cache:
           keys:
-            - go-mod-v1-{{ checksum "go.sum" }}
+            - machine-go-mod-v1-{{ checksum "go.sum" }}
 
       - run:
           name: Test
-          command: |
-            go get -v -d ./...
-            go test -v ./internal/...
-
-      - save_cache:
-          key: go-mod-v1-{{ checksum "go.sum" }}
-          paths:
-            - "/go/pkg/mod"
+          command: make tests
 
       - run:
           name: Check formatting
           command: |
-            go get golang.org/x/tools/cmd/goimports
-            exit $(goimports -l -local "github.com/G-Research/k8s-batch" . | wc -l)
+            exit $(go run golang.org/x/tools/cmd/goimports -l -local "github.com/G-Research/k8s-batch" . | wc -l)
 
       - run:
-          name: Build
-          command: make build-ci
+          name: Build & Run e2e tests
+          command: make build-ci tests-e2e
 
       - store_artifacts:
-          path: armadactl
+          path: bin/armadactl
           destination: armadactl
+
       - run:
           name: Push Image
           command: |
@@ -89,39 +77,9 @@ jobs:
 
             docker tag armada-executor gresearchdev/armada-executor-dev:${TAG}
             docker push gresearchdev/armada-executor-dev:${TAG}
-  e2e:
-    machine:
-      docker_layer_caching: true
-    environment:
-      GO111MODULE: "on"
-      GOPATH: "/home/circleci/go"
-      GOCACHE: "/home/circleci/go/cache"
-    working_directory: ~/go/src/github.com/G-Research/k8s-batch
-    steps:
-      - checkout
-      - run:
-          name: Load relevant docker images
-          command: |
-            docker pull alpine:3.10
-            docker pull gresearchdev/armada-server-dev:branch-${CIRCLE_BRANCH}-${CIRCLE_SHA1}
-            docker pull gresearchdev/armada-executor-dev:branch-${CIRCLE_BRANCH}-${CIRCLE_SHA1}
-      - run: e2e/setup/setup_environment.sh
-      - run: e2e/setup/setup_cluster_ci.sh
-      - run: e2e/setup/setup_kube_config_ci.sh
-      - run: docker run -d --expose=6379 --network=host redis
-      - run: docker run -d --expose=50051 --network=host gresearchdev/armada-server-dev:branch-${CIRCLE_BRANCH}-${CIRCLE_SHA1}
-      - run: docker run -d --network=host -v ~/.kube/config:/kube/config -e KUBECONFIG=/kube/config gresearchdev/armada-executor-dev:branch-${CIRCLE_BRANCH}-${CIRCLE_SHA1}
-      - restore_cache:
-          keys:
-            - machine-go-mod-v1-{{ checksum "go.sum" }}
-      - run:
-          name: Get dependencies
-          command: |
-            go get -d ./...
-      - run:
-          name: Run e2e tests
-          command: |
-            INTEGRATION_ENABLED=true go test ./e2e/test/... -count=1
+
+      - run: cat go.sum
+
       - save_cache:
           key: machine-go-mod-v1-{{ checksum "go.sum" }}
           paths:
@@ -166,12 +124,6 @@ workflows:
   build_and_deploy:
     jobs:
       - build
-      - e2e:
-          requires:
-            - build
-          filters:
-            branches:
-              ignore: master
       - deploy:
           requires:
             - build

diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,6 @@
 .vscode
 .idea
+.kube
 
 # Binaries for programs and plugins
 *.exe

diff --git a/build/armada/Dockerfile b/build/armada/Dockerfile
@@ -3,6 +3,8 @@ FROM alpine:3.10
 COPY ./bin/linux/server /app/
 COPY /config/armada/ /app/config/armada
 
+RUN apk update && apk add --no-cache ca-certificates
+
 WORKDIR /app
 
 ENTRYPOINT ["./server"]
diff --git a/e2e/setup/setup_cluster_ci.sh b/e2e/setup/setup_cluster_ci.sh
@@ -1,4 +1,4 @@
-docker run -d --name kube --privileged -p 8443:8443 -p 10080:10080 bsycorp/kind:v1.15.1
-until curl -s --fail http://127.0.0.1:10080/kubernetes-ready; do
-  sleep 1;
-done
+if curl -s --fail http://127.0.0.1:10080/kubernetes-ready; then
+    exit 0
+fi
+docker run -d --name kube --privileged -p 8443:8443 -p 10080:10080 bsycorp/kind:v1.15.1 || true
diff --git a/e2e/setup/setup_environment.sh b/e2e/setup/setup_environment.sh
@@ -17,8 +17,3 @@ sudo tar -xvf go1.12.6.linux-amd64.tar.gz
 sudo mv go /usr/local
 go version
 
-echo "Installing kind"
-curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/v0.4.0/kind-linux-amd64
-chmod +x kind
-sudo mv kind /usr/local/bin/
-kind version
diff --git a/e2e/setup/setup_kube_config_ci.sh b/e2e/setup/setup_kube_config_ci.sh
@@ -1,3 +1,6 @@
-mkdir ~/.kube
-chmod 777 ~/.kube
-wget -P ~/.kube http://localhost:10080/config
+until curl -s --fail http://localhost:10080/kubernetes-ready; do
+  sleep 1;
+done
+mkdir .kube || true
+rm .kube/config || true
+wget -P .kube http://localhost:10080/config
diff --git a/go.sum b/go.sum
@@ -217,6 +217,12 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/tidwall/gjson v1.3.2 h1:+7p3qQFaH3fOMXAJSrdZwGKcOO/lYdGS0HqGhPqDdTI=
+github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
+github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
+github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
+github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
@@ -225,6 +231,8 @@ github.com/weaveworks/promrus v1.2.0 h1:jOLf6pe6/vss4qGHjXmGz4oDJQA+AOCqEL3FvvZG
 github.com/weaveworks/promrus v1.2.0/go.mod h1:SaE82+OJ91yqjrE1rsvBWVzNZKcHYFtMUyS1+Ogs/KA=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY=
+github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
 github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036 h1:1b6PAtenNyhsmo/NKXVe34h7JEZKva1YB/ne7K7mqKM=
 github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
@@ -295,6 +303,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b h1:mSUCVIwDx4hfXJfWsOPfdzEHxzb2Xjl6BQ8YgPnazQA=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=

diff --git a/internal/armada/repository/job.go b/internal/armada/repository/job.go
@@ -7,7 +7,7 @@ import (
 
 	"github.com/go-redis/redis"
 	"github.com/gogo/protobuf/proto"
-	"github.com/prometheus/common/log"
+	log "github.com/sirupsen/logrus"
 
 	"github.com/G-Research/k8s-batch/internal/armada/api"
 	"github.com/G-Research/k8s-batch/internal/common/util"
@@ -394,9 +394,9 @@ func (repo *RedisJobRepository) leaseJobs(clusterId string, jobs []jobIdentity)
 		if e != nil {
 			log.Error(e)
 		} else if value == alreadyAllocatedByDifferentCluster {
-			log.With("jobId", jobId).Info("Job Already allocated to different cluster")
+			log.WithField("jobId", jobId).Info("Job Already allocated to different cluster")
 		} else if value == jobCancelled {
-			log.With("jobId", jobId).Info("Trying to renew cancelled job")
+			log.WithField("jobId", jobId).Info("Trying to renew cancelled job")
 		} else {
 			leasedJobs = append(leasedJobs, jobId)
 		}

diff --git a/internal/executor/reporter/job_event_reporter.go b/internal/executor/reporter/job_event_reporter.go
@@ -156,7 +156,7 @@ func HasPodBeenInStateForLongerThanGivenDuration(pod *v1.Pod, duration time.Dura
 	deadline := time.Now().Add(-duration)
 	lastStatusChange, err := lastStatusChange(pod)
 
-	if err != nil || lastStatusChange.Before(deadline) {
+	if err == nil && lastStatusChange.Before(deadline) {
 		return true
 	}
 	return false

diff --git a/internal/executor/reporter/job_event_reporter_test.go b/internal/executor/reporter/job_event_reporter_test.go
@@ -79,7 +79,7 @@ func TestHasPodBeenInStateForLongerThanGivenDuration_ReturnsFalse(t *testing.T)
 	assert.False(t, result)
 }
 
-func TestHasPodBeenInStateForLongerThanGivenDuration_ReturnsTrue_WhenNoPodStateChangesCanBeFound(t *testing.T) {
+func TestHasPodBeenInStateForLongerThanGivenDuration_ReturnsFalse_WhenNoPodStateChangesCanBeFound(t *testing.T) {
 	pod := v1.Pod{
 		Status: v1.PodStatus{
 			Conditions: []v1.PodCondition{},
@@ -88,7 +88,7 @@ func TestHasPodBeenInStateForLongerThanGivenDuration_ReturnsTrue_WhenNoPodStateC
 
 	result := HasPodBeenInStateForLongerThanGivenDuration(&pod, 5*time.Second)
 
-	assert.True(t, result)
+	assert.False(t, result)
 }
 
 func TestHasCurrentStateBeenReported_TrueWhenAnnotationExistsForCurrentPhase(t *testing.T) {

diff --git a/makefile b/makefile
@@ -1,3 +1,7 @@
+# use bash for running:
+export SHELL:=/bin/bash
+export SHELLOPTS:=$(if $(SHELLOPTS),$(SHELLOPTS):)pipefail:errexit
+
 gobuildlinux = GOOS=linux GARCH=amd64 CGO_ENABLED=0 go build -ldflags="-s -w"
 gobuild = go build
 
@@ -17,13 +21,49 @@ build: build-server build-executor build-armadactl build-load-tester
 
 build-docker-server:
 	$(gobuildlinux) -o ./bin/linux/server cmd/armada/main.go
-	docker build -t armada -f ./build/armada/Dockerfile .
+	docker build $(dockerFlags) -t armada -f ./build/armada/Dockerfile .
 
 build-docker-executor:
 	$(gobuildlinux) -o ./bin/linux/executor cmd/executor/main.go
-	docker build -t armada-executor -f ./build/executor/Dockerfile .
+	docker build $(dockerFlags) -t armada-executor -f ./build/executor/Dockerfile .
 
 build-docker: build-docker-server build-docker-executor
 
 build-ci: gobuild=$(gobuildlinux)
 build-ci: build-docker build-armadactl build-load-tester
+
+.ONESHELL:
+tests:
+	docker run -d --name=test-redis -p=6379:6379 redis
+	function tearDown {
+		docker stop test-redis
+		docker rm test-redis
+	}
+	trap tearDown EXIT
+	go test -v ./internal/...
+
+e2e-start-cluster:
+	./e2e/setup/setup_cluster_ci.sh
+	./e2e/setup/setup_kube_config_ci.sh
+
+e2e-stop-cluster:
+	docker stop kube
+	docker rm kube
+
+.ONESHELL:
+tests-e2e: build-docker e2e-start-cluster
+	docker run -d --name redis -p=6379:6379 redis
+	docker run -d --name server --network=host -p=50051:50051 armada
+	docker run -d --name executor --network=host -v $(shell pwd)/.kube/config:/kube/config -e KUBECONFIG=/kube/config armada-executor
+	function tearDown {
+		echo -e "\nexecutor logs:"
+		docker logs executor
+		echo -e "\nserver logs:"
+		docker logs server
+		docker stop executor server redis
+		docker rm executor server redis
+	}
+	trap tearDown EXIT
+	echo -e "\nrunning test:"
+	INTEGRATION_ENABLED=true go test ./e2e/test/... -count=1
+