Merge pull request #250 from wagoodman/add-package-tests

Add package installation testing
wagoodman · Nov 20, 2019 · fe158e4 · fe158e4
2 parents 3c8dc62 + 74be19b
commit fe158e4
Show file tree

Hide file tree

Showing 5 changed files with 145 additions and 35 deletions.
diff --git a/.data/.dive-ci b/.data/.dive-ci
@@ -10,6 +10,6 @@ rules:
   highestWastedBytes: 20Mb
 
   # If the amount of wasted space makes up for X% of the image, mark as failed. (fail if the threshold is met or crossed; expressed as a percentage between 0-1)
-  highestUserWastedPercent: 0.10
+  highestUserWastedPercent: 0.5
 
   plugin1/rule1: error
diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
@@ -1,10 +1,8 @@
 name: 'app-pipeline'
-on: [ push, pull_request ]
+on: [ push ]
 env:
   DOCKER_CLI_VERSION: "19.03.1"
 jobs:
-  # This will run on merge to master, and for PRs with folks that also are part of the GHA beta.
-  # Circle ci will also run for redundancy until GHA is out of beta, since circle checks will always run.
   unit-test:
     strategy:
       matrix:
@@ -26,9 +24,8 @@ jobs:
         uses: actions/cache@v1
         with:
           path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
+          key: ${{ runner.os }}-go-${{ matrix.go-version }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ runner.os }}-go-${{ matrix.go-version }}-
 
       - name: Install go dependencies
         if: steps.unit-cache-go-dependencies.outputs.cache-hit != 'true'
@@ -46,17 +43,18 @@ jobs:
 
       - uses: actions/checkout@v1
 
-      - name: Install go tools
-        run: make ci-install-go-tools
+      - name: Install tooling
+        run: |
+          make ci-install-go-tools
+          make ci-install-ci-tools
 
       - name: Cache go dependencies
         id: package-cache-go-dependencies
         uses: actions/cache@v1
         with:
           path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
+          key: ${{ runner.os }}-go-prod-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ runner.os }}-go-prod-
 
       - name: Install dependencies
         if: steps.package-cache-go-dependencies.outputs.cache-hit != 'true'
@@ -66,12 +64,7 @@ jobs:
         run: make ci-static-analyses
 
       - name: Build snapshot artifacts
-        uses: goreleaser/goreleaser-action@v1
-        with:
-          version: latest
-          args: release --snapshot --skip-publish --rm-dist
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: make ci-build-snapshot-packages
 
       - run: docker images wagoodman/dive
 
@@ -84,8 +77,63 @@ jobs:
           name: artifacts
           path: dist
 
+
+  test-linux-artifacts:
+    needs: [ build-artifacts ]
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@master
+
+      - uses: actions/download-artifact@master
+        with:
+          name: artifacts
+          path: dist
+
+      - name: Test linux run
+        run: make ci-test-linux-run
+
+      - name: Test DEB package installation
+        run: make ci-test-deb-package-install
+
+      - name: Test RPM package installation
+        run: make ci-test-rpm-package-install
+
+
+  test-mac-artifacts:
+    needs: [ build-artifacts ]
+    runs-on: macos-latest
+    steps:
+
+      - uses: actions/checkout@master
+
+      - uses: actions/download-artifact@master
+        with:
+          name: artifacts
+          path: dist
+
+      - name: Test darwin run
+        run: make ci-test-mac-run
+
+
+  test-windows-artifacts:
+    needs: [ build-artifacts ]
+    runs-on: windows-latest
+    steps:
+
+      - uses: actions/checkout@master
+
+      - uses: actions/download-artifact@master
+        with:
+          name: artifacts
+          path: dist
+
+      - name: Test windows run
+        run: make ci-test-windows-run
+
+
   release:
-    needs: [ build-artifacts, unit-test  ]
+    needs: [ unit-test, build-artifacts, test-linux-artifacts, test-mac-artifacts, test-windows-artifacts ]
     runs-on: ubuntu-latest
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
@@ -96,14 +144,16 @@ jobs:
 
       - uses: actions/checkout@v1
 
+      - name: Install tooling
+        run: make ci-install-ci-tools
+
       - name: Cache go dependencies
         id: release-cache-go-dependencies
         uses: actions/cache@v1
         with:
           path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
+          key: ${{ runner.os }}-go-prod-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ runner.os }}-go-prod-
 
       - name: Install dependencies
         if: steps.release-cache-go-dependencies.outputs.cache-hit != 'true'
@@ -116,10 +166,7 @@ jobs:
           DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Publish GitHub release
-        uses: goreleaser/goreleaser-action@v1
-        with:
-          version: latest
-          args: release --rm-dist
+        run: make ci-release
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 

diff --git a/Makefile b/Makefile
@@ -21,6 +21,9 @@ ci-static-analyses:
 ci-install-go-tools:
 	curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sudo sh -s -- -b /usr/local/bin/ latest
 
+ci-install-ci-tools:
+	curl -sfL https://install.goreleaser.com/github.com/goreleaser/goreleaser.sh | sudo sh -s -- -b /usr/local/bin/ "v0.122.0"
+
 ci-docker-login:
 	echo '${DOCKER_PASSWORD}' | docker login -u '${DOCKER_USERNAME}' --password-stdin '${PRODUCTION_REGISTRY}'
 
@@ -30,6 +33,15 @@ ci-docker-logout:
 ci-publish-release:
 	goreleaser --rm-dist
 
+ci-build-snapshot-packages:
+	goreleaser \
+		--snapshot \
+		--skip-publish \
+		--rm-dist
+
+ci-release:
+	release --rm-dist
+
 # todo: add --pull=never when supported by host box
 ci-test-production-image:
 	docker run \
@@ -40,6 +52,54 @@ ci-test-production-image:
 			'${TEST_IMAGE}' \
 			--ci
 
+ci-test-deb-package-install:
+	docker run \
+		-v //var/run/docker.sock://var/run/docker.sock \
+		-v /${PWD}://src \
+		-w //src \
+		ubuntu:latest \
+			/bin/bash -x -c "\
+				apt update && \
+				apt install -y curl && \
+				curl -L 'https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_CLI_VERSION}.tgz' | \
+					tar -vxzf - docker/docker --strip-component=1 && \
+					mv docker /usr/local/bin/ &&\
+				docker version && \
+				apt install ./dist/dive_*_linux_amd64.deb -y && \
+				dive --version && \
+				dive '${TEST_IMAGE}' --ci \
+			"
+
+ci-test-rpm-package-install:
+	docker run \
+		-v //var/run/docker.sock://var/run/docker.sock \
+		-v /${PWD}://src \
+		-w //src \
+		fedora:latest \
+			/bin/bash -x -c "\
+				curl -L 'https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_CLI_VERSION}.tgz' | \
+					tar -vxzf - docker/docker --strip-component=1 && \
+					mv docker /usr/local/bin/ &&\
+				docker version && \
+				dnf install ./dist/dive_*_linux_amd64.rpm -y && \
+				dive --version && \
+				dive '${TEST_IMAGE}' --ci \
+			"
+
+ci-test-linux-run:
+	chmod 755 ./dist/dive_linux_amd64/dive && \
+	./dist/dive_linux_amd64/dive '${TEST_IMAGE}'  --ci
+
+# we're not attempting to test docker, just our ability to run on these systems. This avoids setting up docker in CI.
+ci-test-mac-run:
+	chmod 755 ./dist/dive_darwin_amd64/dive && \
+	./dist/dive_darwin_amd64/dive --source docker-archive .data/test-docker-image.tar  --ci --ci-config .data/.dive-ci
+
+# we're not attempting to test docker, just our ability to run on these systems. This avoids setting up docker in CI.
+ci-test-windows-run:
+	./dist/dive_windows_amd64/dive --source docker-archive .data/test-docker-image.tar  --ci --ci-config .data/.dive-ci
+
+
 
 ## For development
 
@@ -65,6 +125,9 @@ build:
 generate-test-data:
 	docker build -t dive-test:latest -f .data/Dockerfile.test-image . && docker image save -o .data/test-docker-image.tar dive-test:latest && echo 'Exported test data!'
 
+test:
+	./.scripts/test-coverage.sh
+
 dev:
 	docker run -ti --rm -v $(PWD):/app -w /app -v dive-pkg:/go/pkg/ golang:1.13 bash
 

diff --git a/runtime/run.go b/runtime/run.go
@@ -71,9 +71,9 @@ func run(enableUi bool, options Options, imageResolver image.Resolver, events ev
 	}
 
 	if options.Ci {
-		events.message(fmt.Sprintf("  efficiency: %2.4f %%\n", analysis.Efficiency*100))
-		events.message(fmt.Sprintf("  wastedBytes: %d bytes (%s)\n", analysis.WastedBytes, humanize.Bytes(analysis.WastedBytes)))
-		events.message(fmt.Sprintf("  userWastedPercent: %2.4f %%\n", analysis.WastedUserPercent*100))
+		events.message(fmt.Sprintf("  efficiency: %2.4f %%", analysis.Efficiency*100))
+		events.message(fmt.Sprintf("  wastedBytes: %d bytes (%s)", analysis.WastedBytes, humanize.Bytes(analysis.WastedBytes)))
+		events.message(fmt.Sprintf("  userWastedPercent: %2.4f %%", analysis.WastedUserPercent*100))
 
 		evaluator := ci.NewCiEvaluator(options.CiConfig)
 		pass := evaluator.Evaluate(analysis)

diff --git a/runtime/run_test.go b/runtime/run_test.go
@@ -189,9 +189,9 @@ func TestRun(t *testing.T) {
 			events: []testEvent{
 				{stdout: "Building image...", stderr: "", errorOnExit: false, errMessage: ""},
 				{stdout: "Analyzing image...", stderr: "", errorOnExit: false, errMessage: ""},
-				{stdout: "  efficiency: 98.4421 %\n", stderr: "", errorOnExit: false, errMessage: ""},
-				{stdout: "  wastedBytes: 32025 bytes (32 kB)\n", stderr: "", errorOnExit: false, errMessage: ""},
-				{stdout: "  userWastedPercent: 48.3491 %\n", stderr: "", errorOnExit: false, errMessage: ""},
+				{stdout: "  efficiency: 98.4421 %", stderr: "", errorOnExit: false, errMessage: ""},
+				{stdout: "  wastedBytes: 32025 bytes (32 kB)", stderr: "", errorOnExit: false, errMessage: ""},
+				{stdout: "  userWastedPercent: 48.3491 %", stderr: "", errorOnExit: false, errMessage: ""},
 				{stdout: "Inefficient Files:\nCount  Wasted Space  File Path\n    2         13 kB  /root/saved.txt\n    2         13 kB  /root/example/somefile1.txt\n    2        6.4 kB  /root/example/somefile3.txt\nResults:\n  FAIL: highestUserWastedPercent: too many bytes wasted, relative to the user bytes added (%-user-wasted-bytes=0.4834911001404049 > threshold=0.1)\n  FAIL: highestWastedBytes: too many bytes wasted (wasted-bytes=32025 > threshold=1000)\n  PASS: lowestEfficiency\nResult:FAIL [Total:3] [Passed:1] [Failed:2] [Warn:0] [Skipped:0]\n", stderr: "", errorOnExit: false, errMessage: ""},
 				{stdout: "", stderr: "", errorOnExit: true, errMessage: ""},
 			},
@@ -209,9 +209,9 @@ func TestRun(t *testing.T) {
 			events: []testEvent{
 				{stdout: "Building image...", stderr: "", errorOnExit: false, errMessage: ""},
 				{stdout: "Analyzing image...", stderr: "", errorOnExit: false, errMessage: ""},
-				{stdout: "  efficiency: 98.4421 %\n", stderr: "", errorOnExit: false, errMessage: ""},
-				{stdout: "  wastedBytes: 32025 bytes (32 kB)\n", stderr: "", errorOnExit: false, errMessage: ""},
-				{stdout: "  userWastedPercent: 48.3491 %\n", stderr: "", errorOnExit: false, errMessage: ""},
+				{stdout: "  efficiency: 98.4421 %", stderr: "", errorOnExit: false, errMessage: ""},
+				{stdout: "  wastedBytes: 32025 bytes (32 kB)", stderr: "", errorOnExit: false, errMessage: ""},
+				{stdout: "  userWastedPercent: 48.3491 %", stderr: "", errorOnExit: false, errMessage: ""},
 				{stdout: "Inefficient Files:\nCount  Wasted Space  File Path\nNone\nResults:\n  MISCONFIGURED: highestUserWastedPercent: invalid config value (''): strconv.ParseFloat: parsing \"\": invalid syntax\n  MISCONFIGURED: highestWastedBytes: invalid config value (''): strconv.ParseFloat: parsing \"\": invalid syntax\n  MISCONFIGURED: lowestEfficiency: invalid config value (''): strconv.ParseFloat: parsing \"\": invalid syntax\nCI Misconfigured\n", stderr: "", errorOnExit: false, errMessage: ""},
 				{stdout: "", stderr: "", errorOnExit: true, errMessage: ""},
 			},