Add go cache for improvement docker build performance (#2297)

* 🐳 Add go cache for improvement docker build performance Signed-off-by: vankichi <kyukawa315@gmail.com> * 🐳 Fix Signed-off-by: vankichi <kyukawa315@gmail.com> * 🐳 Fix Signed-off-by: vankichi <kyukawa315@gmail.com> * 🐳 Add skip clean deps flag Signed-off-by: vankichi <kyukawa315@gmail.com> * 🐳 Fix Signed-off-by: vankichi <kyukawa315@gmail.com> * 🐳 Fix Signed-off-by: vankichi <kyukawa315@gmail.com> * 🐳 remove Signed-off-by: vankichi <kyukawa315@gmail.com> * 🐳 convert /tmp to /home/vankichi Signed-off-by: vankichi <kyukawa315@gmail.com> --------- Signed-off-by: vankichi <kyukawa315@gmail.com>
vdaas · Jan 23, 2024 · f68adfa · f68adfa
1 parent 2ca7187
commit f68adfa
Show file tree

Hide file tree

Showing 18 changed files with 158 additions and 44 deletions.
diff --git a/Makefile b/Makefile
@@ -53,6 +53,7 @@ GOARCH := $(eval GOARCH := $(shell go env GOARCH))$(GOARCH)
 GOBIN := $(eval GOBIN := $(or $(shell go env GOBIN),$(GOPATH)/bin))$(GOBIN)
 GOCACHE := $(eval GOCACHE := $(shell go env GOCACHE))$(GOCACHE)
 GOOS := $(eval GOOS := $(shell go env GOOS))$(GOOS)
+GO_CLEAN_DEPS := true
 GOTEST_TIMEOUT = 30m
 TEST_NOT_IMPL_PLACEHOLDER = NOT IMPLEMENTED BELOW
 

diff --git a/Makefile.d/dependencies.mk b/Makefile.d/dependencies.mk
@@ -47,19 +47,22 @@ go/download:
 .PHONY: go/deps
 ## install Go package dependencies
 go/deps:
-	rm -rf $(ROOTDIR)/vendor \
-		/go/pkg \
-		$(GOCACHE) \
-		$(ROOTDIR)/go.sum \
-		$(ROOTDIR)/go.mod
-	cp $(ROOTDIR)/hack/go.mod.default $(ROOTDIR)/go.mod
-	GOPRIVATE=$(GOPRIVATE) go mod tidy
-	go clean -cache -modcache -testcache -i -r
-	rm -rf $(ROOTDIR)/vendor \
-		/go/pkg \
-		$(GOCACHE) \
-		$(ROOTDIR)/go.sum \
-		$(ROOTDIR)/go.mod
+	if $(GO_CLEAN_DEPS); then \
+        	rm -rf $(ROOTDIR)/vendor \
+        		/go/pkg \
+        		$(GOCACHE) \
+        		$(ROOTDIR)/go.sum \
+        		$(ROOTDIR)/go.mod ; \
+        	cp $(ROOTDIR)/hack/go.mod.default $(ROOTDIR)/go.mod ; \
+        	GOPRIVATE=$(GOPRIVATE) go mod tidy ; \
+        	go clean -cache -modcache -testcache -i -r ; \
+        	rm -rf $(ROOTDIR)/vendor \
+        		/go/pkg \
+        		$(GOCACHE) \
+        		$(ROOTDIR)/go.sum \
+        		$(ROOTDIR)/go.mod ; \
+        	cp $(ROOTDIR)/hack/go.mod.default $(ROOTDIR)/go.mod ; \
+	fi
 	cp $(ROOTDIR)/hack/go.mod.default $(ROOTDIR)/go.mod
 	GOPRIVATE=$(GOPRIVATE) go mod tidy
 	go get -u all 2>/dev/null || true

diff --git a/dockers/agent/core/ngt/Dockerfile b/dockers/agent/core/ngt/Dockerfile
@@ -24,6 +24,9 @@ FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang
 
 FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder
 
+ARG TARGETARCH
+
+ENV ARCH=${TARGETARCH}
 ENV GO111MODULE on
 ENV DEBIAN_FRONTEND noninteractive
 ENV INITRD No
@@ -65,7 +68,9 @@ COPY .git .
 COPY go.mod .
 COPY go.sum .
 
-RUN make go/download
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make go/download
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal
 COPY internal .
@@ -87,7 +92,9 @@ COPY versions .
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
 RUN make ngt/install
-RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \
     && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}"
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG}

diff --git a/dockers/agent/sidecar/Dockerfile b/dockers/agent/sidecar/Dockerfile
@@ -24,6 +24,9 @@ FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang
 
 FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder
 
+ARG TARGETARCH
+
+ENV ARCH=${TARGETARCH}
 ENV GO111MODULE on
 ENV GOPATH /go
 ENV GOROOT /opt/go
@@ -58,7 +61,9 @@ COPY .git .
 COPY go.mod .
 COPY go.sum .
 
-RUN make go/download
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make go/download
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal
 COPY internal .
@@ -79,7 +84,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions
 COPY versions .
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
-RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \
     && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}"
 
 FROM --platform=${BUILDPLATFORM} ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG}

diff --git a/dockers/ci/base/Dockerfile b/dockers/ci/base/Dockerfile
@@ -23,6 +23,9 @@ FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang
 FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder
 LABEL maintainer="${MAINTAINER}"
 
+ARG TARGETARCH
+
+ENV ARCH=${TARGETARCH}
 ENV GO111MODULE on
 ENV DEBIAN_FRONTEND noninteractive
 ENV INITRD No
@@ -71,9 +74,13 @@ COPY apis/proto apis/proto
 COPY versions versions
 COPY hack/go.mod.default hack/go.mod.default
 
-RUN make deps ROOTDIR=$ROOTDIR
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make deps ROOTDIR=$ROOTDIR GO_CLEAN_DEPS=false
 
-RUN make ngt/install \
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make ngt/install \
     && make helm/install \
     && make helm-docs/install \
     && make kind/install \

diff --git a/dockers/dev/Dockerfile b/dockers/dev/Dockerfile
@@ -21,6 +21,10 @@ ARG MAINTAINER="vdaas.org vald team <vald@vdaas.org>"
 FROM --platform=${BUILDPLATFORM} mcr.microsoft.com/vscode/devcontainers/go:1 AS base
 LABEL maintainer="${MAINTAINER}"
 
+ARG TARGETARCH
+
+ENV ARCH=${TARGETARCH}
+
 # skipcq: DOK-DL3008
 RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
@@ -51,7 +55,9 @@ COPY versions versions
 COPY hack/go.mod.default hack/go.mod.default
 
 # basic deps
-RUN make deps \
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make deps GO_CLEAN_DEPS=false \
     && make ngt/install \
     && make helm/install \
     && make helm-docs/install \
@@ -62,7 +68,9 @@ RUN make deps \
     && make kubectl/install
 
 # additional deps
-RUN make k3d/install \
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make k3d/install \
     && make buf/install \
     && make k9s/install \
     && make minikube/install
diff --git a/dockers/discoverer/k8s/Dockerfile b/dockers/discoverer/k8s/Dockerfile
@@ -24,6 +24,9 @@ FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang
 
 FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder
 
+ARG TARGETARCH
+
+ENV ARCH=${TARGETARCH}
 ENV GO111MODULE on
 ENV DEBIAN_FRONTEND noninteractive
 ENV INITRD No
@@ -58,7 +61,9 @@ COPY .git .
 COPY go.mod .
 COPY go.sum .
 
-RUN make go/download
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make go/download
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal
 COPY internal .
@@ -76,7 +81,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions
 COPY versions .
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
-RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \
     && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}"
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG}

diff --git a/dockers/gateway/filter/Dockerfile b/dockers/gateway/filter/Dockerfile
@@ -24,6 +24,9 @@ FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang
 
 FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder
 
+ARG TARGETARCH
+
+ENV ARCH=${TARGETARCH}
 ENV GO111MODULE on
 ENV DEBIAN_FRONTEND noninteractive
 ENV INITRD No
@@ -57,7 +60,9 @@ COPY .git .
 COPY go.mod .
 COPY go.sum .
 
-RUN make go/download
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make go/download
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal
 COPY internal .
@@ -75,7 +80,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions
 COPY versions .
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
-RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \
     && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}"
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG}

diff --git a/dockers/gateway/lb/Dockerfile b/dockers/gateway/lb/Dockerfile
@@ -24,6 +24,9 @@ FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang
 
 FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder
 
+ARG TARGETARCH
+
+ENV ARCH=${TARGETARCH}
 ENV GO111MODULE on
 ENV DEBIAN_FRONTEND noninteractive
 ENV INITRD No
@@ -57,7 +60,9 @@ COPY .git .
 COPY go.mod .
 COPY go.sum .
 
-RUN make go/download
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make go/download
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal
 COPY internal .
@@ -75,7 +80,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions
 COPY versions .
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
-RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \
     && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}"
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG}

diff --git a/dockers/gateway/mirror/Dockerfile b/dockers/gateway/mirror/Dockerfile
@@ -23,6 +23,9 @@ FROM golang:${GO_VERSION} AS golang
 
 FROM ubuntu:devel AS builder
 
+ARG TARGETARCH
+
+ENV ARCH=${TARGETARCH}
 ENV GO111MODULE on
 ENV DEBIAN_FRONTEND noninteractive
 ENV INITRD No
@@ -53,7 +56,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
 COPY go.mod .
 COPY go.sum .
 
-RUN go mod download
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    go mod download
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal
 COPY internal .
@@ -77,7 +82,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
 COPY Makefile .
 COPY .git .
 
-RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \
     && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}"
 
 FROM ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG}

diff --git a/dockers/index/job/correction/Dockerfile b/dockers/index/job/correction/Dockerfile
@@ -24,6 +24,9 @@ FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang
 
 FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder
 
+ARG TARGETARCH
+
+ENV ARCH=${TARGETARCH}
 ENV GO111MODULE on
 ENV DEBIAN_FRONTEND noninteractive
 ENV INITRD No
@@ -57,7 +60,9 @@ COPY .git .
 COPY go.mod .
 COPY go.sum .
 
-RUN make go/download
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make go/download
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal
 COPY internal .
@@ -75,7 +80,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions
 COPY versions .
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
-RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \
     && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}"
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG}

diff --git a/dockers/index/job/creation/Dockerfile b/dockers/index/job/creation/Dockerfile
@@ -24,6 +24,9 @@ FROM --platform=${BUILDPLATFORM} golang:${GO_VERSION} AS golang
 
 FROM --platform=${BUILDPLATFORM} ubuntu:devel AS builder
 
+ARG TARGETARCH
+
+ENV ARCH=${TARGETARCH}
 ENV GO111MODULE on
 ENV DEBIAN_FRONTEND noninteractive
 ENV INITRD No
@@ -57,7 +60,9 @@ COPY .git .
 COPY go.mod .
 COPY go.sum .
 
-RUN make go/download
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make go/download
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal
 COPY internal .
@@ -75,7 +80,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions
 COPY versions .
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
-RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \
     && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}"
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG}

diff --git a/dockers/index/job/readreplica/rotate/Dockerfile b/dockers/index/job/readreplica/rotate/Dockerfile
@@ -24,6 +24,9 @@ FROM golang:${GO_VERSION} AS golang
 
 FROM ubuntu:devel AS builder
 
+ARG TARGETARCH
+
+ENV ARCH=${TARGETARCH}
 ENV GO111MODULE on
 ENV DEBIAN_FRONTEND noninteractive
 ENV INITRD No
@@ -57,7 +60,9 @@ COPY .git .
 COPY go.mod .
 COPY go.sum .
 
-RUN make go/download
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make go/download
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal
 COPY internal .
@@ -75,7 +80,9 @@ WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions
 COPY versions .
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
-RUN make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \
+RUN --mount=type=cache,target="${GOPATH}/pkg",id="go-build-${ARCH}" \
+    --mount=type=cache,target="${HOME}/.cache/go-build",id="go-build-${ARCH}" \
+    make REPO=${ORG} NAME=${REPO} cmd/${PKG}/${APP_NAME} \
     && mv "cmd/${PKG}/${APP_NAME}" "/usr/bin/${APP_NAME}"
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG}