Merge pull request #624 from tonistiigi/update-lint-generate

Prefer buildkit in lint/proto scripts

hack/dockerfiles/generated-files.Dockerfile

@@ -22,12 +22,13 @@ RUN go generate ./...
 # `sys` and `proc` into the source directory. With this we can use
 # `tar --strip-components=1 generated-files` on the output of `docker
 # export`.
+FROM gobuild-base AS generated
 RUN mkdir /generated-files
 RUN find . -name "*.pb.go" ! -path ./vendor/\* | tar -cf - --files-from - | tar -C /generated-files -xf -
 
 FROM scratch AS update
 
-COPY --from=gobuild-base generated-files /generated-files
+COPY --from=generated generated-files /generated-files
 
 FROM gobuild-base AS validate
 

hack/dockerfiles/generated-files.buildkit.Dockerfile

@@ -0,0 +1,39 @@
+# syntax=tonistiigi/dockerfile:runmount20180828
+
+# protoc is dynamically linked to glibc to can't use golang:1.10-alpine
+FROM golang:1.11 AS gobuild-base
+ARG PROTOC_VERSION=3.1.0
+ARG GOGO_VERSION=master
+RUN apt-get update && apt-get install -y \
+ git \
+ unzip \
+ && true
+RUN wget -q https://github.com/google/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip && unzip protoc-${PROTOC_VERSION}-linux-x86_64.zip -d /usr/local
+
+RUN go get -d github.com/gogo/protobuf/protoc-gen-gogofaster \
+  && cd /go/src/github.com/gogo/protobuf \
+        && git checkout -q $GOGO_VERSION \
+        && go install ./protoc-gen-gogo ./protoc-gen-gogofaster ./protoc-gen-gogoslick
+
+WORKDIR /go/src/github.com/moby/buildkit
+
+# Generate into a subdirectory because if it is in the root then the
+# extraction with `docker export` ends up putting `.dockerenv`, `dev`,
+# `sys` and `proc` into the source directory. With this we can use
+# `tar --strip-components=1 generated-files` on the output of `docker
+# export`.
+FROM gobuild-base AS generated
+RUN mkdir /generated-files
+RUN --mount=target=/tmp/src \
+	cp -r /tmp/src/. . && \
+	git add -A && \
+	go generate ./... && \
+	git ls-files -m --others -- **/*.pb.go | tar -cf - --files-from - | tar -C /generated-files -xf -
+
+FROM scratch AS update
+COPY --from=generated generated-files /
+
+FROM gobuild-base AS validate
+RUN --mount=target=/tmp/src \
+	cp -r /tmp/src/. . && \
+	go generate ./... && git diff && ./hack/validate-generated-files check

hack/dockerfiles/lint.Dockerfile

@@ -5,3 +5,4 @@ RUN  go get -u gopkg.in/alecthomas/gometalinter.v1 \
   && gometalinter --install
 WORKDIR /go/src/github.com/moby/buildkit
 COPY . .
+RUN gometalinter --config=gometalinter.json ./...

hack/dockerfiles/lint.buildkit.Dockerfile

@@ -0,0 +1,10 @@
+# syntax=tonistiigi/dockerfile:runmount20180828
+
+FROM golang:1.11-alpine
+RUN  apk add --no-cache git
+RUN  go get -u gopkg.in/alecthomas/gometalinter.v1 \
+  && mv /go/bin/gometalinter.v1 /go/bin/gometalinter \
+  && gometalinter --install
+WORKDIR /go/src/github.com/moby/buildkit
+RUN --mount=target=/go/src/github.com/moby/buildkit \
+	gometalinter --config=gometalinter.json ./...

hack/lint

@@ -1,9 +1,35 @@
 #!/usr/bin/env bash
 
+. $(dirname $0)/util
 set -eu -o pipefail -x
 
-iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
-docker build --iidfile $iidfile -f ./hack/dockerfiles/lint.Dockerfile --force-rm .
-iid=$(cat $iidfile)
-docker run $iid gometalinter --config=gometalinter.json ./...
-rm -f $iidfile
+lintLegacy() {
+  docker build -f ./hack/dockerfiles/lint.Dockerfile --force-rm .
+}
+
+lintDocker() {
+  export DOCKER_BUILDKIT=1
+  iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
+  docker build --iidfile $iidfile -f ./hack/dockerfiles/lint.buildkit.Dockerfile --force-rm .
+  iid=$(cat $iidfile)
+  docker rmi $iid
+  rm -f $iidfile
+}
+
+lint() {
+  buildctl build --frontend=dockerfile.v0 \
+    --local context=. --local dockerfile=. \
+    --frontend-opt filename=./hack/dockerfiles/lint.buildkit.Dockerfile
+}
+
+case $buildmode in
+"buildkit")
+  lint
+  ;;
+"docker-buildkit")
+  lintDocker
+  ;;
+*)
+  lintLegacy
+  ;;
+esac

hack/update-generated-files

@@ -1,15 +1,42 @@
 #!/usr/bin/env bash
 
+. $(dirname $0)/util
 set -eu -o pipefail -x
 
 gogo_version=$(awk '$1 == "github.com/gogo/protobuf" { print $2 }' vendor.conf)
-iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
-docker build --build-arg GOGO_VERSION=$gogo_version --iidfile $iidfile -f ./hack/dockerfiles/generated-files.Dockerfile --target update --force-rm .
-iid=$(cat $iidfile)
-cid=$(docker create $iid noop)
+case $buildmode in
+"buildkit")
+  buildctl build --frontend=dockerfile.v0 --local context=. --local dockerfile=. \
+    --frontend-opt build-arg:GOGO_VERSION=$gogo_version \
+    --frontend-opt target=update \
+    --frontend-opt filename=./hack/dockerfiles/generated-files.buildkit.Dockerfile \
+    --exporter=local --exporter-opt output=.
+  ;;
+*)
+  iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
+  case $buildmode in
+  "docker-buildkit")
+    export DOCKER_BUILDKIT=1
+    docker build --build-arg GOGO_VERSION=$gogo_version --iidfile $iidfile -f ./hack/dockerfiles/generated-files.buildkit.Dockerfile --target update --force-rm .
+    ;;
+  *)
+    docker build --build-arg GOGO_VERSION=$gogo_version --iidfile $iidfile -f ./hack/dockerfiles/generated-files.Dockerfile --target update --force-rm .
+    ;;
+  esac
+  iid=$(cat $iidfile)
+  cid=$(docker create $iid noop)
 
-docker export $cid | tar -xf - --strip-components=1 generated-files
+  case $buildmode in
+  "docker-buildkit")
+    docker export $cid | tar -xf - 
+    ;;
+  *)
+    docker export $cid | tar -xf - --strip-components=1 generated-files
+    ;;
+  esac
 
-docker rm $cid
+  docker rm $cid
 
-rm -f $iidfile
+  rm -f $iidfile
+  ;;
+esac

hack/util

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+: ${PREFER_DOCKER=}
+: ${PREFER_LEGACY=}
+
+newerEqualThan() { # $1=minimum wanted version $2=actual-version
+  [ "$1" = "$(echo -e "$1\n$2" | sort -V | head -n 1)" ]
+}
+
+buildmode="legacy"
+if [ "$PREFER_DOCKER" != "1" ] && [ "$PREFER_LEGACY" != "1" ] && buildctl debug workers 2>/dev/null >/dev/null; then
+  buildmode="buildkit";
+else
+  serverVersion=$(docker info --format '{{.ServerVersion}}')
+  experimental=$(docker info --format '{{.ExperimentalBuild}}')
+  if [ "$PREFER_LEGACY" != "1" ] && ( newerEqualThan "18.09" $serverVersion || \
+    ( newerEqualThan "18.06" $serverVersion && [ "true" = "$experimental" ] ) || \
+    [ "$DOCKER_BUILDKIT" = "1" ]); then
+    buildmode="docker-buildkit";
+  fi
+fi

hack/validate-generated-files

@@ -1,26 +1,40 @@
 #!/usr/bin/env bash
 
-set -eu -o pipefail
-
 case ${1:-} in
-    '')
-	gogo_version=$(awk '$1 == "github.com/gogo/protobuf" { print $2 }' vendor.conf)
-	docker build --build-arg GOGO_VERSION=$gogo_version -f ./hack/dockerfiles/generated-files.Dockerfile --target validate --force-rm .
-	;;
-    check)
-	diffs="$(git status --porcelain -- **/*.pb.go 2>/dev/null)"
-	set +x
-	if [ "$diffs" ] ; then
-	    {
-		echo 'The result of "go generate" differs'
-		echo
-		echo "$diffs"
-		echo
-		echo 'Please update with "make generated-files"'
-		echo
-	    } >&2
-	    false
-	fi
-	echo 'Congratulations! All auto generated files are correct.'
-	;;
+'')
+  . $(dirname $0)/util
+  gogo_version=$(awk '$1 == "github.com/gogo/protobuf" { print $2 }' vendor.conf)
+  case $buildmode in
+  "buildkit")
+    buildctl build --frontend=dockerfile.v0 --local context=. --local dockerfile=. --frontend-opt build-arg:GOGO_VERSION=$gogo_version --frontend-opt filename=./hack/dockerfiles/generated-files.buildkit.Dockerfile
+    ;;
+  "docker-buildkit")
+    export DOCKER_BUILDKIT=1
+    iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
+    docker build --iidfile $iidfile --build-arg GOGO_VERSION=$gogo_version -f ./hack/dockerfiles/generated-files.buildkit.Dockerfile --target validate --force-rm . || exit 1
+    iid=$(cat $iidfile)
+    docker rmi $iid
+    rm -f $iidfile
+    ;;
+  *)
+    docker build --build-arg GOGO_VERSION=$gogo_version -f ./hack/dockerfiles/generated-files.Dockerfile --target validate --force-rm .
+    ;;
+  esac
+  ;;
+check)
+  diffs="$(git status --porcelain -- **/*.pb.go 2>/dev/null)"
+  set +x
+  if [ "$diffs" ] ; then
+      {
+        echo 'The result of "go generate" differs'
+        echo
+        echo "$diffs"
+        echo
+        echo 'Please update with "make generated-files"'
+        echo
+      } >&2
+      exit 1
+  fi
+  echo 'Congratulations! All auto generated files are correct.'
+  ;;
 esac