From 9ec66d1051f3a1d197aff0c23e7b32521a576de1 Mon Sep 17 00:00:00 2001 From: Nitin Garg Date: Tue, 5 Mar 2024 10:25:57 +0000 Subject: [PATCH 1/8] Upgrade to golang 1.22 (without dockerfile fix) Reapply "Upgrade to golang 1.22 (#1753)" (#1766) This reverts commit a63a46fdbecbcfa58591e76a9bf49af6239ae7f1 and fixes merge conflicts with golang 1.21.7 upgrade which went in earlier. --- .github/workflows/ci.yml | 4 ++-- Dockerfile | 2 +- go.mod | 2 +- perfmetrics/scripts/ml_tests/pytorch/run_model.sh | 2 +- .../ml_tests/tf/resnet/setup_scripts/setup_container.sh | 4 ++-- perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh | 4 ++-- perfmetrics/scripts/read_cache/setup.sh | 2 +- perfmetrics/scripts/run_e2e_tests.sh | 4 ++-- tools/cd_scripts/e2e_test.sh | 2 +- tools/containerize_gcsfuse_docker/Dockerfile | 2 +- tools/package_gcsfuse_docker/Dockerfile | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21c8eb402f..b26582df77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: linux-tests: strategy: matrix: - go: [ 1.21.x ] + go: [ 1.22.x ] runs-on: ubuntu-20.04 timeout-minutes: 15 @@ -40,7 +40,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: "1.21" + go-version: "1.22" - name: checkout code uses: actions/checkout@v3 - name: golangci-lint diff --git a/Dockerfile b/Dockerfile index 9ff01e6cd9..42aade0911 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ # Mount the gcsfuse to /mnt/gcs: # > docker run --privileged --device /fuse -v /mnt/gcs:/gcs:rw,rshared gcsfuse -FROM golang:1.21.7-alpine as builder +FROM golang:1.22.0-alpine as builder RUN apk add git diff --git a/go.mod b/go.mod index 336379318d..bb6ef30ff0 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/googlecloudplatform/gcsfuse -go 1.21 +go 1.22.0 require ( cloud.google.com/go/compute/metadata v0.2.3 diff --git a/perfmetrics/scripts/ml_tests/pytorch/run_model.sh b/perfmetrics/scripts/ml_tests/pytorch/run_model.sh index 7efd0cf1f8..a35fc38f3c 100755 --- a/perfmetrics/scripts/ml_tests/pytorch/run_model.sh +++ b/perfmetrics/scripts/ml_tests/pytorch/run_model.sh @@ -18,7 +18,7 @@ NUM_EPOCHS=80 TEST_BUCKET="gcsfuse-ml-data" # Install golang -wget -O go_tar.tar.gz https://go.dev/dl/go1.21.7.linux-amd64.tar.gz -q +wget -O go_tar.tar.gz https://go.dev/dl/go1.22.0.linux-amd64.tar.gz -q rm -rf /usr/local/go && tar -C /usr/local -xzf go_tar.tar.gz export PATH=$PATH:/usr/local/go/bin diff --git a/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh b/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh index df7afd1a7c..8b84438f35 100755 --- a/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh +++ b/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh @@ -1,11 +1,11 @@ #!/bin/bash -# Installs go1.21 on the container, builds gcsfuse using log_rotation file +# Installs go1.22 on the container, builds gcsfuse using log_rotation file # and installs tf-models-official v2.13.2, makes update to include clear_kernel_cache # and epochs functionality, and runs the model # Install go lang -wget -O go_tar.tar.gz https://go.dev/dl/go1.21.7.linux-amd64.tar.gz -q +wget -O go_tar.tar.gz https://go.dev/dl/go1.22.0.linux-amd64.tar.gz -q sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local export PATH=$PATH:/usr/local/go/bin diff --git a/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh b/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh index 0a881ca34f..4cc83868d7 100755 --- a/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh +++ b/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh @@ -34,8 +34,8 @@ set -e sudo apt-get update echo Installing git sudo apt-get install git -echo Installing go-lang 1.21.7 -wget -O go_tar.tar.gz https://go.dev/dl/go1.21.7.linux-amd64.tar.gz -q +echo Installing go-lang 1.22.0 +wget -O go_tar.tar.gz https://go.dev/dl/go1.22.0.linux-amd64.tar.gz -q sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local export PATH=$PATH:/usr/local/go/bin export CGO_ENABLED=0 diff --git a/perfmetrics/scripts/read_cache/setup.sh b/perfmetrics/scripts/read_cache/setup.sh index 7e77b3d985..f25b14300d 100755 --- a/perfmetrics/scripts/read_cache/setup.sh +++ b/perfmetrics/scripts/read_cache/setup.sh @@ -64,7 +64,7 @@ sed -i 's/define \+FIO_IO_U_PLAT_GROUP_NR \+\([0-9]\+\)/define FIO_IO_U_PLAT_GRO cd - # Install and validate go. -version=1.21.7 +version=1.22.0 wget -O go_tar.tar.gz https://go.dev/dl/go${version}.linux-amd64.tar.gz -q sudo rm -rf /usr/local/go tar -xzf go_tar.tar.gz && sudo mv go /usr/local diff --git a/perfmetrics/scripts/run_e2e_tests.sh b/perfmetrics/scripts/run_e2e_tests.sh index b9a6fa731f..21d3c981f4 100755 --- a/perfmetrics/scripts/run_e2e_tests.sh +++ b/perfmetrics/scripts/run_e2e_tests.sh @@ -43,8 +43,8 @@ upgrade_gcloud_version # e.g. architecture=arm64 or amd64 architecture=$(dpkg --print-architecture) -echo "Installing go-lang 1.21.7..." -wget -O go_tar.tar.gz https://go.dev/dl/go1.21.7.linux-${architecture}.tar.gz -q +echo "Installing go-lang 1.22.0..." +wget -O go_tar.tar.gz https://go.dev/dl/go1.22.0.linux-${architecture}.tar.gz -q sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local export PATH=$PATH:/usr/local/go/bin # install python3-setuptools tools. diff --git a/tools/cd_scripts/e2e_test.sh b/tools/cd_scripts/e2e_test.sh index b7f3b7960f..95e91226e4 100755 --- a/tools/cd_scripts/e2e_test.sh +++ b/tools/cd_scripts/e2e_test.sh @@ -111,7 +111,7 @@ else fi # install go -wget -O go_tar.tar.gz https://go.dev/dl/go1.21.7.linux-${architecture}.tar.gz +wget -O go_tar.tar.gz https://go.dev/dl/go1.22.0.linux-${architecture}.tar.gz sudo tar -C /usr/local -xzf go_tar.tar.gz export PATH=${PATH}:/usr/local/go/bin #Write gcsfuse and go version to log file diff --git a/tools/containerize_gcsfuse_docker/Dockerfile b/tools/containerize_gcsfuse_docker/Dockerfile index dd9a0b1af5..5b1668fec3 100644 --- a/tools/containerize_gcsfuse_docker/Dockerfile +++ b/tools/containerize_gcsfuse_docker/Dockerfile @@ -34,7 +34,7 @@ ARG OS_VERSION ARG OS_NAME # Image with gcsfuse installed and its package (.deb) -FROM golang:1.21.7 as gcsfuse-package +FROM golang:1.22.0 as gcsfuse-package RUN apt-get update -qq && apt-get install -y ruby ruby-dev rubygems build-essential rpm fuse && gem install --no-document bundler diff --git a/tools/package_gcsfuse_docker/Dockerfile b/tools/package_gcsfuse_docker/Dockerfile index 4332093bb6..6a786a7f0b 100644 --- a/tools/package_gcsfuse_docker/Dockerfile +++ b/tools/package_gcsfuse_docker/Dockerfile @@ -17,7 +17,7 @@ # Copy the gcsfuse packages to the host: # > docker run -it -v /tmp:/output gcsfuse-release cp -r /packages /output -FROM golang:1.21.7 as builder +FROM golang:1.22.0 as builder RUN apt-get update -qq && apt-get install -y ruby ruby-dev rubygems build-essential rpm && gem install --no-document bundler From d6959a6a13f6ada5a302d803c177ca2e734fd22c Mon Sep 17 00:00:00 2001 From: Nitin Garg Date: Tue, 5 Mar 2024 10:23:17 +0000 Subject: [PATCH 2/8] Fix containerize_gcsfuse/dockerfile for go 1.22 Does the following 1. Create a go module (go.mod) in a directory package_gcsfuse, to bypass the change in go get behaviour introduced in 1.17 and enforced in go 1.22. 2. Fix some paths and commands etc. accordingly in the dockerfile. --- tools/containerize_gcsfuse_docker/Dockerfile | 21 ++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tools/containerize_gcsfuse_docker/Dockerfile b/tools/containerize_gcsfuse_docker/Dockerfile index 5b1668fec3..e48a871b29 100644 --- a/tools/containerize_gcsfuse_docker/Dockerfile +++ b/tools/containerize_gcsfuse_docker/Dockerfile @@ -36,6 +36,11 @@ ARG OS_NAME # Image with gcsfuse installed and its package (.deb) FROM golang:1.22.0 as gcsfuse-package +# Directory for creating a go module +RUN mkdir containerized_gcsfuse +WORKDIR $PWD/containerized_gcsfuse +RUN go mod init containerized_gcsfuse + RUN apt-get update -qq && apt-get install -y ruby ruby-dev rubygems build-essential rpm fuse && gem install --no-document bundler ENV CGO_ENABLED=0 @@ -43,21 +48,17 @@ ENV GOOS=linux ENV GO111MODULE=auto ARG GCSFUSE_VERSION -ARG GCSFUSE_REPO="github.com/googlecloudplatform/gcsfuse/" -ENV GCSFUSE_PATH "$GOPATH/src/$GCSFUSE_REPO" -RUN go get -d ${GCSFUSE_REPO} - -WORKDIR ${GCSFUSE_PATH} -# Branch name for building through a particular branch. -ARG BRANCH_NAME="v${GCSFUSE_VERSION}" -RUN git checkout "${BRANCH_NAME}" +ARG GCSFUSE_REPO="github.com/googlecloudplatform/gcsfuse" +ENV GCSFUSE_PATH "$GOPATH/pkg/mod/$GCSFUSE_REPO@v${GCSFUSE_VERSION}" +RUN go get ${GCSFUSE_REPO}@v${GCSFUSE_VERSION} # Install fpm package using bundle RUN bundle install --gemfile=${GCSFUSE_PATH}/tools/gem_dependency/Gemfile ARG GCSFUSE_BIN="/gcsfuse" WORKDIR ${GOPATH} -RUN go install ${GCSFUSE_REPO}/tools/build_gcsfuse +# The following can be optimizing by building from local code. +RUN go install ${GCSFUSE_REPO}/tools/build_gcsfuse@v${GCSFUSE_VERSION} RUN mkdir -p ${GCSFUSE_BIN} RUN build_gcsfuse ${GCSFUSE_PATH} ${GCSFUSE_BIN} ${GCSFUSE_VERSION} RUN mkdir -p ${GCSFUSE_BIN}/usr && mv ${GCSFUSE_BIN}/bin ${GCSFUSE_BIN}/usr/bin @@ -73,7 +74,7 @@ RUN fpm \ -v ${GCSFUSE_VERSION} \ -d fuse \ --vendor "" \ - --url "https://$GCSFUSE_REPO" \ + --url "https://$GCSFUSE_REPO/" \ --description "A user-space file system for Google Cloud Storage." # distroless image with gcsfuse installed. From 0c3c9b37b653e84b6211db92857d1b502b8ad074 Mon Sep 17 00:00:00 2001 From: Nitin Garg Date: Tue, 5 Mar 2024 10:47:42 +0000 Subject: [PATCH 3/8] Fix package_gcsfuse/dockerfile for go 1.22 Does the following 1. Create a go module (go.mod) in a directory package_gcsfuse, to bypass the change in go get behaviour introduced in 1.17 and enforced in go 1.22. 2. Fix some paths and commands etc. accordingly in the dockerfile. --- tools/package_gcsfuse_docker/Dockerfile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tools/package_gcsfuse_docker/Dockerfile b/tools/package_gcsfuse_docker/Dockerfile index 6a786a7f0b..24449c59ce 100644 --- a/tools/package_gcsfuse_docker/Dockerfile +++ b/tools/package_gcsfuse_docker/Dockerfile @@ -19,6 +19,11 @@ FROM golang:1.22.0 as builder +# Directory for creating a go module +RUN mkdir package_gcsfuse +WORKDIR $PWD/package_gcsfuse +RUN go mod init package_gcsfuse + RUN apt-get update -qq && apt-get install -y ruby ruby-dev rubygems build-essential rpm && gem install --no-document bundler ENV CGO_ENABLED=0 @@ -26,19 +31,14 @@ ENV GOOS=linux ENV GO111MODULE=auto ARG GCSFUSE_VERSION -ARG GCSFUSE_REPO="github.com/googlecloudplatform/gcsfuse/" -ENV GCSFUSE_PATH "$GOPATH/src/$GCSFUSE_REPO" -RUN go get -d ${GCSFUSE_REPO} +ARG GCSFUSE_REPO="github.com/googlecloudplatform/gcsfuse" +ENV GCSFUSE_PATH "$GOPATH/pkg/mod/$GCSFUSE_REPO@v${GCSFUSE_VERSION}" +RUN go get ${GCSFUSE_REPO}@v${GCSFUSE_VERSION} WORKDIR ${GCSFUSE_PATH} ARG DEBEMAIL="gcs-fuse-maintainers@google.com" ARG DEBFULLNAME="GCSFuse Team" -# Build Arg for building through a particular branch/commit. By default, it uses -# the tag corresponding to passed GCSFUSE VERSION -ARG BRANCH_NAME="v${GCSFUSE_VERSION}" -RUN git checkout "${BRANCH_NAME}" - # Install fpm package using bundle RUN bundle install --gemfile=${GCSFUSE_PATH}/tools/gem_dependency/Gemfile @@ -50,13 +50,13 @@ fi ARG GCSFUSE_BIN="/gcsfuse_${GCSFUSE_VERSION}_${ARCHITECTURE}" ARG GCSFUSE_DOC="${GCSFUSE_BIN}/usr/share/doc/gcsfuse" WORKDIR ${GOPATH} -RUN go install ${GCSFUSE_REPO}/tools/build_gcsfuse +RUN go install ${GCSFUSE_REPO}/tools/build_gcsfuse@v${GCSFUSE_VERSION} RUN mkdir -p ${GCSFUSE_BIN} RUN build_gcsfuse ${GCSFUSE_PATH} ${GCSFUSE_BIN} ${GCSFUSE_VERSION} RUN mkdir -p ${GCSFUSE_BIN}/usr && mv ${GCSFUSE_BIN}/bin ${GCSFUSE_BIN}/usr/bin # Creating structure for debian package as we are using 'dpkg-deb --build' to create debian package -RUN mkdir -p ${GCSFUSE_BIN}/DEBIAN && cp $GOPATH/src/$GCSFUSE_REPO/DEBIAN/* ${GCSFUSE_BIN}/DEBIAN/ +RUN mkdir -p ${GCSFUSE_BIN}/DEBIAN && cp ${GCSFUSE_PATH}/DEBIAN/* ${GCSFUSE_BIN}/DEBIAN/ RUN mkdir -p ${GCSFUSE_DOC} RUN mv ${GCSFUSE_BIN}/DEBIAN/copyright ${GCSFUSE_DOC} && \ mv ${GCSFUSE_BIN}/DEBIAN/changelog ${GCSFUSE_DOC} && \ @@ -90,5 +90,5 @@ RUN fpm \ --rpm-digest sha256 \ --license Apache-2.0 \ --vendor "" \ - --url "https://$GCSFUSE_REPO" \ + --url "https://$GCSFUSE_REPO/" \ --description "A user-space file system for Google Cloud Storage." From ce54c3638e571d6731902dee0bdb3f8eb5a22938 Mon Sep 17 00:00:00 2001 From: Nitin Garg Date: Thu, 7 Mar 2024 04:30:39 +0000 Subject: [PATCH 4/8] Switch containerize*/Dockerfile to use git clone --- tools/containerize_gcsfuse_docker/Dockerfile | 21 ++++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tools/containerize_gcsfuse_docker/Dockerfile b/tools/containerize_gcsfuse_docker/Dockerfile index e48a871b29..1f59b12893 100644 --- a/tools/containerize_gcsfuse_docker/Dockerfile +++ b/tools/containerize_gcsfuse_docker/Dockerfile @@ -36,11 +36,6 @@ ARG OS_NAME # Image with gcsfuse installed and its package (.deb) FROM golang:1.22.0 as gcsfuse-package -# Directory for creating a go module -RUN mkdir containerized_gcsfuse -WORKDIR $PWD/containerized_gcsfuse -RUN go mod init containerized_gcsfuse - RUN apt-get update -qq && apt-get install -y ruby ruby-dev rubygems build-essential rpm fuse && gem install --no-document bundler ENV CGO_ENABLED=0 @@ -48,17 +43,21 @@ ENV GOOS=linux ENV GO111MODULE=auto ARG GCSFUSE_VERSION -ARG GCSFUSE_REPO="github.com/googlecloudplatform/gcsfuse" -ENV GCSFUSE_PATH "$GOPATH/pkg/mod/$GCSFUSE_REPO@v${GCSFUSE_VERSION}" -RUN go get ${GCSFUSE_REPO}@v${GCSFUSE_VERSION} +ARG GCSFUSE_REPO="https://github.com/googlecloudplatform/gcsfuse/" +ARG BRANCH_NAME="v${GCSFUSE_VERSION}" +RUN git clone --branch ${BRANCH_NAME} ${GCSFUSE_REPO} + +ARG GCSFUSE_PATH=/go/gcsfuse +WORKDIR ${GCSFUSE_PATH} # Install fpm package using bundle RUN bundle install --gemfile=${GCSFUSE_PATH}/tools/gem_dependency/Gemfile ARG GCSFUSE_BIN="/gcsfuse" WORKDIR ${GOPATH} -# The following can be optimizing by building from local code. -RUN go install ${GCSFUSE_REPO}/tools/build_gcsfuse@v${GCSFUSE_VERSION} + +WORKDIR ${GCSFUSE_PATH}/tools/build_gcsfuse +RUN go install RUN mkdir -p ${GCSFUSE_BIN} RUN build_gcsfuse ${GCSFUSE_PATH} ${GCSFUSE_BIN} ${GCSFUSE_VERSION} RUN mkdir -p ${GCSFUSE_BIN}/usr && mv ${GCSFUSE_BIN}/bin ${GCSFUSE_BIN}/usr/bin @@ -74,7 +73,7 @@ RUN fpm \ -v ${GCSFUSE_VERSION} \ -d fuse \ --vendor "" \ - --url "https://$GCSFUSE_REPO/" \ + --url "https://$GCSFUSE_REPO" \ --description "A user-space file system for Google Cloud Storage." # distroless image with gcsfuse installed. From bb56f3c786a8355ed00da9a610e7503d165c9ad8 Mon Sep 17 00:00:00 2001 From: Nitin Garg Date: Thu, 7 Mar 2024 14:32:50 +0000 Subject: [PATCH 5/8] Switch package*/Dockerfile to use git clone --- tools/package_gcsfuse_docker/Dockerfile | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/tools/package_gcsfuse_docker/Dockerfile b/tools/package_gcsfuse_docker/Dockerfile index 24449c59ce..c859b4a1ac 100644 --- a/tools/package_gcsfuse_docker/Dockerfile +++ b/tools/package_gcsfuse_docker/Dockerfile @@ -19,11 +19,6 @@ FROM golang:1.22.0 as builder -# Directory for creating a go module -RUN mkdir package_gcsfuse -WORKDIR $PWD/package_gcsfuse -RUN go mod init package_gcsfuse - RUN apt-get update -qq && apt-get install -y ruby ruby-dev rubygems build-essential rpm && gem install --no-document bundler ENV CGO_ENABLED=0 @@ -31,11 +26,13 @@ ENV GOOS=linux ENV GO111MODULE=auto ARG GCSFUSE_VERSION -ARG GCSFUSE_REPO="github.com/googlecloudplatform/gcsfuse" -ENV GCSFUSE_PATH "$GOPATH/pkg/mod/$GCSFUSE_REPO@v${GCSFUSE_VERSION}" -RUN go get ${GCSFUSE_REPO}@v${GCSFUSE_VERSION} +ARG GCSFUSE_REPO="https://github.com/googlecloudplatform/gcsfuse/" +ARG BRANCH_NAME="v${GCSFUSE_VERSION}" +RUN git clone --branch ${BRANCH_NAME} ${GCSFUSE_REPO} +ARG GCSFUSE_PATH=/go/gcsfuse WORKDIR ${GCSFUSE_PATH} + ARG DEBEMAIL="gcs-fuse-maintainers@google.com" ARG DEBFULLNAME="GCSFuse Team" @@ -49,8 +46,8 @@ RUN if [ "${ARCHITECTURE}" != "amd64" ] && [ "${ARCHITECTURE}" != "arm64" ]; the fi ARG GCSFUSE_BIN="/gcsfuse_${GCSFUSE_VERSION}_${ARCHITECTURE}" ARG GCSFUSE_DOC="${GCSFUSE_BIN}/usr/share/doc/gcsfuse" -WORKDIR ${GOPATH} -RUN go install ${GCSFUSE_REPO}/tools/build_gcsfuse@v${GCSFUSE_VERSION} +WORKDIR ${GCSFUSE_PATH}/tools/build_gcsfuse +RUN go install RUN mkdir -p ${GCSFUSE_BIN} RUN build_gcsfuse ${GCSFUSE_PATH} ${GCSFUSE_BIN} ${GCSFUSE_VERSION} RUN mkdir -p ${GCSFUSE_BIN}/usr && mv ${GCSFUSE_BIN}/bin ${GCSFUSE_BIN}/usr/bin @@ -90,5 +87,5 @@ RUN fpm \ --rpm-digest sha256 \ --license Apache-2.0 \ --vendor "" \ - --url "https://$GCSFUSE_REPO/" \ + --url "https://$GCSFUSE_REPO" \ --description "A user-space file system for Google Cloud Storage." From a812499f5a75ca60648ec1ad4f8febd993d7014a Mon Sep 17 00:00:00 2001 From: Nitin Garg Date: Fri, 8 Mar 2024 10:29:06 +0000 Subject: [PATCH 6/8] Replace /go with ${GOPATH} --- tools/containerize_gcsfuse_docker/Dockerfile | 2 +- tools/package_gcsfuse_docker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/containerize_gcsfuse_docker/Dockerfile b/tools/containerize_gcsfuse_docker/Dockerfile index 1f59b12893..ec86e5e986 100644 --- a/tools/containerize_gcsfuse_docker/Dockerfile +++ b/tools/containerize_gcsfuse_docker/Dockerfile @@ -47,7 +47,7 @@ ARG GCSFUSE_REPO="https://github.com/googlecloudplatform/gcsfuse/" ARG BRANCH_NAME="v${GCSFUSE_VERSION}" RUN git clone --branch ${BRANCH_NAME} ${GCSFUSE_REPO} -ARG GCSFUSE_PATH=/go/gcsfuse +ARG GCSFUSE_PATH=${GOPATH}/gcsfuse WORKDIR ${GCSFUSE_PATH} # Install fpm package using bundle diff --git a/tools/package_gcsfuse_docker/Dockerfile b/tools/package_gcsfuse_docker/Dockerfile index c859b4a1ac..70751e0a81 100644 --- a/tools/package_gcsfuse_docker/Dockerfile +++ b/tools/package_gcsfuse_docker/Dockerfile @@ -30,7 +30,7 @@ ARG GCSFUSE_REPO="https://github.com/googlecloudplatform/gcsfuse/" ARG BRANCH_NAME="v${GCSFUSE_VERSION}" RUN git clone --branch ${BRANCH_NAME} ${GCSFUSE_REPO} -ARG GCSFUSE_PATH=/go/gcsfuse +ARG GCSFUSE_PATH=${GOPATH}/gcsfuse WORKDIR ${GCSFUSE_PATH} ARG DEBEMAIL="gcs-fuse-maintainers@google.com" From 18e7b7bdc6336003175b55612c0be56ecf65538f Mon Sep 17 00:00:00 2001 From: Nitin Garg Date: Mon, 11 Mar 2024 10:24:07 +0000 Subject: [PATCH 7/8] Minimize run of git clone Fetch only the last commit state in docker files during git clone to reduce its runtime. This reduces the runtime of git clone dramatically. --- tools/containerize_gcsfuse_docker/Dockerfile | 2 +- tools/package_gcsfuse_docker/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/containerize_gcsfuse_docker/Dockerfile b/tools/containerize_gcsfuse_docker/Dockerfile index ec86e5e986..b0d0e068f3 100644 --- a/tools/containerize_gcsfuse_docker/Dockerfile +++ b/tools/containerize_gcsfuse_docker/Dockerfile @@ -45,7 +45,7 @@ ENV GO111MODULE=auto ARG GCSFUSE_VERSION ARG GCSFUSE_REPO="https://github.com/googlecloudplatform/gcsfuse/" ARG BRANCH_NAME="v${GCSFUSE_VERSION}" -RUN git clone --branch ${BRANCH_NAME} ${GCSFUSE_REPO} +RUN git clone --branch ${BRANCH_NAME} --depth 1 ${GCSFUSE_REPO} ARG GCSFUSE_PATH=${GOPATH}/gcsfuse WORKDIR ${GCSFUSE_PATH} diff --git a/tools/package_gcsfuse_docker/Dockerfile b/tools/package_gcsfuse_docker/Dockerfile index 70751e0a81..af451c3c32 100644 --- a/tools/package_gcsfuse_docker/Dockerfile +++ b/tools/package_gcsfuse_docker/Dockerfile @@ -28,7 +28,7 @@ ENV GO111MODULE=auto ARG GCSFUSE_VERSION ARG GCSFUSE_REPO="https://github.com/googlecloudplatform/gcsfuse/" ARG BRANCH_NAME="v${GCSFUSE_VERSION}" -RUN git clone --branch ${BRANCH_NAME} ${GCSFUSE_REPO} +RUN git clone --branch ${BRANCH_NAME} --depth 1 ${GCSFUSE_REPO} ARG GCSFUSE_PATH=${GOPATH}/gcsfuse WORKDIR ${GCSFUSE_PATH} From 1144c0ae82d4028dd3548584e1d54ae5e1f002db Mon Sep 17 00:00:00 2001 From: Nitin Garg Date: Tue, 12 Mar 2024 02:12:34 +0000 Subject: [PATCH 8/8] Upgrade from go 1.22.0 to 1.22.1 --- Dockerfile | 2 +- go.mod | 2 +- perfmetrics/scripts/ml_tests/pytorch/run_model.sh | 2 +- .../ml_tests/tf/resnet/setup_scripts/setup_container.sh | 4 ++-- perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh | 4 ++-- perfmetrics/scripts/read_cache/setup.sh | 2 +- perfmetrics/scripts/run_e2e_tests.sh | 4 ++-- tools/cd_scripts/e2e_test.sh | 2 +- tools/containerize_gcsfuse_docker/Dockerfile | 2 +- tools/package_gcsfuse_docker/Dockerfile | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 42aade0911..b57c83ca0e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ # Mount the gcsfuse to /mnt/gcs: # > docker run --privileged --device /fuse -v /mnt/gcs:/gcs:rw,rshared gcsfuse -FROM golang:1.22.0-alpine as builder +FROM golang:1.22.1-alpine as builder RUN apk add git diff --git a/go.mod b/go.mod index bb6ef30ff0..063c472bab 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/googlecloudplatform/gcsfuse -go 1.22.0 +go 1.22.1 require ( cloud.google.com/go/compute/metadata v0.2.3 diff --git a/perfmetrics/scripts/ml_tests/pytorch/run_model.sh b/perfmetrics/scripts/ml_tests/pytorch/run_model.sh index a35fc38f3c..0e8c004c84 100755 --- a/perfmetrics/scripts/ml_tests/pytorch/run_model.sh +++ b/perfmetrics/scripts/ml_tests/pytorch/run_model.sh @@ -18,7 +18,7 @@ NUM_EPOCHS=80 TEST_BUCKET="gcsfuse-ml-data" # Install golang -wget -O go_tar.tar.gz https://go.dev/dl/go1.22.0.linux-amd64.tar.gz -q +wget -O go_tar.tar.gz https://go.dev/dl/go1.22.1.linux-amd64.tar.gz -q rm -rf /usr/local/go && tar -C /usr/local -xzf go_tar.tar.gz export PATH=$PATH:/usr/local/go/bin diff --git a/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh b/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh index 8b84438f35..d0f70bc2c9 100755 --- a/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh +++ b/perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh @@ -1,11 +1,11 @@ #!/bin/bash -# Installs go1.22 on the container, builds gcsfuse using log_rotation file +# Installs go1.22.1 on the container, builds gcsfuse using log_rotation file # and installs tf-models-official v2.13.2, makes update to include clear_kernel_cache # and epochs functionality, and runs the model # Install go lang -wget -O go_tar.tar.gz https://go.dev/dl/go1.22.0.linux-amd64.tar.gz -q +wget -O go_tar.tar.gz https://go.dev/dl/go1.22.1.linux-amd64.tar.gz -q sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local export PATH=$PATH:/usr/local/go/bin diff --git a/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh b/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh index 4cc83868d7..bef1bd7827 100755 --- a/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh +++ b/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh @@ -34,8 +34,8 @@ set -e sudo apt-get update echo Installing git sudo apt-get install git -echo Installing go-lang 1.22.0 -wget -O go_tar.tar.gz https://go.dev/dl/go1.22.0.linux-amd64.tar.gz -q +echo Installing go-lang 1.22.1 +wget -O go_tar.tar.gz https://go.dev/dl/go1.22.1.linux-amd64.tar.gz -q sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local export PATH=$PATH:/usr/local/go/bin export CGO_ENABLED=0 diff --git a/perfmetrics/scripts/read_cache/setup.sh b/perfmetrics/scripts/read_cache/setup.sh index f25b14300d..2e096cb606 100755 --- a/perfmetrics/scripts/read_cache/setup.sh +++ b/perfmetrics/scripts/read_cache/setup.sh @@ -64,7 +64,7 @@ sed -i 's/define \+FIO_IO_U_PLAT_GROUP_NR \+\([0-9]\+\)/define FIO_IO_U_PLAT_GRO cd - # Install and validate go. -version=1.22.0 +version=1.22.1 wget -O go_tar.tar.gz https://go.dev/dl/go${version}.linux-amd64.tar.gz -q sudo rm -rf /usr/local/go tar -xzf go_tar.tar.gz && sudo mv go /usr/local diff --git a/perfmetrics/scripts/run_e2e_tests.sh b/perfmetrics/scripts/run_e2e_tests.sh index 21d3c981f4..c0a03ede39 100755 --- a/perfmetrics/scripts/run_e2e_tests.sh +++ b/perfmetrics/scripts/run_e2e_tests.sh @@ -43,8 +43,8 @@ upgrade_gcloud_version # e.g. architecture=arm64 or amd64 architecture=$(dpkg --print-architecture) -echo "Installing go-lang 1.22.0..." -wget -O go_tar.tar.gz https://go.dev/dl/go1.22.0.linux-${architecture}.tar.gz -q +echo "Installing go-lang 1.22.1..." +wget -O go_tar.tar.gz https://go.dev/dl/go1.22.1.linux-${architecture}.tar.gz -q sudo rm -rf /usr/local/go && tar -xzf go_tar.tar.gz && sudo mv go /usr/local export PATH=$PATH:/usr/local/go/bin # install python3-setuptools tools. diff --git a/tools/cd_scripts/e2e_test.sh b/tools/cd_scripts/e2e_test.sh index 95e91226e4..fa1a0ebea2 100755 --- a/tools/cd_scripts/e2e_test.sh +++ b/tools/cd_scripts/e2e_test.sh @@ -111,7 +111,7 @@ else fi # install go -wget -O go_tar.tar.gz https://go.dev/dl/go1.22.0.linux-${architecture}.tar.gz +wget -O go_tar.tar.gz https://go.dev/dl/go1.22.1.linux-${architecture}.tar.gz sudo tar -C /usr/local -xzf go_tar.tar.gz export PATH=${PATH}:/usr/local/go/bin #Write gcsfuse and go version to log file diff --git a/tools/containerize_gcsfuse_docker/Dockerfile b/tools/containerize_gcsfuse_docker/Dockerfile index b0d0e068f3..e683cfa04d 100644 --- a/tools/containerize_gcsfuse_docker/Dockerfile +++ b/tools/containerize_gcsfuse_docker/Dockerfile @@ -34,7 +34,7 @@ ARG OS_VERSION ARG OS_NAME # Image with gcsfuse installed and its package (.deb) -FROM golang:1.22.0 as gcsfuse-package +FROM golang:1.22.1 as gcsfuse-package RUN apt-get update -qq && apt-get install -y ruby ruby-dev rubygems build-essential rpm fuse && gem install --no-document bundler diff --git a/tools/package_gcsfuse_docker/Dockerfile b/tools/package_gcsfuse_docker/Dockerfile index af451c3c32..e47b9dff38 100644 --- a/tools/package_gcsfuse_docker/Dockerfile +++ b/tools/package_gcsfuse_docker/Dockerfile @@ -17,7 +17,7 @@ # Copy the gcsfuse packages to the host: # > docker run -it -v /tmp:/output gcsfuse-release cp -r /packages /output -FROM golang:1.22.0 as builder +FROM golang:1.22.1 as builder RUN apt-get update -qq && apt-get install -y ruby ruby-dev rubygems build-essential rpm && gem install --no-document bundler