diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh
index f3466ff76bddd..36c6320f229fa 100755
--- a/hack/lib/golang.sh
+++ b/hack/lib/golang.sh
@@ -447,7 +447,8 @@ kube::golang::set_platform_envs() {
 }
 
 # Create the GOPATH tree under $KUBE_OUTPUT
-kube::golang::create_gopath_tree() {
+#FIXME: remove this when kube::golang::old::setup_env() is removed
+kube::golang::old::create_gopath_tree() {
   local go_pkg_dir="${KUBE_GOPATH}/src/${KUBE_GO_PACKAGE}"
   local go_pkg_basedir
   go_pkg_basedir=$(dirname "${go_pkg_dir}")
@@ -508,7 +509,7 @@ EOF
   fi
 }
 
-# kube::golang::setup_env will check that the `go` commands is available in
+# kube::golang::old::setup_env will check that the `go` commands is available in
 # ${PATH}. It will also check that the Go version is good enough for the
 # Kubernetes build.
 #
@@ -517,7 +518,8 @@ EOF
 #   env-var GOBIN is unset (we want binaries in a predictable place)
 #   env-var GO15VENDOREXPERIMENT=1
 #   current directory is within GOPATH
-kube::golang::setup_env() {
+#FIXME: remove this when all callers are converted
+kube::golang::old::setup_env() {
   kube::golang::verify_go_version
 
   # Set up GOPATH.  We have tools which depend on being in a GOPATH (see
@@ -539,7 +541,7 @@ kube::golang::setup_env() {
   #
   # Eventually, when we no longer rely on run-in-gopath.sh we may be able to
   # simplify this some.
-  kube::golang::create_gopath_tree
+  kube::golang::old::create_gopath_tree
   export GOPATH="${KUBE_GOPATH}"
 
   # If these are not set, set them now.  This ensures that any subsequent
@@ -569,6 +571,55 @@ kube::golang::setup_env() {
   export GO15VENDOREXPERIMENT=1
 }
 
+# kube::golang::new::setup_env will check that the `go` commands is available in
+# ${PATH}. It will also check that the Go version is good enough for the
+# Kubernetes build.
+#
+# Outputs:
+#   env-var GOPATH points to our local output dir
+#   env-var GOBIN is unset (we want binaries in a predictable place)
+#   env-var PATH includes the local GOPATH
+#FIXME: rename this when all callers are converted
+kube::golang::new::setup_env() {
+  kube::golang::verify_go_version
+
+  # Set up GOPATH.  We have tools which depend on being in a GOPATH (see
+  # hack/run-in-gopath.sh).
+  #
+  # Even in module mode, we need to set GOPATH for `go build` and `go install`
+  # to work.  We build various tools (usually via `go install`) from a lot of
+  # scripts.
+  #   * We can't just set GOBIN because that does not work on cross-compiles.
+  #   * We could always use `go build -o <something>`, but it's subtle wrt
+  #     cross-compiles and whether the <something> is a file or a directory,
+  #     and EVERY caller has to get it *just* right.
+  #   * We could leave GOPATH alone and let `go install` write binaries
+  #     wherever the user's GOPATH says (or doesn't say).
+  #
+  # Instead we set it to a phony local path and process the results ourselves.
+  # In particular, GOPATH[0]/bin will be used for `go install`, with
+  # cross-compiles adding an extra directory under that.
+  #
+  # Eventually, when we no longer rely on run-in-gopath.sh we may be able to
+  # simplify this some.
+  export GOPATH="${KUBE_GOPATH}"
+
+  # If these are not set, set them now.  This ensures that any subsequent
+  # scripts we run (which may call this function again) use the same values.
+  export GOCACHE="${GOCACHE:-"${KUBE_GOPATH}/cache/build"}"
+  export GOMODCACHE="${GOMODCACHE:-"${KUBE_GOPATH}/cache/mod"}"
+
+  # Make sure our own Go binaries are in PATH.
+  export PATH="${KUBE_GOPATH}/bin:${PATH}"
+
+  # Unset GOBIN in case it already exists in the current session.
+  # Cross-compiles will not work with it set.
+  unset GOBIN
+
+  # Explicitly turn on modules.
+  export GO111MODULE=on
+}
+
 kube::golang::setup_gomaxprocs() {
   # GOMAXPROCS by default does not reflect the number of cpu(s) available
   # when running in a container, please see https://github.com/golang/go/issues/33803
@@ -737,14 +788,14 @@ kube::golang::build_some_binaries() {
     done
     if [[ "${#uncovered[@]}" != 0 ]]; then
       V=2 kube::log::info "Building ${uncovered[*]} without coverage..."
-      GO111MODULE=on GOPROXY=off go install "${build_args[@]}" "${uncovered[@]}"
+      GOPROXY=off go install "${build_args[@]}" "${uncovered[@]}"
     else
       V=2 kube::log::info "Nothing to build without coverage."
-     fi
-   else
+    fi
+  else
     V=2 kube::log::info "Coverage is disabled."
-    GO111MODULE=on GOPROXY=off go install "${build_args[@]}" "$@"
-   fi
+    GOPROXY=off go install "${build_args[@]}" "$@"
+  fi
 }
 
 # Args:
@@ -856,7 +907,7 @@ kube::golang::build_binaries() {
   # Create a sub-shell so that we don't pollute the outer environment
   (
     # Check for `go` binary and set ${GOPATH}.
-    kube::golang::setup_env
+    kube::golang::new::setup_env
     V=2 kube::log::info "Go version: $(GOFLAGS='' go version)"
 
     local host_platform
@@ -867,6 +918,9 @@ kube::golang::build_binaries() {
     # build_binaries_for_platform.
     local goflags goldflags goasmflags gogcflags gotags
 
+    # FIXME: decide if we need this or not
+    goflags+=("-modcacherw") # since we keep GOMODCACHE locally
+
     # This is $(pwd) because we use run-in-gopath to build.  Once that is
     # excised, this can become ${KUBE_ROOT}.
     local trimroot # two lines to appease shellcheck SC2155
diff --git a/hack/lib/protoc.sh b/hack/lib/protoc.sh
index 82c781cd18a06..74bfd9d6c125b 100644
--- a/hack/lib/protoc.sh
+++ b/hack/lib/protoc.sh
@@ -31,7 +31,7 @@ PROTOC_VERSION=23.4
 # and formats it correctly
 # $1: Full path to the directory where the api.proto file is
 function kube::protoc::generate_proto() {
-  kube::golang::setup_env
+  kube::golang::old::setup_env
   GO111MODULE=on GOPROXY=off go install k8s.io/code-generator/cmd/go-to-protobuf/protoc-gen-gogo
 
   kube::protoc::check_protoc
diff --git a/hack/lib/verify-generated.sh b/hack/lib/verify-generated.sh
index 4770f99ca5c47..044f75c03df00 100755
--- a/hack/lib/verify-generated.sh
+++ b/hack/lib/verify-generated.sh
@@ -35,7 +35,7 @@ kube::verify::generated() {
     kube::util::ensure_clean_working_dir
 
     # This sets up the environment, like GOCACHE, which keeps the worktree cleaner.
-    kube::golang::setup_env
+    kube::golang::old::setup_env
 
     _tmpdir="$(kube::realpath "$(mktemp -d -t "verify-generated-$(basename "$1").XXXXXX")")"
     git worktree add -f -q "${_tmpdir}" HEAD
diff --git a/hack/make-rules/test-e2e-node.sh b/hack/make-rules/test-e2e-node.sh
index 49e3e04ac7183..1f7c1139ba6ef 100755
--- a/hack/make-rules/test-e2e-node.sh
+++ b/hack/make-rules/test-e2e-node.sh
@@ -17,7 +17,7 @@
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
-kube::golang::setup_env
+kube::golang::old::setup_env
 kube::golang::setup_gomaxprocs
 
 # start the cache mutation detector by default so that cache mutators will be found
diff --git a/hack/make-rules/test.sh b/hack/make-rules/test.sh
index 4aa72730d8383..5e7dabe4f85c3 100755
--- a/hack/make-rules/test.sh
+++ b/hack/make-rules/test.sh
@@ -21,7 +21,7 @@ set -o pipefail
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
-kube::golang::setup_env
+kube::golang::old::setup_env
 kube::golang::setup_gomaxprocs
 
 # start the cache mutation detector by default so that cache mutators will be found
diff --git a/hack/run-in-gopath.sh b/hack/run-in-gopath.sh
index 25cb5291671da..c671f302a214e 100755
--- a/hack/run-in-gopath.sh
+++ b/hack/run-in-gopath.sh
@@ -28,7 +28,7 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
 # This sets up a clean GOPATH and makes sure we are currently in it.
-kube::golang::setup_env
+kube::golang::old::setup_env
 
 # Run the user-provided command.
 "${@}"
diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh
index 201aca0b00478..7382338a32c2e 100755
--- a/hack/update-codegen.sh
+++ b/hack/update-codegen.sh
@@ -27,7 +27,7 @@ source "${KUBE_ROOT}/hack/lib/init.sh"
 source "${KUBE_ROOT}/hack/lib/protoc.sh"
 cd "${KUBE_ROOT}"
 
-kube::golang::setup_env
+kube::golang::old::setup_env
 
 DBG_CODEGEN="${DBG_CODEGEN:-0}"
 GENERATED_FILE_PREFIX="${GENERATED_FILE_PREFIX:-zz_generated.}"
diff --git a/hack/update-generated-api-compatibility-data.sh b/hack/update-generated-api-compatibility-data.sh
index 3a0a2c51fc6db..3ef94d72b9012 100755
--- a/hack/update-generated-api-compatibility-data.sh
+++ b/hack/update-generated-api-compatibility-data.sh
@@ -21,7 +21,7 @@ set -o pipefail
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
-kube::golang::setup_env
+kube::golang::old::setup_env
 
 # run in module mode to match test command in readme.md
 export GO111MODULE=on
diff --git a/hack/update-generated-docs.sh b/hack/update-generated-docs.sh
index 220c2dc3daac3..2e15f7dfcf543 100755
--- a/hack/update-generated-docs.sh
+++ b/hack/update-generated-docs.sh
@@ -25,7 +25,7 @@ set -o pipefail
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
-kube::golang::setup_env
+kube::golang::old::setup_env
 
 BINS=(
 	cmd/gendocs
diff --git a/hack/update-generated-protobuf-dockerized.sh b/hack/update-generated-protobuf-dockerized.sh
index 64dda03ec5b44..500c5f387f04b 100755
--- a/hack/update-generated-protobuf-dockerized.sh
+++ b/hack/update-generated-protobuf-dockerized.sh
@@ -28,7 +28,7 @@ source "${KUBE_ROOT}/hack/lib/init.sh"
 source "${KUBE_ROOT}/hack/lib/protoc.sh"
 
 kube::protoc::check_protoc
-kube::golang::setup_env
+kube::golang::old::setup_env
 
 GO111MODULE=on GOPROXY=off go install k8s.io/code-generator/cmd/go-to-protobuf
 GO111MODULE=on GOPROXY=off go install k8s.io/code-generator/cmd/go-to-protobuf/protoc-gen-gogo
diff --git a/hack/update-go-workspace.sh b/hack/update-go-workspace.sh
index f7af3b8250814..51ede4282d55a 100755
--- a/hack/update-go-workspace.sh
+++ b/hack/update-go-workspace.sh
@@ -28,7 +28,7 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
 # This sets up the environment, like GOCACHE, which keeps the worktree cleaner.
-kube::golang::setup_env
+kube::golang::new::setup_env
 export GO111MODULE=on # TODO(thockin): remove this when it's the default
 
 cd "${KUBE_ROOT}"
diff --git a/hack/update-golangci-lint-config.sh b/hack/update-golangci-lint-config.sh
index 172ab07001823..acdfee32e2c47 100755
--- a/hack/update-golangci-lint-config.sh
+++ b/hack/update-golangci-lint-config.sh
@@ -26,7 +26,7 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
 # This sets up the environment, like GOCACHE, which keeps the worktree cleaner.
-kube::golang::setup_env
+kube::golang::old::setup_env
 
 # Remove all files, some of them might be obsolete.
 rm -f hack/golangci*.yaml
diff --git a/hack/update-mocks.sh b/hack/update-mocks.sh
index ce1601a5c5af6..44c0682674611 100755
--- a/hack/update-mocks.sh
+++ b/hack/update-mocks.sh
@@ -27,7 +27,7 @@ source "${KUBE_ROOT}/hack/lib/init.sh"
 export GO111MODULE=on
 
 kube::golang::verify_go_version
-kube::golang::setup_env
+kube::golang::old::setup_env
 
 echo 'installing mockgen'
 pushd "${KUBE_ROOT}/hack/tools" >/dev/null
diff --git a/hack/update-openapi-spec.sh b/hack/update-openapi-spec.sh
index 5bd7c182efb71..bfb53a5b45795 100755
--- a/hack/update-openapi-spec.sh
+++ b/hack/update-openapi-spec.sh
@@ -27,7 +27,7 @@ OPENAPI_ROOT_DIR="${KUBE_ROOT}/api/openapi-spec"
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
 kube::util::require-jq
-kube::golang::setup_env
+kube::golang::old::setup_env
 
 make -C "${KUBE_ROOT}" WHAT=cmd/kube-apiserver
 
@@ -46,7 +46,7 @@ function cleanup()
 
 trap cleanup EXIT SIGINT
 
-kube::golang::setup_env
+kube::golang::old::setup_env
 
 TMP_DIR=${TMP_DIR:-$(kube::realpath "$(mktemp -d -t "$(basename "$0").XXXXXX")")}
 ETCD_HOST=${ETCD_HOST:-127.0.0.1}
diff --git a/hack/update-yamlfmt.sh b/hack/update-yamlfmt.sh
index ca784eb3997dd..b3046df1f7fc7 100755
--- a/hack/update-yamlfmt.sh
+++ b/hack/update-yamlfmt.sh
@@ -21,7 +21,7 @@ set -o pipefail
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
-kube::golang::setup_env
+kube::golang::old::setup_env
 
 cd "${KUBE_ROOT}"
 
diff --git a/hack/verify-cli-conventions.sh b/hack/verify-cli-conventions.sh
index 9a3b7a45f5571..bb8d29cb1fb00 100755
--- a/hack/verify-cli-conventions.sh
+++ b/hack/verify-cli-conventions.sh
@@ -25,7 +25,7 @@ set -o pipefail
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
-kube::golang::setup_env
+kube::golang::old::setup_env
 
 BINS=(
 	cmd/clicheck
diff --git a/hack/verify-description.sh b/hack/verify-description.sh
index 95db1a3d49ac5..5bb8d87edfec4 100755
--- a/hack/verify-description.sh
+++ b/hack/verify-description.sh
@@ -26,7 +26,7 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 source "${KUBE_ROOT}/hack/lib/util.sh"
 
-kube::golang::setup_env
+kube::golang::old::setup_env
 
 make -C "${KUBE_ROOT}" WHAT=cmd/genswaggertypedocs
 
diff --git a/hack/verify-fieldname-docs.sh b/hack/verify-fieldname-docs.sh
index 2997234cf65d4..27899f24c60a3 100755
--- a/hack/verify-fieldname-docs.sh
+++ b/hack/verify-fieldname-docs.sh
@@ -26,7 +26,7 @@ KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 source "${KUBE_ROOT}/hack/lib/util.sh"
 
-kube::golang::setup_env
+kube::golang::old::setup_env
 
 make -C "${KUBE_ROOT}" WHAT=cmd/fieldnamedocscheck
 
diff --git a/hack/verify-generated-docs.sh b/hack/verify-generated-docs.sh
index a38df2e91d42e..9810cf76cd881 100755
--- a/hack/verify-generated-docs.sh
+++ b/hack/verify-generated-docs.sh
@@ -25,7 +25,7 @@ set -o pipefail
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
-kube::golang::setup_env
+kube::golang::old::setup_env
 
 BINS=(
 	cmd/gendocs
diff --git a/hack/verify-gofmt.sh b/hack/verify-gofmt.sh
index dc8d3d7bebecd..90f13dd2f441a 100755
--- a/hack/verify-gofmt.sh
+++ b/hack/verify-gofmt.sh
@@ -30,7 +30,7 @@ source "${KUBE_ROOT}/hack/lib/init.sh"
 
 cd "${KUBE_ROOT}"
 
-kube::golang::setup_env
+kube::golang::old::setup_env
 
 find_files() {
   find . -not \( \
diff --git a/hack/verify-govulncheck.sh b/hack/verify-govulncheck.sh
index e739285d4eef5..cf156ab1a2579 100755
--- a/hack/verify-govulncheck.sh
+++ b/hack/verify-govulncheck.sh
@@ -25,7 +25,7 @@ source "${KUBE_ROOT}/hack/lib/util.sh"
 kube::util::ensure_clean_working_dir
 
 # This sets up the environment, like GOCACHE, which keeps the worktree cleaner.
-kube::golang::setup_env
+kube::golang::old::setup_env
 # Opt into using go modules
 export GO111MODULE=on
 
diff --git a/hack/verify-import-boss.sh b/hack/verify-import-boss.sh
index e1dd50ba1d264..a3b2fddccfe26 100755
--- a/hack/verify-import-boss.sh
+++ b/hack/verify-import-boss.sh
@@ -26,7 +26,7 @@ set -o pipefail
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
-kube::golang::setup_env
+kube::golang::old::setup_env
 
 GO111MODULE=on GOPROXY=off go install k8s.io/code-generator/cmd/import-boss
 
diff --git a/hack/verify-imports.sh b/hack/verify-imports.sh
index 35a2e5caa692e..8941c370640c3 100755
--- a/hack/verify-imports.sh
+++ b/hack/verify-imports.sh
@@ -26,7 +26,7 @@ set -o pipefail
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
-kube::golang::setup_env
+kube::golang::old::setup_env
 
 make -C "${KUBE_ROOT}" WHAT=cmd/importverifier
 
diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh
index 2c0f18e4412ee..ce473ba236fa4 100755
--- a/hack/verify-licenses.sh
+++ b/hack/verify-licenses.sh
@@ -27,7 +27,7 @@ source "${KUBE_ROOT}/hack/lib/init.sh"
 source "${KUBE_ROOT}/hack/lib/util.sh"
 
 # This sets up the environment, like GOCACHE, which keeps the worktree cleaner.
-kube::golang::setup_env
+kube::golang::old::setup_env
 kube::util::ensure-temp-dir
 
 # Creating a new repository tree 
diff --git a/hack/verify-openapi-spec.sh b/hack/verify-openapi-spec.sh
index cfb582bb845a1..e1f43f0850d17 100755
--- a/hack/verify-openapi-spec.sh
+++ b/hack/verify-openapi-spec.sh
@@ -26,7 +26,7 @@ set -o pipefail
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
-kube::golang::setup_env
+kube::golang::old::setup_env
 kube::etcd::install
 
 make -C "${KUBE_ROOT}" WHAT=cmd/kube-apiserver
diff --git a/test/instrumentation/stability-utils.sh b/test/instrumentation/stability-utils.sh
index 1a71c96da3004..85e7eacfd2d70 100755
--- a/test/instrumentation/stability-utils.sh
+++ b/test/instrumentation/stability-utils.sh
@@ -26,7 +26,7 @@ stability_check_setup() {
   kube::golang::verify_go_version
   kube::util::ensure-temp-dir
   cd "${KUBE_ROOT}"
-  kube::golang::setup_env
+  kube::golang::old::setup_env
 }
 
 function find_files_to_check() {
diff --git a/test/integration/ipamperf/test-performance.sh b/test/integration/ipamperf/test-performance.sh
index 20d9832c87d94..fb3fc88f35636 100755
--- a/test/integration/ipamperf/test-performance.sh
+++ b/test/integration/ipamperf/test-performance.sh
@@ -70,7 +70,7 @@ done
 KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../../../
 source "${KUBE_ROOT}/hack/lib/init.sh"
 
-kube::golang::setup_env
+kube::golang::old::setup_env
 
 DIR_BASENAME=$(dirname "${BASH_SOURCE[0]}")
 pushd "${DIR_BASENAME}"