diff --git a/Dockerfile b/Dockerfile
index 1fada2706..ded7298ce 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,10 +3,6 @@
 #
 ARG BUILDER_BASE=quay.io/cilium/cilium-envoy-builder:6.1.0-latest
 
-# Common Builder image used in cilium/cilium
-# We need gcc for cgo cross-compilation at least, we can swap to something smaller later on
-ARG PROXYLIB_BUILDER=quay.io/cilium/cilium-builder:832f86bb0f7c7129c1536d5620174deeec645117@sha256:6dbac9f9eba3e20f8edad4676689aa8c11b172035fe5e25b533552f42dea4e9a
-
 #
 # ARCHIVE_IMAGE defaults to the result of the first stage below,
 # refreshing the build caches from Envoy dependencies before the final
@@ -18,12 +14,14 @@ ARG PROXYLIB_BUILDER=quay.io/cilium/cilium-builder:832f86bb0f7c7129c1536d5620174
 #
 ARG ARCHIVE_IMAGE=builder-fresh
 
-FROM --platform=$BUILDPLATFORM $PROXYLIB_BUILDER as proxylib
+FROM --platform=$BUILDPLATFORM $BUILDER_BASE as proxylib
 WORKDIR /go/src/github.com/cilium/proxy
+COPY --chown=1337:1337 . ./
 ARG TARGETARCH
 ENV TARGETARCH=$TARGETARCH
-RUN --mount=type=bind,readwrite,target=/go/src/github.com/cilium/proxy --mount=mode=0777,target=/cilium/proxy/.cache,type=cache --mount=mode=0777,target=/go/pkg,type=cache \
-    GOARCH=${TARGETARCH} make -C proxylib all && mv proxylib/libcilium.so /tmp/libcilium.so
+RUN --mount=mode=0777,gid=1337,uid=1337,target=/cilium/proxy/.cache,type=cache \
+    --mount=mode=0777,gid=1337,uid=1337,target=/go/pkg,type=cache \
+    PATH=$PATH:/usr/local/go/bin GOARCH=${TARGETARCH} make -C proxylib all && mv proxylib/libcilium.so /tmp/libcilium.so
 
 FROM --platform=$BUILDPLATFORM $BUILDER_BASE as builder-fresh
 LABEL maintainer="maintainer@cilium.io"
diff --git a/proxylib/Makefile b/proxylib/Makefile
index 1ec88201d..dc4b0655c 100644
--- a/proxylib/Makefile
+++ b/proxylib/Makefile
@@ -21,7 +21,7 @@ EXTRA_GO_BUILD_LDFLAGS = -extldflags -Wl,-soname,libcilium.so
 
 TARGET := libcilium.so
 
-.PHONY: all $(TARGET) clean header libcilium.h test
+.PHONY: all $(TARGET) clean test
 
 all: $(TARGET)
 
diff --git a/proxylib/libcilium.h b/proxylib/libcilium.h
old mode 100644
new mode 100755