Backport PR #2577 to release/v1.7 for refactor docker and change buil…

…dkit-syft-scanner reference to ghcr.io (#2578)

Signed-off-by: kpango <kpango@vdaas.org>

.gitfiles

@@ -77,19 +77,15 @@
 .github/workflows/coverage.yml
 .github/workflows/detect-internal-config-changes.yml
 .github/workflows/dockers-agent-faiss-image.yml
-.github/workflows/dockers-agent-faiss-image.yml
-.github/workflows/dockers-agent-faiss-image.yml
 .github/workflows/dockers-agent-image.yml
-.github/workflows/dockers-agent-image.yml
-.github/workflows/dockers-agent-ngt-image.yml
-.github/workflows/dockers-agent-ngt-image.yml
 .github/workflows/dockers-agent-ngt-image.yml
 .github/workflows/dockers-agent-sidecar-image.yml
 .github/workflows/dockers-benchmark-job-image.yml
 .github/workflows/dockers-benchmark-operator-image.yaml
 .github/workflows/dockers-binfmt-image.yaml
 .github/workflows/dockers-buildbase-image.yml
 .github/workflows/dockers-buildkit-image.yaml
+.github/workflows/dockers-buildkit-syft-scanner-image.yaml
 .github/workflows/dockers-ci-container-image.yml
 .github/workflows/dockers-dev-container-image.yml
 .github/workflows/dockers-discoverer-k8s-image.yml
@@ -141,8 +137,6 @@ CODE_OF_CONDUCT.md
 CONTRIBUTING.md
 LICENSE
 Makefile
-Makefile
-Makefile
 Makefile.d/actions.mk
 Makefile.d/bench.mk
 Makefile.d/build.mk
@@ -299,17 +293,11 @@ assets/test/templates/common/call.tmpl
 assets/test/templates/common/fill.tmpl
 assets/test/templates/common/function.tmpl
 assets/test/templates/common/header.tmpl
-assets/test/templates/common/inline.tmpl
-assets/test/templates/common/inputs.tmpl
-assets/test/templates/common/message.tmpl
 assets/test/templates/common/results.tmpl
 assets/test/templates/option/call.tmpl
 assets/test/templates/option/fill.tmpl
 assets/test/templates/option/function.tmpl
 assets/test/templates/option/header.tmpl
-assets/test/templates/option/inline.tmpl
-assets/test/templates/option/inputs.tmpl
-assets/test/templates/option/message.tmpl
 assets/test/templates/option/results.tmpl
 buf.gen.yaml
 buf.work.yaml
@@ -523,31 +511,24 @@ cmd/tools/cli/loadtest/main_test.go
 cmd/tools/cli/loadtest/sample.yaml
 cmd/tools/cli/vdctl/main.go
 cmd/tools/cli/vdctl/main_test.go
-design/.gitkeep
 design/Vald Architecture Assets.drawio
 design/Vald Architecture Dataflow.drawio
 design/Vald Architecture Overview.drawio
 design/vald_architecture_overview.png
 dockers/agent/core/agent/Dockerfile
-dockers/agent/core/agent/Dockerfile
 dockers/agent/core/agent/README.md
 dockers/agent/core/faiss/Dockerfile
-dockers/agent/core/faiss/Dockerfile
-dockers/agent/core/faiss/Dockerfile
 dockers/agent/core/ngt/Dockerfile
 dockers/agent/core/ngt/README.md
 dockers/agent/sidecar/Dockerfile
 dockers/agent/sidecar/README.md
 dockers/binfmt/Dockerfile
 dockers/buildbase/Dockerfile
 dockers/buildkit/Dockerfile
-dockers/ci/base/Dockerfile
-dockers/ci/base/Dockerfile
+dockers/buildkit/syft/scanner/Dockerfile
 dockers/ci/base/Dockerfile
 dockers/ci/base/README.md
 dockers/dev/Dockerfile
-dockers/dev/Dockerfile
-dockers/dev/Dockerfile
 dockers/dev/README.md
 dockers/discoverer/k8s/Dockerfile
 dockers/discoverer/k8s/README.md
@@ -574,7 +555,6 @@ docs/api/README.md
 docs/api/build_proto.md
 docs/api/filter-gateway.md
 docs/api/flush.md
-docs/api/flush.md
 docs/api/insert.md
 docs/api/mirror-gateway.md
 docs/api/object.md
@@ -642,13 +622,8 @@ example/helm/values-with-pyroscope.yaml
 example/helm/values.yaml
 example/manifest/scylla/configmap.yaml
 example/manifest/scylla/job.yaml
-github_info.json
-go.mod
-go.mod
 go.mod
 go.sum
-go.sum
-go.sum
 hack/CHANGELOG.template.md
 hack/benchmark/assets/checksum/fashion-mnist-784-euclidean.md5
 hack/benchmark/assets/checksum/gist-960-euclidean.md5
@@ -724,7 +699,6 @@ hack/benchmark/internal/starter/starter.go
 hack/benchmark/metrics/metrics.go
 hack/benchmark/src/singleflight/singleflight_bench_test.go
 hack/docker/gen/main.go
-hack/docker/gen/main.go
 hack/git/hooks/pre-commit
 hack/go.mod.default
 hack/gorules/rules.go
@@ -1424,85 +1398,41 @@ internal/worker/worker.go
 internal/worker/worker_option.go
 internal/worker/worker_option_test.go
 internal/worker/worker_test.go
-k8s/agent/clusterrole.yaml
-k8s/agent/clusterrolebinding.yaml
-k8s/agent/daemonset.yaml
-k8s/agent/deployment.yaml
-k8s/agent/faiss/configmap.yaml
-k8s/agent/hpa.yaml
-k8s/agent/networkpolicy.yaml
 k8s/agent/ngt/configmap.yaml
 k8s/agent/pdb.yaml
 k8s/agent/priorityclass.yaml
-k8s/agent/serviceaccount.yaml
-k8s/agent/sidecar/configmap.yaml
-k8s/agent/sidecar/svc.yaml
 k8s/agent/statefulset.yaml
 k8s/agent/svc.yaml
 k8s/debug/kind/config.yaml
 k8s/discoverer/clusterrole.yaml
 k8s/discoverer/clusterrolebinding.yaml
 k8s/discoverer/configmap.yaml
-k8s/discoverer/daemonset.yaml
 k8s/discoverer/deployment.yaml
-k8s/discoverer/hpa.yaml
-k8s/discoverer/networkpolicy.yaml
 k8s/discoverer/pdb.yaml
 k8s/discoverer/priorityclass.yaml
 k8s/discoverer/serviceaccount.yaml
 k8s/discoverer/svc.yaml
 k8s/external/minio/deployment.yaml
 k8s/external/minio/mb-job.yaml
 k8s/external/minio/svc.yaml
-k8s/gateway/gateway/filter/configmap.yaml
-k8s/gateway/gateway/filter/daemonset.yaml
-k8s/gateway/gateway/filter/deployment.yaml
-k8s/gateway/gateway/filter/hpa.yaml
-k8s/gateway/gateway/filter/networkpolicy.yaml
-k8s/gateway/gateway/filter/pdb.yaml
-k8s/gateway/gateway/filter/priorityclass.yaml
-k8s/gateway/gateway/filter/svc.yaml
 k8s/gateway/gateway/ing.yaml
 k8s/gateway/gateway/lb/configmap.yaml
-k8s/gateway/gateway/lb/daemonset.yaml
 k8s/gateway/gateway/lb/deployment.yaml
 k8s/gateway/gateway/lb/hpa.yaml
-k8s/gateway/gateway/lb/networkpolicy.yaml
 k8s/gateway/gateway/lb/pdb.yaml
 k8s/gateway/gateway/lb/priorityclass.yaml
 k8s/gateway/gateway/lb/svc.yaml
-k8s/gateway/gateway/mirror/clusterrole.yaml
-k8s/gateway/gateway/mirror/clusterrolebinding.yaml
-k8s/gateway/gateway/mirror/configmap.yaml
-k8s/gateway/gateway/mirror/daemonset.yaml
-k8s/gateway/gateway/mirror/deployment.yaml
-k8s/gateway/gateway/mirror/hpa.yaml
-k8s/gateway/gateway/mirror/networkpolicy.yaml
-k8s/gateway/gateway/mirror/pdb.yaml
-k8s/gateway/gateway/mirror/priorityclass.yaml
-k8s/gateway/gateway/mirror/serviceaccount.yaml
-k8s/gateway/gateway/mirror/svc.yaml
 k8s/index/job/correction/configmap.yaml
 k8s/index/job/correction/cronjob.yaml
-k8s/index/job/correction/networkpolicy.yaml
 k8s/index/job/creation/configmap.yaml
 k8s/index/job/creation/cronjob.yaml
-k8s/index/job/creation/networkpolicy.yaml
-k8s/index/job/readreplica/rotate/clusterrole.yaml
-k8s/index/job/readreplica/rotate/clusterrolebinding.yaml
-k8s/index/job/readreplica/rotate/configmap.yaml
-k8s/index/job/readreplica/rotate/networkpolicy.yaml
-k8s/index/job/readreplica/rotate/serviceaccount.yaml
 k8s/index/job/save/configmap.yaml
 k8s/index/job/save/cronjob.yaml
-k8s/index/job/save/networkpolicy.yaml
 k8s/index/operator/configmap.yaml
 k8s/index/operator/deployment.yaml
 k8s/index/operator/priorityclass.yaml
 k8s/manager/index/configmap.yaml
-k8s/manager/index/daemonset.yaml
 k8s/manager/index/deployment.yaml
-k8s/manager/index/networkpolicy.yaml
 k8s/manager/index/pdb.yaml
 k8s/manager/index/priorityclass.yaml
 k8s/manager/index/svc.yaml
@@ -1561,12 +1491,6 @@ k8s/operator/helm/operator.yaml
 k8s/operator/helm/serviceaccount.yaml
 k8s/operator/helm/svc.yaml
 k8s/readreplica/configmap.yaml
-k8s/readreplica/deployment.yaml
-k8s/readreplica/hpa.yaml
-k8s/readreplica/networkpolicy.yaml
-k8s/readreplica/pvc.yaml
-k8s/readreplica/snapshot.yaml
-k8s/readreplica/svc.yaml
 k8s/tools/benchmark/job/clusterrole.yaml
 k8s/tools/benchmark/job/clusterrolebinding.yaml
 k8s/tools/benchmark/job/serviceaccount.yaml
@@ -1962,12 +1886,8 @@ pkg/tools/cli/loadtest/usecase/load.go
 pkg/tools/cli/loadtest/usecase/load_test.go
 renovate.json
 rust/Cargo.lock
-rust/Cargo.lock
-rust/Cargo.lock
 rust/Cargo.toml
 rust/bin/agent/Cargo.toml
-rust/bin/agent/Cargo.toml
-rust/bin/agent/Cargo.toml
 rust/bin/agent/src/handler.rs
 rust/bin/agent/src/handler/common.rs
 rust/bin/agent/src/handler/index.rs
@@ -1985,29 +1905,21 @@ rust/libs/ngt-rs/src/lib.rs
 rust/libs/ngt/Cargo.toml
 rust/libs/ngt/src/lib.rs
 rust/libs/proto/Cargo.toml
-rust/libs/proto/src/core.v1.rs
 rust/libs/proto/src/core.v1.tonic.rs
-rust/libs/proto/src/discoverer.v1.rs
 rust/libs/proto/src/discoverer.v1.tonic.rs
-rust/libs/proto/src/filter.egress.v1.rs
 rust/libs/proto/src/filter.egress.v1.tonic.rs
-rust/libs/proto/src/filter.ingress.v1.rs
 rust/libs/proto/src/filter.ingress.v1.tonic.rs
 rust/libs/proto/src/lib.rs
-rust/libs/proto/src/mirror.v1.rs
 rust/libs/proto/src/mirror.v1.tonic.rs
 rust/libs/proto/src/payload.v1.rs
 rust/libs/proto/src/rpc.v1.rs
-rust/libs/proto/src/sidecar.v1.rs
 rust/libs/proto/src/sidecar.v1.tonic.rs
-rust/libs/proto/src/vald.v1.rs
 rust/libs/proto/src/vald.v1.tonic.rs
 rust/rust-toolchain
 rust/rust-toolchain.toml
 tests/chaos/chart/.helmignore
 tests/chaos/chart/Chart.yaml
 tests/chaos/chart/README.md
-tests/chaos/chart/templates/NOTES.txt
 tests/chaos/chart/templates/_helpers.tpl
 tests/chaos/chart/templates/network/bandwidth.yaml
 tests/chaos/chart/templates/network/partition.yaml
@@ -2049,8 +1961,6 @@ versions/NGT_VERSION
 versions/OPERATOR_SDK_VERSION
 versions/OTEL_OPERATOR_VERSION
 versions/PROMETHEUS_STACK_VERSION
-versions/PROMETHEUS_STACK_VERSION
-versions/PROMETHEUS_STACK_VERSION
 versions/PROTOBUF_VERSION
 versions/REVIEWDOG_VERSION
 versions/RUST_VERSION

.github/workflows/_docker-image.yaml

@@ -48,8 +48,9 @@ jobs:
           (github.event_name == 'push' &&
               github.ref == 'refs/heads/main') ||
           (github.event_name == 'push' &&
-              startsWith( github.ref, 'refs/heads/release/v')) ||
-          startsWith( github.ref, 'refs/tags/')
+              startsWith(github.ref, 'refs/heads/release/v')) ||
+          startsWith(github.ref, 'refs/tags/') ||
+          (github.event_name == 'schedule')
       }}
     steps:
       - name: Get ref
@@ -80,7 +81,7 @@ jobs:
           driver-opts: |
             image=ghcr.io/vdaas/vald/vald-buildkit:nightly
             network=host
-          buildkitd-flags: "--debug --oci-worker-gc=false"
+          buildkitd-flags: "--debug --oci-worker-gc=false --oci-worker-snapshotter=stargz"
       - name: Login to DockerHub
         uses: docker/login-action@v3
         with:

Makefile

@@ -24,15 +24,15 @@ TAG                            ?= latest
 CRORG                          ?= $(ORG)
 GHCRORG                         = ghcr.io/$(REPO)
 AGENT_IMAGE                     = $(NAME)-agent
-AGENT_NGT_IMAGE                 = $(NAME)-agent-ngt
-AGENT_FAISS_IMAGE               = $(NAME)-agent-faiss
-AGENT_SIDECAR_IMAGE             = $(NAME)-agent-sidecar
+AGENT_NGT_IMAGE                 = $(AGENT_IMAGE)-ngt
+AGENT_FAISS_IMAGE               = $(AGENT_IMAGE)-faiss
+AGENT_SIDECAR_IMAGE             = $(AGENT_IMAGE)-sidecar
 BENCHMARK_JOB_IMAGE             = $(NAME)-benchmark-job
 BENCHMARK_OPERATOR_IMAGE        = $(NAME)-benchmark-operator
 BINFMT_IMAGE                    = $(NAME)-binfmt
 BUILDBASE_IMAGE                 = $(NAME)-buildbase
 BUILDKIT_IMAGE                  = $(NAME)-buildkit
-BUILDKIT_SYFT_SCANNER_IMAGE     = $(NAME)-buildkit-syft-scanner
+BUILDKIT_SYFT_SCANNER_IMAGE     = $(BUILDKIT_IMAGE)-syft-scanner
 CI_CONTAINER_IMAGE              = $(NAME)-ci-container
 DEV_CONTAINER_IMAGE             = $(NAME)-dev-container
 DISCOVERER_IMAGE                = $(NAME)-discoverer-k8s
@@ -49,6 +49,8 @@ MIRROR_GATEWAY_IMAGE            = $(NAME)-mirror-gateway
 READREPLICA_ROTATE_IMAGE        = $(NAME)-readreplica-rotate
 MAINTAINER                      = "$(ORG).org $(NAME) team <$(NAME)@$(ORG).org>"
 
+DEFAULT_BUILDKIT_SYFT_SCANNER_IMAGE = $(GHCRORG)/$(BUILDKIT_SYFT_SCANNER_IMAGE):nightly
+
 VERSION ?= $(eval VERSION := $(shell cat versions/VALD_VERSION))$(VERSION)
 
 NGT_REPO = github.com/yahoojapan/NGT

Makefile.d/docker.mk

@@ -68,7 +68,7 @@ ifeq ($(REMOTE),true)
 		--build-arg GO_VERSION=$(GO_VERSION) \
 		--build-arg RUST_VERSION=$(RUST_VERSION) \
 		--build-arg MAINTAINER=$(MAINTAINER) \
-		--attest type=sbom,generator=docker/buildkit-syft-scanner:edge \
+		--attest type=sbom,generator=$(DEFAULT_BUILDKIT_SYFT_SCANNER_IMAGE) \
 		--provenance=mode=max \
 		-t $(CRORG)/$(IMAGE):$(TAG) \
 		-t $(GHCRORG)/$(IMAGE):$(TAG) \
@@ -229,6 +229,7 @@ docker/name/buildkit-syft-scanner:
 docker/build/buildkit-syft-scanner:
 	@make DOCKERFILE="$(ROOTDIR)/dockers/buildkit/syft/scanner/Dockerfile" \
 		IMAGE=$(BUILDKIT_SYFT_SCANNER_IMAGE) \
+		DEFAULT_BUILDKIT_SYFT_SCANNER_IMAGE="docker/buildkit-syft-scanner:edge" \
 		docker/build/image
 
 .PHONY: docker/name/ci-container

dockers/agent/core/agent/Dockerfile

@@ -15,17 +15,13 @@
 # limitations under the License.
 #
 
-# DO_NOT_EDIT this Dockerfile is generated by hack/docker/gen/main.go
-
+# DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
 ARG UPX_OPTIONS=-9
 # skipcq: DOK-DL3026,DOK-DL3007
 FROM ghcr.io/vdaas/vald/vald-buildbase:nightly AS builder
-ARG MAINTAINER="vdaas.org vald team <vald@vdaas.org>"
-LABEL maintainer="${MAINTAINER}"
-
+LABEL maintainer="vdaas.org vald team <vald@vdaas.org>"
 # skipcq: DOK-DL3002
 USER root:root
-
 ARG TARGETARCH
 ARG TARGETOS
 ARG GO_VERSION
@@ -43,10 +39,9 @@ ENV REPO=vald
 ENV RUST_HOME=/usr/loacl/lib/rust
 ENV TZ=Etc/UTC
 ENV USER=root
-ENV RUSTUP_HOME=${RUST_HOME}/rustup
 ENV CARGO_HOME=${RUST_HOME}/cargo
+ENV RUSTUP_HOME=${RUST_HOME}/rustup
 ENV PATH=${CARGO_HOME}/bin:${RUSTUP_HOME}/bin:/usr/local/bin:${PATH}
-
 WORKDIR ${HOME}/rust/src/github.com/${ORG}/${REPO}
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 #skipcq: DOK-W1001, DOK-SC2046, DOK-SC2086, DOK-DL3008
@@ -92,12 +87,8 @@ RUN --mount=type=bind,target=.,rw \
     && rm -rf rust/target
 # skipcq: DOK-DL3026,DOK-DL3007
 FROM gcr.io/distroless/cc-debian12:nonroot
-ARG MAINTAINER="vdaas.org vald team <vald@vdaas.org>"
-LABEL maintainer="${MAINTAINER}"
-
-ENV APP_NAME=agent
-
-COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
+LABEL maintainer="vdaas.org vald team <vald@vdaas.org>"
+COPY --from=builder /usr/bin/agent /usr/bin/agent
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/agent"]
+ENTRYPOINT ["/usr/bin/agent"]