Backport #2559

.gitfiles

@@ -76,7 +76,12 @@
 .github/workflows/coverage.yml
 .github/workflows/detect-internal-config-changes.yml
 .github/workflows/dockers-agent-faiss-image.yml
+.github/workflows/dockers-agent-faiss-image.yml
+.github/workflows/dockers-agent-faiss-image.yml
 .github/workflows/dockers-agent-image.yml
+.github/workflows/dockers-agent-image.yml
+.github/workflows/dockers-agent-ngt-image.yml
+.github/workflows/dockers-agent-ngt-image.yml
 .github/workflows/dockers-agent-ngt-image.yml
 .github/workflows/dockers-agent-sidecar-image.yml
 .github/workflows/dockers-benchmark-job-image.yml
@@ -133,6 +138,8 @@ CODE_OF_CONDUCT.md
 CONTRIBUTING.md
 LICENSE
 Makefile
+Makefile
+Makefile
 Makefile.d/actions.mk
 Makefile.d/bench.mk
 Makefile.d/build.mk
@@ -519,16 +526,23 @@ design/Vald Architecture Dataflow.drawio
 design/Vald Architecture Overview.drawio
 design/vald_architecture_overview.png
 dockers/agent/core/agent/Dockerfile
+dockers/agent/core/agent/Dockerfile
 dockers/agent/core/agent/README.md
 dockers/agent/core/faiss/Dockerfile
+dockers/agent/core/faiss/Dockerfile
+dockers/agent/core/faiss/Dockerfile
 dockers/agent/core/ngt/Dockerfile
 dockers/agent/core/ngt/README.md
 dockers/agent/sidecar/Dockerfile
 dockers/agent/sidecar/README.md
 dockers/buildbase/Dockerfile
 dockers/ci/base/Dockerfile
+dockers/ci/base/Dockerfile
+dockers/ci/base/Dockerfile
 dockers/ci/base/README.md
 dockers/dev/Dockerfile
+dockers/dev/Dockerfile
+dockers/dev/Dockerfile
 dockers/dev/README.md
 dockers/discoverer/k8s/Dockerfile
 dockers/discoverer/k8s/README.md
@@ -555,6 +569,7 @@ docs/api/README.md
 docs/api/build_proto.md
 docs/api/filter-gateway.md
 docs/api/flush.md
+docs/api/flush.md
 docs/api/insert.md
 docs/api/mirror-gateway.md
 docs/api/object.md
@@ -622,8 +637,13 @@ example/helm/values-with-pyroscope.yaml
 example/helm/values.yaml
 example/manifest/scylla/configmap.yaml
 example/manifest/scylla/job.yaml
+github_info.json
+go.mod
+go.mod
 go.mod
 go.sum
+go.sum
+go.sum
 hack/CHANGELOG.template.md
 hack/benchmark/assets/checksum/fashion-mnist-784-euclidean.md5
 hack/benchmark/assets/checksum/gist-960-euclidean.md5
@@ -699,6 +719,7 @@ hack/benchmark/internal/starter/starter.go
 hack/benchmark/metrics/metrics.go
 hack/benchmark/src/singleflight/singleflight_bench_test.go
 hack/docker/gen/main.go
+hack/docker/gen/main.go
 hack/git/hooks/pre-commit
 hack/go.mod.default
 hack/gorules/rules.go
@@ -1936,8 +1957,12 @@ pkg/tools/cli/loadtest/usecase/load.go
 pkg/tools/cli/loadtest/usecase/load_test.go
 renovate.json
 rust/Cargo.lock
+rust/Cargo.lock
+rust/Cargo.lock
 rust/Cargo.toml
 rust/bin/agent/Cargo.toml
+rust/bin/agent/Cargo.toml
+rust/bin/agent/Cargo.toml
 rust/bin/agent/src/handler.rs
 rust/bin/agent/src/handler/common.rs
 rust/bin/agent/src/handler/index.rs
@@ -2019,6 +2044,8 @@ versions/NGT_VERSION
 versions/OPERATOR_SDK_VERSION
 versions/OTEL_OPERATOR_VERSION
 versions/PROMETHEUS_STACK_VERSION
+versions/PROMETHEUS_STACK_VERSION
+versions/PROMETHEUS_STACK_VERSION
 versions/PROTOBUF_VERSION
 versions/REVIEWDOG_VERSION
 versions/RUST_VERSION

.github/workflows/dockers-agent-faiss-image.yml

@@ -27,42 +27,43 @@ on:
       - "v*.*.*-*"
   pull_request:
     paths:
+      - "!internal/**/*_test.go"
+      - "!internal/db/**"
+      - "!internal/k8s/**"
       - ".github/actions/docker-build/action.yaml"
       - ".github/workflows/_docker-image.yaml"
       - ".github/workflows/dockers-agent-faiss-image.yml"
+      - "Makefile"
+      - "apis/grpc/**"
+      - "cmd/agent/core/faiss/**"
+      - "dockers/agent/core/faiss/Dockerfile"
       - "go.mod"
       - "go.sum"
       - "internal/**"
-      - "!internal/**/*_test.go"
-      - "!internal/db/**"
-      - "!internal/k8s/**"
-      - "apis/grpc/**"
       - "pkg/agent/core/faiss/**"
-      - "cmd/agent/core/faiss/**"
-      - "dockers/agent/core/faiss/Dockerfile"
-      - "versions/GO_VERSION"
       - "versions/FAISS_VERSION"
+      - "versions/GO_VERSION"
   pull_request_target:
     paths:
+      - "!internal/**/*_test.go"
+      - "!internal/db/**"
+      - "!internal/k8s/**"
       - ".github/actions/docker-build/action.yaml"
       - ".github/workflows/_docker-image.yaml"
       - ".github/workflows/dockers-agent-faiss-image.yml"
+      - "Makefile"
+      - "apis/grpc/**"
+      - "cmd/agent/core/faiss/**"
+      - "dockers/agent/core/faiss/Dockerfile"
       - "go.mod"
       - "go.sum"
       - "internal/**"
-      - "!internal/**/*_test.go"
-      - "!internal/db/**"
-      - "!internal/k8s/**"
-      - "apis/grpc/**"
       - "pkg/agent/core/faiss/**"
-      - "cmd/agent/core/faiss/**"
-      - "dockers/agent/core/faiss/Dockerfile"
-      - "versions/GO_VERSION"
       - "versions/FAISS_VERSION"
+      - "versions/GO_VERSION"
 jobs:
   build:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: agent-faiss
-      platforms: linux/amd64
     secrets: inherit

.github/workflows/dockers-agent-image.yml

@@ -30,29 +30,33 @@ on:
       - ".github/actions/docker-build/action.yaml"
       - ".github/workflows/_docker-image.yaml"
       - ".github/workflows/dockers-agent-image.yml"
+      - "Makefile"
+      - "dockers/agent/core/agent/Dockerfile"
       - "rust/Cargo.lock"
       - "rust/Cargo.toml"
       - "rust/bin/agent/**"
-      - "rust/libs/ngt/**"
       - "rust/libs/ngt-rs/**"
+      - "rust/libs/ngt/**"
       - "rust/libs/proto/**"
-      - "dockers/agent/core/agent/Dockerfile"
-      - "versions/RUST_VERSION"
+      - "versions/FAISS_VERSION"
       - "versions/NGT_VERSION"
+      - "versions/RUST_VERSION"
   pull_request_target:
     paths:
       - ".github/actions/docker-build/action.yaml"
       - ".github/workflows/_docker-image.yaml"
       - ".github/workflows/dockers-agent-image.yml"
+      - "Makefile"
+      - "dockers/agent/core/agent/Dockerfile"
       - "rust/Cargo.lock"
       - "rust/Cargo.toml"
       - "rust/bin/agent/**"
-      - "rust/libs/ngt/**"
       - "rust/libs/ngt-rs/**"
+      - "rust/libs/ngt/**"
       - "rust/libs/proto/**"
-      - "dockers/agent/core/agent/Dockerfile"
-      - "versions/RUST_VERSION"
+      - "versions/FAISS_VERSION"
       - "versions/NGT_VERSION"
+      - "versions/RUST_VERSION"
 jobs:
   build:
     uses: ./.github/workflows/_docker-image.yaml

.github/workflows/dockers-agent-ngt-image.yml

@@ -27,38 +27,40 @@ on:
       - "v*.*.*-*"
   pull_request:
     paths:
+      - "!internal/**/*_test.go"
+      - "!internal/db/**"
+      - "!internal/k8s/**"
       - ".github/actions/docker-build/action.yaml"
       - ".github/workflows/_docker-image.yaml"
       - ".github/workflows/dockers-agent-ngt-image.yml"
+      - "Makefile"
+      - "apis/grpc/**"
+      - "cmd/agent/core/ngt/**"
+      - "dockers/agent/core/ngt/Dockerfile"
       - "go.mod"
       - "go.sum"
       - "internal/**"
-      - "!internal/**/*_test.go"
-      - "!internal/db/**"
-      - "internal/k8s/**"
-      - "apis/grpc/**"
       - "pkg/agent/core/ngt/**"
-      - "cmd/agent/core/ngt/**"
-      - "dockers/agent/core/ngt/Dockerfile"
-      - "versions/GO_VERSION"
       - "versions/NGT_VERSION"
+      - "versions/GO_VERSION"
   pull_request_target:
     paths:
+      - "!internal/**/*_test.go"
+      - "!internal/db/**"
+      - "!internal/k8s/**"
       - ".github/actions/docker-build/action.yaml"
       - ".github/workflows/_docker-image.yaml"
       - ".github/workflows/dockers-agent-ngt-image.yml"
+      - "Makefile"
+      - "apis/grpc/**"
+      - "cmd/agent/core/ngt/**"
+      - "dockers/agent/core/ngt/Dockerfile"
       - "go.mod"
       - "go.sum"
       - "internal/**"
-      - "!internal/**/*_test.go"
-      - "!internal/db/**"
-      - "internal/k8s/**"
-      - "apis/grpc/**"
       - "pkg/agent/core/ngt/**"
-      - "cmd/agent/core/ngt/**"
-      - "dockers/agent/core/ngt/Dockerfile"
-      - "versions/GO_VERSION"
       - "versions/NGT_VERSION"
+      - "versions/GO_VERSION"
 jobs:
   build:
     uses: ./.github/workflows/_docker-image.yaml

.github/workflows/update-actions.yaml

@@ -0,0 +1,64 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: Update Actions version
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * *"
+jobs:
+  dump-contexts-to-log:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: vdaas/vald/.github/actions/dump-context@main
+  update-version:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.DISPATCH_TOKEN }}
+      - name: Set Git config
+        run: |
+          git config --global --add safe.directory ${GITHUB_WORKSPACE}
+      - uses: crazy-max/ghaction-import-gpg@v6
+        with:
+          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
+          git_user_signingkey: true
+          git_commit_gpgsign: true
+      - name: Run update command
+        run: |
+          make update/actions
+      - name: Check difference
+        id: check_diff
+        run: |
+          if git diff --quiet --exit-code; then
+            echo "Nothing updated"
+          else
+            git diff && git status
+            echo "HAS_GIT_DIFF=true" >> $GITHUB_OUTPUT
+          fi
+      - name: Create PR
+        if: ${{ steps.check_diff.outputs.HAS_GIT_DIFF == 'true' }}
+        uses: peter-evans/create-pull-request@v6
+        with:
+          author: "${{ secrets.DISPATCH_USER }} <vald@vdaas.org>"
+          token: ${{ secrets.DISPATCH_TOKEN }}
+          committer: "${{ secrets.DISPATCH_USER }} <vald@vdaas.org>"
+          signoff: true
+          delete-branch: true
+          base: main
+          title: "Update Actions dependency"
+          body: "Automated pull request to update Actions."

Makefile

@@ -14,6 +14,7 @@
 # limitations under the License.
 #
 
+SHELL                           = bash
 ORG                             ?= vdaas
 NAME                            = vald
 GOPKG                           = github.com/$(ORG)/$(NAME)
@@ -144,7 +145,7 @@ PBDOCS = apis/docs/v1/docs.md
 LDFLAGS = -static -fPIC -pthread -std=gnu++23 -lstdc++ -lm -z relro -z now -flto=auto -march=native -mtune=native -fno-plt -Ofast -fvisibility=hidden -ffp-contract=fast -fomit-frame-pointer -fmerge-all-constants -funroll-loops -falign-functions=32 -ffunction-sections -fdata-sections
 
 NGT_LDFLAGS = -fopenmp -lopenblas -llapack
-FAISS_LDFLAGS = $(NGT_LDFLAGS) -lgfortran -lquadmath
+FAISS_LDFLAGS = $(NGT_LDFLAGS) -lgfortran
 HDF5_LDFLAGS = -lhdf5 -lhdf5_hl -lsz -laec -lz -ldl
 CGO_LDFLAGS = $(FAISS_LDFLAGS) $(HDF5_LDFLAGS)
 

dockers/agent/core/agent/Dockerfile

@@ -43,8 +43,8 @@ ENV REPO=vald
 ENV RUST_HOME=/usr/loacl/lib/rust
 ENV TZ=Etc/UTC
 ENV USER=root
-ENV RUSTUP_HOME=${RUST_HOME}/rustup
 ENV CARGO_HOME=${RUST_HOME}/cargo
+ENV RUSTUP_HOME=${RUST_HOME}/rustup
 ENV PATH=${CARGO_HOME}/bin:${RUSTUP_HOME}/bin:/usr/local/bin:${PATH}
 
 WORKDIR ${HOME}/rust/src/github.com/${ORG}/${REPO}
@@ -75,7 +75,6 @@ RUN --mount=type=bind,target=.,rw \
     libomp-dev \
     libopenblas-dev \
     gfortran \
-    libquadmath0 \
     && ldconfig \
     && echo "${LANG} UTF-8" > /etc/locale.gen \
     && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \

dockers/agent/core/faiss/Dockerfile

@@ -77,7 +77,6 @@ RUN --mount=type=bind,target=.,rw \
     libomp-dev \
     libopenblas-dev \
     gfortran \
-    libquadmath0 \
     && ldconfig \
     && echo "${LANG} UTF-8" > /etc/locale.gen \
     && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \

dockers/ci/base/Dockerfile

@@ -48,8 +48,8 @@ ENV REPO=vald
 ENV RUST_HOME=/usr/loacl/lib/rust
 ENV TZ=Etc/UTC
 ENV USER=root
-ENV CARGO_HOME=${RUST_HOME}/cargo
 ENV RUSTUP_HOME=${RUST_HOME}/rustup
+ENV CARGO_HOME=${RUST_HOME}/cargo
 ENV PATH=${CARGO_HOME}/bin:${GOPATH}/bin:${GOROOT}/bin:${RUSTUP_HOME}/bin:/usr/local/bin:${PATH}
 
 WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
@@ -83,7 +83,6 @@ RUN --mount=type=bind,target=.,rw \
     libomp-dev \
     libopenblas-dev \
     gfortran \
-    libquadmath0 \
     gawk \
     gnupg2 \
     graphviz \

dockers/dev/Dockerfile

@@ -88,7 +88,6 @@ RUN --mount=type=bind,target=.,rw \
     libomp-dev \
     libopenblas-dev \
     gfortran \
-    libquadmath0 \
     gawk \
     gnupg2 \
     graphviz \