Skip to content

Commit

Permalink
Backport #2559 (#2560)
Browse files Browse the repository at this point in the history
Signed-off-by: kpango <kpango@vdaas.org>
  • Loading branch information
kpango authored Aug 2, 2024
1 parent 5023899 commit 678e98d
Show file tree
Hide file tree
Showing 28 changed files with 561 additions and 72 deletions.
27 changes: 27 additions & 0 deletions .gitfiles
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,12 @@
.github/workflows/coverage.yml
.github/workflows/detect-internal-config-changes.yml
.github/workflows/dockers-agent-faiss-image.yml
.github/workflows/dockers-agent-faiss-image.yml
.github/workflows/dockers-agent-faiss-image.yml
.github/workflows/dockers-agent-image.yml
.github/workflows/dockers-agent-image.yml
.github/workflows/dockers-agent-ngt-image.yml
.github/workflows/dockers-agent-ngt-image.yml
.github/workflows/dockers-agent-ngt-image.yml
.github/workflows/dockers-agent-sidecar-image.yml
.github/workflows/dockers-benchmark-job-image.yml
Expand Down Expand Up @@ -133,6 +138,8 @@ CODE_OF_CONDUCT.md
CONTRIBUTING.md
LICENSE
Makefile
Makefile
Makefile
Makefile.d/actions.mk
Makefile.d/bench.mk
Makefile.d/build.mk
Expand Down Expand Up @@ -519,16 +526,23 @@ design/Vald Architecture Dataflow.drawio
design/Vald Architecture Overview.drawio
design/vald_architecture_overview.png
dockers/agent/core/agent/Dockerfile
dockers/agent/core/agent/Dockerfile
dockers/agent/core/agent/README.md
dockers/agent/core/faiss/Dockerfile
dockers/agent/core/faiss/Dockerfile
dockers/agent/core/faiss/Dockerfile
dockers/agent/core/ngt/Dockerfile
dockers/agent/core/ngt/README.md
dockers/agent/sidecar/Dockerfile
dockers/agent/sidecar/README.md
dockers/buildbase/Dockerfile
dockers/ci/base/Dockerfile
dockers/ci/base/Dockerfile
dockers/ci/base/Dockerfile
dockers/ci/base/README.md
dockers/dev/Dockerfile
dockers/dev/Dockerfile
dockers/dev/Dockerfile
dockers/dev/README.md
dockers/discoverer/k8s/Dockerfile
dockers/discoverer/k8s/README.md
Expand All @@ -555,6 +569,7 @@ docs/api/README.md
docs/api/build_proto.md
docs/api/filter-gateway.md
docs/api/flush.md
docs/api/flush.md
docs/api/insert.md
docs/api/mirror-gateway.md
docs/api/object.md
Expand Down Expand Up @@ -622,8 +637,13 @@ example/helm/values-with-pyroscope.yaml
example/helm/values.yaml
example/manifest/scylla/configmap.yaml
example/manifest/scylla/job.yaml
github_info.json
go.mod
go.mod
go.mod
go.sum
go.sum
go.sum
hack/CHANGELOG.template.md
hack/benchmark/assets/checksum/fashion-mnist-784-euclidean.md5
hack/benchmark/assets/checksum/gist-960-euclidean.md5
Expand Down Expand Up @@ -699,6 +719,7 @@ hack/benchmark/internal/starter/starter.go
hack/benchmark/metrics/metrics.go
hack/benchmark/src/singleflight/singleflight_bench_test.go
hack/docker/gen/main.go
hack/docker/gen/main.go
hack/git/hooks/pre-commit
hack/go.mod.default
hack/gorules/rules.go
Expand Down Expand Up @@ -1936,8 +1957,12 @@ pkg/tools/cli/loadtest/usecase/load.go
pkg/tools/cli/loadtest/usecase/load_test.go
renovate.json
rust/Cargo.lock
rust/Cargo.lock
rust/Cargo.lock
rust/Cargo.toml
rust/bin/agent/Cargo.toml
rust/bin/agent/Cargo.toml
rust/bin/agent/Cargo.toml
rust/bin/agent/src/handler.rs
rust/bin/agent/src/handler/common.rs
rust/bin/agent/src/handler/index.rs
Expand Down Expand Up @@ -2019,6 +2044,8 @@ versions/NGT_VERSION
versions/OPERATOR_SDK_VERSION
versions/OTEL_OPERATOR_VERSION
versions/PROMETHEUS_STACK_VERSION
versions/PROMETHEUS_STACK_VERSION
versions/PROMETHEUS_STACK_VERSION
versions/PROTOBUF_VERSION
versions/REVIEWDOG_VERSION
versions/RUST_VERSION
Expand Down
31 changes: 16 additions & 15 deletions .github/workflows/dockers-agent-faiss-image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,42 +27,43 @@ on:
- "v*.*.*-*"
pull_request:
paths:
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- ".github/actions/docker-build/action.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-agent-faiss-image.yml"
- "Makefile"
- "apis/grpc/**"
- "cmd/agent/core/faiss/**"
- "dockers/agent/core/faiss/Dockerfile"
- "go.mod"
- "go.sum"
- "internal/**"
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- "apis/grpc/**"
- "pkg/agent/core/faiss/**"
- "cmd/agent/core/faiss/**"
- "dockers/agent/core/faiss/Dockerfile"
- "versions/GO_VERSION"
- "versions/FAISS_VERSION"
- "versions/GO_VERSION"
pull_request_target:
paths:
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- ".github/actions/docker-build/action.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-agent-faiss-image.yml"
- "Makefile"
- "apis/grpc/**"
- "cmd/agent/core/faiss/**"
- "dockers/agent/core/faiss/Dockerfile"
- "go.mod"
- "go.sum"
- "internal/**"
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- "apis/grpc/**"
- "pkg/agent/core/faiss/**"
- "cmd/agent/core/faiss/**"
- "dockers/agent/core/faiss/Dockerfile"
- "versions/GO_VERSION"
- "versions/FAISS_VERSION"
- "versions/GO_VERSION"
jobs:
build:
uses: ./.github/workflows/_docker-image.yaml
with:
target: agent-faiss
platforms: linux/amd64
secrets: inherit
16 changes: 10 additions & 6 deletions .github/workflows/dockers-agent-image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,29 +30,33 @@ on:
- ".github/actions/docker-build/action.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-agent-image.yml"
- "Makefile"
- "dockers/agent/core/agent/Dockerfile"
- "rust/Cargo.lock"
- "rust/Cargo.toml"
- "rust/bin/agent/**"
- "rust/libs/ngt/**"
- "rust/libs/ngt-rs/**"
- "rust/libs/ngt/**"
- "rust/libs/proto/**"
- "dockers/agent/core/agent/Dockerfile"
- "versions/RUST_VERSION"
- "versions/FAISS_VERSION"
- "versions/NGT_VERSION"
- "versions/RUST_VERSION"
pull_request_target:
paths:
- ".github/actions/docker-build/action.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-agent-image.yml"
- "Makefile"
- "dockers/agent/core/agent/Dockerfile"
- "rust/Cargo.lock"
- "rust/Cargo.toml"
- "rust/bin/agent/**"
- "rust/libs/ngt/**"
- "rust/libs/ngt-rs/**"
- "rust/libs/ngt/**"
- "rust/libs/proto/**"
- "dockers/agent/core/agent/Dockerfile"
- "versions/RUST_VERSION"
- "versions/FAISS_VERSION"
- "versions/NGT_VERSION"
- "versions/RUST_VERSION"
jobs:
build:
uses: ./.github/workflows/_docker-image.yaml
Expand Down
30 changes: 16 additions & 14 deletions .github/workflows/dockers-agent-ngt-image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,38 +27,40 @@ on:
- "v*.*.*-*"
pull_request:
paths:
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- ".github/actions/docker-build/action.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-agent-ngt-image.yml"
- "Makefile"
- "apis/grpc/**"
- "cmd/agent/core/ngt/**"
- "dockers/agent/core/ngt/Dockerfile"
- "go.mod"
- "go.sum"
- "internal/**"
- "!internal/**/*_test.go"
- "!internal/db/**"
- "internal/k8s/**"
- "apis/grpc/**"
- "pkg/agent/core/ngt/**"
- "cmd/agent/core/ngt/**"
- "dockers/agent/core/ngt/Dockerfile"
- "versions/GO_VERSION"
- "versions/NGT_VERSION"
- "versions/GO_VERSION"
pull_request_target:
paths:
- "!internal/**/*_test.go"
- "!internal/db/**"
- "!internal/k8s/**"
- ".github/actions/docker-build/action.yaml"
- ".github/workflows/_docker-image.yaml"
- ".github/workflows/dockers-agent-ngt-image.yml"
- "Makefile"
- "apis/grpc/**"
- "cmd/agent/core/ngt/**"
- "dockers/agent/core/ngt/Dockerfile"
- "go.mod"
- "go.sum"
- "internal/**"
- "!internal/**/*_test.go"
- "!internal/db/**"
- "internal/k8s/**"
- "apis/grpc/**"
- "pkg/agent/core/ngt/**"
- "cmd/agent/core/ngt/**"
- "dockers/agent/core/ngt/Dockerfile"
- "versions/GO_VERSION"
- "versions/NGT_VERSION"
- "versions/GO_VERSION"
jobs:
build:
uses: ./.github/workflows/_docker-image.yaml
Expand Down
64 changes: 64 additions & 0 deletions .github/workflows/update-actions.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
#
# Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: Update Actions version
on:
workflow_dispatch:
schedule:
- cron: "0 0 * * *"
jobs:
dump-contexts-to-log:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: vdaas/vald/.github/actions/dump-context@main
update-version:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
token: ${{ secrets.DISPATCH_TOKEN }}
- name: Set Git config
run: |
git config --global --add safe.directory ${GITHUB_WORKSPACE}
- uses: crazy-max/ghaction-import-gpg@v6
with:
gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
git_user_signingkey: true
git_commit_gpgsign: true
- name: Run update command
run: |
make update/actions
- name: Check difference
id: check_diff
run: |
if git diff --quiet --exit-code; then
echo "Nothing updated"
else
git diff && git status
echo "HAS_GIT_DIFF=true" >> $GITHUB_OUTPUT
fi
- name: Create PR
if: ${{ steps.check_diff.outputs.HAS_GIT_DIFF == 'true' }}
uses: peter-evans/create-pull-request@v6
with:
author: "${{ secrets.DISPATCH_USER }} <vald@vdaas.org>"
token: ${{ secrets.DISPATCH_TOKEN }}
committer: "${{ secrets.DISPATCH_USER }} <vald@vdaas.org>"
signoff: true
delete-branch: true
base: main
title: "Update Actions dependency"
body: "Automated pull request to update Actions."
3 changes: 2 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
# limitations under the License.
#

SHELL = bash
ORG ?= vdaas
NAME = vald
GOPKG = github.com/$(ORG)/$(NAME)
Expand Down Expand Up @@ -144,7 +145,7 @@ PBDOCS = apis/docs/v1/docs.md
LDFLAGS = -static -fPIC -pthread -std=gnu++23 -lstdc++ -lm -z relro -z now -flto=auto -march=native -mtune=native -fno-plt -Ofast -fvisibility=hidden -ffp-contract=fast -fomit-frame-pointer -fmerge-all-constants -funroll-loops -falign-functions=32 -ffunction-sections -fdata-sections

NGT_LDFLAGS = -fopenmp -lopenblas -llapack
FAISS_LDFLAGS = $(NGT_LDFLAGS) -lgfortran -lquadmath
FAISS_LDFLAGS = $(NGT_LDFLAGS) -lgfortran
HDF5_LDFLAGS = -lhdf5 -lhdf5_hl -lsz -laec -lz -ldl
CGO_LDFLAGS = $(FAISS_LDFLAGS) $(HDF5_LDFLAGS)

Expand Down
3 changes: 1 addition & 2 deletions dockers/agent/core/agent/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ ENV REPO=vald
ENV RUST_HOME=/usr/loacl/lib/rust
ENV TZ=Etc/UTC
ENV USER=root
ENV RUSTUP_HOME=${RUST_HOME}/rustup
ENV CARGO_HOME=${RUST_HOME}/cargo
ENV RUSTUP_HOME=${RUST_HOME}/rustup
ENV PATH=${CARGO_HOME}/bin:${RUSTUP_HOME}/bin:/usr/local/bin:${PATH}

WORKDIR ${HOME}/rust/src/github.com/${ORG}/${REPO}
Expand Down Expand Up @@ -75,7 +75,6 @@ RUN --mount=type=bind,target=.,rw \
libomp-dev \
libopenblas-dev \
gfortran \
libquadmath0 \
&& ldconfig \
&& echo "${LANG} UTF-8" > /etc/locale.gen \
&& ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \
Expand Down
1 change: 0 additions & 1 deletion dockers/agent/core/faiss/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,6 @@ RUN --mount=type=bind,target=.,rw \
libomp-dev \
libopenblas-dev \
gfortran \
libquadmath0 \
&& ldconfig \
&& echo "${LANG} UTF-8" > /etc/locale.gen \
&& ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime \
Expand Down
3 changes: 1 addition & 2 deletions dockers/ci/base/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ ENV REPO=vald
ENV RUST_HOME=/usr/loacl/lib/rust
ENV TZ=Etc/UTC
ENV USER=root
ENV CARGO_HOME=${RUST_HOME}/cargo
ENV RUSTUP_HOME=${RUST_HOME}/rustup
ENV CARGO_HOME=${RUST_HOME}/cargo
ENV PATH=${CARGO_HOME}/bin:${GOPATH}/bin:${GOROOT}/bin:${RUSTUP_HOME}/bin:/usr/local/bin:${PATH}

WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
Expand Down Expand Up @@ -83,7 +83,6 @@ RUN --mount=type=bind,target=.,rw \
libomp-dev \
libopenblas-dev \
gfortran \
libquadmath0 \
gawk \
gnupg2 \
graphviz \
Expand Down
1 change: 0 additions & 1 deletion dockers/dev/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,6 @@ RUN --mount=type=bind,target=.,rw \
libomp-dev \
libopenblas-dev \
gfortran \
libquadmath0 \
gawk \
gnupg2 \
graphviz \
Expand Down
Loading

0 comments on commit 678e98d

Please sign in to comment.