implement ngt-rs and its unit test and add GitHub Actions workflows (#…

…2478) * implement test * update rust dependencies * fix * make format * add Github Actions workflows * fix * fix * fix * fix * fix * fix * fix * fix * fix --------- Co-authored-by: Yusuke Kato <kpango@vdaas.org>
vdaas · Apr 22, 2024 · f062590 · f062590
1 parent 9e091b1
commit f062590
Show file tree

Hide file tree

Showing 15 changed files with 1,036 additions and 52 deletions.
diff --git a/.github/workflows/dockers-agent-image.yml b/.github/workflows/dockers-agent-image.yml
@@ -0,0 +1,74 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: "Build docker image: agent"
+on:
+  push:
+    branches:
+      - "main"
+      - "release/v*.*"
+      - "!release/v*.*.*"
+    tags:
+      - "*.*.*"
+      - "v*.*.*"
+      - "*.*.*-*"
+      - "v*.*.*-*"
+    paths:
+      - ".github/actions/docker-build/action.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-agent-image.yml"
+      - "rust/Cargo.lock"
+      - "rust/Cargo.toml"
+      - "rust/bin/agent/**"
+      - "rust/libs/ngt/**"
+      - "rust/libs/ngt-rs/**"
+      - "rust/libs/proto/**"
+      - "dockers/agent/core/ngt-rust/Dockerfile"
+      - "versions/RUST_VERSION"
+      - "versions/NGT_VERSION"
+  pull_request:
+    paths:
+      - ".github/actions/docker-build/action.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-agent-image.yml"
+      - "rust/Cargo.lock"
+      - "rust/Cargo.toml"
+      - "rust/bin/agent/**"
+      - "rust/libs/ngt/**"
+      - "rust/libs/ngt-rs/**"
+      - "rust/libs/proto/**"
+      - "dockers/agent/core/agent/Dockerfile"
+      - "versions/RUST_VERSION"
+      - "versions/NGT_VERSION"
+  pull_request_target:
+    paths:
+      - ".github/actions/docker-build/action.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-agent-image.yml"
+      - "rust/Cargo.lock"
+      - "rust/Cargo.toml"
+      - "rust/bin/agent/**"
+      - "rust/libs/ngt/**"
+      - "rust/libs/ngt-rs/**"
+      - "rust/libs/proto/**"
+      - "dockers/agent/core/agent/Dockerfile"
+      - "versions/RUST_VERSION"
+      - "versions/NGT_VERSION"
+jobs:
+  build:
+    uses: ./.github/workflows/_docker-image.yaml
+    with:
+      target: agent
+    secrets: inherit
diff --git a/Makefile b/Makefile
@@ -24,6 +24,7 @@ GHCRORG                         = ghcr.io/$(ORG)/$(NAME)
 AGENT_NGT_IMAGE                 = $(NAME)-agent-ngt
 AGENT_FAISS_IMAGE               = $(NAME)-agent-faiss
 AGENT_SIDECAR_IMAGE             = $(NAME)-agent-sidecar
+AGENT_IMAGE                     = $(NAME)-agent
 CI_CONTAINER_IMAGE              = $(NAME)-ci-container
 DEV_CONTAINER_IMAGE             = $(NAME)-dev-container
 DISCOVERER_IMAGE                = $(NAME)-discoverer-k8s

diff --git a/Makefile.d/build.mk b/Makefile.d/build.mk
@@ -95,6 +95,12 @@ cmd/tools/benchmark/operator/operator:
 	$(eval CGO_ENABLED = 0)
 	$(call go-build,tools/benchmark/operator,,-static,,,$@)
 
+rust/target/release/agent:
+	pushd rust && cargo build -p agent --release && popd
+
+rust/target/debug/agent:
+	pushd rust && cargo build -p agent && popd
+
 .PHONY: binary/build/zip
 ## build all binaries and zip them
 binary/build/zip: \

diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk
@@ -108,6 +108,17 @@ docker/build/agent-sidecar:
 		IMAGE=$(AGENT_SIDECAR_IMAGE) \
 		docker/build/image
 
+.PHONY: docker/name/agent
+docker/name/agent:
+	@echo "$(ORG)/$(AGENT_IMAGE)"
+
+.PHONY: docker/build/agent
+docker/build/agent:
+	@make DOCKERFILE="$(ROOTDIR)/dockers/agent/core/agent/Dockerfile" \
+		IMAGE=$(AGENT_IMAGE) \
+		DISTROLESS_IMAGE=gcr.io/distroless/cc-debian12 \
+		docker/build/image
+
 .PHONY: docker/name/discoverer-k8s
 docker/name/discoverer-k8s:
 	@echo "$(ORG)/$(DISCOVERER_IMAGE)"

diff --git a/dockers/agent/core/agent/Dockerfile b/dockers/agent/core/agent/Dockerfile
@@ -0,0 +1,95 @@
+# syntax = docker/dockerfile:latest
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <vald@vdaas.org>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ARG DISTROLESS_IMAGE=gcr.io/distroless/cc-debian12
+ARG DISTROLESS_IMAGE_TAG=nonroot
+ARG MAINTAINER="vdaas.org vald team <vald@vdaas.org>"
+
+FROM --platform=${TARGETPLATFORM} ubuntu:devel AS builder
+
+ARG TARGETARCH
+ARG TARGETOS
+ARG RUST_VERSION
+
+ENV RUST_HOME /usr/local/lib/rust
+ENV RUSTUP_HOME ${RUST_HOME}/rustup
+ENV CARGO_HOME ${RUST_HOME}/cargo
+ENV DEBIAN_FRONTEND noninteractive
+ENV INITRD No
+ENV LANG en_US.UTF-8
+ENV PATH ${PATH}:${RUSTUP_HOME}/bin:${CARGO_HOME}/bin
+ENV ORG vdaas
+ENV REPO vald
+ENV PKG agent/core/ngt
+ENV APP_NAME ngt
+
+# skipcq: DOK-DL3008
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    cmake \
+    curl \
+    g++ \
+    gcc \
+    git \
+    liblapack-dev \
+    libomp-dev \
+    libopenblas-dev \
+    unzip \
+    upx \
+    && ldconfig \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/Makefile.d
+COPY Makefile.d .
+
+WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
+COPY Makefile .
+COPY .git .
+
+RUN make RUST_VERSION=${RUST_VERSION} rust/install
+
+WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd
+COPY cmd .
+
+WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/versions
+COPY versions .
+
+WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/rust
+COPY rust .
+
+WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
+RUN make ngt/install
+
+RUN make rust/target/release/agent \
+    && mv ./rust/target/release/agent "/usr/bin/${APP_NAME}"
+
+WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG}
+RUN cp sample.yaml /tmp/config.yaml
+
+FROM --platform=${TARGETPLATFORM} ${DISTROLESS_IMAGE}:${DISTROLESS_IMAGE_TAG}
+LABEL maintainer="${MAINTAINER}"
+
+ENV APP_NAME ngt
+
+COPY --from=builder /usr/bin/${APP_NAME} /usr/bin/${APP_NAME}
+COPY --from=builder /tmp/config.yaml /etc/server/config.yaml
+
+USER nonroot:nonroot
+
+ENTRYPOINT ["/usr/bin/ngt"]
diff --git a/dockers/agent/core/agent/README.md b/dockers/agent/core/agent/README.md
@@ -0,0 +1,89 @@
+# Vald Agent
+
+<!-- introduction sentence -->
+
+⚠ This image is under developing and runs incorrectly.
+
+`vald-agent` is the docker image for the vald-agent component.
+
+This image is responsible for the following:
+
+- Store index data along with the user request.
+  - The store destination is In-Memory, Volume Mounts, Persistent Volume, or External Storage(⚠).
+- Search the nearest neighbor vectors of the request vector and return the search result.
+
+⚠ When you'd like to use the external storage, it requires [vald-agent-sidecar](https://hub.docker.com/r/vdaas/vald-agent-sidecar/tags?page=1&name=latest) on the Kubernetes cluster.
+
+For more details, please refer to the [component document](https://vald.vdaas.org/docs/overview/component/agent).
+
+<div align="center">
+    <img src="https://github.com/vdaas/vald/blob/main/assets/image/readme.svg?raw=true" width="50%" />
+</div>
+
+[![latest Image](https://img.shields.io/docker/v/vdaas/vald-agent/latest?label=vald-agent)](https://hub.docker.com/r/vdaas/vald-agent/tags?page=1&name=latest)
+[![License: Apache 2.0](https://img.shields.io/github/license/vdaas/vald.svg?style=flat-square)](https://opensource.org/licenses/Apache-2.0)
+[![latest ver.](https://img.shields.io/github/release/vdaas/vald.svg?style=flat-square)](https://github.com/vdaas/vald/releases/latest)
+[![Twitter](https://img.shields.io/badge/twitter-follow-blue?logo=twitter&style=flat-square)](https://twitter.com/vdaas_vald)
+
+## Requirement
+
+<!-- FIXME: If image has some requirements, describe here with :warning: emoji -->
+
+### linux/amd64
+
+- CPU instruction: requires `AVX2` or `AVX512`
+
+### linux/arm64
+
+⚠ This image does NOT support running on M1/M2 Mac.
+
+## Get Started
+
+<!-- Get Started -->
+<!-- Vald Agent requires more chapter Agent Standalone -->
+
+You can use `vald-agent` in 3 ways.
+
+1. One of the components of the Vald cluster
+   - Refer to [Get Started](https://vald.vdaas.org/docs/tutotial/get-started/).
+1. Standalone on the Kubernetes cluster
+   - Refer to [Vald Agent Standalone on Kubernetes](https://vald.vdaas.org/docs/tutorial/vald-agent-standalone-on-k8s/)
+1. Standalone on Docker
+   - Refer to [Vald Agent Standalone on Docker](https://vald.vdaas.org/docs/tutorial/vald-agent-standalone-on-docker/)
+
+## Versions
+
+| tag     | linux/amd64 | linux/arm64 | description                                                                                                                     |
+| :------ | :---------: | :---------: | :------------------------------------------------------------------------------------------------------------------------------ |
+| latest  |     ✅      |     ✅      | the latest image is the same as the latest version of [vdaas/vald](https://github.com/vdaas/vald) repository version.           |
+| nightly |     ✅      |     ✅      | the nightly applies the main branch's source code of the [vdaas/vald](https://github.com/vdaas/vald) repository.                |
+| vX.Y.Z  |     ✅      |     ✅      | the vX.Y.Z image applies the source code of the [vdaas/vald](https://github.com/vdaas/vald) repository.                         |
+| pr-XXX  |     ✅      |     ❌      | the pr-XXX image applies the source code of the pull request XXX of the [vdaas/vald](https://github.com/vdaas/vald) repository. |
+
+## Dockerfile
+
+<!-- FIXME -->
+
+The `Dockerfile` of this image is [here](https://github.com/vdaas/vald/blob/main/dockers/agent/core/agent/Dockerfile).
+
+## About Vald Project
+
+<!-- About Vald Project -->
+<!-- This chapter is static -->
+
+The information about the Vald project, please refer to the following:
+
+- [Official website](https://vald.vdaas.org)
+- [GitHub](https://github.com/vdaas/vald)
+
+## Contacts
+
+We're love to support you!
+Please feel free to contact us anytime with your questions or issue reports.
+
+- [Official Slack WS](https://join.slack.com/t/vald-community/shared_invite/zt-db2ky9o4-R_9p2sVp8xRwztVa8gfnPA)
+- [GitHub Issue](https://github.com/vdaas/vald/issues)
+
+## License
+
+This product is under the terms of the Apache License v2.0; refer [LICENSE](https://github.com/vdaas/vald/blob/main/LICENSE) file.