add loadtest job and container

.github/labeler.yml

@@ -58,6 +58,15 @@ area/meta:
   - cmd/meta/**/*
   - pkg/meta/**/*
 
+area/tools/cli/loadtest:
+  - apis/proto/agent/core/**/*
+  - apis/grpc/agent/core/**/*
+  - apis/proto/vald/**/*
+  - apis/grpc/vald/**/*
+  - cmd/tools/cli/loadtest/**/*
+  - pkg/tools/cli/loadtest/**/*
+
+
 area/internal:
   - internal/**/*
 

.github/workflows/dockers-loadtest-image.yml

@@ -0,0 +1,88 @@
+name: 'Build docker image: loadtest'
+on:
+  push:
+    branches:
+      - master
+    tags:
+       - '*.*.*'
+       - 'v*.*.*'
+       - '*.*.*-*'
+       - 'v*.*.*-*'
+    paths:
+      - 'internal/**'
+      - '!internal/db/**'
+      - '!internal/k8s/**'
+      - 'apis/grpc/**'
+      - 'pkg/tools/cli/loadtest/**'
+      - 'cmd/tools/cli/loadtest/**'
+      - 'dockers/base/Dockerfile'
+      - 'dockers/tools/cli/loadtest/Dockerfile'
+      - 'versions/GO_VERSION'
+  pull_request:
+    paths:
+      - 'internal/**'
+      - '!internal/db/**'
+      - '!internal/k8s/**'
+      - 'apis/grpc/**'
+      - 'pkg/tools/cli/loadtest/**'
+      - 'cmd/tools/cli/loadtest/**'
+      - 'dockers/base/Dockerfile'
+      - 'dockers/tools/cli/loadtest/Dockerfile'
+      - 'versions/GO_VERSION'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+      with:
+        fetch-depth: 10
+    - name: Build the Docker image
+      run: |
+        make docker/build/loadtest
+    - name: login to DockerHub
+      run: |
+        echo ${DOCKERHUB_PASS} | docker login --username ${DOCKERHUB_USER} --password-stdin
+      env:
+        DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+        DOCKERHUB_PASS: ${{ secrets.DOCKERHUB_PASS }}
+    - name: push to DockerHub (master)
+      if: github.ref == 'refs/heads/master'
+      run: |
+        imagename=`make docker/name/loadtest`
+        docker tag ${imagename} ${imagename}:nightly
+        docker push ${imagename}:nightly
+    - name: push to DockerHub (pull request)
+      if: github.event_name == 'pull_request'
+      run: |
+        imagename=`make docker/name/loadtest`
+        pr_num=`cat $GITHUB_EVENT_PATH | jq -r ".number"`
+        docker tag ${imagename} ${imagename}:pr-${pr_num}
+        docker push ${imagename}:pr-${pr_num}
+    - name: push to DockerHub (tags)
+      if: startsWith( github.ref, 'refs/tags/')
+      run: |
+        imagename=`make docker/name/loadtest`
+        docker push ${imagename}:latest
+        tag_name=`echo $GITHUB_REF | sed -e 's:^refs/tags/::'`
+        docker tag ${imagename} ${imagename}:${tag_name}
+        docker push ${imagename}:${tag_name}
+        docker tag ${imagename} ${imagename}:nightly
+        docker push ${imagename}:nightly
+  slack:
+    name: Slack notification
+    needs: build
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/')
+    steps:
+      - uses: technote-space/workflow-conclusion-action@v1
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - uses: 8398a7/action-slack@v2
+        with:
+          author_name: vald-loadtest image build
+          status: ${{ env.WORKFLOW_CONCLUSION }}
+          only_mention_fail: channel
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }}

Makefile

@@ -31,6 +31,7 @@ MANAGER_COMPRESSOR_IMAGE        = $(NAME)-manager-compressor
 MANAGER_INDEX_IMAGE             = $(NAME)-manager-index
 CI_CONTAINER_IMAGE              = $(NAME)-ci-container
 HELM_OPERATOR_IMAGE             = $(NAME)-helm-operator
+LOADTEST_IMAGE                  = $(NAME)-loadtest
 
 VERSION := $(eval VALD_VERSION := $(shell cat versions/VALD_VERSION))$(VALD_VERSION)
 

Makefile.d/docker.mk

@@ -149,6 +149,15 @@ docker/build/operator/helm:
 	    -t $(REPO)/$(HELM_OPERATOR_IMAGE) . \
 	    --build-arg OPERATOR_SDK_VERSION=$(OPERATOR_SDK_VERSION)
 
+.PHONY: docker/name/loadtest
+docker/name/loadtest:
+	@echo "$(REPO)/$(LOADTEST_IMAGE)"
+
+.PHONY: docker/build/loadtest
+## build loadtest image
+docker/build/loadtest: docker/build/base
+	docker build -f dockers/tools/cli/loadtest/Dockerfile -t $(REPO)/$(LOADTEST_IMAGE) .
+
 .PHONY: dockfmt/install
 dockfmt/install: $(BINDIR)/dockfmt
 

README.md

@@ -189,6 +189,12 @@ make init
     <td><a href="https://hub.docker.com/r/vdaas/vald-helm-operator">vdaas/vald-helm-operator</a></td>
     <td><img src="https://github.com/vdaas/vald/workflows/Build%20docker%20image:%20helm-operator/badge.svg"></td>
   </tr>
+  <tr>
+    <td>loadtester</td>
+    <td></td>
+    <td><a href="https://hub.docker.com/r/vdaas/vald-loadtest">vdaas/vald-loadtest</a></td>
+    <td><img src="https://github.com/vdaas/vald/workflows/Build%20docker%20image:%20loadtest/badge.svg"></td>
+  </tr>
 </table>
 
 ## Contribution

dockers/base/Dockerfile

@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 
-FROM ubuntu:latest AS builder
+FROM ubuntu:devel AS builder
 
 ENV GO111MODULE on
 ENV DEBIAN_FRONTEND noninteractive

dockers/tools/cli/loadtest/Dockerfile

@@ -0,0 +1,89 @@
+#
+# Copyright (C) 2019-2020 Vdaas.org Vald team ( kpango, rinx, kmrmt )
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM vdaas/vald-base:latest AS builder
+
+ENV ORG vdaas
+ENV REPO vald
+ENV APP_NAME loadtest
+ENV PKG tools/cli/${APP_NAME}
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libhdf5-dev \
+    && ldconfig \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/internal
+COPY internal .
+
+WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/apis/grpc
+COPY apis/grpc .
+
+WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/pkg/${PKG}
+COPY pkg/${PKG} .
+
+WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}/cmd/${PKG}
+COPY cmd/${PKG} .
+
+WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO}
+COPY versions/GO_VERSION .
+COPY versions/VALD_VERSION .
+COPY .git .
+RUN GO_VERSION="$(cat GO_VERSION)" \
+    && VALD_VERSION="$(cat VALD_VERSION)" \
+    && GIT_COMMIT="$(git rev-list -1 HEAD)" \
+    && CPU_INFO_FLAGS="$(cat /proc/cpuinfo | grep flags | cut -d " " -f 2- | head -1)" \
+    && GOOS="$(go env GOOS)" \
+    && GOARCH="$(go env GOARCH)" \
+    && CGO_ENABLED=1 \
+    && CGO_CXXFLAGS="-g -Ofast -march=native" \
+    CGO_FFLAGS="-g -Ofast -march=native" \
+    CGO_LDFLAGS="-g -Ofast -march=native" \
+    GO111MODULE=on \
+    go build \
+    --ldflags "-s -w -linkmode 'external' \
+    -extldflags '-fPIC -m64 -pthread -fopenmp -std=c++17 -lstdc++ -lm' \
+    -X 'github.com/${ORG}/${REPO}/internal/info.Version=${VALD_VERSION}' \
+    -X 'github.com/${ORG}/${REPO}/internal/info.GitCommit=${GIT_COMMIT}' \
+    -X 'github.com/${ORG}/${REPO}/internal/info.GoVersion=${GO_VERSION}' \
+    -X 'github.com/${ORG}/${REPO}/internal/info.GoOS=${GOOS}' \
+    -X 'github.com/${ORG}/${REPO}/internal/info.GoArch=${GOARCH}' \
+    -X 'github.com/${ORG}/${REPO}/internal/info.CGOEnabled=${CGO_ENABLED}' \
+    -X 'github.com/${ORG}/${REPO}/internal/info.BuildCPUInfoFlags=${CPU_INFO_FLAGS}'" \
+    -a \
+    -tags "cgo netgo" \
+    -trimpath \
+    -installsuffix "cgo netgo" \
+    -o "${APP_NAME}" \
+    "cmd/${PKG}/main.go" \
+    && upx -9 -o "/usr/bin/${APP_NAME}" "${APP_NAME}"
+
+# Start From Scratch For Running Environment
+FROM scratch
+# Start From Alpine For Debug Environment
+# FROM alpine:latest
+
+ENV APP_NAME loadtest
+
+# Copy certificates for SSL/TLS
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+# Copy permissions
+COPY --from=builder /etc/passwd /etc/passwd
+# Copy our static executable
+COPY --from=builder /usr/bin/${APP_NAME} /go/bin/${APP_NAME}
+
+ENTRYPOINT ["/go/bin/loadtest"]

k8s/tools/cli/loadtest/configmap.yaml

@@ -0,0 +1,85 @@
+---
+#
+# Copyright (C) 2019-2020 Vdaas.org Vald team ( kpango, rinx, kmrmt )
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vald-loadtest-config
+  labels:
+    app.kubernetes.io/name: vald
+    app.kubernetes.io/component: loadtest
+data:
+  config.yaml: |
+    ---
+    version: v0.0.0
+    time_zone: JST
+    logging:
+      logger: glg
+      level: debug
+      format: json
+    method: insert
+    dataset: fashion-mnist
+    concurrency: 100
+    addr: "localhost:8081"
+    client:
+      addrs: []
+      health_check_duration: 1s
+      backoff:
+        backoff_factor: 0
+        backoff_time_limit: ""
+        enable_error_log: false
+        initial_duration: ""
+        jitter_limit: ""
+        maximum_duration: ""
+        retry_count: 0
+      call_option:
+        max_recv_msg_size: 0
+        max_retry_rpc_buffer_size: 0
+        max_send_msg_size: 0
+        wait_for_ready: true
+      dial_option:
+        enable_backoff: false
+        initial_connection_window_size: 0
+        initial_window_size: 0
+        insecure: true
+        keep_alive:
+          permit_without_stream: false
+          time: ""
+          timeout: ""
+        max_backoff_delay: ""
+        max_msg_size: 0
+        read_buffer_size: 0
+        tcp:
+          dialer:
+            dual_stack_enabled: true
+            keep_alive: ""
+            timeout: ""
+          dns:
+            cache_enabled: false
+            cache_expiration: ""
+            refresh_duration: ""
+          tls:
+            ca: /path/to/ca
+            cert: /path/to/cert
+            enabled: false
+            key: /path/to/key
+        timeout: ""
+        write_buffer_size: 0
+      tls:
+        ca: /path/to/ca
+        cert: /path/to/cert
+        enabled: false
+        key: /path/to/key

k8s/tools/cli/loadtest/cronjob.yaml

@@ -0,0 +1,58 @@
+---
+#
+# Copyright (C) 2019-2020 Vdaas.org Vald team ( kpango, rinx, kmrmt )
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: vald-loadtest
+  labels:
+    app: vald-loadtest
+    app.kubernetes.io/name: vald
+    app.kubernetes.io/component: loadtest
+spec:
+  schedule: "*/1 * * * *"
+  jobTemplate:
+    spec:
+      parallelism: 1
+      backoffLimit: 10
+      template:
+        metadata:
+          app: vald-loadtest
+          app.kubernetes.io/name: vald
+          app.kubernetes.io/component: loadtest
+        spec:
+          containers:
+            - name: vald-loadtest
+              image: "vdaas/vald-loadtest:nightly"
+              imagePullPolicy: Always
+              resources:
+                limits:
+                  cpu: 300m
+                  memory: 100Mi
+                requests:
+                  cpu: 100m
+                  memory: 30Mi
+              terminationMessagePath: /dev/termination-log
+              terminationMessagePolicy: File
+              volumeMounts:
+                - name: vald-loadtest-config
+                  mountPath: /etc/server/
+          dnsPolicy: ClusterFirst
+          restartPolicy: Never
+          volumes:
+            - name: vald-loadtest-config
+              configMap:
+                name: vald-loadtest-config