Implements vineyard-operator to schedule jobs based on data locality. (…

…v6d-io#223) Vineyard operator is a kubernetes controller to operator the vineyard cluster. Within the vineyard operator, there's a scheduler-plugin called vineyard-scheduler which performs data aware job scheduling based on the locality information of objects in vineyard. Vineyard operator defines two CRDs to represents objects in vineyard: + GlobalObject for global objects + LocalObject for local objects The scheduler plugin works based the CRDs in the cluster. When vineyardd persists objects to etcd, it will create a CRD entry in kubernetes as well. The metadata of CRD has location information and scheduler plugin scores pod based on the data locality of required objects by jobs. For jobs, some label annotation are used to describe which objects are required in the pod spec. ``` labels: app: pytorch scheduling.k8s.v6d.io/job: pytorch scheduling.k8s.v6d.io/required: "o0000022285553e22" scheduling.k8s.v6d.io/replica: "4" ``` Resolves v6d-io#97. Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>
sighingnow · Jun 1, 2021 · b549f9a · b549f9a
1 parent bf019cf
commit b549f9a
Show file tree

Hide file tree

Showing 124 changed files with 6,390 additions and 37 deletions.
diff --git a/.github/workflows/kubernetes-operator.yaml b/.github/workflows/kubernetes-operator.yaml
@@ -0,0 +1,55 @@
+# Copyright 2020-2021 Alibaba Group Holding Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Kubernetes Operator
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    env:
+      IMG: docker.pkg.github.com/v6d-io/v6d/vineyard-controller
+    strategy:
+      matrix:
+        os: [ubuntu-20.04]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.15.0' # The Go version to download (if necessary) and use.
+
+      - name: Login docker registry
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | sudo docker login https://docker.pkg.github.com -u $GITHUB_ACTOR --password-stdin
+
+      - name: Operator
+        run: |
+          cd k8s
+          make
+
+      - name: Docker images
+        run: |
+          cd k8s
+          sudo make docker-build IMG=${IMG}:${{ github.sha }}
+          sudo docker tag ${IMG}:${{ github.sha }} ${IMG}:nightly
+
+      - name: Publish docker images
+        if: ${{ github.ref == 'refs/heads/main' && github.repository == 'v6d-io/v6d' }}
+        run: |
+          sudo make docker-push IMG=${IMG}:${{ github.sha }}
+          sudo docker push ${IMG}:nightly
diff --git a/README.rst b/README.rst
@@ -1,7 +1,7 @@
 .. raw:: html
 
     <h1 align="center">
-        <img src="https://v6d.io/_static/vineyard_logo.png" width="397" alt="vineyard">
+        <img src="https://v6d.io/_static/vineyard-logo.png" width="397" alt="vineyard">
     </h1>
     <p align="center">
         an in-memory immutable data manager

diff --git a/charts/hack/install-ubuntu.sh b/charts/hack/install-ubuntu.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -x
+set -e
+set -o pipefail
+
+ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+
+helm install vineyard $ROOT/vineyard/ -n vineyard-system \
+    --set image.repository="registry-vpc.cn-hongkong.aliyuncs.com/libvineyard/vineyardd" \
+    --set image.tagPrefix=ubuntu \
+    --set serviceAccountName=vineyard-server \
+    --set env[0].name="VINEYARD_SYNC_CRDS",env[0].value="1" \
+    --set vineyard.sharedMemorySize=8Gi
+
+set +x
+set +e
+set +o pipefail
diff --git a/charts/hack/install.sh b/charts/hack/install.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -x
+set -e
+set -o pipefail
+
+ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+
+helm install vineyard $ROOT/vineyard/ -n vineyard-system \
+    --set vineyard.sharedMemorySize=8Gi
+
+set +x
+set +e
+set +o pipefail
diff --git a/charts/hack/uninstall.sh b/charts/hack/uninstall.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+
+helm uninstall vineyard -n vineyard-system
+
+kubectl -n vineyard-system delete localobjects --all 2>/dev/null || true
+kubectl -n vineyard-system delete globalobjects --all 2>/dev/null || true
+
+set +x
+set +e
+set +o pipefail
diff --git a/charts/vineyard/Chart.yaml b/charts/vineyard/Chart.yaml
@@ -36,4 +36,3 @@ version: 0.2.1
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 appVersion: 0.2.1
-
diff --git a/charts/vineyard/templates/daemonset.yaml b/charts/vineyard/templates/daemonset.yaml
@@ -17,6 +17,9 @@ spec:
       labels:
         {{- include "vineyard.selectorLabels" . | nindent 8 }}
     spec:
+      {{- if .Values.serviceAccountName }}
+      serviceAccountName: {{ .Values.serviceAccountName }}
+      {{- end }}
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
@@ -27,11 +30,19 @@ spec:
       {{- end }}
       containers:
         - name: {{ .Chart.Name }}
+          {{- if .Values.image.tagPrefix }}
+          {{- if .Values.image.tag }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tagPrefix }}-{{ .Values.image.tag }}"
+          {{- else }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tagPrefix }}-v{{ default .Chart.AppVersion }}"
+          {{- end }}
+          {{- else }}
           {{- if .Values.image.tag }}
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
           {{- else }}
           image: "{{ .Values.image.repository }}:v{{ default .Chart.AppVersion }}"
           {{- end }}
+          {{- end }}
           {{- if .Values.image.pullPolicy }}
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           {{- end }}
@@ -41,7 +52,7 @@ spec:
           - |
             /usr/bin/wait-for-it.sh -t 60 etcd-for-vineyard.{{ .Release.Namespace }}.svc.cluster.local:2379;
             sleep 1;
-            /usr/bin/vineyardd \
+            /usr/local/bin/vineyardd \
           {{- if .Values.vineyard.sharedMemorySize }}
               --size \
               {{ .Values.vineyard.sharedMemorySize }} \

diff --git a/charts/vineyard/templates/etcd.yaml b/charts/vineyard/templates/etcd.yaml
@@ -37,11 +37,11 @@ spec:
     - --advertise-client-urls
     - http://etcd0:2379
     - --initial-cluster
-    - etcd0=http://etcd0:2380
+    - etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380
     - --initial-cluster-state
     - new
     image: quay.io/coreos/etcd:latest
-    name: etcd0
+    name: etcd
     ports:
     - containerPort: 2379
       name: client
@@ -71,3 +71,121 @@ spec:
     targetPort: 2380
   selector:
     etcd_node: etcd0
+
+---
+
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: etcd
+    etcd_node: etcd1
+  name: etcd1
+spec:
+  containers:
+  - command:
+    - /usr/local/bin/etcd
+    - --name
+    - etcd1
+    - --initial-advertise-peer-urls
+    - http://etcd1:2380
+    - --listen-peer-urls
+    - http://0.0.0.0:2380
+    - --listen-client-urls
+    - http://0.0.0.0:2379
+    - --advertise-client-urls
+    - http://etcd1:2379
+    - --initial-cluster
+    - etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380
+    - --initial-cluster-state
+    - new
+    image: quay.io/coreos/etcd:latest
+    name: etcd
+    ports:
+    - containerPort: 2379
+      name: client
+      protocol: TCP
+    - containerPort: 2380
+      name: server
+      protocol: TCP
+  restartPolicy: Always
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    etcd_node: etcd1
+  name: etcd1
+spec:
+  ports:
+  - name: client
+    port: 2379
+    protocol: TCP
+    targetPort: 2379
+  - name: server
+    port: 2380
+    protocol: TCP
+    targetPort: 2380
+  selector:
+    etcd_node: etcd1
+
+---
+
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: etcd
+    etcd_node: etcd2
+  name: etcd2
+spec:
+  containers:
+  - command:
+    - /usr/local/bin/etcd
+    - --name
+    - etcd2
+    - --initial-advertise-peer-urls
+    - http://etcd2:2380
+    - --listen-peer-urls
+    - http://0.0.0.0:2380
+    - --listen-client-urls
+    - http://0.0.0.0:2379
+    - --advertise-client-urls
+    - http://etcd2:2379
+    - --initial-cluster
+    - etcd0=http://etcd0:2380,etcd1=http://etcd1:2380,etcd2=http://etcd2:2380
+    - --initial-cluster-state
+    - new
+    image: quay.io/coreos/etcd:latest
+    name: etcd
+    ports:
+    - containerPort: 2379
+      name: client
+      protocol: TCP
+    - containerPort: 2380
+      name: server
+      protocol: TCP
+  restartPolicy: Always
+
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    etcd_node: etcd2
+  name: etcd2
+spec:
+  ports:
+  - name: client
+    port: 2379
+    protocol: TCP
+    targetPort: 2379
+  - name: server
+    port: 2380
+    protocol: TCP
+    targetPort: 2380
+  selector:
+    etcd_node: etcd2
diff --git a/charts/vineyard/values.yaml b/charts/vineyard/values.yaml
@@ -2,12 +2,11 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-replicaCount: 1
-
 image:
   repository: libvineyard/vineyardd
-  pullPolicy: IfNotPresent
+  pullPolicy: Always
   # Overrides the image tag whose default is the chart appVersion.
+  tagPrefix: ""
   tag: ""
 
 imagePullSecrets: []
@@ -25,6 +24,7 @@ resources: {}
   # choice for the user. This also increases chances charts run on environments with little
   # resources, such as Minikube. If you do want to specify resources, uncomment the following
   # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  #
   # limits:
   #   cpu: 100m
   #   memory: 128Mi
@@ -39,6 +39,8 @@ affinity: {}
 vineyard:
   sharedMemorySize: ""
 
+serviceAccountName: ""
+
 tolerations: {}
   # this toleration is to have the daemonset runnable on master nodes
   # remove it if your masters can't run pods.

diff --git a/cmake/FindRdkafka.cmake b/cmake/FindRdkafka.cmake
@@ -8,32 +8,32 @@
 # - Try to find librdkafka
 #
 # The following variables are optionally searched for defaults
-#  RDKAFKA_ROOT_DIR:            Base directory where all RDKAFKA components are found
+#  Rdkafka_ROOT_DIR:            Base directory where all rdkafka components are found
 #
 # The following are set after configuration is done:
-#  RDKAFKA_FOUND
-#  RDKAFKA_INCLUDE_DIRS
-#  RDKAFKA_LIBRARIES
-#  RDKAFKA_LIBRARY_DIRS
+#  Rdkafka_FOUND
+#  Rdkafka_INCLUDE_DIRS
+#  Rdkafka_LIBRARIES
+#  Rdkafka_LIBRARY_DIRS
 
 include(FindPackageHandleStandardArgs)
 
-set(RDKAFKA_ROOT_DIR "" CACHE PATH "Folder contains librdkafka")
+set(Rdkafka_ROOT_DIR "" CACHE PATH "Folder contains librdkafka")
 
 # We are testing only a couple of files in the include directories
-find_path(RDKAFKA_INCLUDE_DIR librdkafka PATHS ${RDKAFKA_ROOT_DIR}/include)
+find_path(Rdkafka_INCLUDE_DIR librdkafka PATHS ${Rdkafka_ROOT_DIR}/include)
 
-find_library(RDKAFKA_LIBRARY rdkafka PATHS  ${RDKAFKA_ROOT_DIR}/lib)
-find_library(RDKAFKA++_LIBRARY rdkafka++ PATHS  ${RDKAFKA_ROOT_DIR}/lib)
+find_library(Rdkafka_LIBRARY rdkafka PATHS  ${Rdkafka_ROOT_DIR}/lib)
+find_library(Rdkafka++_LIBRARY rdkafka++ PATHS  ${Rdkafka_ROOT_DIR}/lib)
 
-find_package_handle_standard_args(RDKAFKA DEFAULT_MSG RDKAFKA_INCLUDE_DIR RDKAFKA_LIBRARY)
+find_package_handle_standard_args(Rdkafka DEFAULT_MSG Rdkafka_INCLUDE_DIR Rdkafka_LIBRARY)
 
 
-if(RDKAFKA_FOUND)
-    set(RDKAFKA_INCLUDE_DIRS ${RDKAFKA_INCLUDE_DIR})
-    # The RDKAFKA_LIBRARY comes later, since it is depended by the former.
-    set(RDKAFKA_LIBRARIES ${RDKAFKA++_LIBRARY} ${RDKAFKA_LIBRARY})
-    message(STATUS "Found rdkafka (include: ${RDKAFKA_INCLUDE_DIRS}, library: ${RDKAFKA_LIBRARIES})")
-    mark_as_advanced(RDKAFKA_LIBRARY_DEBUG RDKAFKA_LIBRARY_RELEASE
-                     RDKAFKA_LIBRARY RDKAFKA_INCLUDE_DIR RDKAFKA_ROOT_DIR)
+if(Rdkafka_FOUND)
+    set(Rdkafka_INCLUDE_DIRS ${Rdkafka_INCLUDE_DIR})
+    # The Rdkafka_LIBRARY comes later, since it is depended by the former.
+    set(Rdkafka_LIBRARIES ${Rdkafka++_LIBRARY} ${Rdkafka_LIBRARY})
+    message(STATUS "Found rdkafka (include: ${Rdkafka_INCLUDE_DIRS}, library: ${Rdkafka_LIBRARIES})")
+    mark_as_advanced(Rdkafka_LIBRARY_DEBUG Rdkafka_LIBRARY_RELEASE
+                     Rdkafka_LIBRARY Rdkafka_INCLUDE_DIR Rdkafka_ROOT_DIR)
 endif()
Original file line number	Diff line number	Diff line change
Expand Up		@@ -36,4 +36,3 @@ version: 0.2.1
		# incremented each time you make changes to the application. Versions are not expected to
		# follow Semantic Versioning. They should reflect the version the application is using.
		appVersion: 0.2.1