Enable user to provide spark job template as input for jobservice dep…

…loyment (#1285) * Enable user to provide spark job template as input for jobservice deployment Signed-off-by: Khor Shu Heng <khor.heng@gojek.com> * Peg mypy version for CI Signed-off-by: Khor Shu Heng <khor.heng@gojek.com> Co-authored-by: Khor Shu Heng <khor.heng@gojek.com>
feast-dev · Jan 26, 2021 · 232821d · 232821d
1 parent 5754b74
commit 232821d
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 3 deletions.
diff --git a/infra/charts/feast/charts/feast-jobservice/templates/configmap.yaml b/infra/charts/feast/charts/feast-jobservice/templates/configmap.yaml
@@ -0,0 +1,16 @@
+{{- if .Values.sparkOperator.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "feast-jobservice.fullname" . }}-spark-template
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ template "feast-jobservice.name" . }}
+    component: jobservice
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  jobTemplate.yaml: |
+{{- toYaml .Values.sparkOperator.jobTemplate | nindent 4 }}
+{{- end }}
diff --git a/infra/charts/feast/charts/feast-jobservice/templates/deployment.yaml b/infra/charts/feast/charts/feast-jobservice/templates/deployment.yaml
@@ -37,30 +37,43 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
 
-      {{- if .Values.secrets }}
+      {{- if or .Values.secrets .Values.sparkOperator.enabled }}
       volumes:
+      {{- end }}
       {{- range $secret := .Values.secrets }}
       - name: {{ $secret }}
         secret:
           secretName: {{ $secret }}
       {{- end }}
+      {{- if .Values.sparkOperator.enabled }}
+      - name: {{ template "feast-jobservice.fullname" . }}-spark-template
+        configMap:
+          name: {{ template "feast-jobservice.fullname" . }}-spark-template
       {{- end }}
 
       containers:
       - name: {{ .Chart.Name }}
         image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
         imagePullPolicy: {{ .Values.image.pullPolicy }}
 
-        {{- if .Values.secrets }}
+        {{- if or .Values.secrets .Values.sparkOperator.enabled }}
         volumeMounts:
+        {{- end }}
         {{- range $secret := .Values.secrets }}
         - name: {{ $secret }}
           mountPath: "/etc/secrets/{{ $secret }}"
           readOnly: true
         {{- end }}
+        {{- if .Values.sparkOperator.enabled }}
+        - name: {{ template "feast-jobservice.fullname" . }}-spark-template
+          mountPath: "/etc/configs"
         {{- end }}
 
         env:
+        {{- if .Values.sparkOperator.enabled }}
+        - name: FEAST_SPARK_K8S_JOB_TEMPLATE_PATH
+          value: /etc/configs/jobTemplate.yaml
+        {{- end }}
         {{- range $key, $value := .Values.envOverrides }}
         - name: {{ printf "%s" $key | replace "." "_" | upper | quote }}
           {{- if eq (kindOf $value) "map" }}

diff --git a/infra/charts/feast/charts/feast-jobservice/values.yaml b/infra/charts/feast/charts/feast-jobservice/values.yaml
@@ -21,6 +21,12 @@ gcpServiceAccount:
 # gcpProjectId -- Project ID to use when using Google Cloud services such as BigQuery, Cloud Storage and Dataflow
 gcpProjectId: ""
 
+sparkOperator:
+  # sparkOperator.enabled -- Flag to create and mount custom job template on the jobservice deployment as configmap
+  enabled: false
+  # sparkOperator.jobTemplate -- Content of the job template, in yaml format
+  jobTemplate: {}
+
 prometheus:
   # prometheus.enabled -- Flag to enable scraping of metrics
   enabled: true
@@ -124,3 +130,6 @@ envOverrides: {}
 
 # podLabels -- Labels to be added to Feast Job Service pods
 podLabels: {}
+
+# secrets -- Arbitrary secrets to mount on the job service pod, on /etc/secrets/<secret name>
+secrets: []
diff --git a/sdk/python/requirements-ci.txt b/sdk/python/requirements-ci.txt
@@ -9,7 +9,7 @@ pandas~=1.0.0
 mock==2.0.0
 pandavro==1.5.*
 moto
-mypy
+mypy==0.790
 mypy-protobuf
 avro==1.10.0
 gcsfs