From fe9bf3dbe23a30ba912e9a80c874d91fa7a80a47 Mon Sep 17 00:00:00 2001 From: Prabhu31 Date: Tue, 6 Jul 2021 15:27:00 +0530 Subject: [PATCH 1/7] Removed EFS dependency from zeppelin job server --- .../kubernetesTemplates/zeppelinServer.yaml | 76 +++++-------------- .../kubernetesTemplates/zeppelinService.yaml | 2 +- 2 files changed, 20 insertions(+), 58 deletions(-) diff --git a/api/utils/kubernetesTemplates/zeppelinServer.yaml b/api/utils/kubernetesTemplates/zeppelinServer.yaml index e6186b44..c863da68 100644 --- a/api/utils/kubernetesTemplates/zeppelinServer.yaml +++ b/api/utils/kubernetesTemplates/zeppelinServer.yaml @@ -9,18 +9,23 @@ metadata: spec: serviceAccountName: zeppelin-server volumes: - - name: zeppelin-server-notebook-volume - persistentVolumeClaim: - claimName: cuelake-notebook-efs-pvc - - name: zeppelin-server-conf - persistentVolumeClaim: - claimName: cuelake-conf-efs-pvc - - name: nginx-conf - configMap: - name: zeppelin-server-conf - items: - - key: nginx.conf - path: nginx.conf + - name: shared-notebook + emptyDir: + - name: shared-conf + emptyDir: + initContainers: + - name: init + image: bitnami/kubectl + command: ['sh', '-c'] + volumeMounts: + - name: shared-notebook + mountPath: /shared-notebook + - name: shared-conf + mountPath: /shared-conf + args: + - >- + kubectl cp $(kubectl get pods | grep zeppelin-server | awk '{print $1}' ):/zeppelin/conf /shared-conf && + kubectl cp $(kubectl get pods | grep zeppelin-server | awk '{print $1}' ):/zeppelin/notebook /shared-notebook containers: - name: zeppelin-server image: 'cuebook/zeppelin-server:0.9.0' @@ -58,9 +63,9 @@ spec: apiVersion: v1 fieldPath: metadata.name volumeMounts: - - name: zeppelin-server-notebook-volume + - name: shared-notebook mountPath: /zeppelin/notebook - - name: zeppelin-server-conf + - name: shared-conf mountPath: /zeppelin/conf lifecycle: preStop: @@ -78,46 +83,3 @@ spec: requests: memory: "1024Mi" cpu: "250m" - - name: zeppelin-server-gateway - image: 'nginx:1.14.0' - command: - - /bin/sh - - '-c' - args: - - >- - cp -f /tmp/conf/nginx.conf /etc/nginx/nginx.conf; sed -i -e - "s/SERVICE_DOMAIN/$SERVICE_DOMAIN/g" /etc/nginx/nginx.conf; sed -i -e - "s/NAMESPACE/$(cat - /var/run/secrets/kubernetes.io/serviceaccount/namespace)/g" - /etc/nginx/nginx.conf; cat /etc/nginx/nginx.conf; /usr/sbin/nginx - env: - - name: SERVICE_DOMAIN - valueFrom: - configMapKeyRef: - name: zeppelin-server-conf-map - key: SERVICE_DOMAIN - volumeMounts: - - name: nginx-conf - mountPath: /tmp/conf - lifecycle: - preStop: - exec: - command: - - /usr/sbin/nginx - - '-s' - - quit - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - imagePullPolicy: IfNotPresent - - name: dnsmasq - image: 'janeczku/go-dnsmasq:release-1.0.5' - args: - - '--listen' - - '127.0.0.1:53' - - '--default-resolver' - - '--append-search-domains' - - '--hostsfile=/etc/hosts' - - '--verbose' - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - imagePullPolicy: IfNotPresent \ No newline at end of file diff --git a/api/utils/kubernetesTemplates/zeppelinService.yaml b/api/utils/kubernetesTemplates/zeppelinService.yaml index 4d3c6929..cdd03344 100644 --- a/api/utils/kubernetesTemplates/zeppelinService.yaml +++ b/api/utils/kubernetesTemplates/zeppelinService.yaml @@ -8,7 +8,7 @@ spec: - name: http protocol: TCP port: 80 - targetPort: 80 + targetPort: 8080 - name: rpc protocol: TCP port: 12320 From 02b127426c438e40faee726848aca93077331ddd Mon Sep 17 00:00:00 2001 From: Prabhu31 Date: Tue, 6 Jul 2021 16:35:51 +0530 Subject: [PATCH 2/7] Deployment fixes --- Dockerfile | 2 +- api/app/settings.py | 2 +- api/utils/kubernetesTemplates/zeppelinServer.yaml | 2 +- cuelake.yaml | 13 ++++++++++--- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index c1a50942..f9f2c5ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM node:12-alpine as builder WORKDIR /app ENV PATH /app/node_modules/.bin:$PATH COPY ui/package.json /app/package.json -RUN npm install --silent +RUN npm install COPY ui /app RUN npm run build diff --git a/api/app/settings.py b/api/app/settings.py index b2a227a9..1811fa24 100644 --- a/api/app/settings.py +++ b/api/app/settings.py @@ -156,7 +156,7 @@ REDIS_BROKER_URL = os.environ.get("REDIS_BROKER_URL", "redis://redis:6379/0") # s3 file settings -S3_BUCKET_NAME = os.environ.get("S3_BUCKET_NAME", None) +S3_BUCKET_NAME = os.environ.get("S3_BUCKET_NAME", "") S3_FILES_PREFIX = os.environ.get("S3_FILES_PREFIX", "files/") # Postgres configuration for Schema Browser diff --git a/api/utils/kubernetesTemplates/zeppelinServer.yaml b/api/utils/kubernetesTemplates/zeppelinServer.yaml index c863da68..12dc6e4b 100644 --- a/api/utils/kubernetesTemplates/zeppelinServer.yaml +++ b/api/utils/kubernetesTemplates/zeppelinServer.yaml @@ -28,7 +28,7 @@ spec: kubectl cp $(kubectl get pods | grep zeppelin-server | awk '{print $1}' ):/zeppelin/notebook /shared-notebook containers: - name: zeppelin-server - image: 'cuebook/zeppelin-server:0.9.0' + image: 'cuebook/zeppelin-server:test' command: - sh - '-c' diff --git a/cuelake.yaml b/cuelake.yaml index 6809917b..d7a78f35 100644 --- a/cuelake.yaml +++ b/cuelake.yaml @@ -117,7 +117,7 @@ spec: path: nginx.conf containers: - name: zeppelin-server - image: cuebook/zeppelin-server:0.9.0 + image: cuebook/zeppelin-server:test command: ["sh", "-c"] args: - curl https://raw.githubusercontent.com/cuebook/cuelake/main/zeppelinConf/interpreter.json -o $(ZEPPELIN_HOME)/conf/interpreter.json && curl https://raw.githubusercontent.com/cuebook/cuelake/main/zeppelinConf/zeppelin-env.sh -o $(ZEPPELIN_HOME)/conf/zeppelin-env.sh && curl https://raw.githubusercontent.com/cuebook/cuelake/main/zeppelinConf/zeppelin-site.xml -o $(ZEPPELIN_HOME)/conf/zeppelin-site.xml && $(ZEPPELIN_HOME)/bin/zeppelin.sh @@ -211,7 +211,7 @@ metadata: name: zeppelin-server-role rules: - apiGroups: [""] - resources: ["pods", "services", "configmaps"] + resources: ["pods", "pods/exec", "services", "configmaps"] verbs: ["create", "get", "update", "patch", "list", "delete", "watch"] - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings"] @@ -300,16 +300,23 @@ spec: labels: app.kubernetes.io/name: lakehouse spec: + serviceAccountName: zeppelin-server volumes: - name: lakehouse-db-volume persistentVolumeClaim: claimName: lakehouse-db-volume-pvc containers: - name: lakehouse - image: cuebook/lakehouse:latest + image: cuebook/lakehouse:0.2 volumeMounts: - name: lakehouse-db-volume # configure this to persist db mountPath: /code/db + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace envFrom: - configMapRef: name: cuelake-conf-map From fb0e03eb772b83bccfa097b95a5f00932f481009 Mon Sep 17 00:00:00 2001 From: Vikrant Dubey <55645115+vikrantcue@users.noreply.github.com> Date: Wed, 7 Jul 2021 13:50:43 +0530 Subject: [PATCH 3/7] Temporary fix for crontab schedule --- api/seeddata/crontabschedule.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/seeddata/crontabschedule.json b/api/seeddata/crontabschedule.json index 0bc3acc3..baaeaca2 100644 --- a/api/seeddata/crontabschedule.json +++ b/api/seeddata/crontabschedule.json @@ -6,8 +6,8 @@ "minute": "0", "hour": "0", "day_of_week": "0", - "day_of_month": "30", - "month_of_year": "2", + "day_of_month": "1", + "month_of_year": "1", "timezone": "Asia/Kolkata" } }, @@ -18,4 +18,4 @@ "name": "No Schedule" } } -] \ No newline at end of file +] From 46732d6920630fd11f3dc92f5dde67aeaf83844a Mon Sep 17 00:00:00 2001 From: Vikrant Dubey <55645115+vikrantcue@users.noreply.github.com> Date: Wed, 7 Jul 2021 13:53:35 +0530 Subject: [PATCH 4/7] Change zeppelin server image with the zeppelin server lite --- cuelake.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuelake.yaml b/cuelake.yaml index d7a78f35..3acc3367 100644 --- a/cuelake.yaml +++ b/cuelake.yaml @@ -117,7 +117,7 @@ spec: path: nginx.conf containers: - name: zeppelin-server - image: cuebook/zeppelin-server:test + image: cuebook/zeppelin-server-lite:0.2 command: ["sh", "-c"] args: - curl https://raw.githubusercontent.com/cuebook/cuelake/main/zeppelinConf/interpreter.json -o $(ZEPPELIN_HOME)/conf/interpreter.json && curl https://raw.githubusercontent.com/cuebook/cuelake/main/zeppelinConf/zeppelin-env.sh -o $(ZEPPELIN_HOME)/conf/zeppelin-env.sh && curl https://raw.githubusercontent.com/cuebook/cuelake/main/zeppelinConf/zeppelin-site.xml -o $(ZEPPELIN_HOME)/conf/zeppelin-site.xml && $(ZEPPELIN_HOME)/bin/zeppelin.sh From fe10fab2855579797128263d8e1494fbcf76d7f2 Mon Sep 17 00:00:00 2001 From: Vikrant Dubey <55645115+vikrantcue@users.noreply.github.com> Date: Wed, 7 Jul 2021 13:54:37 +0530 Subject: [PATCH 5/7] Change zeppelin server image to zeppelin server lite --- api/utils/kubernetesTemplates/zeppelinServer.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/utils/kubernetesTemplates/zeppelinServer.yaml b/api/utils/kubernetesTemplates/zeppelinServer.yaml index 12dc6e4b..399fbdf7 100644 --- a/api/utils/kubernetesTemplates/zeppelinServer.yaml +++ b/api/utils/kubernetesTemplates/zeppelinServer.yaml @@ -28,7 +28,7 @@ spec: kubectl cp $(kubectl get pods | grep zeppelin-server | awk '{print $1}' ):/zeppelin/notebook /shared-notebook containers: - name: zeppelin-server - image: 'cuebook/zeppelin-server:test' + image: 'cuebook/zeppelin-server-lite:0.2' command: - sh - '-c' From 15775d3520417ae42fb3d6107a58719ae89507cf Mon Sep 17 00:00:00 2001 From: Vikrant Dubey Date: Wed, 7 Jul 2021 14:20:16 +0530 Subject: [PATCH 6/7] Adding resource request for lakehouse --- cuelake.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cuelake.yaml b/cuelake.yaml index 3acc3367..f5ac3c7d 100644 --- a/cuelake.yaml +++ b/cuelake.yaml @@ -308,6 +308,10 @@ spec: containers: - name: lakehouse image: cuebook/lakehouse:0.2 + resources: + requests: + memory: "2560Mi" + cpu: "250m" volumeMounts: - name: lakehouse-db-volume # configure this to persist db mountPath: /code/db From f22293680e8de96697197a93152f793c1847ed91 Mon Sep 17 00:00:00 2001 From: Vikrant Dubey Date: Wed, 7 Jul 2021 14:34:51 +0530 Subject: [PATCH 7/7] Fix for health check API --- api/utils/zeppelinAPI.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/api/utils/zeppelinAPI.py b/api/utils/zeppelinAPI.py index dc08edf4..7fd79286 100644 --- a/api/utils/zeppelinAPI.py +++ b/api/utils/zeppelinAPI.py @@ -149,8 +149,11 @@ def restartInterpreter(self, interpreterName: str): return self.__parseResponse(response) def healthCheck(self): - response = requests.get(f"{self.ZEPPELIN_ADDR}/{ZEPPELIN_VERSION_ENDPOINT}") - return self.__parseResponse(response) + try: + response = requests.get(f"{self.ZEPPELIN_ADDR}/{ZEPPELIN_VERSION_ENDPOINT}") + return self.__parseResponse(response) + except Exception as ex: + return False def __parseResponse(self, response): """