From 0b6f5e5f0cc01fc63686c3dc2aac39493ee98bb5 Mon Sep 17 00:00:00 2001 From: lvsi <15239928381@163.com> Date: Thu, 11 Apr 2024 16:08:19 +0800 Subject: [PATCH] feat: add azkaban support --- .env.template | 2 +- .work/config/.env.amd64 | 2 +- .work/config/.env.arm64 | 2 +- azkaban/.env | 8 +++--- azkaban/docker-compose.yml | 3 +- azkaban/hooks/after_start.sh | 2 ++ azkaban/make_app_image/Dockerfile | 46 +++++++++++++++++++++++++++++++ sparrowtool | 2 +- 8 files changed, 58 insertions(+), 9 deletions(-) diff --git a/.env.template b/.env.template index e0818fd..5a0d4c9 100644 --- a/.env.template +++ b/.env.template @@ -22,7 +22,7 @@ DOCKERHUB_REPO=docker.io/lvsid # change to your own docker hub. ### Services ############################################## # the enable service list -ENABLE_SERVICE_LIST=("zookeeper" "go" "phpfpm" "python" "mysql" "redis" "postgres" "etcd" "nginx" "kafka" "kafkaui" "etcdkeeper" "jupyter" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "difylocal" "django") +ENABLE_SERVICE_LIST=("zookeeper" "go" "phpfpm" "python" "mysql" "redis" "postgres" "etcd" "nginx" "kafka" "kafkaui" "etcdkeeper" "jupyter" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "difylocal" "django" "azkaban") ########################################################### ################ Containers Customization ################# diff --git a/.work/config/.env.amd64 b/.work/config/.env.amd64 index 1e953d4..ae61942 100644 --- a/.work/config/.env.amd64 +++ b/.work/config/.env.amd64 @@ -22,7 +22,7 @@ DOCKERHUB_REPO=docker.io/lvsid # change to your own docker hub. ### Services ############################################## # the enable service list -ENABLE_SERVICE_LIST=("zookeeper" "go" "phpfpm" "python" "mysql" "redis" "postgres" "etcd" "nginx" "kafka" "kafkaui" "etcdkeeper" "jupyter" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "difylocal" "django") +ENABLE_SERVICE_LIST=("zookeeper" "go" "phpfpm" "python" "mysql" "redis" "postgres" "etcd" "nginx" "kafka" "kafkaui" "etcdkeeper" "jupyter" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "difylocal" "django" "azkaban") ########################################################### ################ Containers Customization ################# diff --git a/.work/config/.env.arm64 b/.work/config/.env.arm64 index 1c58182..ca1dec9 100644 --- a/.work/config/.env.arm64 +++ b/.work/config/.env.arm64 @@ -22,7 +22,7 @@ DOCKERHUB_REPO=docker.io/lvsid # change to your own docker hub. ### Services ############################################## # the enable service list -ENABLE_SERVICE_LIST=("zookeeper" "go" "phpfpm" "python" "mysql" "redis" "postgres" "etcd" "nginx" "kafka" "kafkaui" "etcdkeeper" "jupyter" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "difylocal" "django") +ENABLE_SERVICE_LIST=("zookeeper" "go" "phpfpm" "python" "mysql" "redis" "postgres" "etcd" "nginx" "kafka" "kafkaui" "etcdkeeper" "jupyter" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "difylocal" "django" "azkaban") ########################################################### ################ Containers Customization ################# diff --git a/azkaban/.env b/azkaban/.env index d46b3b5..0d1863c 100644 --- a/azkaban/.env +++ b/azkaban/.env @@ -2,12 +2,12 @@ # -------------- image config --------------- # official image -IMAGE_OFFICIAL_AZKABAN_NAME=test/test -IMAGE_OFFICIAL_AZKABAN_VERSION=4.0.0 +IMAGE_OFFICIAL_AZKABAN_NAME=centos +IMAGE_OFFICIAL_AZKABAN_VERSION=8.4.2105 # basic image -IMAGE_BASIC_AZKABAN_VERSION=4.0.0 +IMAGE_BASIC_AZKABAN_VERSION=0.0.1 # app image IMAGE_APP_AZKABAN_VERSION=latest # -------------- service config -------------- AZKABAN_HOST_PORT=4500 -AZKABAN_CONTAINER_PORT=4500 +AZKABAN_CONTAINER_PORT=8081 diff --git a/azkaban/docker-compose.yml b/azkaban/docker-compose.yml index 6f231aa..bc11846 100644 --- a/azkaban/docker-compose.yml +++ b/azkaban/docker-compose.yml @@ -19,4 +19,5 @@ services: networks: - backend # only for debug - # command: "tail -f /dev/null" + # Since it(/bin/start-solo.sh) will start a process in the background and exit directly, resulting in no program running in the foreground, you can enable a command like tail -f to ensure that it will not exit. + command: "tail -f /dev/null" diff --git a/azkaban/hooks/after_start.sh b/azkaban/hooks/after_start.sh index ec8131c..16142d6 100644 --- a/azkaban/hooks/after_start.sh +++ b/azkaban/hooks/after_start.sh @@ -2,3 +2,5 @@ # include sdk of sparrow. . .work/include/sdk.sh + +docker exec -d sparrow_container_${CONTAINER_NAMESPACE}_azkaban bash /home/azkaban/azkaban-solo-server/build/install/azkaban-solo-server/bin/start-solo.sh diff --git a/azkaban/make_app_image/Dockerfile b/azkaban/make_app_image/Dockerfile index 0e17aa3..62c4b10 100644 --- a/azkaban/make_app_image/Dockerfile +++ b/azkaban/make_app_image/Dockerfile @@ -1,3 +1,49 @@ ARG FROM_PLATFORM ARG IMAGE_BASIC_AZKABAN_VERSION FROM --platform=${FROM_PLATFORM} sparrow-basic-azkaban:${IMAGE_BASIC_AZKABAN_VERSION} + +# change source +# fix error: https://stackoverflow.com/questions/70963985/error-failed-to-download-metadata-for-repo-appstream-cannot-prepare-internal +RUN cd /etc/yum.repos.d/ +RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* +RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* +RUN yum install wget -y + +# change yum source +RUN cd /etc/yum.repos.d/ && ls -alh && cp CentOS-Linux-BaseOS.repo CentOS-Linux-BaseOS.repo.bk && wget http://mirrors.aliyun.com/repo/Centos-8.repo +RUN yum clean all && cd /etc/yum.repos.d/ && ls -alh && mv Centos-8.repo CentOS-Linux-BaseOS.repo && yum makecache + +# install git +RUN yum update -y && yum install git -y && git --version + +# install java +RUN yum -y list java* && yum install java-1.8.0-openjdk.x86_64 -y + +# install nodejs +RUN curl --silent --location https://rpm.nodesource.com/setup_18.x | bash +RUN yum install -y nodejs && node -v + +# change npm source +# you don't need yum install -y npm, because npm has already installed after nodejs is installed. +# if you yum install -y npm, may cause long-time wait and error +RUN npm config set registry https://registry.npmmirror.com +RUN npm config get registry + +# clone azkaban +RUN cd /home && git clone https://github.com/azkaban/azkaban.git + +# modify azkaban +# error: Execution failed for task ':azkaban-web-server:nodeSetup'. https://blog.csdn.net/liuandefeng/article/details/130983335 | https://www.cnblogs.com/wuzhiblog/p/13620906.html | https://github.com/azkaban/azkaban/issues/3310 +RUN sed -i 's/download = true/download = false/g' /home/azkaban/azkaban-web-server/build.gradle + +# build azkaban +RUN cd /home/azkaban/ && ./gradlew build installDist -x test + +# run azkaban +# Since it will start a process in the background and exit directly, resulting in no program running in the foreground, you can enable a command like tail -f to ensure that it will not exit. +# RUN cd /home/azkaban/azkaban-solo-server/build/install/azkaban-solo-server && ./bin/start-solo.sh +WORKDIR /home/azkaban/azkaban-solo-server/build/install/azkaban-solo-server + +# after start, you need start ./bin/start-solo.sh +# There will be problems if you start it directly here. The current implementation plan is to execute the startup command in the after_start hook. +# ./hooks/after_start.sh diff --git a/sparrowtool b/sparrowtool index 60ad984..1aea9f2 100755 --- a/sparrowtool +++ b/sparrowtool @@ -169,7 +169,7 @@ search_env() { done # search these directory - subdirectories=("etcd" "etcdkeeper" "go" "jupyter" "kafka" "kafkaui" "mysql" "nginx" "phpfpm" "postgres" "python" "redis" "zookeeper" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "difylocal" "django") # when you create a new service, then append it to the subdirectories variable + subdirectories=("etcd" "etcdkeeper" "go" "jupyter" "kafka" "kafkaui" "mysql" "nginx" "phpfpm" "postgres" "python" "redis" "zookeeper" "langchain" "nodejs" "mongodb" "ssdb" "prometheus" "grafana" "elasticsearch" "kibana" "prompthub" "nacos" "difylocal" "django" "azkaban") # when you create a new service, then append it to the subdirectories variable for dir in "${subdirectories[@]}"; do # show the service information if [ "$start_port" == "" ]; then