Update to Flink 1.12.0 #9345

xintongsong · 2020-12-23T05:45:37Z

This is a replacement of #9249.

cc @rogaha @tianon @yosifkit @rmetzger @dianfu @wangyang0918

github-actions · 2020-12-23T06:54:21Z

Diff for 54196e5:

diff --git a/_bashbrew-cat b/_bashbrew-cat
index 2817ce3..2c145f8 100644
--- a/_bashbrew-cat
+++ b/_bashbrew-cat
@@ -1,14 +1,6 @@
 Maintainers: Patrick Lucas <me@patricklucas.com> (@patricklucas), Ismaël Mejía <iemejia@gmail.com> (@iemejia)
 GitRepo: https://github.com/apache/flink-docker.git
 
-Tags: 1.10.2-scala_2.11, 1.10-scala_2.11
-GitCommit: 58a29fca7c6ff05999fad4371638d16335f7e93e
-Directory: 1.10/scala_2.11-debian
-
-Tags: 1.10.2-scala_2.12, 1.10-scala_2.12, 1.10.2, 1.10
-GitCommit: 58a29fca7c6ff05999fad4371638d16335f7e93e
-Directory: 1.10/scala_2.12-debian
-
 Tags: 1.11.3-scala_2.11-java8, 1.11-scala_2.11-java8, 1.11.3-scala_2.11, 1.11-scala_2.11
 GitCommit: 7035f03679b11352f2fdecd9f6a9bb0ec8bc2022
 Directory: 1.11/scala_2.11-java8-debian
@@ -24,3 +16,19 @@ Directory: 1.11/scala_2.12-java8-debian
 Tags: 1.11.3-scala_2.12-java11, 1.11-scala_2.12-java11, 1.11.3-java11, 1.11-java11
 GitCommit: 7035f03679b11352f2fdecd9f6a9bb0ec8bc2022
 Directory: 1.11/scala_2.12-java11-debian
+
+Tags: 1.12.0-scala_2.11-java8, 1.12-scala_2.11-java8, scala_2.11-java8, 1.12.0-scala_2.11, 1.12-scala_2.11, scala_2.11
+GitCommit: fb04fbd8471473ce33513f118df64f0daf759022
+Directory: 1.12/scala_2.11-java8-debian
+
+Tags: 1.12.0-scala_2.11-java11, 1.12-scala_2.11-java11, scala_2.11-java11
+GitCommit: fb04fbd8471473ce33513f118df64f0daf759022
+Directory: 1.12/scala_2.11-java11-debian
+
+Tags: 1.12.0-scala_2.12-java8, 1.12-scala_2.12-java8, scala_2.12-java8, 1.12.0-scala_2.12, 1.12-scala_2.12, scala_2.12, 1.12.0-java8, 1.12-java8, java8, 1.12.0, 1.12, latest
+GitCommit: fb04fbd8471473ce33513f118df64f0daf759022
+Directory: 1.12/scala_2.12-java8-debian
+
+Tags: 1.12.0-scala_2.12-java11, 1.12-scala_2.12-java11, scala_2.12-java11, 1.12.0-java11, 1.12-java11, java11
+GitCommit: fb04fbd8471473ce33513f118df64f0daf759022
+Directory: 1.12/scala_2.12-java11-debian
diff --git a/_bashbrew-list b/_bashbrew-list
index f4a0459..9fe01b8 100644
--- a/_bashbrew-list
+++ b/_bashbrew-list
@@ -1,9 +1,3 @@
-flink:1.10
-flink:1.10-scala_2.11
-flink:1.10-scala_2.12
-flink:1.10.2
-flink:1.10.2-scala_2.11
-flink:1.10.2-scala_2.12
 flink:1.11
 flink:1.11-java8
 flink:1.11-java11
@@ -22,3 +16,30 @@ flink:1.11.3-scala_2.11-java11
 flink:1.11.3-scala_2.12
 flink:1.11.3-scala_2.12-java8
 flink:1.11.3-scala_2.12-java11
+flink:1.12
+flink:1.12-java8
+flink:1.12-java11
+flink:1.12-scala_2.11
+flink:1.12-scala_2.11-java8
+flink:1.12-scala_2.11-java11
+flink:1.12-scala_2.12
+flink:1.12-scala_2.12-java8
+flink:1.12-scala_2.12-java11
+flink:1.12.0
+flink:1.12.0-java8
+flink:1.12.0-java11
+flink:1.12.0-scala_2.11
+flink:1.12.0-scala_2.11-java8
+flink:1.12.0-scala_2.11-java11
+flink:1.12.0-scala_2.12
+flink:1.12.0-scala_2.12-java8
+flink:1.12.0-scala_2.12-java11
+flink:java8
+flink:java11
+flink:latest
+flink:scala_2.11
+flink:scala_2.11-java8
+flink:scala_2.11-java11
+flink:scala_2.12
+flink:scala_2.12-java8
+flink:scala_2.12-java11
diff --git a/flink_1.11-java11/Dockerfile b/flink_java11/Dockerfile
similarity index 92%
copy from flink_1.11-java11/Dockerfile
copy to flink_java11/Dockerfile
index 8b27b61..48c2b59 100644
--- a/flink_1.11-java11/Dockerfile
+++ b/flink_java11/Dockerfile
@@ -21,7 +21,7 @@ FROM openjdk:11-jre
 # Install dependencies
 RUN set -ex; \
   apt-get update; \
-  apt-get -y install libsnappy1v5 gettext-base; \
+  apt-get -y install libsnappy1v5 gettext-base libjemalloc-dev; \
   rm -rf /var/lib/apt/lists/*
 
 # Grab gosu for easy step-down from root
@@ -44,9 +44,9 @@ RUN set -ex; \
   gosu nobody true
 
 # Configure Flink version
-ENV FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.11.3/flink-1.11.3-bin-scala_2.12.tgz \
-    FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.11.3/flink-1.11.3-bin-scala_2.12.tgz.asc \
-    GPG_KEY=F8E419AA0B60C28879E876859DFF40967ABFC5A4 \
+ENV FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.12.0/flink-1.12.0-bin-scala_2.12.tgz \
+    FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.12.0/flink-1.12.0-bin-scala_2.12.tgz.asc \
+    GPG_KEY=D9839159 \
     CHECK_GPG=true
 
 # Prepare environment
diff --git a/flink_1.10-scala_2.11/docker-entrypoint.sh b/flink_java11/docker-entrypoint.sh
similarity index 62%
copy from flink_1.10-scala_2.11/docker-entrypoint.sh
copy to flink_java11/docker-entrypoint.sh
index 6084f06..65f8fc8 100755
--- a/flink_1.10-scala_2.11/docker-entrypoint.sh
+++ b/flink_java11/docker-entrypoint.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 ###############################################################################
 #  Licensed to the Apache Software Foundation (ASF) under one
@@ -19,6 +19,10 @@
 ###############################################################################
 
 COMMAND_STANDALONE="standalone-job"
+# Deprecated, should be remove in Flink release 1.13
+COMMAND_NATIVE_KUBERNETES="native-k8s"
+COMMAND_HISTORY_SERVER="history-server"
+COMMAND_DISABLE_JEMALLOC="disable-jemalloc"
 
 # If unspecified, the hostname of the container is taken as the JobManager address
 JOB_MANAGER_RPC_ADDRESS=${JOB_MANAGER_RPC_ADDRESS:-$(hostname -f)}
@@ -91,21 +95,54 @@ prepare_job_manager_start() {
     envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}"
 }
 
+disable_jemalloc_env() {
+  # use nameref '_args' to update the passed 'args' within function
+  local -n _args=$1
+  if [ "${_args[0]}" = ${COMMAND_DISABLE_JEMALLOC} ]; then
+      echo "Disable Jemalloc as the memory allocator"
+      _args=("${_args[@]:1}")
+  else
+      export LD_PRELOAD=$LD_PRELOAD:/usr/lib/x86_64-linux-gnu/libjemalloc.so
+  fi
+}
+
+args=("$@")
 if [ "$1" = "help" ]; then
-    echo "Usage: $(basename "$0") (jobmanager|${COMMAND_STANDALONE}|taskmanager|help)"
+    printf "Usage: $(basename "$0") (jobmanager|${COMMAND_STANDALONE}|taskmanager|${COMMAND_HISTORY_SERVER}) [${COMMAND_DISABLE_JEMALLOC}]\n"
+    printf "    Or $(basename "$0") help\n\n"
+    printf "By default, Flink image adopts jemalloc as default memory allocator and will disable jemalloc if option '${COMMAND_DISABLE_JEMALLOC}' given.\n"
     exit 0
 elif [ "$1" = "jobmanager" ]; then
-    shift 1
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
     prepare_job_manager_start
 
-    exec $(drop_privs_cmd) "$FLINK_HOME/bin/jobmanager.sh" start-foreground "$@"
+    exec $(drop_privs_cmd) "$FLINK_HOME/bin/jobmanager.sh" start-foreground "${args[@]}"
 elif [ "$1" = ${COMMAND_STANDALONE} ]; then
-    shift 1
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
     prepare_job_manager_start
 
-    exec $(drop_privs_cmd) "$FLINK_HOME/bin/standalone-job.sh" start-foreground "$@"
+    exec $(drop_privs_cmd) "$FLINK_HOME/bin/standalone-job.sh" start-foreground "${args[@]}"
+elif [ "$1" = ${COMMAND_HISTORY_SERVER} ]; then
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
+    echo "Starting History Server"
+    copy_plugins_if_required
+
+    if [ -n "${FLINK_PROPERTIES}" ]; then
+        echo "${FLINK_PROPERTIES}" >> "${CONF_FILE}"
+    fi
+    envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}"
+
+    exec $(drop_privs_cmd) "$FLINK_HOME/bin/historyserver.sh" start-foreground "${args[@]}"
 elif [ "$1" = "taskmanager" ]; then
-    shift 1
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
     echo "Starting Task Manager"
     copy_plugins_if_required
 
@@ -119,7 +156,31 @@ elif [ "$1" = "taskmanager" ]; then
     fi
     envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}"
 
-    exec $(drop_privs_cmd) "$FLINK_HOME/bin/taskmanager.sh" start-foreground "$@"
+    exec $(drop_privs_cmd) "$FLINK_HOME/bin/taskmanager.sh" start-foreground "${args[@]}"
+elif [ "$1" = "$COMMAND_NATIVE_KUBERNETES" ]; then
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
+    copy_plugins_if_required
+
+    export _FLINK_HOME_DETERMINED=true
+    . $FLINK_HOME/bin/config.sh
+    export FLINK_CLASSPATH="`constructFlinkClassPath`:$INTERNAL_HADOOP_CLASSPATHS"
+    # Start commands for jobmanager and taskmanager are generated by Flink internally.
+    echo "Start command: ${args[@]}"
+    exec $(drop_privs_cmd) bash -c "${args[@]}"
 fi
 
-exec "$@"
+args=("${args[@]}")
+
+disable_jemalloc_env args
+
+copy_plugins_if_required
+
+# Set the Flink related environments
+export _FLINK_HOME_DETERMINED=true
+. $FLINK_HOME/bin/config.sh
+export FLINK_CLASSPATH="`constructFlinkClassPath`:$INTERNAL_HADOOP_CLASSPATHS"
+
+# Running command in pass-through mode
+exec $(drop_privs_cmd) "${args[@]}"
diff --git a/flink_1.10/Dockerfile b/flink_latest/Dockerfile
similarity index 92%
rename from flink_1.10/Dockerfile
rename to flink_latest/Dockerfile
index 29afb1a..ceb8bb1 100644
--- a/flink_1.10/Dockerfile
+++ b/flink_latest/Dockerfile
@@ -21,7 +21,7 @@ FROM openjdk:8-jre
 # Install dependencies
 RUN set -ex; \
   apt-get update; \
-  apt-get -y install libsnappy1v5 gettext-base; \
+  apt-get -y install libsnappy1v5 gettext-base libjemalloc-dev; \
   rm -rf /var/lib/apt/lists/*
 
 # Grab gosu for easy step-down from root
@@ -44,9 +44,9 @@ RUN set -ex; \
   gosu nobody true
 
 # Configure Flink version
-ENV FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.10.2/flink-1.10.2-bin-scala_2.12.tgz \
-    FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.10.2/flink-1.10.2-bin-scala_2.12.tgz.asc \
-    GPG_KEY=C63E230EFFF519A5BBF2C9AE6767487CD505859C \
+ENV FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.12.0/flink-1.12.0-bin-scala_2.12.tgz \
+    FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.12.0/flink-1.12.0-bin-scala_2.12.tgz.asc \
+    GPG_KEY=D9839159 \
     CHECK_GPG=true
 
 # Prepare environment
diff --git a/flink_1.10/docker-entrypoint.sh b/flink_latest/docker-entrypoint.sh
similarity index 62%
rename from flink_1.10/docker-entrypoint.sh
rename to flink_latest/docker-entrypoint.sh
index 6084f06..65f8fc8 100755
--- a/flink_1.10/docker-entrypoint.sh
+++ b/flink_latest/docker-entrypoint.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 ###############################################################################
 #  Licensed to the Apache Software Foundation (ASF) under one
@@ -19,6 +19,10 @@
 ###############################################################################
 
 COMMAND_STANDALONE="standalone-job"
+# Deprecated, should be remove in Flink release 1.13
+COMMAND_NATIVE_KUBERNETES="native-k8s"
+COMMAND_HISTORY_SERVER="history-server"
+COMMAND_DISABLE_JEMALLOC="disable-jemalloc"
 
 # If unspecified, the hostname of the container is taken as the JobManager address
 JOB_MANAGER_RPC_ADDRESS=${JOB_MANAGER_RPC_ADDRESS:-$(hostname -f)}
@@ -91,21 +95,54 @@ prepare_job_manager_start() {
     envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}"
 }
 
+disable_jemalloc_env() {
+  # use nameref '_args' to update the passed 'args' within function
+  local -n _args=$1
+  if [ "${_args[0]}" = ${COMMAND_DISABLE_JEMALLOC} ]; then
+      echo "Disable Jemalloc as the memory allocator"
+      _args=("${_args[@]:1}")
+  else
+      export LD_PRELOAD=$LD_PRELOAD:/usr/lib/x86_64-linux-gnu/libjemalloc.so
+  fi
+}
+
+args=("$@")
 if [ "$1" = "help" ]; then
-    echo "Usage: $(basename "$0") (jobmanager|${COMMAND_STANDALONE}|taskmanager|help)"
+    printf "Usage: $(basename "$0") (jobmanager|${COMMAND_STANDALONE}|taskmanager|${COMMAND_HISTORY_SERVER}) [${COMMAND_DISABLE_JEMALLOC}]\n"
+    printf "    Or $(basename "$0") help\n\n"
+    printf "By default, Flink image adopts jemalloc as default memory allocator and will disable jemalloc if option '${COMMAND_DISABLE_JEMALLOC}' given.\n"
     exit 0
 elif [ "$1" = "jobmanager" ]; then
-    shift 1
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
     prepare_job_manager_start
 
-    exec $(drop_privs_cmd) "$FLINK_HOME/bin/jobmanager.sh" start-foreground "$@"
+    exec $(drop_privs_cmd) "$FLINK_HOME/bin/jobmanager.sh" start-foreground "${args[@]}"
 elif [ "$1" = ${COMMAND_STANDALONE} ]; then
-    shift 1
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
     prepare_job_manager_start
 
-    exec $(drop_privs_cmd) "$FLINK_HOME/bin/standalone-job.sh" start-foreground "$@"
+    exec $(drop_privs_cmd) "$FLINK_HOME/bin/standalone-job.sh" start-foreground "${args[@]}"
+elif [ "$1" = ${COMMAND_HISTORY_SERVER} ]; then
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
+    echo "Starting History Server"
+    copy_plugins_if_required
+
+    if [ -n "${FLINK_PROPERTIES}" ]; then
+        echo "${FLINK_PROPERTIES}" >> "${CONF_FILE}"
+    fi
+    envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}"
+
+    exec $(drop_privs_cmd) "$FLINK_HOME/bin/historyserver.sh" start-foreground "${args[@]}"
 elif [ "$1" = "taskmanager" ]; then
-    shift 1
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
     echo "Starting Task Manager"
     copy_plugins_if_required
 
@@ -119,7 +156,31 @@ elif [ "$1" = "taskmanager" ]; then
     fi
     envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}"
 
-    exec $(drop_privs_cmd) "$FLINK_HOME/bin/taskmanager.sh" start-foreground "$@"
+    exec $(drop_privs_cmd) "$FLINK_HOME/bin/taskmanager.sh" start-foreground "${args[@]}"
+elif [ "$1" = "$COMMAND_NATIVE_KUBERNETES" ]; then
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
+    copy_plugins_if_required
+
+    export _FLINK_HOME_DETERMINED=true
+    . $FLINK_HOME/bin/config.sh
+    export FLINK_CLASSPATH="`constructFlinkClassPath`:$INTERNAL_HADOOP_CLASSPATHS"
+    # Start commands for jobmanager and taskmanager are generated by Flink internally.
+    echo "Start command: ${args[@]}"
+    exec $(drop_privs_cmd) bash -c "${args[@]}"
 fi
 
-exec "$@"
+args=("${args[@]}")
+
+disable_jemalloc_env args
+
+copy_plugins_if_required
+
+# Set the Flink related environments
+export _FLINK_HOME_DETERMINED=true
+. $FLINK_HOME/bin/config.sh
+export FLINK_CLASSPATH="`constructFlinkClassPath`:$INTERNAL_HADOOP_CLASSPATHS"
+
+# Running command in pass-through mode
+exec $(drop_privs_cmd) "${args[@]}"
diff --git a/flink_1.11-java11/Dockerfile b/flink_scala_2.11-java11/Dockerfile
similarity index 92%
copy from flink_1.11-java11/Dockerfile
copy to flink_scala_2.11-java11/Dockerfile
index 8b27b61..50c635e 100644
--- a/flink_1.11-java11/Dockerfile
+++ b/flink_scala_2.11-java11/Dockerfile
@@ -21,7 +21,7 @@ FROM openjdk:11-jre
 # Install dependencies
 RUN set -ex; \
   apt-get update; \
-  apt-get -y install libsnappy1v5 gettext-base; \
+  apt-get -y install libsnappy1v5 gettext-base libjemalloc-dev; \
   rm -rf /var/lib/apt/lists/*
 
 # Grab gosu for easy step-down from root
@@ -44,9 +44,9 @@ RUN set -ex; \
   gosu nobody true
 
 # Configure Flink version
-ENV FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.11.3/flink-1.11.3-bin-scala_2.12.tgz \
-    FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.11.3/flink-1.11.3-bin-scala_2.12.tgz.asc \
-    GPG_KEY=F8E419AA0B60C28879E876859DFF40967ABFC5A4 \
+ENV FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.12.0/flink-1.12.0-bin-scala_2.11.tgz \
+    FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.12.0/flink-1.12.0-bin-scala_2.11.tgz.asc \
+    GPG_KEY=D9839159 \
     CHECK_GPG=true
 
 # Prepare environment
diff --git a/flink_1.10-scala_2.11/docker-entrypoint.sh b/flink_scala_2.11-java11/docker-entrypoint.sh
similarity index 62%
copy from flink_1.10-scala_2.11/docker-entrypoint.sh
copy to flink_scala_2.11-java11/docker-entrypoint.sh
index 6084f06..65f8fc8 100755
--- a/flink_1.10-scala_2.11/docker-entrypoint.sh
+++ b/flink_scala_2.11-java11/docker-entrypoint.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 ###############################################################################
 #  Licensed to the Apache Software Foundation (ASF) under one
@@ -19,6 +19,10 @@
 ###############################################################################
 
 COMMAND_STANDALONE="standalone-job"
+# Deprecated, should be remove in Flink release 1.13
+COMMAND_NATIVE_KUBERNETES="native-k8s"
+COMMAND_HISTORY_SERVER="history-server"
+COMMAND_DISABLE_JEMALLOC="disable-jemalloc"
 
 # If unspecified, the hostname of the container is taken as the JobManager address
 JOB_MANAGER_RPC_ADDRESS=${JOB_MANAGER_RPC_ADDRESS:-$(hostname -f)}
@@ -91,21 +95,54 @@ prepare_job_manager_start() {
     envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}"
 }
 
+disable_jemalloc_env() {
+  # use nameref '_args' to update the passed 'args' within function
+  local -n _args=$1
+  if [ "${_args[0]}" = ${COMMAND_DISABLE_JEMALLOC} ]; then
+      echo "Disable Jemalloc as the memory allocator"
+      _args=("${_args[@]:1}")
+  else
+      export LD_PRELOAD=$LD_PRELOAD:/usr/lib/x86_64-linux-gnu/libjemalloc.so
+  fi
+}
+
+args=("$@")
 if [ "$1" = "help" ]; then
-    echo "Usage: $(basename "$0") (jobmanager|${COMMAND_STANDALONE}|taskmanager|help)"
+    printf "Usage: $(basename "$0") (jobmanager|${COMMAND_STANDALONE}|taskmanager|${COMMAND_HISTORY_SERVER}) [${COMMAND_DISABLE_JEMALLOC}]\n"
+    printf "    Or $(basename "$0") help\n\n"
+    printf "By default, Flink image adopts jemalloc as default memory allocator and will disable jemalloc if option '${COMMAND_DISABLE_JEMALLOC}' given.\n"
     exit 0
 elif [ "$1" = "jobmanager" ]; then
-    shift 1
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
     prepare_job_manager_start
 
-    exec $(drop_privs_cmd) "$FLINK_HOME/bin/jobmanager.sh" start-foreground "$@"
+    exec $(drop_privs_cmd) "$FLINK_HOME/bin/jobmanager.sh" start-foreground "${args[@]}"
 elif [ "$1" = ${COMMAND_STANDALONE} ]; then
-    shift 1
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
     prepare_job_manager_start
 
-    exec $(drop_privs_cmd) "$FLINK_HOME/bin/standalone-job.sh" start-foreground "$@"
+    exec $(drop_privs_cmd) "$FLINK_HOME/bin/standalone-job.sh" start-foreground "${args[@]}"
+elif [ "$1" = ${COMMAND_HISTORY_SERVER} ]; then
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
+    echo "Starting History Server"
+    copy_plugins_if_required
+
+    if [ -n "${FLINK_PROPERTIES}" ]; then
+        echo "${FLINK_PROPERTIES}" >> "${CONF_FILE}"
+    fi
+    envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}"
+
+    exec $(drop_privs_cmd) "$FLINK_HOME/bin/historyserver.sh" start-foreground "${args[@]}"
 elif [ "$1" = "taskmanager" ]; then
-    shift 1
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
     echo "Starting Task Manager"
     copy_plugins_if_required
 
@@ -119,7 +156,31 @@ elif [ "$1" = "taskmanager" ]; then
     fi
     envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}"
 
-    exec $(drop_privs_cmd) "$FLINK_HOME/bin/taskmanager.sh" start-foreground "$@"
+    exec $(drop_privs_cmd) "$FLINK_HOME/bin/taskmanager.sh" start-foreground "${args[@]}"
+elif [ "$1" = "$COMMAND_NATIVE_KUBERNETES" ]; then
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
+    copy_plugins_if_required
+
+    export _FLINK_HOME_DETERMINED=true
+    . $FLINK_HOME/bin/config.sh
+    export FLINK_CLASSPATH="`constructFlinkClassPath`:$INTERNAL_HADOOP_CLASSPATHS"
+    # Start commands for jobmanager and taskmanager are generated by Flink internally.
+    echo "Start command: ${args[@]}"
+    exec $(drop_privs_cmd) bash -c "${args[@]}"
 fi
 
-exec "$@"
+args=("${args[@]}")
+
+disable_jemalloc_env args
+
+copy_plugins_if_required
+
+# Set the Flink related environments
+export _FLINK_HOME_DETERMINED=true
+. $FLINK_HOME/bin/config.sh
+export FLINK_CLASSPATH="`constructFlinkClassPath`:$INTERNAL_HADOOP_CLASSPATHS"
+
+# Running command in pass-through mode
+exec $(drop_privs_cmd) "${args[@]}"
diff --git a/flink_1.10-scala_2.11/Dockerfile b/flink_scala_2.11/Dockerfile
similarity index 92%
rename from flink_1.10-scala_2.11/Dockerfile
rename to flink_scala_2.11/Dockerfile
index 7c07b15..4d3b0f2 100644
--- a/flink_1.10-scala_2.11/Dockerfile
+++ b/flink_scala_2.11/Dockerfile
@@ -21,7 +21,7 @@ FROM openjdk:8-jre
 # Install dependencies
 RUN set -ex; \
   apt-get update; \
-  apt-get -y install libsnappy1v5 gettext-base; \
+  apt-get -y install libsnappy1v5 gettext-base libjemalloc-dev; \
   rm -rf /var/lib/apt/lists/*
 
 # Grab gosu for easy step-down from root
@@ -44,9 +44,9 @@ RUN set -ex; \
   gosu nobody true
 
 # Configure Flink version
-ENV FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.10.2/flink-1.10.2-bin-scala_2.11.tgz \
-    FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.10.2/flink-1.10.2-bin-scala_2.11.tgz.asc \
-    GPG_KEY=C63E230EFFF519A5BBF2C9AE6767487CD505859C \
+ENV FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.12.0/flink-1.12.0-bin-scala_2.11.tgz \
+    FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.12.0/flink-1.12.0-bin-scala_2.11.tgz.asc \
+    GPG_KEY=D9839159 \
     CHECK_GPG=true
 
 # Prepare environment
diff --git a/flink_1.10-scala_2.11/docker-entrypoint.sh b/flink_scala_2.11/docker-entrypoint.sh
similarity index 62%
rename from flink_1.10-scala_2.11/docker-entrypoint.sh
rename to flink_scala_2.11/docker-entrypoint.sh
index 6084f06..65f8fc8 100755
--- a/flink_1.10-scala_2.11/docker-entrypoint.sh
+++ b/flink_scala_2.11/docker-entrypoint.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 ###############################################################################
 #  Licensed to the Apache Software Foundation (ASF) under one
@@ -19,6 +19,10 @@
 ###############################################################################
 
 COMMAND_STANDALONE="standalone-job"
+# Deprecated, should be remove in Flink release 1.13
+COMMAND_NATIVE_KUBERNETES="native-k8s"
+COMMAND_HISTORY_SERVER="history-server"
+COMMAND_DISABLE_JEMALLOC="disable-jemalloc"
 
 # If unspecified, the hostname of the container is taken as the JobManager address
 JOB_MANAGER_RPC_ADDRESS=${JOB_MANAGER_RPC_ADDRESS:-$(hostname -f)}
@@ -91,21 +95,54 @@ prepare_job_manager_start() {
     envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}"
 }
 
+disable_jemalloc_env() {
+  # use nameref '_args' to update the passed 'args' within function
+  local -n _args=$1
+  if [ "${_args[0]}" = ${COMMAND_DISABLE_JEMALLOC} ]; then
+      echo "Disable Jemalloc as the memory allocator"
+      _args=("${_args[@]:1}")
+  else
+      export LD_PRELOAD=$LD_PRELOAD:/usr/lib/x86_64-linux-gnu/libjemalloc.so
+  fi
+}
+
+args=("$@")
 if [ "$1" = "help" ]; then
-    echo "Usage: $(basename "$0") (jobmanager|${COMMAND_STANDALONE}|taskmanager|help)"
+    printf "Usage: $(basename "$0") (jobmanager|${COMMAND_STANDALONE}|taskmanager|${COMMAND_HISTORY_SERVER}) [${COMMAND_DISABLE_JEMALLOC}]\n"
+    printf "    Or $(basename "$0") help\n\n"
+    printf "By default, Flink image adopts jemalloc as default memory allocator and will disable jemalloc if option '${COMMAND_DISABLE_JEMALLOC}' given.\n"
     exit 0
 elif [ "$1" = "jobmanager" ]; then
-    shift 1
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
     prepare_job_manager_start
 
-    exec $(drop_privs_cmd) "$FLINK_HOME/bin/jobmanager.sh" start-foreground "$@"
+    exec $(drop_privs_cmd) "$FLINK_HOME/bin/jobmanager.sh" start-foreground "${args[@]}"
 elif [ "$1" = ${COMMAND_STANDALONE} ]; then
-    shift 1
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
     prepare_job_manager_start
 
-    exec $(drop_privs_cmd) "$FLINK_HOME/bin/standalone-job.sh" start-foreground "$@"
+    exec $(drop_privs_cmd) "$FLINK_HOME/bin/standalone-job.sh" start-foreground "${args[@]}"
+elif [ "$1" = ${COMMAND_HISTORY_SERVER} ]; then
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
+    echo "Starting History Server"
+    copy_plugins_if_required
+
+    if [ -n "${FLINK_PROPERTIES}" ]; then
+        echo "${FLINK_PROPERTIES}" >> "${CONF_FILE}"
+    fi
+    envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}"
+
+    exec $(drop_privs_cmd) "$FLINK_HOME/bin/historyserver.sh" start-foreground "${args[@]}"
 elif [ "$1" = "taskmanager" ]; then
-    shift 1
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
     echo "Starting Task Manager"
     copy_plugins_if_required
 
@@ -119,7 +156,31 @@ elif [ "$1" = "taskmanager" ]; then
     fi
     envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}"
 
-    exec $(drop_privs_cmd) "$FLINK_HOME/bin/taskmanager.sh" start-foreground "$@"
+    exec $(drop_privs_cmd) "$FLINK_HOME/bin/taskmanager.sh" start-foreground "${args[@]}"
+elif [ "$1" = "$COMMAND_NATIVE_KUBERNETES" ]; then
+    args=("${args[@]:1}")
+    disable_jemalloc_env args
+
+    copy_plugins_if_required
+
+    export _FLINK_HOME_DETERMINED=true
+    . $FLINK_HOME/bin/config.sh
+    export FLINK_CLASSPATH="`constructFlinkClassPath`:$INTERNAL_HADOOP_CLASSPATHS"
+    # Start commands for jobmanager and taskmanager are generated by Flink internally.
+    echo "Start command: ${args[@]}"
+    exec $(drop_privs_cmd) bash -c "${args[@]}"
 fi
 
-exec "$@"
+args=("${args[@]}")
+
+disable_jemalloc_env args
+
+copy_plugins_if_required
+
+# Set the Flink related environments
+export _FLINK_HOME_DETERMINED=true
+. $FLINK_HOME/bin/config.sh
+export FLINK_CLASSPATH="`constructFlinkClassPath`:$INTERNAL_HADOOP_CLASSPATHS"
+
+# Running command in pass-through mode
+exec $(drop_privs_cmd) "${args[@]}"

xintongsong · 2020-12-31T02:28:31Z

@rogaha @tianon @yosifkit
Could someone please help take a look at this PR?

renatomariscal · 2021-01-11T22:05:55Z

Any updates on this? My team is anxious to get the goodies of 1.12 😬

tianon · 2021-01-11T22:21:09Z

There's still ongoing discussion in #9249.

yosifkit · 2021-01-28T22:52:07Z

#9249 was fixed and merged, so closing this one.

github-actions bot added the library/flink label Dec 23, 2020

xintongsong mentioned this pull request Dec 23, 2020

[flink] add 1.12 ; remove 1.10 #9249

Merged

Update to Flink 1.12.0

54196e5

xintongsong force-pushed the flink-1.12.0 branch from 88361b5 to 54196e5 Compare December 23, 2020 06:52

yosifkit closed this Jan 28, 2021

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Update to Flink 1.12.0 #9345

Update to Flink 1.12.0 #9345

xintongsong commented Dec 23, 2020

github-actions bot commented Dec 23, 2020

xintongsong commented Dec 31, 2020

renatomariscal commented Jan 11, 2021

tianon commented Jan 11, 2021

yosifkit commented Jan 28, 2021

Update to Flink 1.12.0 #9345

Update to Flink 1.12.0 #9345

Conversation

xintongsong commented Dec 23, 2020

github-actions bot commented Dec 23, 2020

xintongsong commented Dec 31, 2020

renatomariscal commented Jan 11, 2021

tianon commented Jan 11, 2021

yosifkit commented Jan 28, 2021