diff --git a/docs/deployment/resource-providers/standalone/docker.md b/docs/deployment/resource-providers/standalone/docker.md index e23fb5b6b64f7..9c827ab8c9bf4 100644 --- a/docs/deployment/resource-providers/standalone/docker.md +++ b/docs/deployment/resource-providers/standalone/docker.md @@ -326,10 +326,10 @@ $ docker build --tag pyflink:latest . Flink introduced `jemalloc` as default memory allocator to resolve memory fragmentation problem (please refer to [FLINK-19125](https://issues.apache.org/jira/browse/FLINK-19125)). You could switch back to use `glibc` as memory allocator to restore the old behavior or if any unexpected memory consumption or problem observed -(and please report the issue via JIRA or mailing list if you found any), by passing `disable-jemalloc` parameter: +(and please report the issue via JIRA or mailing list if you found any), by setting the `DISABLE_JEMALLOC` environment variable to `true`. ```sh - $ docker run disable-jemalloc + $ docker run --env DISABLED_JEMALLOC=true disable-jemalloc ``` ### Advanced customization diff --git a/docs/deployment/resource-providers/standalone/docker.zh.md b/docs/deployment/resource-providers/standalone/docker.zh.md index 4893b12204342..1a6ae2a021d64 100644 --- a/docs/deployment/resource-providers/standalone/docker.zh.md +++ b/docs/deployment/resource-providers/standalone/docker.zh.md @@ -326,10 +326,10 @@ $ docker build --tag pyflink:latest . Flink introduced `jemalloc` as default memory allocator to resolve memory fragmentation problem (please refer to [FLINK-19125](https://issues.apache.org/jira/browse/FLINK-19125)). You could switch back to use `glibc` as memory allocator to restore the old behavior or if any unexpected memory consumption or problem observed -(and please report the issue via JIRA or mailing list if you found any), by passing `disable-jemalloc` parameter: +(and please report the issue via JIRA or mailing list if you found any), by setting the `DISABLE_JEMALLOC` environment variable to `true`. ```sh - $ docker run disable-jemalloc + $ docker run --env DISABLED_JEMALLOC=true disable-jemalloc ``` ### Advanced customization diff --git a/flink-dist/src/main/flink-bin/bin/docker-entrypoint.sh b/flink-dist/src/main/flink-bin/bin/docker-entrypoint.sh new file mode 100644 index 0000000000000..ec93985f281df --- /dev/null +++ b/flink-dist/src/main/flink-bin/bin/docker-entrypoint.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash + +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +####################################### +# This script is called by the Flink docker images when starting a process. +# It contains docker-specific features, and hard-codes a few options. +# +# Globals: +# FLINK_HOME - (required) path to the Flink home directory +# ENABLE_BUILT_IN_PLUGINS - semi-colon (;) separated list of plugins to enable, e.g., "flink-plugin1.jar;flink-plugin2.jar" +# FLINK_PROPERTIES - additional flink-conf.yaml entries as a multi-line string +# JOB_MANAGER_RPC_ADDRESS - RPC address of the job manager +# TASK_MANAGER_NUMBER_OF_TASK_SLOTS - number of slots for task executors +####################################### + +COMMAND_STANDALONE="standalone-job" +# Deprecated, should be remove in Flink release 1.13 +COMMAND_NATIVE_KUBERNETES="native-k8s" +COMMAND_HISTORY_SERVER="history-server" + +args=("$@") + +bin=`dirname "$0"` +bin=`cd "$bin"; pwd` + +# FLINK_HOME is set by the docker image +if [ -z "${FLINK_HOME}" ]; then + echo "FLINK_HOME must be set" + exit 1 +fi +export _FLINK_HOME_DETERMINED=true +. ${FLINK_HOME}/bin/config.sh + +# If unspecified, the hostname of the container is taken as the JobManager address +JOB_MANAGER_RPC_ADDRESS=${JOB_MANAGER_RPC_ADDRESS:-$(hostname -f)} +CONF_FILE="${FLINK_CONF_DIR}/flink-conf.yaml" + +copy_plugins_if_required() { + if [ -z "$ENABLE_BUILT_IN_PLUGINS" ]; then + return 0 + fi + + echo "Enabling required built-in plugins" + for target_plugin in $(echo "$ENABLE_BUILT_IN_PLUGINS" | tr ';' ' '); do + echo "Linking ${target_plugin} to plugin directory" + plugin_name=${target_plugin%.jar} + + mkdir -p "${FLINK_PLUGINS_DIR}/${plugin_name}" + if [ ! -e "${FLINK_OPT_DIR}/${target_plugin}" ]; then + echo "Plugin ${target_plugin} does not exist. Exiting." + exit 1 + else + ln -fs "${FLINK_OPT_DIR}/${target_plugin}" "${FLINK_HOME}/plugins/${plugin_name}" + echo "Successfully enabled ${target_plugin}" + fi + done +} + +set_config_option() { + local option=$1 + local value=$2 + + # escape periods for usage in regular expressions + local escaped_option=$(echo ${option} | sed -e "s/\./\\\./g") + + # either override an existing entry, or append a new one + if grep -E "^${escaped_option}:.*" "${CONF_FILE}" > /dev/null; then + sed -i -e "s/${escaped_option}:.*/$option: $value/g" "${CONF_FILE}" + else + echo "${option}: ${value}" >> "${CONF_FILE}" + fi +} + +set_common_options() { + set_config_option jobmanager.rpc.address ${JOB_MANAGER_RPC_ADDRESS} + set_config_option blob.server.port 6124 + set_config_option query.server.port 6125 +} + +prepare_job_manager_start() { + echo "Starting Job Manager" + copy_plugins_if_required + + set_common_options + + if [ -n "${FLINK_PROPERTIES}" ]; then + echo "${FLINK_PROPERTIES}" >> "${CONF_FILE}" + fi + envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}" +} + +if [ "$1" = "help" ]; then + printf "Usage: $(basename "$0") (jobmanager|${COMMAND_STANDALONE}|taskmanager|${COMMAND_HISTORY_SERVER}) [${COMMAND_DISABLE_JEMALLOC}]\n" + printf " Or $(basename "$0") help\n\n" + printf "By default, Flink image adopts jemalloc as default memory allocator and will disable jemalloc if option '${COMMAND_DISABLE_JEMALLOC}' given.\n" + exit 0 +elif [ "$1" = "jobmanager" ]; then + args=("${args[@]:1}") + + prepare_job_manager_start + + exec "${FLINK_BIN_DIR}/jobmanager.sh" start-foreground "${args[@]}" +elif [ "$1" = ${COMMAND_STANDALONE} ]; then + args=("${args[@]:1}") + + prepare_job_manager_start + + exec "${FLINK_BIN_DIR}/standalone-job.sh" start-foreground "${args[@]}" +elif [ "$1" = ${COMMAND_HISTORY_SERVER} ]; then + args=("${args[@]:1}") + + echo "Starting History Server" + copy_plugins_if_required + + if [ -n "${FLINK_PROPERTIES}" ]; then + echo "${FLINK_PROPERTIES}" >> "${CONF_FILE}" + fi + envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}" + + exec "${FLINK_BIN_DIR}/historyserver.sh" start-foreground "${args[@]}" +elif [ "$1" = "taskmanager" ]; then + args=("${args[@]:1}") + + echo "Starting Task Manager" + copy_plugins_if_required + + TASK_MANAGER_NUMBER_OF_TASK_SLOTS=${TASK_MANAGER_NUMBER_OF_TASK_SLOTS:-$(grep -c ^processor /proc/cpuinfo)} + + set_common_options + set_config_option taskmanager.numberOfTaskSlots ${TASK_MANAGER_NUMBER_OF_TASK_SLOTS} + + if [ -n "${FLINK_PROPERTIES}" ]; then + echo "${FLINK_PROPERTIES}" >> "${CONF_FILE}" + fi + envsubst < "${CONF_FILE}" > "${CONF_FILE}.tmp" && mv "${CONF_FILE}.tmp" "${CONF_FILE}" + + exec "${FLINK_BIN_DIR}/taskmanager.sh" start-foreground "${args[@]}" +elif [ "$1" = "$COMMAND_NATIVE_KUBERNETES" ]; then + args=("${args[@]:1}") + + copy_plugins_if_required + + # Override classpath since it was assembled manually + export FLINK_CLASSPATH="`constructFlinkClassPath`:${INTERNAL_HADOOP_CLASSPATHS}" + # Start commands for jobmanager and taskmanager are generated by Flink internally. + echo "Start command: ${args[@]}" + exec bash -c "${args[@]}" +fi + +copy_plugins_if_required + +# Override classpath since it was assembled manually +export FLINK_CLASSPATH="`constructFlinkClassPath`:${INTERNAL_HADOOP_CLASSPATHS}" + +# Running command in pass-through mode +exec "${args[@]}" diff --git a/flink-end-to-end-tests/test-scripts/common_docker.sh b/flink-end-to-end-tests/test-scripts/common_docker.sh index 0682b152cbcda..23b43c15a6a16 100644 --- a/flink-end-to-end-tests/test-scripts/common_docker.sh +++ b/flink-end-to-end-tests/test-scripts/common_docker.sh @@ -47,8 +47,8 @@ function build_image() { start_file_server local server_pid=$! - echo "Preparing Dockeriles" - git clone https://github.com/apache/flink-docker.git --branch dev-master --single-branch + echo "Preparing Dockerfiles" + git clone https://github.com/zentol/flink-docker.git --branch 20915 --single-branch cd flink-docker ./add-custom.sh -u ${file_server_address}:9999/flink.tgz -n ${image_name}