Fix bug: precommands blocked when running tensorboard plugin (#41)

Fix bug: precommands blocked when running tensorboard plugin

src/plugins/tensorboard/tensorboard.sh.template

@@ -22,13 +22,14 @@ set -o pipefail
 
 TENSORFLOW_VERSION=$(tensorboard --version_tb)
 MAJOR_VERSION=${TENSORFLOW_VERSION:0:1}
+
 if [[ "$MAJOR_VERSION" = "1" ]]; then
     tensorboard --logdir={{ logdir }} --port={{ port }} &
 elif [[ "$MAJOR_VERSION" = "2" ]]; then
     {%- if multi_path is sameas false %}
-    tensorboard --logdir={{ logdir_v2 }} --port={{ port }} --bind_all &
+    tensorboard --logdir={{ logdir_v2 }} --port={{ port }} --bind_all 2>&1 > RUNTIME_LOG_PIPE &
     {%- else %}
-    tensorboard --logdir_spec={{ logdir_spec }} --port={{ port }} --bind_all &
+    tensorboard --logdir_spec={{ logdir_spec }} --port={{ port }} --bind_all 2>&1 > RUNTIME_LOG_PIPE &
     {%- endif %}
 else
     echo "Tensorflow version is ${TENSORFLOW_VERSION}, not support"

src/runtime.d/runtime_entry.sh

@@ -120,14 +120,24 @@ chmod a+rx ${USER_STDOUT_LOG_DIR} ${USER_STDERR_LOG_DIR} ${USER_ALL_LOG_DIR}
 
 # execute preCommands generated by plugin
 log "[INFO] Starting to exec precommands"
-${RUNTIME_SCRIPT_DIR}/precommands.sh 2>&1 | ${PROCESS_RUNTIME_LOG} ${RUNTIME_LOG}
+
+# export for all plugins
+export RUNTIME_LOG_PIPE=${RUNTIME_WORK_DIR}/runtime.d/runtime_log_pipe
+mkfifo ${RUNTIME_LOG_PIPE}
+${RUNTIME_SCRIPT_DIR}/precommands.sh 2>&1 > ${RUNTIME_LOG_PIPE} &
+PRECOMMAND_PID=$!
+${PROCESS_RUNTIME_LOG} ${RUNTIME_LOG} < ${RUNTIME_LOG_PIPE} &
+
+# Since precommands may run some processes in background, we do not wait log process finished
+wait ${PRECOMMAND_PID}
+
 log "[INFO] Precommands finished"
 
 # Put verbose output to user.pai.all, stdout to user.pai.stdout, stderr to user.pai.stderr
 # execute user commands
 # priority=100
 log "[INFO] USER COMMAND START"
-LOG_PIPE=${RUNTIME_WORK_DIR}/runtime.d/log_pip
+LOG_PIPE=${RUNTIME_WORK_DIR}/runtime.d/user_log_pipe
 mkfifo ${LOG_PIPE}
 ${RUNTIME_SCRIPT_DIR}/user.sh \
   2> >(tee >(${RUNTIME_SCRIPT_DIR}/multilog s${LOCAL_LOG_MAX_SIZE} n${LOCAL_LOG_MAX_FILES} ${USER_STDERR_LOG_DIR}) | tee -a ${LOG_PIPE} >&2) \