From bd42753271afdacf26325b959cfd043f28c9176e Mon Sep 17 00:00:00 2001
From: Unai Sarasola <pianista215@users.noreply.github.com>
Date: Wed, 17 Jan 2018 12:36:38 +0100
Subject: [PATCH] [SPK-187] Fix: separate stderr stdout and checking
 SPARK_LOG_LEVEL presence (#127)

* Separating stderr from stdout and checking if SPARK_LOG_LEVEL is being provided

* Solved in dispatcher

* Removed tests

* Empty commit

* Changed condition

* Uncommented tests

* Go Jenkins!

* Go Jenkins!
---
 conf/log4j.properties.template | 23 +++++++++++++++++------
 docker/dispatcher/service      |  8 ++++++--
 docker/dispatcher/spark-env.sh |  9 +++++++--
 3 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/conf/log4j.properties.template b/conf/log4j.properties.template
index ec1aa187dfb32..e55449ef319a9 100644
--- a/conf/log4j.properties.template
+++ b/conf/log4j.properties.template
@@ -15,12 +15,23 @@
 # limitations under the License.
 #
 
-# Set everything to be logged to the console
-log4j.rootCategory=INFO, console
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
+log4j.rootCategory=INFO, stdout, stderr
+
+# Stdout configuraton
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
+log4j.appender.stdout.filter.filter1.levelMin=TRACE
+log4j.appender.stdout.filter.filter1.levelMax=INFO
+log4j.appender.stdout.filter.filter1=org.apache.log4j.varia.LevelRangeFilter
+
+# Stderr configuration
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Threshold = WARN
+log4j.appender.stderr.target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
 
 # Set the default spark-shell log level to WARN. When running the spark-shell, the
 # log level for this class is used to overwrite the root logger's log level, so that
diff --git a/docker/dispatcher/service b/docker/dispatcher/service
index 0dc93c739fe05..265f05b5b15b1 100644
--- a/docker/dispatcher/service
+++ b/docker/dispatcher/service
@@ -14,8 +14,12 @@ function export_daemon_opts() {
 }
 
 function set_log_level() {
-    sed "s,log4j.rootCategory=INFO,log4j.rootCategory=${SPARK_LOG_LEVEL}," \
-        /opt/spark/dist/conf/log4j.properties.template >/opt/spark/dist/conf/log4j.properties
+    if [ ! -z "$SPARK_LOG_LEVEL" ]; then
+        sed "s,log4j.rootCategory=INFO,log4j.rootCategory=${SPARK_LOG_LEVEL}," \
+            /opt/spark/dist/conf/log4j.properties.template >/opt/spark/dist/conf/log4j.properties
+    else
+        echo "No SPARK_LOG_LEVEL provided. Leaving as default"
+    fi
 }
 
 function add_if_non_empty() {
diff --git a/docker/dispatcher/spark-env.sh b/docker/dispatcher/spark-env.sh
index 91bdc1588c0ea..cd63c3f8f93ea 100644
--- a/docker/dispatcher/spark-env.sh
+++ b/docker/dispatcher/spark-env.sh
@@ -15,10 +15,15 @@ fi
 
 
 function set_log_level() {
-    sed "s,log4j.rootCategory=INFO,log4j.rootCategory=${SPARK_LOG_LEVEL}," \
-        /opt/spark/dist/conf/log4j.properties.template >/opt/spark/dist/conf/log4j.properties
+    if [ ! -z "$SPARK_LOG_LEVEL" ]; then
+        sed "s,log4j.rootCategory=INFO,log4j.rootCategory=${SPARK_LOG_LEVEL}," \
+            /opt/spark/dist/conf/log4j.properties.template >/opt/spark/dist/conf/log4j.properties
+    else
+        cp /opt/spark/dist/conf/log4j.properties.template /opt/spark/dist/conf/log4j.properties
+    fi
 }
 
+# Separating stderr from stdout (already applied in the template) and applying SPARK_LOG_LEVEL if provided
 set_log_level
 
 # I first set this to MESOS_SANDBOX, as a Workaround for MESOS-5866