[SPK-187] Fix: separate stderr stdout and checking SPARK_LOG_LEVEL pr…

…esence (apache#127) * Separating stderr from stdout and checking if SPARK_LOG_LEVEL is being provided * Solved in dispatcher * Removed tests * Empty commit * Changed condition * Uncommented tests * Go Jenkins! * Go Jenkins!
jlopezmalla · Jan 17, 2018 · bd42753 · bd42753
1 parent f39e4ff
commit bd42753
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 10 deletions.
diff --git a/conf/log4j.properties.template b/conf/log4j.properties.template
@@ -15,12 +15,23 @@
 # limitations under the License.
 #
 
-# Set everything to be logged to the console
-log4j.rootCategory=INFO, console
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
+log4j.rootCategory=INFO, stdout, stderr
+
+# Stdout configuraton
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
+log4j.appender.stdout.filter.filter1.levelMin=TRACE
+log4j.appender.stdout.filter.filter1.levelMax=INFO
+log4j.appender.stdout.filter.filter1=org.apache.log4j.varia.LevelRangeFilter
+
+# Stderr configuration
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Threshold = WARN
+log4j.appender.stderr.target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
 
 # Set the default spark-shell log level to WARN. When running the spark-shell, the
 # log level for this class is used to overwrite the root logger's log level, so that

diff --git a/docker/dispatcher/service b/docker/dispatcher/service
@@ -14,8 +14,12 @@ function export_daemon_opts() {
 }
 
 function set_log_level() {
-    sed "s,log4j.rootCategory=INFO,log4j.rootCategory=${SPARK_LOG_LEVEL}," \
-        /opt/spark/dist/conf/log4j.properties.template >/opt/spark/dist/conf/log4j.properties
+    if [ ! -z "$SPARK_LOG_LEVEL" ]; then
+        sed "s,log4j.rootCategory=INFO,log4j.rootCategory=${SPARK_LOG_LEVEL}," \
+            /opt/spark/dist/conf/log4j.properties.template >/opt/spark/dist/conf/log4j.properties
+    else
+        echo "No SPARK_LOG_LEVEL provided. Leaving as default"
+    fi
 }
 
 function add_if_non_empty() {

diff --git a/docker/dispatcher/spark-env.sh b/docker/dispatcher/spark-env.sh
@@ -15,10 +15,15 @@ fi
 
 
 function set_log_level() {
-    sed "s,log4j.rootCategory=INFO,log4j.rootCategory=${SPARK_LOG_LEVEL}," \
-        /opt/spark/dist/conf/log4j.properties.template >/opt/spark/dist/conf/log4j.properties
+    if [ ! -z "$SPARK_LOG_LEVEL" ]; then
+        sed "s,log4j.rootCategory=INFO,log4j.rootCategory=${SPARK_LOG_LEVEL}," \
+            /opt/spark/dist/conf/log4j.properties.template >/opt/spark/dist/conf/log4j.properties
+    else
+        cp /opt/spark/dist/conf/log4j.properties.template /opt/spark/dist/conf/log4j.properties
+    fi
 }
 
+# Separating stderr from stdout (already applied in the template) and applying SPARK_LOG_LEVEL if provided
 set_log_level
 
 # I first set this to MESOS_SANDBOX, as a Workaround for MESOS-5866