Merge pull request #3 from ThalesGroup/release/guavus-v3.0.0-1

Release/guavus v3.0.0 1

.VERSION

@@ -0,0 +1 @@
+3.0.0-1,1.0

Dockerfile

@@ -0,0 +1,49 @@
+ARG spark_image_tag=3.0.0-1-hadoop3.2-1.0
+
+FROM artifacts.ggn.in.guavus.com:4244/spark:${spark_image_tag}
+
+ARG spark_uid=185
+
+USER root
+
+RUN apt-get update && \
+    apt-get -y install curl && \
+    curl -fSL http://artifacts.ggn.in.guavus.com:8081/artifactory/libs-release-local/org/elasticsearch/elasticsearch-hadoop-core/7.8.1_3.0.0/elasticsearch-hadoop-core-7.8.1_3.0.0.jar -o elasticsearch-hadoop-core-7.8.1_3.0.0.jar && \
+    curl -fSL http://artifacts.ggn.in.guavus.com:8081/artifactory/libs-release-local/org/elasticsearch/elasticsearch-hadoop-mr/7.8.1_3.0.0/elasticsearch-hadoop-mr-7.8.1_3.0.0.jar -o elasticsearch-hadoop-mr-7.8.1_3.0.0.jar && \
+    curl -fSL http://artifacts.ggn.in.guavus.com:8081/artifactory/libs-release-local/org/elasticsearch/elasticsearch-hadoop-sql/7.8.1_3.0.0/elasticsearch-hadoop-sql-7.8.1_3.0.0.jar -o elasticsearch-hadoop-sql-7.8.1_3.0.0.jar && \
+    curl -fSL https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-s3/1.11.832/aws-java-sdk-s3-1.11.832.jar -o aws-java-sdk-s3-1.11.832.jar && \
+    curl -fSL https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk/1.11.832/aws-java-sdk-1.11.832.jar -o aws-java-sdk-1.11.832.jar && \
+    curl -fSL https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-core/1.11.832/aws-java-sdk-core-1.11.832.jar -o aws-java-sdk-core-1.11.832.jar && \
+    curl -fSL https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-dynamodb/1.11.832/aws-java-sdk-dynamodb-1.11.832.jar -o aws-java-sdk-dynamodb-1.11.832.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/commons/commons-pool2/2.8.0/commons-pool2-2.8.0.jar -o commons-pool2-2.8.0.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.2.0/hadoop-aws-3.2.0.jar -o hadoop-aws-3.2.0.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/spark/spark-avro_2.12/3.0.0/spark-avro_2.12-3.0.0.jar -o spark-avro_2.12-3.0.0.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/spark/spark-token-provider-kafka-0-10_2.12/3.0.0/spark-token-provider-kafka-0-10_2.12-3.0.0.jar -o spark-token-provider-kafka-0-10_2.12-3.0.0.jar && \
+    curl -fSL https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar -o jets3t-0.9.0.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/spark/spark-sql-kafka-0-10_2.12/3.0.0/spark-sql-kafka-0-10_2.12-3.0.0.jar -o spark-sql-kafka-0-10_2.12-3.0.0.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/kafka/kafka-clients/2.2.0/kafka-clients-2.2.0.jar -o kafka-clients-2.2.0.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/spark/spark-streaming-kafka-0-10_2.12/3.0.0/spark-streaming-kafka-0-10_2.12-3.0.0.jar -o spark-streaming-kafka-0-10_2.12-3.0.0.jar && \
+    curl -fSL https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.13.0/jmx_prometheus_javaagent-0.13.0.jar -o jmx_prometheus_javaagent-0.13.0.jar && \
+    mv elasticsearch-hadoop-core-7.8.1_3.0.0.jar /opt/spark/jars/ && \
+    mv elasticsearch-hadoop-mr-7.8.1_3.0.0.jar /opt/spark/jars/ && \
+    mv elasticsearch-hadoop-sql-7.8.1_3.0.0.jar /opt/spark/jars/ && \
+    mv aws-java-sdk-s3-1.11.832.jar /opt/spark/jars/ && \
+    mv aws-java-sdk-1.11.832.jar /opt/spark/jars/ && \
+    mv aws-java-sdk-core-1.11.832.jar /opt/spark/jars/ && \
+    mv aws-java-sdk-dynamodb-1.11.832.jar /opt/spark/jars/ && \
+    mv commons-pool2-2.8.0.jar /opt/spark/jars/ && \
+    mv hadoop-aws-3.2.0.jar /opt/spark/jars/ && \
+    mv spark-avro_2.12-3.0.0.jar /opt/spark/jars/ && \
+    mv spark-token-provider-kafka-0-10_2.12-3.0.0.jar /opt/spark/jars/ && \
+    mv jets3t-0.9.0.jar /opt/spark/jars/ && \
+    mv spark-sql-kafka-0-10_2.12-3.0.0.jar /opt/spark/jars/ && \
+    mv kafka-clients-2.2.0.jar /opt/spark/jars/ && \
+    mv spark-streaming-kafka-0-10_2.12-3.0.0.jar /opt/spark/jars/ && \
+    mv jmx_prometheus_javaagent-0.13.0.jar /opt/spark/jars/
+
+
+
+ENTRYPOINT [ "/opt/entrypoint.sh" ]
+
+# Specify the User that the actual main process will run as
+USER ${spark_uid}

Jenkinsfile

@@ -0,0 +1,105 @@
+@Library('jenkins_lib')_
+pipeline
+ {
+    agent {label 'slave'}
+
+    environment {
+
+        project = "apache-spark";
+        buildNum = currentBuild.getNumber() ;
+        //ex. like feat, release, fix
+        buildType = BRANCH_NAME.split("/").first();
+        //ex. like OP-<User-Story ID>
+        branchVersion = BRANCH_NAME.split("/").last().toUpperCase();
+        // Define global environment variables in this section
+    }
+
+    stages {
+        stage("Define Release version") {
+            steps {
+                script {
+                    //Global Lib for Environment Versions Definition
+                    versionDefine()
+                    env.GUAVUS_SPARK_VERSION = "${VERSION}".split(",").first();
+                    env.GUAVUS_DOCKER_VERSION = "${VERSION}".split(",").last();
+                    env.dockerTag = "${GUAVUS_SPARK_VERSION}-hadoop3.2-${GUAVUS_DOCKER_VERSION}-${RELEASE}" 
+                    echo "GUAVUS_SPARK_VERSION : ${GUAVUS_SPARK_VERSION}"
+                    echo "GUAVUS_DOCKER_VERSION : ${GUAVUS_DOCKER_VERSION}"
+                    echo "DOCKER TAG : ${dockerTag}"
+                }
+            }
+        }
+
+        stage("Versioning") {
+            steps {
+                echo "GUAVUS_SPARK_VERSION : ${GUAVUS_SPARK_VERSION}"
+                echo "GUAVUS_DOCKER_VERSION : ${GUAVUS_DOCKER_VERSION}"
+                sh 'mvn versions:set -DnewVersion=${GUAVUS_SPARK_VERSION}'
+            }
+        }
+
+        stage("Initialize Variable") {
+            steps {
+                script {
+                    PUSH_JAR = false;
+                    PUSH_DOCKER = false;
+                    DOCKER_IMAGE_NAME = "spark-opsiq";
+                    longCommit = sh(returnStdout: true, script: "git rev-parse HEAD").trim()
+
+                    if( env.buildType in ['release'] )
+                    {
+                        PUSH_JAR = true;
+                    }
+                    else if ( env.buildType ==~ /PR-.*/ ) {
+                        PUSH_DOCKER = true
+                    }
+
+                }
+
+            }
+        }
+
+        stage("Push JAR to Maven Artifactory") {
+            when {
+                expression { PUSH_JAR == true }
+            }
+            steps {
+                script {
+                    echo "Pushing JAR to Maven Artifactory"
+                    sh "mvn deploy -U -Dcheckstyle.skip=true -Denforcer.skip=true -DskipTests=true;"
+                }
+            }
+        }
+
+        stage("Build and Push Docker") {
+                    when {
+                        expression { PUSH_DOCKER == true }
+                        }
+                    stages {
+                        stage("Create Docker Image") {
+                            steps {
+                                script {
+                                    echo "Creating docker build..."
+                                    sh "./dev/make-distribution.sh --name guavus_spark-${GUAVUS_SPARK_VERSION}-3.2.0 -Phive -Phive-thriftserver -Pkubernetes -Phadoop-3.2 -Dhadoop.version=3.2.0"
+                                    sh "./dist/bin/docker-image-tool.sh -r artifacts.ggn.in.guavus.com:4244 -t ${GUAVUS_SPARK_VERSION}-hadoop3.2-${GUAVUS_DOCKER_VERSION} build"
+                                    sh "./dist/bin/docker-image-tool.sh -r artifacts.ggn.in.guavus.com:4244 -t ${GUAVUS_SPARK_VERSION}-hadoop3.2-${GUAVUS_DOCKER_VERSION} push"
+                                    sh "docker build -t ${DOCKER_IMAGE_NAME} --build-arg GIT_HEAD=${longCommit} --build-arg GIT_BRANCH=${env.BRANCH_NAME} --build-arg VERSION=${dockerTag} --build-arg BUILD_NUMBER=${env.BUILD_NUMBER} ."
+                                }
+                            }
+                        }
+
+                        stage("PUSH Docker") {
+                            steps {
+                                script {
+                                    echo "Docker PUSH..."
+                                    docker_push( buildType, DOCKER_IMAGE_NAME )
+                                }
+                            }
+                        }
+                    }
+                }
+
+    }
+
+ }
+

assembly/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

common/kvstore/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

common/network-common/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

common/network-shuffle/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

common/network-yarn/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

common/sketch/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

common/tags/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

common/unsafe/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

core/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

docs/structured-streaming-programming-guide.md

@@ -1674,6 +1674,11 @@ Any of the stateful operation(s) after any of below stateful operations can have
 As Spark cannot check the state function of `mapGroupsWithState`/`flatMapGroupsWithState`, Spark assumes that the state function
 emits late rows if the operator uses Append mode.
 
+Spark provides two ways to check the number of late rows on stateful operators which would help you identify the issue:
+
+1. On Spark UI: check the metrics in "CountLateRows" node in query execution details page in SQL tab
+2. On Streaming Query Listener: check "numLateInputRows" in "stateOperators" in QueryProcessEvent
+
 There's a known workaround: split your streaming query into multiple queries per stateful operator, and ensure
 end-to-end exactly once per query. Ensuring end-to-end exactly once for the last query is optional.
 

examples/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

external/avro/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/docker-integration-tests/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kafka-0-10-assembly/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kafka-0-10-sql/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kafka-0-10-token-provider/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kafka-0-10/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kinesis-asl-assembly/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/kinesis-asl/pom.xml

@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

external/spark-ganglia-lgpl/pom.xml

@@ -20,7 +20,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
 

graphx/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

hadoop-cloud/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

launcher/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

mllib-local/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
 

mllib/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-parent_2.12</artifactId>
-    <version>3.0.0</version>
+    <version>3.0.0-1</version>
     <relativePath>../pom.xml</relativePath>
   </parent>