Skip to content

Commit

Permalink
Update hive example to avoid conflict (apache#38)
Browse files Browse the repository at this point in the history
  • Loading branch information
Baohe Zhang authored and Dhruve Ashar committed Aug 25, 2020
1 parent b62ce89 commit 7514aee
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 5 deletions.
1 change: 1 addition & 0 deletions src/main/resources/oozie/spark_hive/workflow.xml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
<spark-opts>--num-executors 3 --executor-memory 2g --executor-cores 1 --queue default --conf spark.yarn.security.tokens.hive.enabled=false</spark-opts>
<arg>hdfs:///user/${wf:conf('user.name')}/${wfRoot}/output/hive_location</arg>
<arg>hdfs:///user/${wf:conf('user.name')}/${wfRoot}/output/kv1.txt</arg>
<arg>${sparkTag}_hive_test</arg>
</spark>
<ok to="end" />
<error to="fail" />
Expand Down
11 changes: 6 additions & 5 deletions src/main/scala/com/yahoo/spark/starter/SparkHiveExample.scala
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,14 @@ case class Record(key: Int, value: String)
object SparkHiveExample {
def main(args: Array[String]) {

if (args.length < 2) {
System.err.println("Usage: SparkHiveExample <databaseDir> <inputFile>")
if (args.length < 3) {
System.err.println("Usage: SparkHiveExample <databaseDir> <inputFile> <database>")
System.exit(1)
}

val databaseDir = args(0)
val inputFile = args(1)
val database = args(2)

val spark = SparkSession
.builder()
Expand All @@ -27,9 +28,9 @@ object SparkHiveExample {
import spark.sql

// Create a database under hdfs home directory for testing purpose
sql(s"CREATE DATABASE IF NOT EXISTS spark_hive_test LOCATION '$databaseDir'")
sql(s"CREATE DATABASE IF NOT EXISTS $database LOCATION '$databaseDir'")

sql("USE spark_hive_test")
sql(s"USE $database")

// Create the table and load data from a text file on hdfs
sql("CREATE TABLE IF NOT EXISTS src (key INT, value STRING) USING hive")
Expand Down Expand Up @@ -113,7 +114,7 @@ object SparkHiveExample {
// ...

// Drop the database
sql("DROP DATABASE IF EXISTS spark_hive_test CASCADE")
sql(s"DROP DATABASE IF EXISTS $database CASCADE")

spark.stop()
}
Expand Down

0 comments on commit 7514aee

Please sign in to comment.