Skip to content

Commit

Permalink
[SPARK-4774] [SQL] Makes HiveFromSpark more portable
Browse files Browse the repository at this point in the history
HiveFromSpark read the kv1.txt file from
SPARK_HOME/examples/src/main/resources/kv1.txt which assumed
you had a source tree checked out. Now we copy the
kv1.txt file to a temporary file and delete it when
the jvm shutsdown. This allows us to run this example
outside of a spark source tree.
  • Loading branch information
Kostas Sakellis committed Dec 6, 2014
1 parent 6eb1b6f commit 6770f83
Showing 1 changed file with 11 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,17 +17,26 @@

package org.apache.spark.examples.sql.hive

import com.google.common.io.{ByteStreams, Files}

import java.io.File

import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql._
import org.apache.spark.sql.hive.HiveContext

object HiveFromSpark {
case class Record(key: Int, value: String)

// Copy kv1.txt file from classpath to temporary directory
val kv1Stream = HiveFromSpark.getClass.getResourceAsStream("/kv1.txt")
val kv1File = File.createTempFile("kv1", "txt")
kv1File.deleteOnExit()
ByteStreams.copy(kv1Stream, Files.newOutputStreamSupplier(kv1File))

def main(args: Array[String]) {
val sparkConf = new SparkConf().setAppName("HiveFromSpark")
val sc = new SparkContext(sparkConf)
val path = s"${System.getenv("SPARK_HOME")}/examples/src/main/resources/kv1.txt"

// A hive context adds support for finding tables in the MetaStore and writing queries
// using HiveQL. Users who do not have an existing Hive deployment can still create a
Expand All @@ -37,7 +46,7 @@ object HiveFromSpark {
import hiveContext._

sql("CREATE TABLE IF NOT EXISTS src (key INT, value STRING)")
sql(s"LOAD DATA LOCAL INPATH '$path' INTO TABLE src")
sql(s"LOAD DATA LOCAL INPATH '${kv1File.getAbsolutePath}' INTO TABLE src")

// Queries are expressed in HiveQL
println("Result of 'SELECT *': ")
Expand Down

0 comments on commit 6770f83

Please sign in to comment.