Skip to content

Commit

Permalink
Explicitly set the implicit parameters
Browse files Browse the repository at this point in the history
  • Loading branch information
zsxwing committed Dec 19, 2014
1 parent ca03559 commit 734bac9
Showing 1 changed file with 14 additions and 7 deletions.
21 changes: 14 additions & 7 deletions core/src/main/scala/org/apache/spark/rdd/RDD.scala
Original file line number Diff line number Diff line change
Expand Up @@ -1176,13 +1176,17 @@ abstract class RDD[T: ClassTag](
def saveAsTextFile(path: String) {
// https://issues.apache.org/jira/browse/SPARK-2075
// NullWritable is a Comparable rather than Comparable[NullWritable] in Hadoop 1.+,
// so the compiler cannot find an implicit Ordering for it. It will generate different
// anonymous classes for `saveAsTextFile` in Hadoop 1.+ and Hadoop 2.+. Therefore, here we
// provide an Ordering for NullWritable so that the compiler will generate same codes.
implicit val nullWritableOrdering = new Ordering[NullWritable] {
// so the compiler cannot find an implicit Ordering for it and will use the default `null`.
// It will generate different anonymous classes for `saveAsTextFile` in Hadoop 1.+ and
// Hadoop 2.+. Therefore, here we provide an Ordering for NullWritable so that the compiler
// will generate same bytecode.
val nullWritableOrdering = new Ordering[NullWritable] {
override def compare(x: NullWritable, y: NullWritable): Int = 0
}
this.map(x => (NullWritable.get(), new Text(x.toString)))
val nullWritableClassTag = implicitly[ClassTag[NullWritable]]
val textClassTag = implicitly[ClassTag[Text]]
val r = this.map(x => (NullWritable.get(), new Text(x.toString)))
RDD.rddToPairRDDFunctions(r)(nullWritableClassTag, textClassTag, nullWritableOrdering)
.saveAsHadoopFile[TextOutputFormat[NullWritable, Text]](path)
}

Expand All @@ -1191,10 +1195,13 @@ abstract class RDD[T: ClassTag](
*/
def saveAsTextFile(path: String, codec: Class[_ <: CompressionCodec]) {
// https://issues.apache.org/jira/browse/SPARK-2075
implicit val nullWritableOrdering = new Ordering[NullWritable] {
val nullWritableOrdering = new Ordering[NullWritable] {
override def compare(x: NullWritable, y: NullWritable): Int = 0
}
this.map(x => (NullWritable.get(), new Text(x.toString)))
val nullWritableClassTag = implicitly[ClassTag[NullWritable]]
val textClassTag = implicitly[ClassTag[Text]]
val r = this.map(x => (NullWritable.get(), new Text(x.toString)))
RDD.rddToPairRDDFunctions(r)(nullWritableClassTag, textClassTag, nullWritableOrdering)
.saveAsHadoopFile[TextOutputFormat[NullWritable, Text]](path, codec)
}

Expand Down

0 comments on commit 734bac9

Please sign in to comment.