Skip to content

Commit

Permalink
Use null for the implicit Ordering
Browse files Browse the repository at this point in the history
  • Loading branch information
zsxwing committed Dec 20, 2014
1 parent 734bac9 commit e4ad8b5
Showing 1 changed file with 11 additions and 13 deletions.
24 changes: 11 additions & 13 deletions core/src/main/scala/org/apache/spark/rdd/RDD.scala
Original file line number Diff line number Diff line change
Expand Up @@ -1175,18 +1175,19 @@ abstract class RDD[T: ClassTag](
*/
def saveAsTextFile(path: String) {
// https://issues.apache.org/jira/browse/SPARK-2075
// NullWritable is a Comparable rather than Comparable[NullWritable] in Hadoop 1.+,
// so the compiler cannot find an implicit Ordering for it and will use the default `null`.
// It will generate different anonymous classes for `saveAsTextFile` in Hadoop 1.+ and
// Hadoop 2.+. Therefore, here we provide an Ordering for NullWritable so that the compiler
// will generate same bytecode.
val nullWritableOrdering = new Ordering[NullWritable] {
override def compare(x: NullWritable, y: NullWritable): Int = 0
}
//
// NullWritable is a `Comparable` in Hadoop 1.+, so the compiler cannot find an implicit
// Ordering for it and will use the default `null`. However, it's a `Comparable[NullWritable]`
// in Hadoop 2.+, so the compiler will call the implicit `Ordering.ordered` method to create an
// Ordering for `NullWritable`. That's why the compiler will generate different anonymous
// classes for `saveAsTextFile` in Hadoop 1.+ and Hadoop 2.+.
//
// Therefore, here we provide an explicit Ordering `null` to make sure the compiler generate
// same bytecodes for `saveAsTextFile`.
val nullWritableClassTag = implicitly[ClassTag[NullWritable]]
val textClassTag = implicitly[ClassTag[Text]]
val r = this.map(x => (NullWritable.get(), new Text(x.toString)))
RDD.rddToPairRDDFunctions(r)(nullWritableClassTag, textClassTag, nullWritableOrdering)
RDD.rddToPairRDDFunctions(r)(nullWritableClassTag, textClassTag, null)
.saveAsHadoopFile[TextOutputFormat[NullWritable, Text]](path)
}

Expand All @@ -1195,13 +1196,10 @@ abstract class RDD[T: ClassTag](
*/
def saveAsTextFile(path: String, codec: Class[_ <: CompressionCodec]) {
// https://issues.apache.org/jira/browse/SPARK-2075
val nullWritableOrdering = new Ordering[NullWritable] {
override def compare(x: NullWritable, y: NullWritable): Int = 0
}
val nullWritableClassTag = implicitly[ClassTag[NullWritable]]
val textClassTag = implicitly[ClassTag[Text]]
val r = this.map(x => (NullWritable.get(), new Text(x.toString)))
RDD.rddToPairRDDFunctions(r)(nullWritableClassTag, textClassTag, nullWritableOrdering)
RDD.rddToPairRDDFunctions(r)(nullWritableClassTag, textClassTag, null)
.saveAsHadoopFile[TextOutputFormat[NullWritable, Text]](path, codec)
}

Expand Down

0 comments on commit e4ad8b5

Please sign in to comment.