Support INF and +Infinity when allowNonNumericNumbers is off.

apache · May 23, 2016 · af1e3a1 · af1e3a1
1 parent 025edea
commit af1e3a1
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 12 deletions.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JacksonParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JacksonParser.scala
@@ -131,9 +131,10 @@ object JacksonParser extends Logging {
         val value = parser.getText
         if (value.equals("NaN") ||
           value.equals("Infinity") ||
+          value.equals("+Infinity") ||
           value.equals("-Infinity")) {
           value.toFloat
-        } else if (value.equals("+INF")) {
+        } else if (value.equals("+INF") || value.equals("INF")) {
           Float.PositiveInfinity
         } else if (value.equals("-INF")) {
           Float.NegativeInfinity
@@ -149,9 +150,10 @@ object JacksonParser extends Logging {
         val value = parser.getText
         if (value.equals("NaN") ||
           value.equals("Infinity") ||
+          value.equals("+Infinity") ||
           value.equals("-Infinity")) {
           value.toDouble
-        } else if (value.equals("+INF")) {
+        } else if (value.equals("+INF") || value.equals("INF")) {
           Double.PositiveInfinity
         } else if (value.equals("-INF")) {
           Double.NegativeInfinity

diff --git a/.../test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala b/.../test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala
@@ -97,7 +97,8 @@ class JsonParsingOptionsSuite extends QueryTest with SharedSQLContext {
   test("allowNonNumericNumbers off") {
     // non-quoted non-numeric numbers don't work if allowNonNumericNumbers is off.
     var testCases: Seq[String] = Seq("""{"age": NaN}""", """{"age": Infinity}""",
-      """{"age": -Infinity}""", """{"age": +INF}""", """{"age": -INF}""")
+      """{"age": +Infinity}""", """{"age": -Infinity}""", """{"age": INF}""",
+      """{"age": +INF}""", """{"age": -INF}""")
     testCases.foreach { str =>
       val rdd = spark.sparkContext.parallelize(Seq(str))
       val df = spark.read.option("allowNonNumericNumbers", "false").json(rdd)
@@ -106,10 +107,11 @@ class JsonParsingOptionsSuite extends QueryTest with SharedSQLContext {
     }
 
     // quoted non-numeric numbers should still work even allowNonNumericNumbers is off.
-    testCases = Seq("""{"age": "NaN"}""", """{"age": "Infinity"}""", """{"age": "-Infinity"}""",
-      """{"age": "+INF"}""", """{"age": "-INF"}""")
-    val tests: Seq[Double => Boolean] = Seq(_.isNaN, _.isPosInfinity, _.isNegInfinity,
-      _.isPosInfinity, _.isNegInfinity)
+    testCases = Seq("""{"age": "NaN"}""", """{"age": "Infinity"}""", """{"age": "+Infinity"}""",
+      """{"age": "-Infinity"}""", """{"age": "INF"}""", """{"age": "+INF"}""",
+      """{"age": "-INF"}""")
+    val tests: Seq[Double => Boolean] = Seq(_.isNaN, _.isPosInfinity, _.isPosInfinity,
+      _.isNegInfinity, _.isPosInfinity, _.isPosInfinity, _.isNegInfinity)
     val schema = StructType(StructField("age", DoubleType, true) :: Nil)
 
     testCases.zipWithIndex.foreach { case (str, idx) =>
@@ -123,11 +125,12 @@ class JsonParsingOptionsSuite extends QueryTest with SharedSQLContext {
 
   test("allowNonNumericNumbers on") {
     val testCases: Seq[String] = Seq("""{"age": NaN}""", """{"age": Infinity}""",
-      """{"age": -Infinity}""", """{"age": +INF}""", """{"age": -INF}""", """{"age": "NaN"}""",
-      """{"age": "Infinity"}""", """{"age": "-Infinity"}""")
-    val tests: Seq[Double => Boolean] = Seq(_.isNaN, _.isPosInfinity, _.isNegInfinity,
-      _.isPosInfinity, _.isNegInfinity, _.isNaN, _.isPosInfinity, _.isNegInfinity,
-      _.isPosInfinity, _.isNegInfinity)
+      """{"age": +Infinity}""", """{"age": -Infinity}""", """{"age": +INF}""",
+      """{"age": -INF}""", """{"age": "NaN"}""", """{"age": "Infinity"}""",
+      """{"age": "-Infinity"}""")
+    val tests: Seq[Double => Boolean] = Seq(_.isNaN, _.isPosInfinity, _.isPosInfinity,
+      _.isNegInfinity, _.isPosInfinity, _.isNegInfinity, _.isNaN, _.isPosInfinity,
+      _.isNegInfinity, _.isPosInfinity, _.isNegInfinity)
     val schema = StructType(StructField("age", DoubleType, true) :: Nil)
     testCases.zipWithIndex.foreach { case (str, idx) =>
       val rdd = spark.sparkContext.parallelize(Seq(str))