WIP

YanTangZhai · Sep 20, 2014 · bd3ba53 · bd3ba53
1 parent 5c04a5f
commit bd3ba53
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 3 deletions.
diff --git a/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala b/core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala
@@ -350,6 +350,8 @@ private[spark] object PythonRDD extends Logging {
     } catch {
       case eof: EOFException => {}
     }
+    println("RDDDD ==================")
+    println(objs)
     JavaRDD.fromRDD(sc.sc.parallelize(objs, parallelism))
   }
 

diff --git a/examples/src/main/python/streaming/test_oprations.py b/examples/src/main/python/streaming/test_oprations.py
@@ -9,11 +9,15 @@
     conf = SparkConf()
     conf.setAppName("PythonStreamingNetworkWordCount")
     ssc = StreamingContext(conf=conf, duration=Seconds(1))
+    ssc.checkpoint("/tmp/spark_ckp")
 
-    test_input = ssc._testInputStream([1,1,1,1])
-    mapped = test_input.map(lambda x: (x, 1))
-    mapped.pyprint()
+    test_input = ssc._testInputStream([[1],[1],[1]])
+#    ssc.checkpoint("/tmp/spark_ckp")
+    fm_test = test_input.flatMap(lambda x: x.split(" "))
+    mapped_test = fm_test.map(lambda x: (x, 1))
 
+
+    mapped_test.print_()
     ssc.start()
 #    ssc.awaitTermination()
 #    ssc.stop()