From b692868af13a24b5e75a300d4a0ad9907b9a2512 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Wed, 26 Mar 2014 15:12:19 -0700 Subject: [PATCH] Revert --- .../org/apache/spark/mllib/linalg/SVD.scala | 14 ----------- .../apache/spark/mllib/linalg/SVDSuite.scala | 25 +++++++++---------- 2 files changed, 12 insertions(+), 27 deletions(-) diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/SVD.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/SVD.scala index 9157eb9db7791..d5b7d9f855f03 100644 --- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/SVD.scala +++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/SVD.scala @@ -118,7 +118,6 @@ class SVD { val m = matrix.m val n = matrix.n - println("matrix "+matrix) if (m < n || m <= 0 || n <= 0) { throw new IllegalArgumentException("Expecting a tall and skinny matrix m=$m n=$n") } @@ -130,7 +129,6 @@ class SVD { val rowIndices = matrix.rows.map(_.i) // compute SVD - println("Computing SVD") val (u, sigma, v) = denseSVD(matrix.rows.map(_.data)) if (computeU) { @@ -288,8 +286,6 @@ class SVD { // Compute A^T A, assuming rows are sparse enough to fit in memory val rows = data.map(entry => (entry.i, (entry.j, entry.mval))).groupByKey() - println("rows "+rows) - rows.foreach{e => println("r "+e)} val emits = rows.flatMap { case (rowind, colsI) => val cols = colsI.toList @@ -301,26 +297,18 @@ class SVD { } } }.reduceByKey(_ + _) - println("emits ") - emits.foreach{e => println("e "+e)} // Construct jblas A^T A locally val ata = DoubleMatrix.zeros(n, n) for (entry <- emits.collect()) { ata.put(entry._1._1, entry._1._2, entry._2) } - println("ata "+ata) // Since A^T A is small, we can compute its SVD directly val svd = Singular.sparseSVD(ata) - println("init svd") - svd.foreach{x => println(x)} - println("done") val V = svd(0) - println("V" + V) // This will be updated to rcond val sigmas = MatrixFunctions.sqrt(svd(1)).toArray.filter(x => x > 1e-9) - println("Sigmas "+sigmas) if (sigmas.size < k) { throw new Exception("Not enough singular values to return k=" + k + " s=" + sigmas.size) @@ -354,8 +342,6 @@ class SVD { // Multiply A by VS^-1 val aCols = data.map(entry => (entry.j, (entry.i, entry.mval))) val bRows = vsirdd.map(entry => (entry._1._1, (entry._1._2, entry._2))) - aCols.foreach{x => println("a "+x)} - bRows.foreach{x => println("b "+x)} val retUdata = aCols.join(bRows).map { case (key, ((rowInd, rowVal), (colInd, colVal))) => ((rowInd, colInd), rowVal * colVal) diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/SVDSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/SVDSuite.scala index 3a9811a72240b..20e2b0f84be06 100644 --- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/SVDSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/SVDSuite.scala @@ -87,17 +87,17 @@ class SVDSuite extends FunSuite with BeforeAndAfterAll { val retu = getDenseMatrix(u) val rets = getDenseMatrix(s) val retv = getDenseMatrix(v) - - - // check individual decomposition + + + // check individual decomposition assertMatrixApproximatelyEquals(retu, svd(0)) assertMatrixApproximatelyEquals(rets, DoubleMatrix.diag(svd(1))) assertMatrixApproximatelyEquals(retv, svd(2)) // check multiplication guarantee - assertMatrixApproximatelyEquals(retu.mmul(rets).mmul(retv.transpose), denseA) + assertMatrixApproximatelyEquals(retu.mmul(rets).mmul(retv.transpose), denseA) } -/* + test("dense full rank matrix svd") { val m = 10 val n = 3 @@ -120,7 +120,7 @@ class SVDSuite extends FunSuite with BeforeAndAfterAll { val retv = new DoubleMatrix(v) - // check individual decomposition + // check individual decomposition assertMatrixApproximatelyEquals(retu, svd(0)) assertMatrixApproximatelyEquals(rets, DoubleMatrix.diag(svd(1))) assertMatrixApproximatelyEquals(retv, svd(2)) @@ -131,7 +131,7 @@ class SVDSuite extends FunSuite with BeforeAndAfterAll { test("rank one matrix svd") { val m = 10 - val n = 3 + val n = 3 val data = sc.makeRDD(Array.tabulate(m, n){ (a,b) => MatrixEntry(a, b, 1.0) }.flatten ) val k = 1 @@ -153,13 +153,13 @@ class SVDSuite extends FunSuite with BeforeAndAfterAll { val rets = getDenseMatrix(s) val retv = getDenseMatrix(v) - // check individual decomposition + // check individual decomposition assertMatrixApproximatelyEquals(retu, svd(0).getColumn(0)) assertMatrixApproximatelyEquals(rets, DoubleMatrix.diag(svd(1).getRow(0))) assertMatrixApproximatelyEquals(retv, svd(2).getColumn(0)) // check multiplication guarantee - assertMatrixApproximatelyEquals(retu.mmul(rets).mmul(retv.transpose), denseA) + assertMatrixApproximatelyEquals(retu.mmul(rets).mmul(retv.transpose), denseA) } test("truncated with k") { @@ -168,7 +168,7 @@ class SVDSuite extends FunSuite with BeforeAndAfterAll { val data = sc.makeRDD(Array.tabulate(m,n){ (a, b) => MatrixEntry(a, b, (a + 2).toDouble * (b + 1)/(1 + a + b)) }.flatten ) val a = SparseMatrix(data, m, n) - + val k = 1 // only one svalue above this val decomposed = new SVD().setK(k).compute(a) @@ -185,11 +185,10 @@ class SVDSuite extends FunSuite with BeforeAndAfterAll { val retv = getDenseMatrix(v) assert(retrank == 1, "rank returned not one") - - // check individual decomposition + + // check individual decomposition assertMatrixApproximatelyEquals(retu, svd(0).getColumn(0)) assertMatrixApproximatelyEquals(rets, DoubleMatrix.diag(svd(1).getRow(0))) assertMatrixApproximatelyEquals(retv, svd(2).getColumn(0)) } - */ }