Skip to content

Commit

Permalink
Assert diff app filters to be diff value
Browse files Browse the repository at this point in the history
  • Loading branch information
EnricoMi committed Sep 21, 2023
1 parent cd8705d commit 7a46941
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 0 deletions.
5 changes: 5 additions & 0 deletions src/main/scala/uk/co/gresearch/spark/diff/App.scala
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,11 @@ object App {
case Some(options) => options
case None => sys.exit(1)
}
val unknownFilters = options.filter.filter(filter => !options.diffOptions.diffValues.contains(filter))
if (unknownFilters.nonEmpty) {
throw new RuntimeException(s"Filter ${unknownFilters.mkString("'", "', '", "'")} not allowed, " +
s"these are the configured diff values: ${options.diffOptions.diffValues.mkString("'", "', '", "'")}")
}

// create spark session
val spark = SparkSession.builder()
Expand Down
25 changes: 25 additions & 0 deletions src/test/scala/uk/co/gresearch/spark/diff/AppSuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -83,4 +83,29 @@ class AppSuite extends AnyFunSuite with SparkTestSession {
}
}
}

test(s"run app with unknown filter") {
withTempPath { path =>
// write left dataframe as parquet
val leftPath = new File(path, "left.parquet").getAbsolutePath
DiffSuite.left(spark).write.parquet(leftPath)

// write right dataframe as csv
val rightPath = new File(path, "right.parquet").getAbsolutePath
DiffSuite.right(spark).write.parquet(rightPath)

// launch app
val outputPath = new File(path, "diff.parquet").getAbsolutePath
assertThrows[RuntimeException](
App.main(Array(
"--format", "parquet",
"--id", "id",
"--filter", "A",
leftPath,
rightPath,
outputPath
))
)
}
}
}

0 comments on commit 7a46941

Please sign in to comment.