Skip to content

Commit

Permalink
[MINOR] Add detailed error logs in RunCompactionProcedure (apache#10070)
Browse files Browse the repository at this point in the history
* add detailed error logs in RunCompactionProcedure
* only print 100 error file paths into logs
  • Loading branch information
beyond1920 authored Nov 15, 2023
1 parent 2185abb commit f218e54
Showing 1 changed file with 9 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -146,10 +146,15 @@ class RunCompactionProcedure extends BaseProcedure with ProcedureBuilder with Sp

private def handleResponse(metadata: HoodieCommitMetadata): Unit = {
// Handle error
val writeStats = metadata.getPartitionToWriteStats.entrySet().flatMap(e => e.getValue).toList
val errorsCount = writeStats.map(state => state.getTotalWriteErrors).sum
if (errorsCount > 0) {
throw new HoodieException(s" Found $errorsCount when writing record")
val writeStatsHasErrors = metadata.getPartitionToWriteStats.entrySet()
.flatMap(e => e.getValue)
.filter(_.getTotalWriteErrors > 0)
if (writeStatsHasErrors.nonEmpty) {
val errorsCount = writeStatsHasErrors.map(_.getTotalWriteErrors).sum
log.error(s"Found $errorsCount when writing record.\n Printing out the top 100 file path with errors.")
writeStatsHasErrors.take(100).foreach(state =>
log.error(s"Error occurred while writing the file: ${state.getPath}."))
throw new HoodieException(s"Found $errorsCount when writing record")
}
}

Expand Down

0 comments on commit f218e54

Please sign in to comment.