Skip to content

Commit

Permalink
Add comment
Browse files Browse the repository at this point in the history
  • Loading branch information
CTTY committed Nov 11, 2023
1 parent 42ce7b6 commit dc97350
Showing 1 changed file with 2 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,7 @@ private List<String> getPartitionPathWithPathPrefixUsingFilterExpression(String

// List all directories in parallel
engineContext.setJobStatus(this.getClass().getSimpleName(), "Listing all partitions with prefix " + relativePathPrefix);
// Need to use serializable file status here, see HUDI-5936
List<HoodieSerializableFileStatus> dirToFileListing = engineContext.flatMap(pathsToList, path -> {
FileSystem fileSystem = path.getFileSystem(hadoopConf.get());
return Arrays.stream(HoodieSerializableFileStatus.fromFileStatuses(fileSystem.listStatus(path)));
Expand Down Expand Up @@ -232,6 +233,7 @@ public Map<String, FileStatus[]> getAllFilesInPartitions(Collection<String> part
int parallelism = Math.min(DEFAULT_LISTING_PARALLELISM, partitionPaths.size());

engineContext.setJobStatus(this.getClass().getSimpleName(), "Listing all files in " + partitionPaths.size() + " partitions");
// Need to use serializable file status here, see HUDI-5936
List<Pair<String, HoodieSerializableFileStatus[]>> partitionToFiles = engineContext.map(new ArrayList<>(partitionPaths), partitionPathStr -> {
Path partitionPath = new Path(partitionPathStr);
FileSystem fs = partitionPath.getFileSystem(hadoopConf.get());
Expand Down

0 comments on commit dc97350

Please sign in to comment.