Find Unlinked files should ignore Thumbs.db, etc (#8800)

Co-authored-by: Christoph <siedlerkiller@gmail.com>
Co-authored-by: ThiloteE <73715071+ThiloteE@users.noreply.github.com>
Co-authored-by: “zhaoqingying123“ <710870305@qq.com>
Co-authored-by: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com>
Co-authored-by: Oliver Kopp <kopp.dev@gmail.com>

CHANGELOG.md

@@ -13,6 +13,7 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve
 
 - We added a fetcher for [Biodiversity Heritage Library](https://www.biodiversitylibrary.org/). [8539](https://github.com/JabRef/jabref/issues/8539)
 - We added support for multiple messages in the snackbar. [#7340](https://github.com/JabRef/jabref/issues/7340)
+- We added an extra option in the 'Find Unlinked Files' dialog view to ignore unnecessary files like Thumbs.db, DS_Store, etc. [koppor#373](https://github.com/koppor/jabref/issues/373)
 - JabRef now writes log files. Linux: `$home/.cache/jabref/logs/version`, Windows: `%APPDATA%\..\Local\harawata\jabref\version\logs`, Mac: `Users/.../Library/Logs/jabref/version`
 - We added an importer for Citavi backup files, support ".ctv5bak" and ".ctv6bak" file formats. [#8322](https://github.com/JabRef/jabref/issues/8322)
 

src/main/java/org/jabref/gui/externalfiles/ChainedFilters.java

@@ -0,0 +1,35 @@
+package org.jabref.gui.externalfiles;
+
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Path;
+import java.util.Arrays;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Chains the given filters - if ALL of them accept, the result is also accepted
+ */
+public class ChainedFilters implements DirectoryStream.Filter<Path> {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(ChainedFilters.class);
+
+    private DirectoryStream.Filter<Path>[] filters;
+
+    public ChainedFilters(DirectoryStream.Filter<Path>... filters) {
+        this.filters = filters;
+    }
+
+    @Override
+    public boolean accept(Path entry) throws IOException {
+        return Arrays.stream(filters).allMatch(filter -> {
+            try {
+                return filter.accept(entry);
+            } catch (IOException e) {
+                LOGGER.error("Could not apply filter", e);
+                return true;
+            }
+        });
+    }
+}

src/main/java/org/jabref/gui/externalfiles/FileFilterUtils.java

@@ -16,10 +16,10 @@
 public class FileFilterUtils {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(FileFilterUtils.class);
-    
+
     /* Returns the last edited time of a file as LocalDateTime. */
     public static LocalDateTime getFileTime(Path path) {
-        FileTime lastEditedTime = null;
+        FileTime lastEditedTime;
         try {
             lastEditedTime = Files.getLastModifiedTime(path);
         } catch (IOException e) {
@@ -33,28 +33,28 @@ public static LocalDateTime getFileTime(Path path) {
         return localDateTime;
     }
 
-    /* Returns true if a file with a specific path 
+    /* Returns true if a file with a specific path
      * was edited during the last 24 hours. */
     public boolean isDuringLastDay(LocalDateTime fileEditTime) {
         LocalDateTime NOW = LocalDateTime.now(ZoneId.systemDefault());
         return fileEditTime.isAfter(NOW.minusHours(24));
     }
 
-    /* Returns true if a file with a specific path 
+    /* Returns true if a file with a specific path
      * was edited during the last 7 days. */
     public boolean isDuringLastWeek(LocalDateTime fileEditTime) {
         LocalDateTime NOW = LocalDateTime.now(ZoneId.systemDefault());
         return fileEditTime.isAfter(NOW.minusDays(7));
     }
 
-    /* Returns true if a file with a specific path 
+    /* Returns true if a file with a specific path
      * was edited during the last 30 days. */
     public boolean isDuringLastMonth(LocalDateTime fileEditTime) {
         LocalDateTime NOW = LocalDateTime.now(ZoneId.systemDefault());
         return fileEditTime.isAfter(NOW.minusDays(30));
     }
 
-    /* Returns true if a file with a specific path 
+    /* Returns true if a file with a specific path
      * was edited during the last 365 days. */
     public boolean isDuringLastYear(LocalDateTime fileEditTime) {
         LocalDateTime NOW = LocalDateTime.now(ZoneId.systemDefault());
@@ -75,8 +75,10 @@ public static boolean filterByDate(Path path, DateRange filter) {
         return isInDateRange;
     }
 
-    /* Sorts a list of Path objects according to the last edited date 
-     * of their corresponding files, from newest to oldest. */
+    /**
+     * Sorts a list of Path objects according to the last edited date
+     * of their corresponding files, from newest to oldest.
+     */
     public List<Path> sortByDateAscending(List<Path> files) {
         return files.stream()
                 .sorted(Comparator.comparingLong(file -> FileFilterUtils.getFileTime(file)
@@ -86,8 +88,10 @@ public List<Path> sortByDateAscending(List<Path> files) {
                 .collect(Collectors.toList());
     }
 
-    /* Sorts a list of Path objects according to the last edited date 
-     * of their corresponding files, from oldest to newest. */
+    /**
+     * Sorts a list of Path objects according to the last edited date
+     * of their corresponding files, from oldest to newest.
+     */
     public List<Path> sortByDateDescending(List<Path> files) {
         return files.stream()
                 .sorted(Comparator.comparingLong(file -> -FileFilterUtils.getFileTime(file)
@@ -97,8 +101,10 @@ public List<Path> sortByDateDescending(List<Path> files) {
                 .collect(Collectors.toList());
     }
 
-    /* Sorts a list of Path objects according to the last edited date
-     * the order depends on the specified sorter type. */
+    /**
+     * Sorts a list of Path objects according to the last edited date
+     * the order depends on the specified sorter type.
+     */
     public static List<Path> sortByDate(List<Path> files, ExternalFileSorter sortType) {
         FileFilterUtils fileFilter = new FileFilterUtils();
         List<Path> sortedFiles = switch (sortType) {

src/main/java/org/jabref/gui/externalfiles/GitIgnoreFileFilter.java

@@ -0,0 +1,66 @@
+package org.jabref.gui.externalfiles;
+
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.PathMatcher;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.util.function.Predicate.not;
+
+public class GitIgnoreFileFilter implements DirectoryStream.Filter<Path> {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(GitIgnoreFileFilter.class);
+
+    private Set<PathMatcher> gitIgnorePatterns;
+
+    public GitIgnoreFileFilter(Path path) {
+        Path currentPath = path;
+        while ((currentPath != null) && !Files.exists(currentPath.resolve(".gitignore"))) {
+            currentPath = currentPath.getParent();
+        }
+        if (currentPath == null) {
+            // we did not find any gitignore, lets use the default
+            gitIgnorePatterns = Set.of(".git", ".DS_Store", "desktop.ini", "Thumbs.db").stream()
+                                   // duplicate code as below
+                                   .map(line -> "glob:" + line)
+                                   .map(matcherString -> FileSystems.getDefault().getPathMatcher(matcherString))
+                                   .collect(Collectors.toSet());
+        } else {
+            Path gitIgnore = currentPath.resolve(".gitignore");
+            try {
+                Set<PathMatcher> plainGitIgnorePatternsFromGitIgnoreFile = Files.readAllLines(gitIgnore).stream()
+                                                                                .map(line -> line.trim())
+                                                                                .filter(not(String::isEmpty))
+                                                                                .filter(line -> !line.startsWith("#"))
+                                                                                // convert to Java syntax for Glob patterns
+                                                                                .map(line -> "glob:" + line)
+                                                                                .map(matcherString -> FileSystems.getDefault().getPathMatcher(matcherString))
+                                                                                .collect(Collectors.toSet());
+                gitIgnorePatterns = new HashSet<>(plainGitIgnorePatternsFromGitIgnoreFile);
+                // we want to ignore ".gitignore" itself
+                gitIgnorePatterns.add(FileSystems.getDefault().getPathMatcher("glob:.gitignore"));
+            } catch (IOException e) {
+                LOGGER.info("Could not read .gitignore from {}", gitIgnore, e);
+                gitIgnorePatterns = Set.of();
+            }
+        }
+    }
+
+    @Override
+    public boolean accept(Path path) throws IOException {
+        // We assume that git does not stop at a patern, but tries all. We implement that behavior
+        return gitIgnorePatterns.stream().noneMatch(filter ->
+                // we need this one for "*.png"
+                filter.matches(path.getFileName()) ||
+                // we need this one for "**/*.png"
+                filter.matches(path));
+    }
+}

src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesCrawler.java

@@ -66,52 +66,76 @@ protected FileNodeViewModel call() throws IOException {
      * 'state' must be set to 1, to keep the recursion running. When the states value changes, the method will resolve
      * its recursion and return what it has saved so far.
      * <br>
-     * The files are filtered according to the {@link DateRange} filter value 
+     * The files are filtered according to the {@link DateRange} filter value
      * and then sorted according to the {@link ExternalFileSorter} value.
      *
+     * @param unlinkedPDFFileFilter contains a BibDatabaseContext which is used to determine whether the file is linked
+     *
+     * @return FileNodeViewModel containing the data of the current directory and all subdirectories
      * @throws IOException if directory is not a directory or empty
      */
-    private FileNodeViewModel searchDirectory(Path directory, UnlinkedPDFFileFilter fileFilter) throws IOException {
+    FileNodeViewModel searchDirectory(Path directory, UnlinkedPDFFileFilter unlinkedPDFFileFilter) throws IOException {
         // Return null if the directory is not valid.
         if ((directory == null) || !Files.isDirectory(directory)) {
             throw new IOException(String.format("Invalid directory for searching: %s", directory));
         }
 
-        FileNodeViewModel parent = new FileNodeViewModel(directory);
-        Map<Boolean, List<Path>> fileListPartition;
+        FileNodeViewModel fileNodeViewModelForCurrentDirectory = new FileNodeViewModel(directory);
 
-        try (Stream<Path> filesStream = StreamSupport.stream(Files.newDirectoryStream(directory, fileFilter).spliterator(), false)) {
-            fileListPartition = filesStream.collect(Collectors.partitioningBy(Files::isDirectory));
+        // Map from isDirectory (true/false) to full path
+        // Result: Contains only files not matching the filter (i.e., PDFs not linked and files not ignored)
+        // Filters:
+        //   1. UnlinkedPDFFileFilter
+        //   2. GitIgnoreFilter
+        ChainedFilters filters = new ChainedFilters(unlinkedPDFFileFilter, new GitIgnoreFileFilter(directory));
+        Map<Boolean, List<Path>> directoryAndFilePartition;
+        try (Stream<Path> filesStream = StreamSupport.stream(Files.newDirectoryStream(directory, filters).spliterator(), false)) {
+            directoryAndFilePartition = filesStream.collect(Collectors.partitioningBy(Files::isDirectory));
         } catch (IOException e) {
-            LOGGER.error(String.format("%s while searching files: %s", e.getClass().getName(), e.getMessage()));
-            return parent;
+            LOGGER.error("Error while searching files", e);
+            return fileNodeViewModelForCurrentDirectory;
         }
+        List<Path> subDirectories = directoryAndFilePartition.get(true);
+        List<Path> files = directoryAndFilePartition.get(false);
 
-        List<Path> subDirectories = fileListPartition.get(true);
-        List<Path> files = new ArrayList<>(fileListPartition.get(false));
-        int fileCount = 0;
+        // at this point, only unlinked PDFs AND unignored files are contained
 
-        for (Path subDirectory : subDirectories) {
-            FileNodeViewModel subRoot = searchDirectory(subDirectory, fileFilter);
+        // initially, we find no files at all
+        int fileCountOfSubdirectories = 0;
 
+        // now we crawl into the found subdirectories first (!)
+        for (Path subDirectory : subDirectories) {
+            FileNodeViewModel subRoot = searchDirectory(subDirectory, unlinkedPDFFileFilter);
             if (!subRoot.getChildren().isEmpty()) {
-                fileCount += subRoot.getFileCount();
-                parent.getChildren().add(subRoot);
+                fileCountOfSubdirectories += subRoot.getFileCount();
+                fileNodeViewModelForCurrentDirectory.getChildren().add(subRoot);
             }
         }
+        // now we have the data of all subdirectories
+        // it is stored in fileNodeViewModelForCurrentDirectory.getChildren()
+
+        // now we handle the files in the current directory
+
         // filter files according to last edited date.
-        List<Path> filteredFiles = new ArrayList<Path>();
+        // Note that we do not use the "StreamSupport.stream" filtering functionality, because refactoring the code to that would lead to more code
+        List<Path> resultingFiles = new ArrayList<>();
         for (Path path : files) {
             if (FileFilterUtils.filterByDate(path, dateFilter)) {
-                filteredFiles.add(path);
+                resultingFiles.add(path);
             }
         }
+
         // sort files according to last edited date.
-        filteredFiles = FileFilterUtils.sortByDate(filteredFiles, sorter);
-        parent.setFileCount(filteredFiles.size() + fileCount);
-        parent.getChildren().addAll(filteredFiles.stream()
+        resultingFiles = FileFilterUtils.sortByDate(resultingFiles, sorter);
+
+        // the count of all files is the count of the found files in current directory plus the count of all files in the subdirectories
+        fileNodeViewModelForCurrentDirectory.setFileCount(resultingFiles.size() + fileCountOfSubdirectories);
+
+        // create and add FileNodeViewModel to the FileNodeViewModel for the current directory
+        fileNodeViewModelForCurrentDirectory.getChildren().addAll(resultingFiles.stream()
                 .map(FileNodeViewModel::new)
                 .collect(Collectors.toList()));
-        return parent;
+
+        return fileNodeViewModelForCurrentDirectory;
     }
 }

src/main/java/org/jabref/gui/externalfiles/UnlinkedPDFFileFilter.java

@@ -32,7 +32,6 @@ public UnlinkedPDFFileFilter(DirectoryStream.Filter<Path> fileFilter, BibDatabas
 
     @Override
     public boolean accept(Path pathname) throws IOException {
-
         if (Files.isDirectory(pathname)) {
             return true;
         } else {

src/main/java/org/jabref/gui/util/FileNodeViewModel.java

@@ -7,6 +7,7 @@
 import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
+import java.util.Objects;
 
 import javafx.beans.property.ReadOnlyListWrapper;
 import javafx.collections.FXCollections;
@@ -94,4 +95,21 @@ public String toString() {
                 this.children,
                 this.fileCount);
     }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(children, fileCount, path);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof FileNodeViewModel)) {
+            return false;
+        }
+        FileNodeViewModel other = (FileNodeViewModel) obj;
+        return Objects.equals(children, other.children) && (fileCount == other.fileCount) && Objects.equals(path, other.path);
+    }
 }