Find Unlinked files should ignore Thumbs.db, etc

.gitignore

@@ -418,7 +418,6 @@ Thumbs.db
 Thumbs.db:encryptable
 ehthumbs.db
 ehthumbs_vista.db
-
 # Dump file
 *.stackdump
 

CHANGELOG.md

@@ -13,6 +13,7 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve
 
 - We added a fetcher for [Biodiversity Heritage Library)](https://www.biodiversitylibrary.org/) [8539](https://github.com/JabRef/jabref/issues/8539)
 - We added support for multiple messages in the snackbar. [#7340](https://github.com/JabRef/jabref/issues/7340)
+- We added an extra option in the 'Find Unlinked Files' dialog view to ignore unnecessary files like Thumbs.db, DS_Store, etc. [koppor#373](https://github.com/koppor/jabref/issues/373)
 - JabRef now writes log files. Linux: `$home/.cache/jabref/logs/version`, Windows: `%APPDATA%\..\Local\harawata\jabref\version\logs`, Mac: `Users/.../Library/Logs/jabref/version`
 
 ### Changed

src/main/java/org/jabref/gui/externalfiles/FileFilterUtils.java

@@ -8,15 +8,19 @@
 import java.time.ZoneId;
 import java.util.Comparator;
 import java.util.List;
+import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.jabref.logic.util.io.FileUtil;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class FileFilterUtils {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(FileFilterUtils.class);
-
+    private static Set<String> FILEIGNORESET = null;
+
     /* Returns the last edited time of a file as LocalDateTime. */
     public static LocalDateTime getFileTime(Path path) {
         FileTime lastEditedTime = null;
@@ -33,28 +37,28 @@ public static LocalDateTime getFileTime(Path path) {
         return localDateTime;
     }
 
-    /* Returns true if a file with a specific path 
+    /* Returns true if a file with a specific path
      * was edited during the last 24 hours. */
     public boolean isDuringLastDay(LocalDateTime fileEditTime) {
         LocalDateTime NOW = LocalDateTime.now(ZoneId.systemDefault());
         return fileEditTime.isAfter(NOW.minusHours(24));
     }
 
-    /* Returns true if a file with a specific path 
+    /* Returns true if a file with a specific path
      * was edited during the last 7 days. */
     public boolean isDuringLastWeek(LocalDateTime fileEditTime) {
         LocalDateTime NOW = LocalDateTime.now(ZoneId.systemDefault());
         return fileEditTime.isAfter(NOW.minusDays(7));
     }
 
-    /* Returns true if a file with a specific path 
+    /* Returns true if a file with a specific path
      * was edited during the last 30 days. */
     public boolean isDuringLastMonth(LocalDateTime fileEditTime) {
         LocalDateTime NOW = LocalDateTime.now(ZoneId.systemDefault());
         return fileEditTime.isAfter(NOW.minusDays(30));
     }
 
-    /* Returns true if a file with a specific path 
+    /* Returns true if a file with a specific path
      * was edited during the last 365 days. */
     public boolean isDuringLastYear(LocalDateTime fileEditTime) {
         LocalDateTime NOW = LocalDateTime.now(ZoneId.systemDefault());
@@ -75,7 +79,24 @@ public static boolean filterByDate(Path path, DateRange filter) {
         return isInDateRange;
     }
 
-    /* Sorts a list of Path objects according to the last edited date 
+    /* Returns true if a file should be ignored by .gitignore */
+    public static boolean filterForUnlinkedFiles(Path path, FileIgnoreUnlinkedFiles unlinkedFileFilter) {
+        if (unlinkedFileFilter.equals(FileIgnoreUnlinkedFiles.DEFAULT)) {
+            try {
+                FILEIGNORESET = UnlinkedFilesDialogViewModel.getIgnoreFileSet();
+                String fileExtension = FileUtil.getFileExtension(path).orElse("");
+                if (!fileExtension.equals("") && FILEIGNORESET.contains(fileExtension)) {
+                    return false;
+                }
+            } catch (Exception e) {
+                LOGGER.info("Not file");
+            }
+            return !FILEIGNORESET.contains(path.getFileName().toString());
+        }
+        return true;
+    }
+
+    /* Sorts a list of Path objects according to the last edited date
      * of their corresponding files, from newest to oldest. */
     public List<Path> sortByDateAscending(List<Path> files) {
         return files.stream()
@@ -86,7 +107,7 @@ public List<Path> sortByDateAscending(List<Path> files) {
                 .collect(Collectors.toList());
     }
 
-    /* Sorts a list of Path objects according to the last edited date 
+    /* Sorts a list of Path objects according to the last edited date
      * of their corresponding files, from oldest to newest. */
     public List<Path> sortByDateDescending(List<Path> files) {
         return files.stream()

src/main/java/org/jabref/gui/externalfiles/FileIgnoreUnlinkedFiles.java

@@ -0,0 +1,18 @@
+package org.jabref.gui.externalfiles;
+
+import org.jabref.logic.l10n.Localization;
+
+public enum FileIgnoreUnlinkedFiles {
+    DEFAULT(Localization.lang("Default")),
+    INCLUDE_ALL(Localization.lang("Include All Files"));
+
+    private final String fileIgnoreOption;
+
+    FileIgnoreUnlinkedFiles(String fileIgnoreOption) {
+        this.fileIgnoreOption = fileIgnoreOption;
+    }
+
+    public String getFileIgnoreOption() {
+        return fileIgnoreOption;
+    }
+}

src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesCrawler.java

@@ -36,14 +36,16 @@ public class UnlinkedFilesCrawler extends BackgroundTask<FileNodeViewModel> {
     private final ExternalFileSorter sorter;
     private final BibDatabaseContext databaseContext;
     private final FilePreferences filePreferences;
+    private final FileIgnoreUnlinkedFiles unlinkedFileFilter;
 
-    public UnlinkedFilesCrawler(Path directory, Filter<Path> fileFilter, DateRange dateFilter, ExternalFileSorter sorter, BibDatabaseContext databaseContext, FilePreferences filePreferences) {
+    public UnlinkedFilesCrawler(Path directory, Filter<Path> fileFilter, DateRange dateFilter, ExternalFileSorter sorter, BibDatabaseContext databaseContext, FilePreferences filePreferences, FileIgnoreUnlinkedFiles unlinkedFileFilter) {
         this.directory = directory;
         this.fileFilter = fileFilter;
         this.dateFilter = dateFilter;
         this.sorter = sorter;
         this.databaseContext = databaseContext;
         this.filePreferences = filePreferences;
+        this.unlinkedFileFilter = unlinkedFileFilter;
     }
 
     @Override
@@ -66,7 +68,7 @@ protected FileNodeViewModel call() throws IOException {
      * 'state' must be set to 1, to keep the recursion running. When the states value changes, the method will resolve
      * its recursion and return what it has saved so far.
      * <br>
-     * The files are filtered according to the {@link DateRange} filter value 
+     * The files are filtered according to the {@link DateRange} filter value & the {@link FileIgnoreUnlinkedFiles} filter value
      * and then sorted according to the {@link ExternalFileSorter} value.
      *
      * @throws IOException if directory is not a directory or empty
@@ -102,7 +104,7 @@ private FileNodeViewModel searchDirectory(Path directory, UnlinkedPDFFileFilter
         // filter files according to last edited date.
         List<Path> filteredFiles = new ArrayList<Path>();
         for (Path path : files) {
-            if (FileFilterUtils.filterByDate(path, dateFilter)) {
+            if (FileFilterUtils.filterByDate(path, dateFilter) && FileFilterUtils.filterForUnlinkedFiles(path, unlinkedFileFilter)) {
                 filteredFiles.add(path);
             }
         }

src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialog.fxml

@@ -52,12 +52,14 @@
 
                 <Label text="%File type" GridPane.columnIndex="0" GridPane.rowIndex="1"/>
                 <ComboBox fx:id="fileTypeCombo" maxWidth="Infinity" GridPane.columnIndex="1" GridPane.rowIndex="1"/>
-                <Label text="%Last edited:" GridPane.columnIndex="0" GridPane.rowIndex="2"/>
-                <ComboBox fx:id="fileDateCombo" maxWidth="Infinity" GridPane.columnIndex="1" GridPane.rowIndex="2"/>
-                <Label text="%Sort by:" GridPane.columnIndex="0" GridPane.rowIndex="3"/>
-                <ComboBox fx:id="fileSortCombo" maxWidth="Infinity" GridPane.columnIndex="1" GridPane.rowIndex="3"/>
+                <Label text="%Ignore File Extensions" GridPane.columnIndex="0" GridPane.rowIndex="2"/>
+                <ComboBox fx:id="unlinkedFileCombo" maxWidth="Infinity" GridPane.columnIndex="1" GridPane.rowIndex="2"/>
+                <Label text="%Last edited:" GridPane.columnIndex="0" GridPane.rowIndex="3"/>
+                <ComboBox fx:id="fileDateCombo" maxWidth="Infinity" GridPane.columnIndex="1" GridPane.rowIndex="3"/>
+                <Label text="%Sort by:" GridPane.columnIndex="0" GridPane.rowIndex="4"/>
+                <ComboBox fx:id="fileSortCombo" maxWidth="Infinity" GridPane.columnIndex="1" GridPane.rowIndex="4"/>
                 <Button fx:id="scanButton" onAction="#scanFiles" text="%Search"
-                        GridPane.columnIndex="2" GridPane.rowIndex="3">
+                        GridPane.columnIndex="2" GridPane.rowIndex="4">
                     <tooltip>
                         <Tooltip text="%Searches the selected directory for unlinked files."/>
                     </tooltip>

src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialogView.java

@@ -55,6 +55,7 @@ public class UnlinkedFilesDialogView extends BaseDialog<Void> {
 
     @FXML private TextField directoryPathField;
     @FXML private ComboBox<FileExtensionViewModel> fileTypeCombo;
+    @FXML private ComboBox<FileIgnoreUnlinkedFiles> unlinkedFileCombo;
     @FXML private ComboBox<DateRange> fileDateCombo;
     @FXML private ComboBox<ExternalFileSorter> fileSortCombo;
     @FXML private CheckTreeView<FileNodeViewModel> unlinkedFilesList;
@@ -159,6 +160,12 @@ private void initDirectorySelection() {
         fileSortCombo.setItems(viewModel.getSorters());
         fileSortCombo.valueProperty().bindBidirectional(viewModel.selectedSortProperty());
         fileSortCombo.getSelectionModel().selectFirst();
+        new ViewModelListCellFactory<FileIgnoreUnlinkedFiles>()
+                .withText(FileIgnoreUnlinkedFiles::getFileIgnoreOption)
+                .install(unlinkedFileCombo);
+        unlinkedFileCombo.setItems(viewModel.unlinkedFileFilter());
+        unlinkedFileCombo.valueProperty().bindBidirectional(viewModel.selectedFileIgnoreProperty());
+        unlinkedFileCombo.getSelectionModel().selectFirst();
     }
 
     private void initUnlinkedFilesList() {

src/main/java/org/jabref/gui/externalfiles/UnlinkedFilesDialogViewModel.java

@@ -1,14 +1,20 @@
 package org.jabref.gui.externalfiles;
 
+import java.io.BufferedReader;
 import java.io.BufferedWriter;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.DirectoryStream.Filter;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Optional;
+import java.util.Set;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
@@ -55,6 +61,7 @@ public class UnlinkedFilesDialogViewModel {
     private final ImportHandler importHandler;
     private final StringProperty directoryPath = new SimpleStringProperty("");
     private final ObjectProperty<FileExtensionViewModel> selectedExtension = new SimpleObjectProperty<>();
+    private final ObjectProperty<FileIgnoreUnlinkedFiles> selectedFileIgnore = new SimpleObjectProperty<>();
     private final ObjectProperty<DateRange> selectedDate = new SimpleObjectProperty<>();
     private final ObjectProperty<ExternalFileSorter> selectedSort = new SimpleObjectProperty<>();
 
@@ -69,6 +76,7 @@ public class UnlinkedFilesDialogViewModel {
     private final ObservableList<FileExtensionViewModel> fileFilterList;
     private final ObservableList<DateRange> dateFilterList;
     private final ObservableList<ExternalFileSorter> fileSortList;
+    private final ObservableList<FileIgnoreUnlinkedFiles> unlinkedFileFilter;
 
     private final DialogService dialogService;
     private final PreferencesService preferences;
@@ -105,6 +113,8 @@ public UnlinkedFilesDialogViewModel(DialogService dialogService,
                 new FileExtensionViewModel(StandardFileType.BIBTEX_DB, externalFileTypes),
                 new FileExtensionViewModel(StandardFileType.PDF, externalFileTypes));
 
+        this.unlinkedFileFilter = FXCollections.observableArrayList(FileIgnoreUnlinkedFiles.values());
+
         this.dateFilterList = FXCollections.observableArrayList(DateRange.values());
 
         this.fileSortList = FXCollections.observableArrayList(ExternalFileSorter.values());
@@ -119,12 +129,13 @@ public UnlinkedFilesDialogViewModel(DialogService dialogService,
     public void startSearch() {
         Path directory = this.getSearchDirectory();
         Filter<Path> selectedFileFilter = selectedExtension.getValue().dirFilter();
+        FileIgnoreUnlinkedFiles selectedUnlinkedFileIgnoreFilter = selectedFileIgnore.getValue();
         DateRange selectedDateFilter = selectedDate.getValue();
         ExternalFileSorter selectedSortFilter = selectedSort.getValue();
         progressValueProperty.unbind();
         progressTextProperty.unbind();
 
-        findUnlinkedFilesTask = new UnlinkedFilesCrawler(directory, selectedFileFilter, selectedDateFilter, selectedSortFilter, bibDatabase, preferences.getFilePreferences())
+        findUnlinkedFilesTask = new UnlinkedFilesCrawler(directory, selectedFileFilter, selectedDateFilter, selectedSortFilter, bibDatabase, preferences.getFilePreferences(), selectedUnlinkedFileIgnoreFilter)
                 .onRunning(() -> {
                     progressValueProperty.set(ProgressIndicator.INDETERMINATE_PROGRESS);
                     progressTextProperty.setValue(Localization.lang("Searching file system..."));
@@ -205,6 +216,10 @@ public ObservableList<FileExtensionViewModel> getFileFilters() {
         return this.fileFilterList;
     }
 
+    public ObservableList<FileIgnoreUnlinkedFiles> unlinkedFileFilter() {
+        return this.unlinkedFileFilter;
+    }
+
     public ObservableList<DateRange> getDateFilters() {
         return this.dateFilterList;
     }
@@ -266,6 +281,10 @@ public ObjectProperty<ExternalFileSorter> selectedSortProperty() {
         return this.selectedSort;
     }
 
+    public ObjectProperty<FileIgnoreUnlinkedFiles> selectedFileIgnoreProperty() {
+        return this.selectedFileIgnore;
+    }
+
     public StringProperty directoryPathProperty() {
         return this.directoryPath;
     }
@@ -289,4 +308,25 @@ public BooleanProperty taskActiveProperty() {
     public SimpleListProperty<TreeItem<FileNodeViewModel>> checkedFileListProperty() {
         return checkedFileListProperty;
     }
+
+    public static Set<String> getIgnoreFileSet() {
+        String gitIgnorePath = ".gitignore";
+        Set<String> IgnoreFileSet = new HashSet<>();
+        try (BufferedReader br = new BufferedReader(new InputStreamReader(
+                new FileInputStream(gitIgnorePath)
+        ))) {
+            String line = br.readLine();
+            while (line != null) {
+                line = br.readLine();
+                if (line != null && line.length() > 2 && line.charAt(0) == '*') {
+                    IgnoreFileSet.add(line.substring(2).toLowerCase(Locale.ROOT));
+                } else if (line != null && line.length() > 1 && !line.contains("/")) {
+                    IgnoreFileSet.add(line);
+                }
+            }
+        } catch (IOException e) {
+            LOGGER.error("No such file.");
+        }
+        return IgnoreFileSet;
+    }
 }

src/main/resources/l10n/JabRef_en.properties

@@ -2487,3 +2487,5 @@ Success\!\ Finished\ writing\ metadata.=Success! Finished writing metadata.
 
 Custom\ API\ key=Custom API key
 Check\ %0\ API\ Key\ Setting=Check %0 API Key Setting
+Ignore\ File\ Extensions=Ignore File Extensions
+Include\ All\ Files=Include All Files