Added support for voidtools everything DB

.gitattributes

@@ -10,3 +10,4 @@ tests/_data/plugins/os/windows/notifications/wpndatabase.db filter=lfs diff=lfs
 tests/_data/volumes/bde/enc-volume.bin filter=lfs diff=lfs merge=lfs -text
 tests/_data/volumes/md/md-nested.bin.gz filter=lfs diff=lfs merge=lfs -text
 tests/_data/loaders/tar/test-anon-filesystems.tar filter=lfs diff=lfs merge=lfs -text
+tests/_data/plugins/os/windows/everything/*.db filter=lfs diff=lfs merge=lfs -text

dissect/target/plugins/os/windows/everything.py

@@ -0,0 +1,101 @@
+from typing import Iterator, Union
+
+from dissect.target import Target
+from dissect.target.exceptions import UnsupportedPluginError
+from dissect.target.helpers.locate.everything import (
+    EverythingDirectory,
+    EverythingFile,
+    EverythingDBParser,
+)
+from dissect.target.helpers.record import TargetRecordDescriptor
+from dissect.target.plugin import export
+from dissect.target.plugin import Plugin
+
+EverythingDirectoryRecord = TargetRecordDescriptor(
+    "windows/everything/everything_directory",
+    [
+        ("string", "path"),
+        ("filesize", "size"),
+        ("datetime", "date_created"),
+        ("datetime", "date_modified"),
+        ("datetime", "date_accessed"),
+        ("uint32", "attributes"),
+        ("string", "source"),
+    ],
+)
+
+EverythingFileRecord = TargetRecordDescriptor(
+    "windows/everything/everything_file",
+    [
+        ("string", "path"),
+        ("filesize", "size"),
+        ("datetime", "date_created"),
+        ("datetime", "date_modified"),
+        ("datetime", "date_accessed"),
+        ("uint32", "attributes"),
+        ("string", "source"),
+    ],
+)
+
+EverythingRecord = Union[
+    EverythingFileRecord,
+    EverythingDirectoryRecord,
+]
+
+
+class EverythingPlugin(Plugin):
+    __namespace__ = "everything"
+
+    PATH_GLOBS = [
+        "C:\\Program Files\\Everything\\Everything*.db",
+        "C:\\Program Files (x86)\\Everything\\Everything*.db",
+    ]
+    USER_PATH = "AppData\\Local\\Everything\\Everything*.db"
+
+    def __init__(self, target: Target):
+        super().__init__(target)
+        self.configs = []
+
+    def find_user_files(self):
+        for user_details in self.target.user_details.all_with_home():
+            for db in user_details.home_path.glob(self.USER_PATH):
+                if db.exists():
+                    yield db
+
+    def check_compatible(self) -> None:
+        for path_option in self.PATH_GLOBS:
+            for path in self.target.fs.path().glob(path_option):
+                if path.exists():
+                    self.configs.append(path)
+
+        for path in self.find_user_files():
+            self.configs.append(path)
+
+        if not self.configs:
+            raise UnsupportedPluginError("No everything.db files found")
+
+    @export(record=EverythingRecord)
+    def locate(self) -> Iterator[EverythingRecord]:
+        """Yield file and directory names from everything.db file.
+        """
+        for path in self.configs:
+            everything_fh = self.target.fs.path(path).open()
+            everything_file = EverythingDBParser(everything_fh)
+
+            for item in everything_file:
+                if isinstance(item, EverythingDirectory):
+                    typ = EverythingDirectoryRecord
+                elif isinstance(item, EverythingFile):
+                    typ = EverythingFileRecord
+                else:
+                    raise NotImplementedError(f"type {type(item)} is not Recordable")
+                yield typ(
+                    path=item.file_path,
+                    size=item.size,
+                    date_created=item.date_created,
+                    date_modified=item.date_modified,
+                    date_accessed=item.date_accessed,
+                    attributes=item.attributes,
+                    source=path,
+                    _target=self.target
+                )

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dissect.target"
-description = "This module ties all other Dissect modules together, it provides a programming API and command line tools which allow easy access to various data sources inside disk images or file collections (a.k.a. targets)"
+1description = "This module ties all other Dissect modules together, it provides a programming API and command line tools which allow easy access to various data sources inside disk images or file collections (a.k.a. targets)"
 readme = "README.md"
 requires-python = "~=3.9"
 license.text = "Affero General Public License v3"

tests/plugins/os/windows/test_everything.py

@@ -0,0 +1,154 @@
+import pytest
+from datetime import datetime, timezone
+
+from dissect.target.filesystem import VirtualFilesystem
+from dissect.target.plugins.os.windows.everything import (
+    EverythingDirectoryRecord,
+    EverythingFileRecord,
+    EverythingPlugin,
+)
+from dissect.target.target import Target
+from tests._utils import absolute_path
+
+
+def test_everything_ntfs_only(target_win: Target, fs_win: VirtualFilesystem) -> None:
+    fs_win.map_file("\\Program Files\\Everything\\Everything.db",
+                    absolute_path("_data/plugins/os/windows/everything/Everything_NTFS_ONLY.db"))
+    target_win.add_plugin(EverythingPlugin)
+
+    records = list(target_win.everything.locate())
+    assert len(records) == 126828
+
+    recycle_bin = records[0]
+    assert isinstance(recycle_bin, type(EverythingDirectoryRecord()))
+    assert recycle_bin.path == "C:\\$Recycle.Bin"
+    assert recycle_bin.date_modified == datetime(2023, 12, 16, 9, 8, 42, 726189, tzinfo=timezone.utc)
+
+    entry = records[-40000]
+    assert isinstance(entry, type(EverythingFileRecord()))
+    assert entry.path == "C:\\Windows\\System32\\msidntld.dll"
+
+    assert entry.date_modified == datetime(2021, 5, 8, 8, 14, 34, 642525, tzinfo=timezone.utc)
+
+
+def test_everything_ntfs_and_folder(target_win: Target, fs_win: VirtualFilesystem) -> None:
+    fs_win.map_file("\\Program Files\\Everything\\Everything.db",
+                    absolute_path("_data/plugins/os/windows/everything/Everything_NTFS_AND_FOLDER.db"))
+    target_win.add_plugin(EverythingPlugin)
+
+    records = list(target_win.everything.locate())
+    assert len(records) == 127042
+    recycle_bin_c = records[0]
+    assert isinstance(recycle_bin_c, type(EverythingDirectoryRecord()))
+    assert recycle_bin_c.path == "C:\\$Recycle.Bin"
+    assert recycle_bin_c.date_modified == datetime(2023, 12, 16, 9, 8, 42, 726189, tzinfo=timezone.utc)
+
+    recycle_bin_e = records[1]
+    assert isinstance(recycle_bin_e, type(EverythingDirectoryRecord()))
+    assert recycle_bin_e.path == "E:\\$RECYCLE.BIN"
+    assert recycle_bin_e.date_modified == datetime(2024, 1, 27, 9, 54, 32, tzinfo=timezone.utc)
+
+    entry = records[94783]
+    assert isinstance(entry, type(EverythingFileRecord()))
+    assert entry.path == "E:\\potato.txt"
+    assert entry.date_modified == datetime(2024, 1, 27, 9, 54, 32, tzinfo=timezone.utc)
+
+
+def test_everything_ntfs_and_refs(target_win: Target, fs_win: VirtualFilesystem) -> None:
+    fs_win.map_file("\\Program Files\\Everything\\Everything.db",
+                    absolute_path("_data/plugins/os/windows/everything/Everything_NTFS_AND_REFS.db"))
+    target_win.add_plugin(EverythingPlugin)
+
+    records = list(target_win.everything.locate())
+    assert len(records) == 127227
+    recycle_bin_c = records[0]
+    assert isinstance(recycle_bin_c, type(EverythingDirectoryRecord()))
+    assert recycle_bin_c.path == "C:\\$Recycle.Bin"
+    assert recycle_bin_c.date_modified == datetime(2023, 12, 16, 9, 8, 42, 726189, tzinfo=timezone.utc)
+
+    recycle_bin_e = records[1]
+    assert isinstance(recycle_bin_e, type(EverythingDirectoryRecord()))
+    assert recycle_bin_e.path == "E:\\$RECYCLE.BIN"
+    assert recycle_bin_e.date_modified == datetime(2024, 1, 27, 21, 37, 6, 675776, tzinfo=timezone.utc)
+
+    entry = records[94937]
+    assert isinstance(entry, type(EverythingFileRecord()))
+    assert entry.path == "E:\\potato.txt"
+    assert entry.date_modified == datetime(2024, 1, 27, 21, 37, 13, 425810, tzinfo=timezone.utc)
+
+    entry = records[108011]
+    assert isinstance(entry, type(EverythingFileRecord()))
+    assert entry.path == "E:\\test.txt"
+    assert entry.date_modified == datetime(2024, 1, 27, 21, 56, 57, 597593, tzinfo=timezone.utc)
+
+
+def test_everything_refs_with_disabled_ntfs(target_win: Target, fs_win: VirtualFilesystem) -> None:
+    # Just making sure that disabling a drive doesn't mess up parsing
+    fs_win.map_file("\\Program Files\\Everything\\Everything.db",
+                    absolute_path("_data/plugins/os/windows/everything/Everything_REFS_WITH_DISABLED_NTFS.db"))
+    target_win.add_plugin(EverythingPlugin)
+
+    records = list(target_win.everything.locate())
+    assert len(records) == 7
+    recycle_bin_e = records[0]
+    assert isinstance(recycle_bin_e, type(EverythingDirectoryRecord()))
+    assert recycle_bin_e.path == "E:\\$RECYCLE.BIN"
+    assert recycle_bin_e.date_modified == datetime(2024, 1, 27, 21, 37, 6, 675776, tzinfo=timezone.utc)
+
+    entry = records[5]
+    assert isinstance(entry, type(EverythingFileRecord()))
+    assert entry.path == "E:\\potato.txt"
+    assert entry.date_modified == datetime(2024, 1, 27, 21, 37, 13, 425810, tzinfo=timezone.utc)
+
+
+def test_everything_refs_include_only(target_win: Target, fs_win: VirtualFilesystem) -> None:
+    # Just making sure that including specific subfolders doesn't mess up parsing
+    fs_win.map_file("\\Program Files\\Everything\\Everything.db",
+                    absolute_path("_data/plugins/os/windows/everything/Everything_REFS_INCLUDE_ONLY.db"))
+    target_win.add_plugin(EverythingPlugin)
+
+    records = list(target_win.everything.locate())
+    assert len(records) == 4
+    recycle_bin_e = records[0]
+    assert isinstance(recycle_bin_e, type(EverythingDirectoryRecord()))
+    assert recycle_bin_e.path == "E:\\test"
+    assert recycle_bin_e.date_modified == datetime(2024, 1, 27, 22, 6, 9, 597820, tzinfo=timezone.utc)
+
+    entry = records[3]
+    assert isinstance(entry, type(EverythingFileRecord()))
+    assert entry.path == "E:\\test\\test.txt"
+    assert entry.date_modified == datetime(2024, 1, 27, 21, 56, 57, 597593, tzinfo=timezone.utc)
+
+
+def test_everything_efu(target_win: Target, fs_win: VirtualFilesystem) -> None:
+    fs_win.map_file("\\Program Files\\Everything\\Everything.db",
+                    absolute_path("_data/plugins/os/windows/everything/Everything_FILE_LIST.db"))
+    target_win.add_plugin(EverythingPlugin)
+
+    records = list(target_win.everything.locate())
+    assert len(records) == 5
+    entry = records[0]
+    assert isinstance(entry, type(EverythingDirectoryRecord()))
+    assert entry.path == "E:"
+    assert entry.date_modified is None
+
+    entry = records[1]
+    assert isinstance(entry, type(EverythingDirectoryRecord()))
+    assert entry.path == "E:\\test"
+    assert entry.date_modified == datetime(2024, 1, 27, 22, 6, 9, 597820, tzinfo=timezone.utc)
+
+
+@pytest.mark.parametrize(
+    "map_path",
+    [
+        "\\Program Files\\Everything\\Everything.db",
+        "\\Program Files\\Everything\\Everything.COMPNAME.USERNAME.db",
+        "\\Program Files (x86)\\Everything\\Everything.db",
+        "\\Users\\John\\AppData\\Local\\Everything\\Everything.db",
+        "\\Users\\John\\AppData\\Local\\Everything\\Everything.COMPNAME.John.db",
+    ],
+)
+def test_everything_path_mapper(target_win_users: Target, fs_win: VirtualFilesystem, map_path: str) -> None:
+    fs_win.map_file(map_path,
+                    absolute_path("_data/plugins/os/windows/everything/Everything_NTFS_ONLY.db"))
+    assert EverythingPlugin(target_win_users).check_compatible() is None